Commit f7f3419b authored by Vasyl Bodnaruk's avatar Vasyl Bodnaruk

Change mixin to base spider class

parent 642b019e
from .company_maker import CompanyMaker
from .db import Database
from .mixins import CommonMixin
# -*- coding: utf-8 -*-
import scrapy
import dateparser
from ..helpers import CommonMixin
from .base import BaseSpider
from ..items import ExaItem
class AitopSpider(CommonMixin, scrapy.Spider):
class AitopSpider(BaseSpider):
name = "aitop"
allowed_domains = ["aitopics.org"]
......
import scrapy
from scrapy.utils.project import get_project_settings
from ..helpers import CompanyMaker, Database
......@@ -5,9 +6,10 @@ db_settings = get_project_settings().get('DB')
db = Database(**db_settings)
class CommonMixin:
def __init__(self, *args, **kwargs):
self.condition = kwargs.get('query')
class BaseSpider(scrapy.Spider):
def __init__(self, query=None, *args, **kwargs):
super(BaseSpider, self).__init__(*args, **kwargs)
self.condition = query
self.query = "SELECT id, name, country FROM wp_esi_entity WHERE 1 and id=13"
if self.condition:
print(self.condition)
......
......@@ -2,11 +2,11 @@
import dateparser
import scrapy
import traceback
from ..helpers import CommonMixin
from .base import BaseSpider
from ..items import ExaItem
class MobiHealthNewsSpider(CommonMixin, scrapy.Spider):
class MobiHealthNewsSpider(BaseSpider):
name = "mhn"
allowed_domains = ["www.mobihealthnews.com"]
......
# -*- coding: utf-8 -*-
import scrapy
import traceback
from ..helpers import CommonMixin
from .base import BaseSpider
from ..items import ExaItem
class TechcrunchSpider(CommonMixin, scrapy.Spider):
class TechcrunchSpider(BaseSpider):
name = "tc"
allowed_domains = ["techcrunch.com"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment