Commit f7f3419b authored by Vasyl Bodnaruk's avatar Vasyl Bodnaruk

Change mixin to base spider class

parent 642b019e
from .company_maker import CompanyMaker from .company_maker import CompanyMaker
from .db import Database from .db import Database
from .mixins import CommonMixin
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import scrapy import scrapy
import dateparser import dateparser
from ..helpers import CommonMixin from .base import BaseSpider
from ..items import ExaItem from ..items import ExaItem
class AitopSpider(CommonMixin, scrapy.Spider): class AitopSpider(BaseSpider):
name = "aitop" name = "aitop"
allowed_domains = ["aitopics.org"] allowed_domains = ["aitopics.org"]
......
import scrapy
from scrapy.utils.project import get_project_settings from scrapy.utils.project import get_project_settings
from ..helpers import CompanyMaker, Database from ..helpers import CompanyMaker, Database
...@@ -5,9 +6,10 @@ db_settings = get_project_settings().get('DB') ...@@ -5,9 +6,10 @@ db_settings = get_project_settings().get('DB')
db = Database(**db_settings) db = Database(**db_settings)
class CommonMixin: class BaseSpider(scrapy.Spider):
def __init__(self, *args, **kwargs): def __init__(self, query=None, *args, **kwargs):
self.condition = kwargs.get('query') super(BaseSpider, self).__init__(*args, **kwargs)
self.condition = query
self.query = "SELECT id, name, country FROM wp_esi_entity WHERE 1 and id=13" self.query = "SELECT id, name, country FROM wp_esi_entity WHERE 1 and id=13"
if self.condition: if self.condition:
print(self.condition) print(self.condition)
......
...@@ -2,11 +2,11 @@ ...@@ -2,11 +2,11 @@
import dateparser import dateparser
import scrapy import scrapy
import traceback import traceback
from ..helpers import CommonMixin from .base import BaseSpider
from ..items import ExaItem from ..items import ExaItem
class MobiHealthNewsSpider(CommonMixin, scrapy.Spider): class MobiHealthNewsSpider(BaseSpider):
name = "mhn" name = "mhn"
allowed_domains = ["www.mobihealthnews.com"] allowed_domains = ["www.mobihealthnews.com"]
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import scrapy import scrapy
import traceback import traceback
from ..helpers import CommonMixin from .base import BaseSpider
from ..items import ExaItem from ..items import ExaItem
class TechcrunchSpider(CommonMixin, scrapy.Spider): class TechcrunchSpider(BaseSpider):
name = "tc" name = "tc"
allowed_domains = ["techcrunch.com"] allowed_domains = ["techcrunch.com"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment