Add functional for use Company structure

exa/exa/spiders/aitop.py

@@ -12,7 +12,6 @@ db = Database(**db_settings)
 class AitopSpider(scrapy.Spider):
     name = "aitop"
     allowed_domains = ["aitopics.org"]
-    start_urls = ['https://aitopics.org/search?view=&filters=&sort=score+desc&q=x.ai']
 
     def __init__(self, *args, **kwargs):
         self.condition = kwargs.get('query')
@@ -32,7 +31,9 @@ class AitopSpider(scrapy.Spider):
                 pass
 
     def parse(self, response):
-        name = 'Uber'
+
+        company = response.meta['company']
+        name = company.name
         url = 'https://aitopics.org'
 
         def tags(res):
@@ -45,12 +46,13 @@ class AitopSpider(scrapy.Spider):
 
         url_for_parse_tag = tags(response)
         if url_for_parse_tag:
-            return scrapy.Request(url_for_parse_tag, callback=self.parse_by_tag)
+            return scrapy.Request(url_for_parse_tag, callback=self.parse_by_tag, meta=response.meta)
         else:
-            return scrapy.Request(response.url, callback=self.parse_by_title_description)
+            return scrapy.Request(response.url, callback=self.parse_by_title_description, meta=response.meta)
 
     def parse_by_title_description(self, response):
-        name = 'x.ai'
+        company = response.meta['company']
+        name = company.name
 
         def is_company_in_item(item):
             if name.lower() in item['title'].lower():
@@ -65,7 +67,7 @@ class AitopSpider(scrapy.Spider):
                 yield i
         next_url = self.next_url(response)
         if next_url:
-            yield scrapy.Request(next_url, callback=self.parse_by_title_description)
+            yield scrapy.Request(next_url, callback=self.parse_by_title_description, meta=response.meta)
 
     def parse_by_tag(self, response):
         try:
@@ -73,7 +75,7 @@ class AitopSpider(scrapy.Spider):
                 yield i
             next_url = self.next_url(response)
             if next_url:
-                yield scrapy.Request(next_url, callback=self.parse_by_tag)
+                yield scrapy.Request(next_url, callback=self.parse_by_tag, meta=response.meta)
         except:
             pass