Commit 80fff8c6 authored by Vasyl Bodnaruk's avatar Vasyl Bodnaruk

Add functional for use Company structure

parent 9e8d051b
...@@ -12,7 +12,6 @@ db = Database(**db_settings) ...@@ -12,7 +12,6 @@ db = Database(**db_settings)
class AitopSpider(scrapy.Spider): class AitopSpider(scrapy.Spider):
name = "aitop" name = "aitop"
allowed_domains = ["aitopics.org"] allowed_domains = ["aitopics.org"]
start_urls = ['https://aitopics.org/search?view=&filters=&sort=score+desc&q=x.ai']
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.condition = kwargs.get('query') self.condition = kwargs.get('query')
...@@ -32,7 +31,9 @@ class AitopSpider(scrapy.Spider): ...@@ -32,7 +31,9 @@ class AitopSpider(scrapy.Spider):
pass pass
def parse(self, response): def parse(self, response):
name = 'Uber'
company = response.meta['company']
name = company.name
url = 'https://aitopics.org' url = 'https://aitopics.org'
def tags(res): def tags(res):
...@@ -45,12 +46,13 @@ class AitopSpider(scrapy.Spider): ...@@ -45,12 +46,13 @@ class AitopSpider(scrapy.Spider):
url_for_parse_tag = tags(response) url_for_parse_tag = tags(response)
if url_for_parse_tag: if url_for_parse_tag:
return scrapy.Request(url_for_parse_tag, callback=self.parse_by_tag) return scrapy.Request(url_for_parse_tag, callback=self.parse_by_tag, meta=response.meta)
else: else:
return scrapy.Request(response.url, callback=self.parse_by_title_description) return scrapy.Request(response.url, callback=self.parse_by_title_description, meta=response.meta)
def parse_by_title_description(self, response): def parse_by_title_description(self, response):
name = 'x.ai' company = response.meta['company']
name = company.name
def is_company_in_item(item): def is_company_in_item(item):
if name.lower() in item['title'].lower(): if name.lower() in item['title'].lower():
...@@ -65,7 +67,7 @@ class AitopSpider(scrapy.Spider): ...@@ -65,7 +67,7 @@ class AitopSpider(scrapy.Spider):
yield i yield i
next_url = self.next_url(response) next_url = self.next_url(response)
if next_url: if next_url:
yield scrapy.Request(next_url, callback=self.parse_by_title_description) yield scrapy.Request(next_url, callback=self.parse_by_title_description, meta=response.meta)
def parse_by_tag(self, response): def parse_by_tag(self, response):
try: try:
...@@ -73,7 +75,7 @@ class AitopSpider(scrapy.Spider): ...@@ -73,7 +75,7 @@ class AitopSpider(scrapy.Spider):
yield i yield i
next_url = self.next_url(response) next_url = self.next_url(response)
if next_url: if next_url:
yield scrapy.Request(next_url, callback=self.parse_by_tag) yield scrapy.Request(next_url, callback=self.parse_by_tag, meta=response.meta)
except: except:
pass pass
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment