Commit 10fa7994 authored by Vasyl Bodnaruk's avatar Vasyl Bodnaruk

Add function for extend item by common info

parent 80fff8c6
...@@ -89,6 +89,7 @@ class AitopSpider(scrapy.Spider): ...@@ -89,6 +89,7 @@ class AitopSpider(scrapy.Spider):
item['title'] = ''.join(i.xpath(".//div[contains(@class, 'col-xs-12')]/h3/a//text()").extract()) item['title'] = ''.join(i.xpath(".//div[contains(@class, 'col-xs-12')]/h3/a//text()").extract())
item['description'] = ''.join(i.xpath(".//div[@class='summary-content']/p/text()").extract()) item['description'] = ''.join(i.xpath(".//div[@class='summary-content']/p/text()").extract())
item['url'] = i.xpath(".//div[contains(@class, 'col-xs-12')]/h3/a/@href").extract_first() item['url'] = i.xpath(".//div[contains(@class, 'col-xs-12')]/h3/a/@href").extract_first()
item.update(self.get_common_items(response.meta['company']))
items.append(item) items.append(item)
return items return items
except: except:
...@@ -100,3 +101,7 @@ class AitopSpider(scrapy.Spider): ...@@ -100,3 +101,7 @@ class AitopSpider(scrapy.Spider):
return 'https://aitopics.org/search' + has_next return 'https://aitopics.org/search' + has_next
else: else:
return None return None
def get_common_items(self, company):
return {'region_id': company.region_id, 'type_id': company.type_id,
'media_id': company.media_id, 'company_id': company.id}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment