Commit 10fa7994 authored by Vasyl Bodnaruk's avatar Vasyl Bodnaruk

Add function for extend item by common info

parent 80fff8c6
......@@ -89,6 +89,7 @@ class AitopSpider(scrapy.Spider):
item['title'] = ''.join(i.xpath(".//div[contains(@class, 'col-xs-12')]/h3/a//text()").extract())
item['description'] = ''.join(i.xpath(".//div[@class='summary-content']/p/text()").extract())
item['url'] = i.xpath(".//div[contains(@class, 'col-xs-12')]/h3/a/@href").extract_first()
item.update(self.get_common_items(response.meta['company']))
items.append(item)
return items
except:
......@@ -99,4 +100,8 @@ class AitopSpider(scrapy.Spider):
if has_next and has_next != '#':
return 'https://aitopics.org/search' + has_next
else:
return None
\ No newline at end of file
return None
def get_common_items(self, company):
return {'region_id': company.region_id, 'type_id': company.type_id,
'media_id': company.media_id, 'company_id': company.id}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment