Commit 16651415 authored by Vasyl Bodnaruk's avatar Vasyl Bodnaruk

Change response parameter

parent f7f3419b
...@@ -39,7 +39,7 @@ class MobiHealthNewsSpider(BaseSpider): ...@@ -39,7 +39,7 @@ class MobiHealthNewsSpider(BaseSpider):
"..//div[contains(@class, 'text-center')]/ul/li[contains(@class, 'next')]/a/@href").extract_first() "..//div[contains(@class, 'text-center')]/ul/li[contains(@class, 'next')]/a/@href").extract_first()
if has_next: if has_next:
yield scrapy.Request(next_url, callback=self.parse, meta={'company': response.meta['company'], 'post_id': 0}) yield scrapy.Request(next_url, callback=self.parse, meta=response.meta)
except BaseException as e: except BaseException as e:
print('We had error') print('We had error')
......
...@@ -44,7 +44,7 @@ class TechcrunchSpider(BaseSpider): ...@@ -44,7 +44,7 @@ class TechcrunchSpider(BaseSpider):
if has_next: if has_next:
next_url = 'https://techcrunch.com' + has_next + '/' next_url = 'https://techcrunch.com' + has_next + '/'
yield scrapy.Request(next_url, callback=self.parse_tag, yield scrapy.Request(next_url, callback=self.parse_tag,
meta={'company': response.meta['company'], 'post_id': 0}) meta=response.meta)
except BaseException as e: except BaseException as e:
print('We had error') print('We had error')
traceback.print_exc() traceback.print_exc()
...@@ -71,7 +71,7 @@ class TechcrunchSpider(BaseSpider): ...@@ -71,7 +71,7 @@ class TechcrunchSpider(BaseSpider):
"//div[contains(@class, 'river-nav')]//li[contains(@class, 'next')]/a/@href").extract_first() "//div[contains(@class, 'river-nav')]//li[contains(@class, 'next')]/a/@href").extract_first()
if has_next: if has_next:
yield scrapy.Request(has_next + '/', callback=self.parse_company, yield scrapy.Request(has_next + '/', callback=self.parse_company,
meta={'company': response.meta['company'], 'post_id': 0}) meta=response.meta)
except BaseException as e: except BaseException as e:
print('We had error') print('We had error')
traceback.print_exc() traceback.print_exc()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment