Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
E
esi-table-data
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
esi-data-scrapping
esi-table-data
Commits
16651415
Commit
16651415
authored
Jul 17, 2017
by
Vasyl Bodnaruk
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Change response parameter
parent
f7f3419b
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
4 additions
and
4 deletions
+4
-4
mobihealthnews.py
exa/exa/spiders/mobihealthnews.py
+1
-1
techcrunch.py
exa/exa/spiders/techcrunch.py
+3
-3
No files found.
exa/exa/spiders/mobihealthnews.py
View file @
16651415
...
@@ -39,7 +39,7 @@ class MobiHealthNewsSpider(BaseSpider):
...
@@ -39,7 +39,7 @@ class MobiHealthNewsSpider(BaseSpider):
"..//div[contains(@class, 'text-center')]/ul/li[contains(@class, 'next')]/a/@href"
)
.
extract_first
()
"..//div[contains(@class, 'text-center')]/ul/li[contains(@class, 'next')]/a/@href"
)
.
extract_first
()
if
has_next
:
if
has_next
:
yield
scrapy
.
Request
(
next_url
,
callback
=
self
.
parse
,
meta
=
{
'company'
:
response
.
meta
[
'company'
],
'post_id'
:
0
}
)
yield
scrapy
.
Request
(
next_url
,
callback
=
self
.
parse
,
meta
=
response
.
meta
)
except
BaseException
as
e
:
except
BaseException
as
e
:
print
(
'We had error'
)
print
(
'We had error'
)
...
...
exa/exa/spiders/techcrunch.py
View file @
16651415
...
@@ -44,7 +44,7 @@ class TechcrunchSpider(BaseSpider):
...
@@ -44,7 +44,7 @@ class TechcrunchSpider(BaseSpider):
if
has_next
:
if
has_next
:
next_url
=
'https://techcrunch.com'
+
has_next
+
'/'
next_url
=
'https://techcrunch.com'
+
has_next
+
'/'
yield
scrapy
.
Request
(
next_url
,
callback
=
self
.
parse_tag
,
yield
scrapy
.
Request
(
next_url
,
callback
=
self
.
parse_tag
,
meta
=
{
'company'
:
response
.
meta
[
'company'
],
'post_id'
:
0
}
)
meta
=
response
.
meta
)
except
BaseException
as
e
:
except
BaseException
as
e
:
print
(
'We had error'
)
print
(
'We had error'
)
traceback
.
print_exc
()
traceback
.
print_exc
()
...
@@ -71,7 +71,7 @@ class TechcrunchSpider(BaseSpider):
...
@@ -71,7 +71,7 @@ class TechcrunchSpider(BaseSpider):
"//div[contains(@class, 'river-nav')]//li[contains(@class, 'next')]/a/@href"
)
.
extract_first
()
"//div[contains(@class, 'river-nav')]//li[contains(@class, 'next')]/a/@href"
)
.
extract_first
()
if
has_next
:
if
has_next
:
yield
scrapy
.
Request
(
has_next
+
'/'
,
callback
=
self
.
parse_company
,
yield
scrapy
.
Request
(
has_next
+
'/'
,
callback
=
self
.
parse_company
,
meta
=
{
'company'
:
response
.
meta
[
'company'
],
'post_id'
:
0
}
)
meta
=
response
.
meta
)
except
BaseException
as
e
:
except
BaseException
as
e
:
print
(
'We had error'
)
print
(
'We had error'
)
traceback
.
print_exc
()
traceback
.
print_exc
()
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment