Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
E
esi-table-data
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
esi-data-scrapping
esi-table-data
Commits
c3d696f4
Commit
c3d696f4
authored
7 years ago
by
Vasyl Bodnaruk
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add function for check exist company in news
parent
93293705
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
14 additions
and
1 deletion
+14
-1
aitop.py
exa/exa/spiders/aitop.py
+14
-1
No files found.
exa/exa/spiders/aitop.py
View file @
c3d696f4
...
...
@@ -28,6 +28,15 @@ class AitopSpider(scrapy.Spider):
return
scrapy
.
Request
(
response
.
url
,
callback
=
self
.
parse_by_title_description
)
def
parse_by_title_description
(
self
,
response
):
name
=
'x.ai'
def
is_company_in_item
(
item
):
if
name
.
lower
()
in
item
[
'title'
]
.
lower
():
return
True
elif
name
.
lower
()
in
item
[
'description'
]
.
lower
():
return
True
else
:
return
False
rows
=
response
.
xpath
(
".//div[contains(@class, 'summaries')]//div[@class='row']"
)
for
i
in
rows
:
item
=
dict
()
...
...
@@ -35,7 +44,11 @@ class AitopSpider(scrapy.Spider):
item
[
'title'
]
=
i
.
xpath
(
".//div[contains(@class, 'col-xs-12')]/h3/a/text()"
)
.
extract_first
()
item
[
'description'
]
=
i
.
xpath
(
".//div[@class='summary-content']/p/text()"
)
.
extract_first
()
item
[
'url'
]
=
i
.
xpath
(
".//div[contains(@class, 'col-xs-12')]/h3/a/@href"
)
.
extract_first
()
print
(
item
)
# print(item)
has_next
=
response
.
xpath
(
".//ul[@class='pagination']//li/a/@href"
)
.
extract
()[
-
2
]
if
has_next
and
has_next
!=
'#'
:
next_url
=
'https://aitopics.org/search'
+
has_next
return
scrapy
.
Request
(
next_url
,
callback
=
self
.
parse_by_tag
)
def
parse_by_tag
(
self
,
response
):
try
:
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment