Clean parse function

exa/exa/spiders/aitop.py

@@ -63,11 +63,11 @@ class AitopSpider(BaseSpider):
             item['post_id'] = response.meta['post_id']
             item['tags'] = i.xpath(
                 ".//div[@class='row hidden-xs']//div[@title='Concept Tags']//a[@class='filter btn btn-link']/text()").extract()
-            yield item
-            if is_company_in_item(i):
+            if is_company_in_item(item):
                 yield i
+
         next_url = self.next_url(response)
-        if self.can_follow(next_url, is_duplicate):
+        if next_url:
             yield scrapy.Request(next_url, callback=self.parse_by_title_description, meta=response.meta)
         else:
             print("DUPLICATE NEWS")
@@ -93,8 +93,7 @@ class AitopSpider(BaseSpider):
 
             next_url = self.next_url(response)
             if next_url:
-                print('FOLLOW')
-                # yield scrapy.Request(next_url, callback=self.parse_by_tag, meta=response.meta)
+                yield scrapy.Request(next_url, callback=self.parse_by_tag, meta=response.meta)
 
         except:
             pass