changed module DB for reuse

exa/exa/helpers/company_maker.py

@@ -6,12 +6,17 @@ Company = namedtuple('Company', 'id, url, media_id, type_id, region_id')
 # 'http://www.mobihealthnews.com/tag/MedTronic'
 
 
+Entity = namedtuple('Entity', 'id, name, country')
+
 class CompanyMaker:
     def __init__(self, companies=None):
         # self.in_site = ['http://www.mobihealthnews.com/tag/Twitter', 'http://www.mobihealthnews.com/tag/intel',
         #                 'http://www.mobihealthnews.com/tag/ibm', 'http://www.mobihealthnews.com/tag/Salesforce',
         #                 'http://www.mobihealthnews.com/tag/google']
-        self.in_site = companies
+        self.in_site = list()
+        if companies:
+            for i in companies:
+                self.in_site.append(Entity(i[0], i[1], i[2]))
         self.companies = list()
 
     def make_companies(self):

exa/exa/helpers/db.py

@@ -2,7 +2,6 @@ from collections import namedtuple
 import traceback
 import MySQLdb
 
-Entity = namedtuple('Entity', 'id, name, country')
 
 
 class Database:
@@ -36,10 +35,7 @@ class Database:
     def select(self, item):
         try:
             self.cursor.execute(item)
-            result = list()
-            for i in self.cursor.fetchall():
-                result.append(Entity(i[0], i[1], i[2]))
-            return result
+            return self.cursor.fetchall()
         except:
             self.cursor.rollback()
 

exa/exa/pipelines.py

@@ -15,12 +15,9 @@ class ExaPipeline(object):
     def __init__(self):
         # self.out = open('out/out{}.txt'.format(datetime.now()), 'w', newline='\n')
         self.db = Database(**db)
+        self.urls = set(self.db.select('select url from wp_esi_accept'))
         super(ExaPipeline, self).__init__()
 
-    def __del__(self):
-        pass
-        # self.out.close()
-
     def process_item(self, item, spider):
         print(item)
         item['description'] = ''.join(item['description']).replace('\n', ' ')
@@ -29,7 +26,11 @@ class ExaPipeline(object):
         query = """INSERT INTO wp_esi_news_accept (title, description, URL, media_id, type_id, region_id, post_id,
         publish_date, record_date, company_id, is_accepted) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);\n"""
         print(item)
-        news = self.db.insert(query, data)
+        n = self.db.select("select url from wp_esi_news_accept where url={}".format(item['url']))
+        if len(n) == 0:
+            news = self.db.insert(query, data)
+        else:
+            print('Duplicate')
         # self._insert_news_entiry(news, item['company_id'])
         # self.out.write(query)
 

exa/exa/spiders/mobihealthnews.py

@@ -19,7 +19,8 @@ class MobiHealthNewsSpider(scrapy.Spider):
         self.condition = kwargs.get('query')
         self.query = "SELECT id, name, country FROM wp_esi_entity WHERE 1 and id=3"
         if self.condition:
-            self.query += ' and {}'.format(self.condition)
+            print(self.condition)
+            self.query += ' or {}'.format(self.condition)
         self.comp = db.select(self.query)
         super(MobiHealthNewsSpider, self).__init__()