update.py

import json
from scrapy.utils.project import get_project_settings
from newspaper import Article
from exa.helpers import Database
from exa.esi_news_classification.news_classify_tag import Classifier

db_cred = get_project_settings().get('DB')


class NewsUpdater:
    def __init__(self):
        self.classifier = Classifier()
        self.classifier.teach_model()
        self.db = Database(**db_cred)

    def select_news(self, query):
        return self.db.select(query)

    def load_text(self, url):
        article = Article(url)
        article.download()
        article.parse()
        return article.text

    def get_tags(self, text):
        self.classifier.classify(text)
        tags = list()
        for i in self.classifier.teg_accordance:
            tags.append(i[0])
        return json.dumps(tags)

    def update_news(self, query, data):
        self.db.update(query, data)

    # this bad way
    def update_all_tags(self, condition='where 1'):
        for i in self.select_news('select id, url from wp_esi_news_accept ' + condition):
            try:
                text = self.load_text(i[1])
                tags = self.get_tags(text)
                if len(tags) == 0:
                    tags = None
                self.update_news('update wp_esi_news_accept set tags_id=%s where id=%s', (tags, i[0]))
                print('News id={} was updated'.format(i[0]))
            except BaseException as e:
                print(e.with_traceback())

    def update_all_text(self, condition='where 1'):
        for i in self.select_news('select id, url from wp_esi_news_accept ' + condition):
            # try:
                text = self.load_text(i[1])
                data = (text.encode('ascii', 'ignore'), i[0])
                self.update_news('update wp_esi_news_accept set text=%s where id=%s', data)
                print('News id={} was updated'.format(i[0]))
            # except BaseException as e:
            #     print(e.with_traceback())


if __name__ == '__main__':
    ml = NewsUpdater()
    ml.update_all_tags()