update.py

import json
from scrapy.utils.project import get_project_settings
from newspaper import Article
from exa.helpers import Database
from exa.helpers import MLDataMaker, News
from exa.classificator.news_classify_tag import Classifier

db = get_project_settings().get('DB')


class Updater:
    def __init__(self):
        self.classifier = None
        self.db = Database(**db)

    def select_news(self, query):
        return self.db.select(query)

    def load_text(self, url):
        article = Article(url)
        article.download()
        article.parse()
        return article.text

    def get_tags(self, text):

        self.classifier.classify(text)
        tags = list()
        for i in self.classifier.tag_accordance:
            tags.append(i[0])
        return json.dumps(tags)

    def update_news(self, query, data):
        self.db.update(query, data)

    def update_all_tags(self, condition='where 1'):
        self.classifier = Classifier(host=db['host'], user=db['user'], password=db['pwd'], db=db['database'], port=3306)
        self.classifier.teach_model()
        for i in self.select_news('select id, url from wp_esi_news_accept ' + condition):
            try:
                text = self.load_text(i[1])
                tags = self.get_tags(text)
                if len(tags) == 0:
                    tags = None
                self.update_news('update wp_esi_news_accept set tags_id=%s where id=%s', (tags, i[0]))
                print('News id={} was updated'.format(i[0]))
            except BaseException as e:
                print(e.with_traceback(e))

    def update_all_text(self, condition='where 1'):
        for i in self.select_news('select id, url from wp_esi_news_accept ' + condition):
            try:
                text = self.load_text(i[1])
                data = (text.encode('ascii', 'ignore'), i[0])
                self.update_news('update wp_esi_news_accept set text=%s where id=%s', data)
                print('News id={} was updated'.format(i[0]))
            except BaseException as e:
                print(e.with_traceback(e))

    def update_ml_data(self, query='where 1'):
        m = MLDataMaker(self.db)
        n = self.select_news('select id, temp_tags from wp_esi_news_accept {}'.format(query))

        def build_data(news_list):
            res = []
            for i in (News(i, j) for i, j in news_list):
                temp = m.make_ml_data(i)
                if temp:
                    res.extend(temp)
            return res
        data = build_data(n)
        query = 'insert into wp_esi_ml_data(news_id, tag_id) values(%s, %s)'
        data = [(i.news_id, i.tags_id) for i in data]
        self.db.insert_many(query, data)

        print(len(data))

    def update_radar_ids(self):
        tag_radar = {i[0]: i[1] for i in self.db.select('select tag_id, radar_id from wp_esi_radar_tag')}
        news = self.db.select('select id, tags_id from wp_esi_news_accept')
        for i, tags in news:
            print(i, tags)
            if tags:
                radar = [tag_radar[i] for i in json.loads(tags) if i in tag_radar]


if __name__ == '__main__':
    ml = Updater()
    ml.update_radar_ids()