add settings and requirements for make crawler server

8ca4045b · Vasyl Bodnaruk · 0793811f · 8ca4045b · 8ca4045b · 0793811f
Commit 8ca4045b authored Jun 13, 2017 by Vasyl Bodnaruk
7 changed files
--- a/exa/exa/spiders/techcrunch.py
+++ b/exa/exa/spiders/techcrunch.py
@@ -13,7 +13,6 @@ db = Database(**db_settings)
 class TechcrunchSpider(scrapy.Spider):
    name = "tc"
    allowed_domains = ["techcrunch.com"]
-    # start_urls = ['https://techcrunch.com/tag/Ericsson/']

    def __init__(self, *args, **kwargs):
        self.condition = kwargs.get('query')

--- a/exa/scrapy.cfg
+++ b/exa/scrapy.cfg
@@ -6,6 +6,6 @@
 [settings]
 default = exa.settings

-[deploy]
-#url = http://localhost:6800/
+[deploy:esi]
+url = http://localhost:6800/
 project = exa
--- a/exa/t.csv
+++ b/exa/t.csv
--- a/exa/wp_esi_news.sql
+++ b/exa/wp_esi_news.sql
--- a/exa/wp_esi_news_entity.sql
+++ b/exa/wp_esi_news_entity.sql
-- phpMyAdmin SQL Dump
-- version 4.2.12deb2+deb8u2
-- http://www.phpmyadmin.net
--
-- Хост: localhost
-- Час створення: Трв 22 2017 р., 17:36
-- Версія сервера: 5.5.54-0+deb8u1
-- Версія PHP: 5.6.29-0+deb8u1
-
-SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";
-SET time_zone = "+00:00";
-
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
-
--
-- База даних: `esi`
--
-
--
-- Дамп даних таблиці `wp_esi_news_entity`
--
-
-INSERT INTO `wp_esi_news_entity` (`news_id`, `entity_id`) VALUES
-(277, 3),
-(278, 3),
-(279, 3),
-(280, 3),
-(281, 3),
-(282, 3),
-(283, 3),
-(284, 3),
-(285, 3),
-(286, 3),
-(287, 20),
-(288, 21),
-(289, 21),
-(290, 21),
-(291, 21),
-(292, 26),
-(293, 21),
-(294, 26),
-(295, 26),
-(296, 33),
-(297, 33),
-(298, 21),
-(299, 33),
-(300, 21),
-(301, 35),
-(302, 35),
-(303, 21),
-(304, 21),
-(305, 39),
-(306, 35),
-(307, 40),
-(308, 21),
-(309, 40),
-(310, 35),
-(311, 40),
-(312, 39),
-(313, 35),
-(314, 40),
-(315, 39),
-(316, 21),
-(317, 45),
-(318, 21),
-(319, 43),
-(320, 40),
-(321, 47),
-(322, 35),
-(323, 21),
-(324, 40),
-(325, 35),
-(326, 21),
-(327, 43),
-(328, 45),
-(329, 35),
-(330, 49),
-(331, 43),
-(332, 45),
-(333, 47),
-(334, 43),
-(335, 45),
-(336, 47),
-(337, 40),
-(338, 51),
-(339, 35),
-(340, 21),
-(341, 45),
-(342, 47),
-(343, 39),
-(344, 40),
-(345, 51),
-(346, 21),
-(347, 40),
-(348, 51),
-(349, 35),
-(350, 21),
-(351, 39),
-(352, 40),
-(353, 51),
-(354, 47),
-(355, 45),
-(356, 43),
-(357, 39),
-(358, 51),
-(359, 45),
-(360, 43),
-(361, 39),
-(362, 51),
-(363, 35),
-(364, 21),
-(365, 45),
-(366, 43),
-(367, 39),
-(368, 40),
-(369, 51),
-(370, 47),
-(371, 21),
-(372, 45),
-(373, 43),
-(374, 40),
-(375, 51),
-(376, 47),
-(377, 45),
-(378, 43),
-(379, 39),
-(380, 40),
-(381, 45),
-(382, 43),
-(383, 40),
-(384, 35),
-(385, 21),
-(386, 45),
-(387, 51),
-(388, 35),
-(389, 21),
-(390, 45),
-(391, 43),
-(392, 51),
-(393, 35),
-(394, 21),
-(395, 45),
-(396, 43),
-(397, 40),
-(398, 39),
-(399, 51),
-(400, 35),
-(401, 21),
-(402, 45),
-(403, 43),
-(404, 40),
-(405, 39),
-(406, 51),
-(407, 35),
-(408, 21),
-(409, 40),
-(410, 39),
-(411, 51),
-(412, 35),
-(413, 40),
-(414, 39),
-(415, 51),
-(416, 35),
-(417, 43),
-(418, 45),
-(419, 40),
-(420, 39),
-(421, 51),
-(422, 21),
-(423, 43),
-(424, 45),
-(425, 40),
-(426, 39),
-(427, 51),
-(428, 21),
-(429, 43),
-(430, 45),
-(431, 40),
-(432, 39),
-(433, 21),
-(434, 43),
-(435, 45),
-(436, 40),
-(437, 39),
-(438, 35),
-(439, 21),
-(440, 43),
-(441, 45),
-(442, 40),
-(443, 39),
-(444, 35),
-(445, 43),
-(446, 35),
-(447, 21),
-(448, 43),
-(449, 35),
-(450, 21),
-(451, 43),
-(452, 39),
-(453, 40),
-(454, 35),
-(455, 43),
-(456, 39),
-(457, 40),
-(458, 35),
-(459, 21),
-(460, 43),
-(461, 39),
-(462, 40),
-(463, 35),
-(464, 39),
-(465, 40),
-(466, 35),
-(467, 39),
-(468, 40),
-(469, 43),
-(470, 21),
-(471, 39),
-(472, 40),
-(473, 43),
-(474, 21),
-(475, 39),
-(476, 40),
-(477, 35),
-(478, 43),
-(479, 39),
-(480, 40),
-(481, 35),
-(482, 43),
-(483, 39),
-(484, 40),
-(485, 43),
-(486, 21),
-(487, 39),
-(488, 35),
-(489, 43),
-(490, 21),
-(491, 35),
-(492, 43),
-(493, 21),
-(494, 40),
-(495, 35),
-(496, 43),
-(497, 21),
-(498, 39),
-(499, 40),
-(500, 35),
-(501, 43),
-(502, 21),
-(503, 39),
-(504, 40),
-(505, 35),
-(506, 39),
-(507, 40),
-(508, 39),
-(509, 40),
-(510, 39),
-(511, 40),
-(512, 43),
-(513, 39),
-(514, 40),
-(515, 35),
-(516, 43),
-(517, 35),
-(518, 43),
-(519, 21),
-(520, 39),
-(521, 40),
-(522, 35),
-(523, 43),
-(524, 21),
-(525, 39),
-(526, 40),
-(527, 35),
-(528, 43),
-(529, 21),
-(530, 39),
-(531, 35),
-(532, 43),
-(533, 21),
-(534, 35),
-(535, 43),
-(536, 21),
-(537, 40),
-(538, 103),
-(539, 35),
-(540, 43),
-(541, 21),
-(542, 39),
-(543, 40),
-(544, 103),
-(545, 35),
-(546, 43),
-(547, 39),
-(548, 105),
-(549, 40),
-(550, 35),
-(551, 43),
-(552, 39),
-(553, 40),
-(554, 103),
-(555, 21),
-(556, 39),
-(557, 40),
-(558, 21),
-(559, 108),
-(560, 39),
-(561, 40),
-(562, 43),
-(563, 35),
-(564, 21),
-(565, 39),
-(566, 40),
-(567, 43),
-(568, 35),
-(569, 21),
-(570, 39),
-(571, 40),
-(572, 43),
-(573, 35),
-(574, 39),
-(575, 40),
-(576, 43),
-(577, 35),
-(578, 21),
-(579, 40),
-(580, 43),
-(581, 35),
-(582, 21),
-(583, 43),
-(584, 35),
-(585, 21),
-(586, 43),
-(587, 35),
-(588, 21),
-(589, 40),
-(590, 43),
-(591, 35),
-(592, 40),
-(593, 43),
-(594, 35),
-(595, 35),
-(596, 21),
-(597, 21),
-(598, 43),
-(599, 21),
-(600, 35),
-(601, 43),
-(602, 21),
-(603, 43),
-(604, 21),
-(605, 35),
-(606, 43),
-(607, 21),
-(608, 35),
-(609, 43),
-(610, 21),
-(611, 35),
-(612, 43),
-(613, 21),
-(614, 35),
-(615, 43),
-(616, 21),
-(617, 35),
-(618, 43),
-(619, 21),
-(620, 35),
-(621, 43),
-(622, 35),
-(623, 43),
-(624, 21),
-(625, 43),
-(626, 35),
-(627, 21),
-(628, 43),
-(629, 35),
-(630, 21),
-(631, 43),
-(632, 35),
-(633, 21),
-(634, 43),
-(635, 35),
-(636, 21),
-(637, 43),
-(638, 35),
-(639, 21),
-(640, 43),
-(641, 35),
-(642, 21),
-(643, 43),
-(644, 35),
-(645, 21),
-(646, 43),
-(647, 35),
-(648, 43),
-(649, 35),
-(650, 43),
-(651, 35),
-(652, 21),
-(653, 21),
-(654, 21),
-(655, 35),
-(656, 43),
-(657, 21),
-(658, 35),
-(659, 43),
-(660, 21),
-(661, 43),
-(662, 21),
-(663, 43),
-(664, 21),
-(665, 43),
-(666, 21),
-(667, 43),
-(668, 21),
-(669, 43),
-(670, 21),
-(671, 43),
-(672, 21),
-(673, 21),
-(674, 21),
-(675, 21),
-(676, 21),
-(677, 21),
-(678, 21),
-(679, 21),
-(680, 21),
-(681, 21),
-(682, 21),
-(683, 21),
-(684, 21),
-(685, 21),
-(686, 21),
-(687, 21),
-(688, 21),
-(689, 21),
-(690, 21),
-(691, 21),
-(692, 21),
-(693, 21),
-(694, 185),
-(695, 21),
-(696, 21),
-(697, 21),
-(698, 189),
-(699, 21),
-(700, 21),
-(701, 21),
-(702, 193),
-(703, 21),
-(704, 21),
-(705, 21),
-(706, 21),
-(707, 21),
-(708, 21),
-(709, 21),
-(710, 21),
-(711, 21),
-(712, 21),
-(713, 21),
-(714, 21),
-(715, 21),
-(716, 21),
-(717, 21),
-(718, 21),
-(719, 21),
-(720, 218),
-(721, 218),
-(722, 21),
-(723, 218),
-(724, 21),
-(725, 218),
-(726, 21),
-(727, 218),
-(728, 21),
-(729, 218),
-(730, 21),
-(731, 21),
-(732, 21),
-(733, 227),
-(734, 21),
-(735, 21),
-(736, 21),
-(737, 21),
-(738, 21),
-(739, 21),
-(740, 21),
-(741, 21),
-(742, 21),
-(743, 21),
-(744, 21),
-(745, 21),
-(746, 21),
-(747, 21),
-(748, 21),
-(749, 21),
-(750, 21),
-(751, 21),
-(752, 21),
-(753, 21),
-(754, 21),
-(755, 21),
-(756, 21),
-(757, 21),
-(758, 21),
-(759, 21),
-(760, 21),
-(761, 21),
-(762, 21),
-(763, 21),
-(764, 21),
-(765, 21),
-(766, 21),
-(767, 21),
-(768, 21),
-(769, 21),
-(770, 21),
-(771, 21),
-(772, 21),
-(773, 21),
-(774, 21),
-(775, 21),
-(776, 282),
-(777, 282),
-(778, 21),
-(779, 21),
-(780, 21),
-(781, 21),
-(782, 21),
-(783, 21),
-(784, 21),
-(785, 21),
-(786, 21);
-
-/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
-/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
--- a/prod_requirements.txt
+++ b/prod_requirements.txt
+cryptography==1.9
+PyDispatcher==2.0.5
\ No newline at end of file
--- a/requirements.txt
+++ b/requirements.txt
 Scrapy==1.3.3
+PyVirtualDisplay==0.2.1
 selenium==3.4.1
 dateparser==0.6.0
-Django==1.11.1
 mysqlclient==1.3.4
-PyVirtualDisplay==0.2.1
+Django==1.11.1
 django-el-pagination==3.1.0