Add dotenv to project

exa/exa/settings.py

@@ -9,6 +9,19 @@
 #     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
 #     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
 
+import os
+from os.path import join, dirname
+from dotenv import load_dotenv
+
+dotenv_path = join(dirname(__file__), '.env')
+load_dotenv(dotenv_path)
+
+def get_env(name):
+    return os.environ[name]
+
+def get_bool_env(name):
+    return True if os.environ[name] == 'True' else False
+
 BOT_NAME = 'exa'
 
 SPIDER_MODULES = ['exa.spiders']
@@ -100,25 +113,19 @@ ITEM_PIPELINES = {
 
 DB = {
     'host': 'localhost',
-    'user': 'root',
-    'pwd': 'Q!W@q1w2mysql',
+    'user': get_env('DB_USER'),
+    'pwd': get_env('DB_PASSWORD'),
     'database': 'esi'
 }
 
+CRAWLERA_MAXBANS = 3
+
 MIDDLEWARE = {
     'display': True,
     'driver': 'Chrome'
 }
 
 
-# PROXY
-import os
-PROXY_LIST_DIR = os.path.dirname(os.path.abspath(__file__))
-RETRY_TIMES = 10
-RETRY_HTTP_CODES = [416]
-PROXY_MODE = 0
-PROXY_LIST = PROXY_LIST_DIR + '/proxy.txt'
-
 try:
     from .local_settings import *
 except ImportError:

requirements.txt

@@ -15,5 +15,5 @@ django-cors-headers==2.1.0
 celery==4.0.2
 flower==0.9.2
 django-celery-beat==1.0.1
-
+python-dotenv==0.6.4
 redis==2.10.5
\ No newline at end of file