Add proxy middleware to spider

exa/exa/settings.py

@@ -59,9 +59,9 @@ DOWNLOADER_MIDDLEWARES = {
     'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
     'scrapy_fake_useragent.middleware.RandomUserAgentMiddleware': 400,
     # 'exa.middlewares.SeleniumDownloadMiddleware': 543,
-    # 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 90,
-    # 'scrapy_proxies.RandomProxy': 100,
-    # 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,
+    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 90,
+    'scrapy_proxies.RandomProxy': 100,
+    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,
 }
 
 # Enable or disable extensions
@@ -109,6 +109,13 @@ MIDDLEWARE = {
     'driver': 'Chrome'
 }
 
+
+# PROXY
+
+RETRY_TIMES = 10
+RETRY_HTTP_CODES = [416]
+PROXY_MODE = 0
+
 try:
     from .local_settings import *
 except ImportError:

requirements.txt

@@ -2,6 +2,7 @@ Scrapy==1.3.3
 scrapy-fake-useragent==1.1.0
 python-scrapyd-api==2.0.1
 scrapyd-client==1.1.0
+scrapy-proxies==0.3
 PyVirtualDisplay==0.2.1
 selenium==3.4.1
 dateparser==0.6.0