add additional settings for selenium

exa/exa/middlewares.py

@@ -24,19 +24,15 @@ class SeleniumDownloadMiddleware(object):
 
     def spider_opened(self, spider):
         spider.logger.info('Spider opened: %s' % spider.name)
-        from selenium.webdriver.chrome.options import Options
-        opts = Options()
-        opts.add_argument("user-agent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) coc_coc_browser/45.0 Chrome/39.0.2171.98 Safari/537.36")
-
-
-        self.driver = webdriver.Chrome(chrome_options=opts)
-        self.driver.maximize_window()
+        self._get_driver()
 
     def spider_closed(self, spider):
         if self.driver:
             self.driver.quit()
         else:
             print('Driver closed by exception or error')
+        if self.display:
+            self.display.stop()
 
     def process_request(self, request, spider):
         self.driver.set_page_load_timeout(60)
@@ -48,4 +44,23 @@ class SeleniumDownloadMiddleware(object):
             return None
 
         body = str.encode(self.driver.page_source)
-        return HtmlResponse(self.driver.current_url, body=body, encoding='utf-8', request=request)
\ No newline at end of file
+        return HtmlResponse(self.driver.current_url, body=body, encoding='utf-8', request=request)
+
+    def _get_driver(self):
+        from scrapy.utils.project import get_project_settings
+        middleware = get_project_settings().get('MIDDLEWARE')
+        if middleware['display']:
+            from pyvirtualdisplay import Display
+            self.display = Display()
+            self.display.start()
+
+        if middleware['driver'] == 'Chrome':
+            from selenium.webdriver.chrome.options import Options
+            opts = Options()
+            opts.add_argument("user-agent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) coc_coc_browser/45.0 Chrome/39.0.2171.98 Safari/537.36")
+
+            self.driver = webdriver.Chrome(chrome_options=opts)
+            self.driver.maximize_window()
+        if middleware['driver'] == 'Phantom':
+            self.driver = webdriver.PhantomJS()
+            self.driver.maximize_window()
\ No newline at end of file

exa/exa/settings.py

@@ -103,6 +103,11 @@ DB = {
     'database': 'esi'
 }
 
+MIDDLEWARE = {
+    'display': True,
+    'driver': 'Chrome'
+}
+
 try:
     from .local_settings import *
 except ImportError:

requirements.txt

@@ -3,3 +3,5 @@ selenium==3.4.1
 dateparser==0.6.0
 Django==1.11.1
 mysqlclient==1.3.4
+PyVirtualDisplay==0.2.1
+