Commit e75c2c21 authored by Vasyl Bodnaruk's avatar Vasyl Bodnaruk

Get crawlera key

parent 4874adac
...@@ -109,6 +109,11 @@ CRAWLERA_MAXBANS = 3 ...@@ -109,6 +109,11 @@ CRAWLERA_MAXBANS = 3
try: try:
from .local_settings import * from .local_settings import *
except ImportError:
pass
# Import .env like DB pass, CRAWLERA_APIKEY
try:
from .env import * from .env import *
except ImportError: except ImportError:
pass pass
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import dateparser import dateparser
import scrapy import scrapy
from scrapy.utils.project import get_project_settings
from .base import BaseSpider from .base import BaseSpider
from ..items import ExaItem from ..items import ExaItem
from ..settings import get_env
class CbSpider(BaseSpider): class CbSpider(BaseSpider):
name = "cb" name = "cb"
allowed_domains = ["www.crunchbase.com"] allowed_domains = ["www.crunchbase.com"]
crawlera_enabled = True crawlera_enabled = True
crawlera_apikey = get_env('CRAWLERA_APIKEY') crawlera_apikey = get_project_settings().get('CRAWLERA_APIKEY')
# start_urls = ['http://www.crunchbase.com/organization/sense-ly/press/'] # start_urls = ['http://www.crunchbase.com/organization/sense-ly/press/']
co = 0 co = 0
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment