Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Contribute to GitLab
Sign in / Register
Toggle navigation
E
esi-table-data
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
esi-data-scrapping
esi-table-data
Commits
730ea62f
Commit
730ea62f
authored
Jul 04, 2017
by
Vasyl Bodnaruk
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
added functional for creating task
parent
daca3d40
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
26 additions
and
21 deletions
+26
-21
models.py
esi/scraper/models.py
+7
-5
periodic_task_creator.py
esi/scraper/periodic_task_creator.py
+5
-4
views.py
esi/scraper/views.py
+14
-12
No files found.
esi/scraper/models.py
View file @
730ea62f
...
...
@@ -56,13 +56,15 @@ class Job(models.Model):
(
RUNNING
,
'running'
),
(
PENDING
,
'pending'
)
)
DAY
=
'DAY'
,
WEEK
=
'WEEK'
,
DAY
=
'DAY'
WEEK
=
'WEEK'
MONTH
=
'MONTH'
NON
=
'NON'
INTERVAL
=
(
(
DAY
,
'day'
),
(
WEEK
,
'week'
),
(
MONTH
,
'month'
)
(
MONTH
,
'month'
),
(
NON
,
'nonstop'
)
)
name
=
models
.
CharField
(
max_length
=
50
,
default
=
None
,
null
=
True
)
...
...
@@ -74,9 +76,9 @@ class Job(models.Model):
query
=
models
.
CharField
(
max_length
=
255
,
default
=
None
,
null
=
True
,
blank
=
True
)
spider
=
models
.
ForeignKey
(
Spider
,
default
=
None
,
null
=
True
)
schedule_job
=
models
.
OneToOneField
(
PeriodicTask
,
blank
=
True
,
null
=
True
,
default
=
None
)
schedule_job
=
models
.
OneToOneField
(
PeriodicTask
,
on_delete
=
models
.
SET_NULL
,
blank
=
True
,
null
=
True
,
default
=
None
)
interval
=
models
.
CharField
(
max_length
=
5
,
choices
=
INTERVAL
,
default
=
None
,
null
=
True
,
blank
=
True
)
count
=
models
.
IntegerField
(
default
=
None
,
blank
=
Non
e
,
null
=
True
)
count
=
models
.
IntegerField
(
default
=
None
,
blank
=
Tru
e
,
null
=
True
)
objects
=
JobManager
()
...
...
esi/scraper/periodic_task_creator.py
View file @
730ea62f
...
...
@@ -3,11 +3,11 @@ from django_celery_beat.models import PeriodicTask, CrontabSchedule
def
make_data
(
data
):
every
=
lambda
x
:
'*/{}'
.
format
(
x
)
every
=
(
lambda
x
:
'*/{}'
.
format
(
x
)
)
result
=
{
'minute'
:
data
.
get
(
'minute'
,
0
),
'hour'
:
data
.
get
(
'hour'
,
0
),
'day_of_week'
:
data
.
get
(
'day_of_week'
,
'*'
)
,
'day_of_week'
:
'*'
,
'day_of_month'
:
data
.
get
(
'day_of_month'
,
'*'
),
'month_of_year'
:
data
.
get
(
'month_of_year'
,
'*'
)
}
...
...
@@ -18,9 +18,10 @@ def make_data(data):
def
make_periodic_task
(
job_id
,
data
):
schedule
,
_
=
CrontabSchedule
.
objects
.
get_or_create
(
**
data
)
PeriodicTask
.
objects
.
create
(
task
=
PeriodicTask
.
objects
.
create
(
crontab
=
schedule
,
name
=
'
Importing contacts'
,
name
=
'
Task for job {}'
.
format
(
job_id
)
,
task
=
'scraper.tasks.run_job'
,
args
=
[
job_id
]
)
return
task
esi/scraper/views.py
View file @
730ea62f
...
...
@@ -2,12 +2,13 @@ from datetime import datetime, time
from
django.shortcuts
import
render
,
HttpResponseRedirect
,
reverse
,
HttpResponse
from
django.http
import
JsonResponse
from
django.views.generic
import
ListView
,
CreateView
,
UpdateView
from
django.contrib.auth.decorators
import
login_required
,
permission_required
from
django.contrib.auth.decorators
import
login_required
from
django.utils.decorators
import
method_decorator
from
.models
import
Spider
,
Job
from
.service
import
Service
from
.forms
import
CreateJobForm
from
.periodic_task_creator
import
make_periodic_task
@
method_decorator
(
login_required
,
name
=
'post'
)
...
...
@@ -70,20 +71,21 @@ class NewJobCreateView(CreateView):
name
=
request
.
POST
.
get
(
'name'
)
spider
=
Spider
.
objects
.
get
(
pk
=
request
.
POST
.
get
(
'spider'
))
query
=
request
.
POST
.
get
(
'query'
)
date
=
self
.
_date_format
(
request
.
POST
.
get
(
'date'
))
interval
=
request
.
POST
.
get
(
'interval'
)
count
=
request
.
POST
.
get
(
'count'
)
form
=
CreateJobForm
(
request
.
POST
)
# if form.is_valid():
# form.create_job()
job
=
Job
.
objects
.
create
(
name
=
name
,
spider
=
spider
,
query
=
query
)
if
request
.
POST
.
get
(
'date'
):
date
=
self
.
_date_format
(
request
.
POST
.
get
(
'date'
))
data
=
{
'minute'
:
date
.
minute
,
'hour'
:
date
.
hour
,
'day_of_month'
:
date
.
day
,
'month_of_year'
:
date
.
month
}
job
.
schedule_job
=
make_periodic_task
(
job
.
id
,
data
)
job
.
save
()
print
(
request
.
POST
)
if
query
==
'j'
:
query
=
''
print
(
date
)
# Job.objects.create(name=name, spider=spider, query=query)
if
count
:
pass
return
HttpResponseRedirect
(
reverse
(
'job_list'
))
def
_date_format
(
self
,
date
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment