Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
MJU-POPS
ckanext-archiver
Commits
d570c348
Commit
d570c348
authored
Mar 01, 2016
by
David Read
Browse files
Merge pull request #21 from ckan/20-config
Config was not being read by tasks.py
parents
9640a4d2
aad8db48
Changes
2
Hide whitespace changes
Inline
Side-by-side
README.rst
View file @
d570c348
...
@@ -269,16 +269,17 @@ Config settings
...
@@ -269,16 +269,17 @@ Config settings
root /www/resource_cache;
root /www/resource_cache;
}
}
Legacy settings:
Legacy settings
~~~~~~~~~~~~~~~
Older versions of ckanext-archiver put these settings in
Older versions of ckanext-archiver put these settings in
ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH
ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH
but this is
deprecated as of ckanext-archiver 2.0
.
but this is
no longer available
.
There used to be an option DATA_FORMATS for filtering the resources
There used to be an option DATA_FORMATS for filtering the resources
archived, but that has now been removed in ckanext-archiver v2.0, since it
archived, but that has now been removed in ckanext-archiver v2.0, since it
is now not only caching files, but is seen as a broken link checker, which
is now not only caching files, but is seen as a broken link checker, which
applies whatever the format.
applies whatever the format.
Using Archiver
Using Archiver
...
...
ckanext/archiver/tasks.py
View file @
d570c348
...
@@ -16,10 +16,6 @@ from requests.packages import urllib3
...
@@ -16,10 +16,6 @@ from requests.packages import urllib3
from
ckan.lib.celery_app
import
celery
from
ckan.lib.celery_app
import
celery
from
ckan
import
plugins
as
p
from
ckan
import
plugins
as
p
try
:
from
ckanext.archiver
import
settings
except
ImportError
:
from
ckanext.archiver
import
default_settings
as
settings
from
ckanext.archiver
import
interfaces
as
archiver_interfaces
from
ckanext.archiver
import
interfaces
as
archiver_interfaces
toolkit
=
p
.
toolkit
toolkit
=
p
.
toolkit
...
@@ -201,6 +197,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
...
@@ -201,6 +197,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
from
ckan
import
model
from
ckan
import
model
from
pylons
import
config
from
pylons
import
config
from
ckan.plugins
import
toolkit
from
ckan.plugins
import
toolkit
from
ckanext.archiver
import
default_settings
as
settings
get_action
=
toolkit
.
get_action
get_action
=
toolkit
.
get_action
...
@@ -301,7 +298,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
...
@@ -301,7 +298,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
def
download
(
context
,
resource
,
url_timeout
=
30
,
def
download
(
context
,
resource
,
url_timeout
=
30
,
max_content_length
=
settings
.
MAX_CONTENT_LENGTH
,
max_content_length
=
'default'
,
method
=
'GET'
):
method
=
'GET'
):
'''Given a resource, tries to download it.
'''Given a resource, tries to download it.
...
@@ -324,8 +321,12 @@ def download(context, resource, url_timeout=30,
...
@@ -324,8 +321,12 @@ def download(context, resource, url_timeout=30,
Returns a dict of results of a successful download:
Returns a dict of results of a successful download:
mimetype, size, hash, headers, saved_file, url_redirected_to
mimetype, size, hash, headers, saved_file, url_redirected_to
'''
'''
from
ckanext.archiver
import
default_settings
as
settings
log
=
update_resource
.
get_logger
()
log
=
update_resource
.
get_logger
()
if
max_content_length
==
'default'
:
max_content_length
=
settings
.
MAX_CONTENT_LENGTH
url
=
resource
[
'url'
]
url
=
resource
[
'url'
]
url
=
tidy_url
(
url
)
url
=
tidy_url
(
url
)
...
@@ -441,6 +442,7 @@ def archive_resource(context, resource, log, result=None, url_timeout=30):
...
@@ -441,6 +442,7 @@ def archive_resource(context, resource, log, result=None, url_timeout=30):
Returns: {cache_filepath, cache_url}
Returns: {cache_filepath, cache_url}
"""
"""
from
ckanext.archiver
import
default_settings
as
settings
relative_archive_path
=
os
.
path
.
join
(
resource
[
'id'
][:
2
],
resource
[
'id'
])
relative_archive_path
=
os
.
path
.
join
(
resource
[
'id'
][:
2
],
resource
[
'id'
])
archive_dir
=
os
.
path
.
join
(
settings
.
ARCHIVE_DIR
,
relative_archive_path
)
archive_dir
=
os
.
path
.
join
(
settings
.
ARCHIVE_DIR
,
relative_archive_path
)
if
not
os
.
path
.
exists
(
archive_dir
):
if
not
os
.
path
.
exists
(
archive_dir
):
...
@@ -517,6 +519,7 @@ def _set_user_agent_string(headers):
...
@@ -517,6 +519,7 @@ def _set_user_agent_string(headers):
Update the passed headers object with a `User-Agent` key, if there is a
Update the passed headers object with a `User-Agent` key, if there is a
USER_AGENT_STRING option in settings.
USER_AGENT_STRING option in settings.
'''
'''
from
ckanext.archiver
import
default_settings
as
settings
ua_str
=
settings
.
USER_AGENT_STRING
ua_str
=
settings
.
USER_AGENT_STRING
if
ua_str
is
not
None
:
if
ua_str
is
not
None
:
headers
[
'User-Agent'
]
=
ua_str
headers
[
'User-Agent'
]
=
ua_str
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment