Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
MJU-POPS
ckanext-archiver
Commits
e6d7e99c
Commit
e6d7e99c
authored
Mar 29, 2016
by
David Read
Browse files
Merge branch 'master' of github.com:ckan/ckanext-archiver
parents
dcea053f
d570c348
Changes
3
Hide whitespace changes
Inline
Side-by-side
README.rst
View file @
e6d7e99c
...
...
@@ -2,8 +2,8 @@
these badges work. The necessary Travis and Coverage config files have been
generated for you.
.. image:: https://travis-ci.org/
datagovuk
/ckanext-archiver.svg?branch=master
:target: https://travis-ci.org/
datagovuk
/ckanext-archiver
.. image:: https://travis-ci.org/
ckan
/ckanext-archiver.svg?branch=master
:target: https://travis-ci.org/
ckan
/ckanext-archiver
=============
ckanext-archiver
...
...
@@ -269,16 +269,17 @@ Config settings
root /www/resource_cache;
}
Legacy settings:
Legacy settings
~~~~~~~~~~~~~~~
Older versions of ckanext-archiver put these settings in
ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH
but this is
deprecated as of ckanext-archiver 2.0
.
Older versions of ckanext-archiver put these settings in
ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH
but this is
no longer available
.
There used to be an option DATA_FORMATS for filtering the resources
archived, but that has now been removed in ckanext-archiver v2.0, since it
is now not only caching files, but is seen as a broken link checker, which
applies whatever the format.
There used to be an option DATA_FORMATS for filtering the resources
archived, but that has now been removed in ckanext-archiver v2.0, since it
is now not only caching files, but is seen as a broken link checker, which
applies whatever the format.
Using Archiver
...
...
ckanext/archiver/tasks.py
View file @
e6d7e99c
...
...
@@ -16,10 +16,6 @@ from requests.packages import urllib3
from
ckan.lib.celery_app
import
celery
from
ckan
import
plugins
as
p
try
:
from
ckanext.archiver
import
settings
except
ImportError
:
from
ckanext.archiver
import
default_settings
as
settings
from
ckanext.archiver
import
interfaces
as
archiver_interfaces
toolkit
=
p
.
toolkit
...
...
@@ -201,6 +197,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
from
ckan
import
model
from
pylons
import
config
from
ckan.plugins
import
toolkit
from
ckanext.archiver
import
default_settings
as
settings
get_action
=
toolkit
.
get_action
...
...
@@ -301,7 +298,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
def
download
(
context
,
resource
,
url_timeout
=
30
,
max_content_length
=
settings
.
MAX_CONTENT_LENGTH
,
max_content_length
=
'default'
,
method
=
'GET'
):
'''Given a resource, tries to download it.
...
...
@@ -324,8 +321,12 @@ def download(context, resource, url_timeout=30,
Returns a dict of results of a successful download:
mimetype, size, hash, headers, saved_file, url_redirected_to
'''
from
ckanext.archiver
import
default_settings
as
settings
log
=
update_resource
.
get_logger
()
if
max_content_length
==
'default'
:
max_content_length
=
settings
.
MAX_CONTENT_LENGTH
url
=
resource
[
'url'
]
url
=
tidy_url
(
url
)
...
...
@@ -441,6 +442,7 @@ def archive_resource(context, resource, log, result=None, url_timeout=30):
Returns: {cache_filepath, cache_url}
"""
from
ckanext.archiver
import
default_settings
as
settings
relative_archive_path
=
os
.
path
.
join
(
resource
[
'id'
][:
2
],
resource
[
'id'
])
archive_dir
=
os
.
path
.
join
(
settings
.
ARCHIVE_DIR
,
relative_archive_path
)
if
not
os
.
path
.
exists
(
archive_dir
):
...
...
@@ -517,6 +519,7 @@ def _set_user_agent_string(headers):
Update the passed headers object with a `User-Agent` key, if there is a
USER_AGENT_STRING option in settings.
'''
from
ckanext.archiver
import
default_settings
as
settings
ua_str
=
settings
.
USER_AGENT_STRING
if
ua_str
is
not
None
:
headers
[
'User-Agent'
]
=
ua_str
...
...
tests/test_archiver.py
View file @
e6d7e99c
...
...
@@ -12,14 +12,15 @@ from nose.tools import assert_raises, assert_equal
from
ckan
import
model
from
ckan
import
plugins
from
ckan.tests
import
BaseCase
from
ckan.logic
import
get_action
try
:
from
ckan.tests.helpers
import
reset_db
from
ckan.tests
import
factories
as
ckan_factories
from
ckan.tests.legacy
import
BaseCase
except
ImportError
:
from
ckan.new_tests.helpers
import
reset_db
from
ckan.new_tests
import
factories
as
ckan_factories
from
ckan.tests
import
BaseCase
from
ckanext.archiver
import
model
as
archiver_model
from
ckanext.archiver.model
import
Archival
...
...
@@ -375,6 +376,7 @@ class TestArchiver(BaseCase):
assert
params
.
get
(
'package_id'
)
==
pkg
[
'id'
]
assert
params
.
get
(
'resource_id'
)
==
None
class
TestDownload
(
BaseCase
):
'''Tests of the download method (and things it calls).
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment