Upgrade is finished. Everything should be working now.

Commit e6d7e99c authored by David Read's avatar David Read
Browse files

Merge branch 'master' of github.com:ckan/ckanext-archiver

parents dcea053f d570c348
...@@ -2,8 +2,8 @@ ...@@ -2,8 +2,8 @@
these badges work. The necessary Travis and Coverage config files have been these badges work. The necessary Travis and Coverage config files have been
generated for you. generated for you.
.. image:: https://travis-ci.org/datagovuk/ckanext-archiver.svg?branch=master .. image:: https://travis-ci.org/ckan/ckanext-archiver.svg?branch=master
:target: https://travis-ci.org/datagovuk/ckanext-archiver :target: https://travis-ci.org/ckan/ckanext-archiver
============= =============
ckanext-archiver ckanext-archiver
...@@ -269,16 +269,17 @@ Config settings ...@@ -269,16 +269,17 @@ Config settings
root /www/resource_cache; root /www/resource_cache;
} }
Legacy settings: Legacy settings
~~~~~~~~~~~~~~~
Older versions of ckanext-archiver put these settings in Older versions of ckanext-archiver put these settings in
ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH
but this is deprecated as of ckanext-archiver 2.0. but this is no longer available.
There used to be an option DATA_FORMATS for filtering the resources There used to be an option DATA_FORMATS for filtering the resources
archived, but that has now been removed in ckanext-archiver v2.0, since it archived, but that has now been removed in ckanext-archiver v2.0, since it
is now not only caching files, but is seen as a broken link checker, which is now not only caching files, but is seen as a broken link checker, which
applies whatever the format. applies whatever the format.
Using Archiver Using Archiver
......
...@@ -16,10 +16,6 @@ from requests.packages import urllib3 ...@@ -16,10 +16,6 @@ from requests.packages import urllib3
from ckan.lib.celery_app import celery from ckan.lib.celery_app import celery
from ckan import plugins as p from ckan import plugins as p
try:
from ckanext.archiver import settings
except ImportError:
from ckanext.archiver import default_settings as settings
from ckanext.archiver import interfaces as archiver_interfaces from ckanext.archiver import interfaces as archiver_interfaces
toolkit = p.toolkit toolkit = p.toolkit
...@@ -201,6 +197,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue): ...@@ -201,6 +197,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
from ckan import model from ckan import model
from pylons import config from pylons import config
from ckan.plugins import toolkit from ckan.plugins import toolkit
from ckanext.archiver import default_settings as settings
get_action = toolkit.get_action get_action = toolkit.get_action
...@@ -301,7 +298,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue): ...@@ -301,7 +298,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
def download(context, resource, url_timeout=30, def download(context, resource, url_timeout=30,
max_content_length=settings.MAX_CONTENT_LENGTH, max_content_length='default',
method='GET'): method='GET'):
'''Given a resource, tries to download it. '''Given a resource, tries to download it.
...@@ -324,8 +321,12 @@ def download(context, resource, url_timeout=30, ...@@ -324,8 +321,12 @@ def download(context, resource, url_timeout=30,
Returns a dict of results of a successful download: Returns a dict of results of a successful download:
mimetype, size, hash, headers, saved_file, url_redirected_to mimetype, size, hash, headers, saved_file, url_redirected_to
''' '''
from ckanext.archiver import default_settings as settings
log = update_resource.get_logger() log = update_resource.get_logger()
if max_content_length == 'default':
max_content_length = settings.MAX_CONTENT_LENGTH
url = resource['url'] url = resource['url']
url = tidy_url(url) url = tidy_url(url)
...@@ -441,6 +442,7 @@ def archive_resource(context, resource, log, result=None, url_timeout=30): ...@@ -441,6 +442,7 @@ def archive_resource(context, resource, log, result=None, url_timeout=30):
Returns: {cache_filepath, cache_url} Returns: {cache_filepath, cache_url}
""" """
from ckanext.archiver import default_settings as settings
relative_archive_path = os.path.join(resource['id'][:2], resource['id']) relative_archive_path = os.path.join(resource['id'][:2], resource['id'])
archive_dir = os.path.join(settings.ARCHIVE_DIR, relative_archive_path) archive_dir = os.path.join(settings.ARCHIVE_DIR, relative_archive_path)
if not os.path.exists(archive_dir): if not os.path.exists(archive_dir):
...@@ -517,6 +519,7 @@ def _set_user_agent_string(headers): ...@@ -517,6 +519,7 @@ def _set_user_agent_string(headers):
Update the passed headers object with a `User-Agent` key, if there is a Update the passed headers object with a `User-Agent` key, if there is a
USER_AGENT_STRING option in settings. USER_AGENT_STRING option in settings.
''' '''
from ckanext.archiver import default_settings as settings
ua_str = settings.USER_AGENT_STRING ua_str = settings.USER_AGENT_STRING
if ua_str is not None: if ua_str is not None:
headers['User-Agent'] = ua_str headers['User-Agent'] = ua_str
......
...@@ -12,14 +12,15 @@ from nose.tools import assert_raises, assert_equal ...@@ -12,14 +12,15 @@ from nose.tools import assert_raises, assert_equal
from ckan import model from ckan import model
from ckan import plugins from ckan import plugins
from ckan.tests import BaseCase
from ckan.logic import get_action from ckan.logic import get_action
try: try:
from ckan.tests.helpers import reset_db from ckan.tests.helpers import reset_db
from ckan.tests import factories as ckan_factories from ckan.tests import factories as ckan_factories
from ckan.tests.legacy import BaseCase
except ImportError: except ImportError:
from ckan.new_tests.helpers import reset_db from ckan.new_tests.helpers import reset_db
from ckan.new_tests import factories as ckan_factories from ckan.new_tests import factories as ckan_factories
from ckan.tests import BaseCase
from ckanext.archiver import model as archiver_model from ckanext.archiver import model as archiver_model
from ckanext.archiver.model import Archival from ckanext.archiver.model import Archival
...@@ -375,6 +376,7 @@ class TestArchiver(BaseCase): ...@@ -375,6 +376,7 @@ class TestArchiver(BaseCase):
assert params.get('package_id') == pkg['id'] assert params.get('package_id') == pkg['id']
assert params.get('resource_id') == None assert params.get('resource_id') == None
class TestDownload(BaseCase): class TestDownload(BaseCase):
'''Tests of the download method (and things it calls). '''Tests of the download method (and things it calls).
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment