Merge branch 'master' of github.com:ckan/ckanext-archiver

README.rst

@@ -2,8 +2,8 @@
    these badges work. The necessary Travis and Coverage config files have been
    generated for you.
 
-.. image:: https://travis-ci.org/datagovuk/ckanext-archiver.svg?branch=master
-    :target: https://travis-ci.org/datagovuk/ckanext-archiver
+.. image:: https://travis-ci.org/ckan/ckanext-archiver.svg?branch=master
+    :target: https://travis-ci.org/ckan/ckanext-archiver
 
 =============
 ckanext-archiver
@@ -269,16 +269,17 @@ Config settings
                 root /www/resource_cache;
             }
 
-Legacy settings:
+Legacy settings
+~~~~~~~~~~~~~~~
 
-   Older versions of ckanext-archiver put these settings in
-   ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH
-   but this is deprecated as of ckanext-archiver 2.0.
+Older versions of ckanext-archiver put these settings in
+ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH
+but this is no longer available.
 
-   There used to be an option DATA_FORMATS for filtering the resources
-   archived, but that has now been removed in ckanext-archiver v2.0, since it
-   is now not only caching files, but is seen as a broken link checker, which
-   applies whatever the format.
+There used to be an option DATA_FORMATS for filtering the resources
+archived, but that has now been removed in ckanext-archiver v2.0, since it
+is now not only caching files, but is seen as a broken link checker, which
+applies whatever the format.
 
 
 Using Archiver

ckanext/archiver/tasks.py

@@ -16,10 +16,6 @@ from requests.packages import urllib3
 
 from ckan.lib.celery_app import celery
 from ckan import plugins as p
-try:
-    from ckanext.archiver import settings
-except ImportError:
-    from ckanext.archiver import default_settings as settings
 from ckanext.archiver import interfaces as archiver_interfaces
 
 toolkit = p.toolkit
@@ -201,6 +197,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
     from ckan import model
     from pylons import config
     from ckan.plugins import toolkit
+    from ckanext.archiver import default_settings as settings
 
     get_action = toolkit.get_action
 
@@ -301,7 +298,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
 
 
 def download(context, resource, url_timeout=30,
-             max_content_length=settings.MAX_CONTENT_LENGTH,
+             max_content_length='default',
              method='GET'):
     '''Given a resource, tries to download it.
 
@@ -324,8 +321,12 @@ def download(context, resource, url_timeout=30,
     Returns a dict of results of a successful download:
       mimetype, size, hash, headers, saved_file, url_redirected_to
     '''
+    from ckanext.archiver import default_settings as settings
     log = update_resource.get_logger()
 
+    if max_content_length == 'default':
+        max_content_length = settings.MAX_CONTENT_LENGTH
+
     url = resource['url']
 
     url = tidy_url(url)
@@ -441,6 +442,7 @@ def archive_resource(context, resource, log, result=None, url_timeout=30):
 
     Returns: {cache_filepath, cache_url}
     """
+    from ckanext.archiver import default_settings as settings
     relative_archive_path = os.path.join(resource['id'][:2], resource['id'])
     archive_dir = os.path.join(settings.ARCHIVE_DIR, relative_archive_path)
     if not os.path.exists(archive_dir):
@@ -517,6 +519,7 @@ def _set_user_agent_string(headers):
     Update the passed headers object with a `User-Agent` key, if there is a
     USER_AGENT_STRING option in settings.
     '''
+    from ckanext.archiver import default_settings as settings
     ua_str = settings.USER_AGENT_STRING
     if ua_str is not None:
         headers['User-Agent'] = ua_str

tests/test_archiver.py

@@ -12,14 +12,15 @@ from nose.tools import assert_raises, assert_equal
 
 from ckan import model
 from ckan import plugins
-from ckan.tests import BaseCase
 from ckan.logic import get_action
 try:
     from ckan.tests.helpers import reset_db
     from ckan.tests import factories as ckan_factories
+    from ckan.tests.legacy import BaseCase
 except ImportError:
     from ckan.new_tests.helpers import reset_db
     from ckan.new_tests import factories as ckan_factories
+    from ckan.tests import BaseCase
 
 from ckanext.archiver import model as archiver_model
 from ckanext.archiver.model import Archival
@@ -375,6 +376,7 @@ class TestArchiver(BaseCase):
         assert params.get('package_id') == pkg['id']
         assert params.get('resource_id') == None
 
+
 class TestDownload(BaseCase):
     '''Tests of the download method (and things it calls).