Merge pull request #21 from ckan/20-config

Config was not being read by tasks.py

README.rst

@@ -269,16 +269,17 @@ Config settings
                 root /www/resource_cache;
             }
 
-Legacy settings:
+Legacy settings
+~~~~~~~~~~~~~~~
 
-   Older versions of ckanext-archiver put these settings in
-   ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH
-   but this is deprecated as of ckanext-archiver 2.0.
+Older versions of ckanext-archiver put these settings in
+ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH
+but this is no longer available.
 
-   There used to be an option DATA_FORMATS for filtering the resources
-   archived, but that has now been removed in ckanext-archiver v2.0, since it
-   is now not only caching files, but is seen as a broken link checker, which
-   applies whatever the format.
+There used to be an option DATA_FORMATS for filtering the resources
+archived, but that has now been removed in ckanext-archiver v2.0, since it
+is now not only caching files, but is seen as a broken link checker, which
+applies whatever the format.
 
 
 Using Archiver

ckanext/archiver/tasks.py

@@ -16,10 +16,6 @@ from requests.packages import urllib3
 
 from ckan.lib.celery_app import celery
 from ckan import plugins as p
-try:
-    from ckanext.archiver import settings
-except ImportError:
-    from ckanext.archiver import default_settings as settings
 from ckanext.archiver import interfaces as archiver_interfaces
 
 toolkit = p.toolkit
@@ -201,6 +197,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
     from ckan import model
     from pylons import config
     from ckan.plugins import toolkit
+    from ckanext.archiver import default_settings as settings
 
     get_action = toolkit.get_action
 
@@ -301,7 +298,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
 
 
 def download(context, resource, url_timeout=30,
-             max_content_length=settings.MAX_CONTENT_LENGTH,
+             max_content_length='default',
              method='GET'):
     '''Given a resource, tries to download it.
 
@@ -324,8 +321,12 @@ def download(context, resource, url_timeout=30,
     Returns a dict of results of a successful download:
       mimetype, size, hash, headers, saved_file, url_redirected_to
     '''
+    from ckanext.archiver import default_settings as settings
     log = update_resource.get_logger()
 
+    if max_content_length == 'default':
+        max_content_length = settings.MAX_CONTENT_LENGTH
+
     url = resource['url']
 
     url = tidy_url(url)
@@ -441,6 +442,7 @@ def archive_resource(context, resource, log, result=None, url_timeout=30):
 
     Returns: {cache_filepath, cache_url}
     """
+    from ckanext.archiver import default_settings as settings
     relative_archive_path = os.path.join(resource['id'][:2], resource['id'])
     archive_dir = os.path.join(settings.ARCHIVE_DIR, relative_archive_path)
     if not os.path.exists(archive_dir):
@@ -517,6 +519,7 @@ def _set_user_agent_string(headers):
     Update the passed headers object with a `User-Agent` key, if there is a
     USER_AGENT_STRING option in settings.
     '''
+    from ckanext.archiver import default_settings as settings
     ua_str = settings.USER_AGENT_STRING
     if ua_str is not None:
         headers['User-Agent'] = ua_str