Commit d570c348 authored by David Read's avatar David Read
Browse files

Merge pull request #21 from ckan/20-config

Config was not being read by tasks.py
parents 9640a4d2 aad8db48
...@@ -269,16 +269,17 @@ Config settings ...@@ -269,16 +269,17 @@ Config settings
root /www/resource_cache; root /www/resource_cache;
} }
Legacy settings: Legacy settings
~~~~~~~~~~~~~~~
Older versions of ckanext-archiver put these settings in Older versions of ckanext-archiver put these settings in
ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH ckanext/archiver/settings.py as variables ARCHIVE_DIR and MAX_CONTENT_LENGTH
but this is deprecated as of ckanext-archiver 2.0. but this is no longer available.
There used to be an option DATA_FORMATS for filtering the resources There used to be an option DATA_FORMATS for filtering the resources
archived, but that has now been removed in ckanext-archiver v2.0, since it archived, but that has now been removed in ckanext-archiver v2.0, since it
is now not only caching files, but is seen as a broken link checker, which is now not only caching files, but is seen as a broken link checker, which
applies whatever the format. applies whatever the format.
Using Archiver Using Archiver
......
...@@ -16,10 +16,6 @@ from requests.packages import urllib3 ...@@ -16,10 +16,6 @@ from requests.packages import urllib3
from ckan.lib.celery_app import celery from ckan.lib.celery_app import celery
from ckan import plugins as p from ckan import plugins as p
try:
from ckanext.archiver import settings
except ImportError:
from ckanext.archiver import default_settings as settings
from ckanext.archiver import interfaces as archiver_interfaces from ckanext.archiver import interfaces as archiver_interfaces
toolkit = p.toolkit toolkit = p.toolkit
...@@ -201,6 +197,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue): ...@@ -201,6 +197,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
from ckan import model from ckan import model
from pylons import config from pylons import config
from ckan.plugins import toolkit from ckan.plugins import toolkit
from ckanext.archiver import default_settings as settings
get_action = toolkit.get_action get_action = toolkit.get_action
...@@ -301,7 +298,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue): ...@@ -301,7 +298,7 @@ def _update_resource(ckan_ini_filepath, resource_id, queue):
def download(context, resource, url_timeout=30, def download(context, resource, url_timeout=30,
max_content_length=settings.MAX_CONTENT_LENGTH, max_content_length='default',
method='GET'): method='GET'):
'''Given a resource, tries to download it. '''Given a resource, tries to download it.
...@@ -324,8 +321,12 @@ def download(context, resource, url_timeout=30, ...@@ -324,8 +321,12 @@ def download(context, resource, url_timeout=30,
Returns a dict of results of a successful download: Returns a dict of results of a successful download:
mimetype, size, hash, headers, saved_file, url_redirected_to mimetype, size, hash, headers, saved_file, url_redirected_to
''' '''
from ckanext.archiver import default_settings as settings
log = update_resource.get_logger() log = update_resource.get_logger()
if max_content_length == 'default':
max_content_length = settings.MAX_CONTENT_LENGTH
url = resource['url'] url = resource['url']
url = tidy_url(url) url = tidy_url(url)
...@@ -441,6 +442,7 @@ def archive_resource(context, resource, log, result=None, url_timeout=30): ...@@ -441,6 +442,7 @@ def archive_resource(context, resource, log, result=None, url_timeout=30):
Returns: {cache_filepath, cache_url} Returns: {cache_filepath, cache_url}
""" """
from ckanext.archiver import default_settings as settings
relative_archive_path = os.path.join(resource['id'][:2], resource['id']) relative_archive_path = os.path.join(resource['id'][:2], resource['id'])
archive_dir = os.path.join(settings.ARCHIVE_DIR, relative_archive_path) archive_dir = os.path.join(settings.ARCHIVE_DIR, relative_archive_path)
if not os.path.exists(archive_dir): if not os.path.exists(archive_dir):
...@@ -517,6 +519,7 @@ def _set_user_agent_string(headers): ...@@ -517,6 +519,7 @@ def _set_user_agent_string(headers):
Update the passed headers object with a `User-Agent` key, if there is a Update the passed headers object with a `User-Agent` key, if there is a
USER_AGENT_STRING option in settings. USER_AGENT_STRING option in settings.
''' '''
from ckanext.archiver import default_settings as settings
ua_str = settings.USER_AGENT_STRING ua_str = settings.USER_AGENT_STRING
if ua_str is not None: if ua_str is not None:
headers['User-Agent'] = ua_str headers['User-Agent'] = ua_str
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment