PDF datasets

ckanext/dgu/lib/reports.py

@@ -752,10 +752,81 @@ def licence_combinations():
 licence_report_info = {
     'name': 'licence',
     'title': 'Licences',
-    'description': 'Licenses for datasets, reported by publisher.',
+    'description': 'Licenses for datasets.',
     'option_defaults': OrderedDict((('organization', None),
                                     ('include_sub_organizations', False))),
     'option_combinations': licence_combinations,
     'generate': licence_report,
     'template': 'report/licence_report.html',
     }
+
+
+# Datasets only in PDF
+
+def pdf_datasets_report():
+    '''
+    Returns datasets that have data in PDF format, by organization.
+    '''
+    # Get packages
+    pkgs = model.Session.query(model.Package)\
+                .filter_by(state='active')
+
+    # See if PDF
+    num_datasets_published = 0
+    num_datasets_only_pdf = 0
+    datasets_by_publisher_only_pdf = collections.defaultdict(list)
+    # use yield_per, otherwise memory use just goes up til the script is killed
+    # by the os.
+    for pkg in pkgs.yield_per(100):
+        if p.toolkit.asbool(pkg.extras.get('unpublished')):
+            continue
+        num_datasets_published += 1
+
+        formats = set([res.format.lower() for res in pkg.resources
+                       if res.resource_type != 'documentation'])
+        if 'pdf' not in formats:
+            continue
+        org = pkg.get_organization().name
+
+        data_formats = formats - set(('html', '', None))
+        if data_formats == set(('pdf',)):
+            num_datasets_only_pdf += 1
+            datasets_by_publisher_only_pdf[org].append((pkg.name, pkg.title))
+
+    rows = []
+    for org_name, datasets_only_pdf in sorted(
+            datasets_by_publisher_only_pdf.iteritems(),
+            key=lambda x: -len(x[1])):
+        org = model.Session.query(model.Group) \
+                   .filter_by(name=org_name) \
+                   .first()
+        top_org = list(go_up_tree(org))[-1]
+
+        row = OrderedDict((
+            ('organization title', org.title),
+            ('organization name', org.name),
+            ('top-level organization title', top_org.title),
+            ('top-level organization name', top_org.name),
+            ('num datasets only pdf', len(datasets_only_pdf)),
+            ('name datasets only pdf',
+             ' '.join(d[0] for d in datasets_only_pdf)),
+            ('title datasets only pdf',
+             '|'.join(d[1] for d in datasets_only_pdf)),
+            ))
+        rows.append(row)
+
+    return {'table': rows,
+            'num_datasets_published': num_datasets_published,
+            'num_datasets_only_pdf': num_datasets_only_pdf,
+            }
+
+
+pdf_datasets_report_info = {
+    'name': 'pdf_datasets',
+    'title': 'PDF Datasets',
+    'description': 'Datasets with data only in PDF format.',
+    'option_defaults': None,
+    'option_combinations': None,
+    'generate': pdf_datasets_report,
+    'template': 'report/pdf_datasets_report.html',
+    }

ckanext/dgu/plugin.py

@@ -378,6 +378,7 @@ class PublisherPlugin(p.SingletonPlugin):
                 reports.admin_editor_info,
                 reports.licence_report_info,
                 reports.la_schemas_info,
+                reports.pdf_datasets_report_info,
                 ]
 
 

ckanext/dgu/theme/templates/report/pdf_datasets_report.html

+<div class="widget-container">
+  <ul>
+    <li>Datasets only in PDF: {{data['num_datasets_only_pdf']}} / {{data['num_datasets_published']}} ({{ '{:.0%}'.format(data['num_datasets_only_pdf']/data['num_datasets_published']) }})</li>
+    <li>Publishers with datasets only in PDF: {{table|length}}</li>
+  </ul>
+  <p>NB The full lists of datasets are available in the CSV &amp; JSON downloads</p>
+
+  <table class="table table-bordered table-condensed tablesorter" id="report-table" style="width: 100%;table-layout:fixed; margin-top: 8px;">
+    <thead>
+      <tr>
+        <th style="width: 80px" rowspan="2">Publisher</th>
+        <th style="width: 80px" rowspan="2">Top-level Publisher</th>
+        <th style="width: 300px" rowspan="2">Datasets only in PDF</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for row in table %}
+        <tr>
+          <td><a href="/publisher/{{row['organization name']}}">{{row['organization title']}}</a></td>
+          <td><a href="/publisher/{{row['top-level organization name']}}">{{row['top-level organization title']}}</a></td>
+          <td>
+      {% set names = row['name datasets only pdf'].split() %}
+      {% set titles = row['title datasets only pdf'].split('|') %}
+      {% set truncate = 5 %}
+      {{row['num datasets only pdf']}}:
+      {% for name in names[:truncate] %}
+        <a href="{{ h.url_for(controller='package', action='read', id=name) }}">{{ titles[loop.index0] }}</a> &nbsp;
+      {% endfor %}
+      {% if names|length > truncate %}
+         ...
+      {% endif %}
+          </td>
+        </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+
+</div>
+
+