From d87ca3815721d0e61b70ef2b4fe51a34793c075f Mon Sep 17 00:00:00 2001
From: Matevz Markovic <matevz.markovic@xlab.si>
Date: Thu, 20 Oct 2016 15:02:42 +0200
Subject: [PATCH] =?UTF-8?q?Prevodi=20v=20Sloven=C5=A1=C4=8Dino.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
ckanext/qa/plugin.py | 121 +++++++++++++++++-
.../qa/resource_format_openness_scores.json | 10 +-
ckanext/qa/tasks.py | 9 +-
ckanext/qa/templates/qa/openness_stars.html | 8 +-
.../qa/templates/qa/openness_stars_brief.html | 4 +-
ckanext/qa/templates/qa/stars_explained.html | 16 ++-
ckanext/qa/templates/report/openness.html | 40 +++---
7 files changed, 168 insertions(+), 40 deletions(-)
diff --git a/ckanext/qa/plugin.py b/ckanext/qa/plugin.py
index 7f74a98..2109488 100644
--- a/ckanext/qa/plugin.py
+++ b/ckanext/qa/plugin.py
@@ -25,6 +25,123 @@ class QAPlugin(p.SingletonPlugin, p.toolkit.DefaultDatasetForm):
p.implements(p.ITemplateHelpers)
p.implements(p.IPackageController, inherit=True)
+ #++++++++++++++++++++++++++++++++ MJU-POPS change +++++++++++++++++++++++++++++++++++++
+ #Ckanext-QA notifies itself on package update, and not through archiver
+ p.implements(p.IDomainObjectModification, inherit=True)
+
+ # IDomainObjectModification
+
+ def notify(self, entity, operation=None):
+ if not isinstance(entity, model.Package):
+ return
+
+ log.debug('Ckanext-QA: Notified of package event: %s %s', entity.name, operation)
+
+ run_notify = self._is_it_sufficient_change_to_run_notifyQA(entity, operation)
+ if not run_notify:
+ return
+
+ log.debug('Creating notifier task: %s', entity.name)
+
+ #!!!!!!!!!!
+ #lib.create_archiver_package_task(entity, 'priority')
+ #?????????? --> za posamezen vir (resorce) Celeryd ni potrebno obveščati.
+ lib.create_qa_package_task(entity, 'priority')
+
+
+ def _is_it_sufficient_change_to_run_notifyQA(self, package, operation):
+ ''' Returns True if in this revision any of these happened:
+ * it is a new dataset
+ * dataset licence changed (affects qa)
+ * there are resources that have been added or deleted
+ * resources have changed their URL or format (affects qa)
+ '''
+ if operation == 'new':
+ log.debug('New package - will archive')
+ # even if it has no resources, QA needs to show 0 stars against it
+ return True
+ elif operation == 'deleted':
+ log.debug('Deleted package - won\'t notify QA')
+ return False
+ # therefore operation=changed
+
+ # check to see if resources are added, deleted or URL changed
+
+ # look for the latest revision
+ rev_list = package.all_related_revisions
+ if not rev_list:
+ log.debug('No sign of previous revisions - will notify QA')
+ return True
+ # I am not confident we can rely on the info about the current
+ # revision, because we are still in the 'before_commit' stage. So
+ # simply ignore that if it's returned.
+ if rev_list[0][0].id == model.Session.revision.id:
+ rev_list = rev_list[1:]
+ if not rev_list:
+ log.warn('No sign of previous revisions - will notify QA')
+ return True
+ previous_revision = rev_list[0][0]
+ log.debug('Comparing with revision: %s %s',
+ previous_revision.timestamp, previous_revision.id)
+
+ # get the package as it was at that previous revision
+ context = {'model': model, 'session': model.Session,
+ #'user': c.user or c.author,
+ 'ignore_auth': True,
+ 'revision_id': previous_revision.id}
+ data_dict = {'id': package.id}
+ try:
+ old_pkg_dict = p.toolkit.get_action('package_show')(
+ context, data_dict)
+ except p.toolkit.NotFound:
+ log.warn('No sign of previous package - will notify QA anyway')
+ return True
+
+ # has the licence changed?
+ old_licence = (old_pkg_dict['license_id'],
+ lib.get_extra_from_pkg_dict(old_pkg_dict, 'licence')
+ or None)
+ new_licence = (package.license_id,
+ package.extras.get('licence') or None)
+ if old_licence != new_licence:
+ log.debug('Licence has changed - will notify QA: %r->%r',
+ old_licence, new_licence)
+ return True
+
+ # have any resources been added or deleted?
+ old_resources = dict((res['id'], res)
+ for res in old_pkg_dict['resources'])
+ old_res_ids = set(old_resources.keys())
+ new_res_ids = set((res.id for res in package.resources))
+ deleted_res_ids = old_res_ids - new_res_ids
+ if deleted_res_ids:
+ log.debug('Deleted resources - will notify QA. res_ids=%r',
+ deleted_res_ids)
+ return True
+ added_res_ids = new_res_ids - old_res_ids
+ if added_res_ids:
+ log.debug('Added resources - will notify QA. res_ids=%r',
+ added_res_ids)
+ return True
+
+ # have any resources' url/format changed?
+ for res in package.resources:
+ for key in ('url', 'format'):
+ old_res_value = old_resources[res.id][key]
+ new_res_value = getattr(res, key)
+ if old_res_value != new_res_value:
+ log.debug('Resource %s changed - will archive. '
+ 'id=%s pos=%s url="%s"->"%s"',
+ key, res.id[:4], res.position,
+ old_res_value, new_res_value)
+ return True
+ log.debug('Resource unchanged. pos=%s id=%s',
+ res.position, res.id[:4])
+
+ log.debug('No new, deleted or changed resources - won\'t notify QA')
+ return False
+ #-------------------------------- MJU-POPS change -------------------------------------
+
# IConfigurer
def update_config(self, config):
@@ -46,8 +163,8 @@ class QAPlugin(p.SingletonPlugin, p.toolkit.DefaultDatasetForm):
def receive_data(self, operation, queue, **params):
'''Receive notification from ckan-archiver that a dataset has been
archived.'''
- if not operation == 'package-archived':
- return
+ #if not operation == 'package-archived':
+ # return
dataset_id = params['package_id']
dataset = model.Package.get(dataset_id)
diff --git a/ckanext/qa/resource_format_openness_scores.json b/ckanext/qa/resource_format_openness_scores.json
index 864e750..7742942 100644
--- a/ckanext/qa/resource_format_openness_scores.json
+++ b/ckanext/qa/resource_format_openness_scores.json
@@ -1,10 +1,10 @@
[
["_comment",
- "JSON field order as follows:",
- ["Format", "Score"],
- "where:",
- " * Format - the short name for it, corresponding to the name in ckan's resource_formats.json.",
- " * Score - openness score to award it. Integer from 0 to 5. 0 means it is likely not data e.g. a web page is probably just a link to the data, an OWL file is an ontology rather than data, an EXE is probably not data, ZIP will be looked inside but we can't be sure its data so gets 1, images may be satellite weather images so give benefit of doubt. RDFa is likely basic document info for a web page added automatically by a CMS, not real data."
+ "Zaporedje zapisov JSON:",
+ ["Format", "Ocena"],
+ "kjer:",
+ " * Format - kratko ime formata, v skladu z zapisom v ckanovem resource_formats.json.",
+ " * Ocena - ocena odprtosti. Celo število od 0 do 5. 0 pomeni, da verjetno ne gre za podatke, na primer spletna stran, ki le kaže na podatke, datoteka OWL (le ontologija in ne sami podatki), izvršljiva datoteka EXE, datoteka ZIP, v katero se bo pogledalo vendar pa ne moremo biti zagotovi, ali gre za podatke (zaradi tega dobi oceno 1), slike so lahke satelitske slike vremena. RDFa je najverjetneje dokument z osnovnimi podatki o spletni strani, ki je bila avtomatsko dodana preko CSM in ne vsebuje pravih podatkov."
],
["N3", 5],
["SPARQL", 5],
diff --git a/ckanext/qa/tasks.py b/ckanext/qa/tasks.py
index 0a185c7..20d6226 100644
--- a/ckanext/qa/tasks.py
+++ b/ckanext/qa/tasks.py
@@ -20,15 +20,22 @@ class QAError(Exception):
# Description of each score, used elsewhere
OPENNESS_SCORE_DESCRIPTION = {
+'''
0: 'Not obtainable or license is not open',
1: 'Obtainable and open license',
2: 'Machine readable format',
3: 'Open and standardized format',
4: 'Ontologically represented',
5: 'Fully Linked Open Data as appropriate',
+'''
+ 0: 'Ni dosegljivo ali pa licenca ni odprta',
+ 1: 'Dosegljivo ter odprta licenca',
+ 2: 'Strojno-berljiv format',
+ 3: 'Odprt ter standardiziran format',
+ 4: 'Ontološko predstavljiv',
+ 5: 'Popolnoma povezani odprti podatki',
}
-
def load_config(ckan_ini_filepath):
import paste.deploy
config_abs_path = os.path.abspath(ckan_ini_filepath)
diff --git a/ckanext/qa/templates/qa/openness_stars.html b/ckanext/qa/templates/qa/openness_stars.html
index 33dbd87..6e3ebe9 100644
--- a/ckanext/qa/templates/qa/openness_stars.html
+++ b/ckanext/qa/templates/qa/openness_stars.html
@@ -20,11 +20,11 @@ OR the package_show's package['qa'] e.g.
#}
<div class="qa openness-{{ openness_score }}">
{% if openness_score != None %}
- Openness: {% snippet "qa/stars.html", stars=openness_score %} <br>
- Reason: {{ openness_score_reason }} <br>
- <span>Checked: {{ h.render_datetime(updated) }}</span><br>
+ Odprtost: {% snippet "qa/stars.html", stars=openness_score %} <br>
+ Razlog: {{ openness_score_reason }} <br>
+ <span>Preverjeno: {{ h.render_datetime(updated) }}</span><br>
{% else %}
- No openness information yet. Update expected soon.
+ Ni še informacij glede odprtosti. Le-te se pričakujejo v kratkem.
{% endif %}
</div>
diff --git a/ckanext/qa/templates/qa/openness_stars_brief.html b/ckanext/qa/templates/qa/openness_stars_brief.html
index 47c59b0..4aa3f59 100644
--- a/ckanext/qa/templates/qa/openness_stars_brief.html
+++ b/ckanext/qa/templates/qa/openness_stars_brief.html
@@ -20,10 +20,10 @@ OR the package_show's package['qa'] e.g.
#}
<div class="qa openness-{{ openness_score }}">
{% if openness_score != None %}
- <span title="Reason: {{ openness_score_reason }} Checked: {{ h.render_datetime(updated) }}">
+ <span title="Reason: {{ openness_score_reason }} Preverjeno: {{ h.render_datetime(updated) }}">
{% snippet "qa/stars.html", stars=openness_score %} <br>
{% else %}
- No openness information yet
+ Informacij glede odprtosti še ni
{% endif %}
</div>
diff --git a/ckanext/qa/templates/qa/stars_explained.html b/ckanext/qa/templates/qa/stars_explained.html
index e6c1cb8..27e0bca 100644
--- a/ckanext/qa/templates/qa/stars_explained.html
+++ b/ckanext/qa/templates/qa/stars_explained.html
@@ -4,13 +4,17 @@ Pass in:
openness_score - number of stars (or None if you don't want to highlight a particular one)
#}
-<div class="star-openness_score-entry {% if openness_score != None and openness_score < 5 %}fail{% endif %}">★★★★★ Linked data - data URIs and linked to other data (e.g. RDF)</div>
+{# <div class="star-openness_score-entry {% if openness_score != None and openness_score < 5 %}fail{% endif %}">★★★★★ Linked data - data URIs and linked to other data (e.g. RDF)</div> #}
+<div class="star-openness_score-entry {% if openness_score != None and openness_score < 5 %}fail{% endif %}">★★★★★ Povezani podatki - podatkovni URI-ji ter povezave na ostale podatke (na primer RDF)</div>
-<div class="star-openness_score-entry {% if openness_score != None and openness_score < 4 %}fail{% endif %}">★★★★☆ Linkable data - served at URIs (e.g. RDF)</div>
+{# <div class="star-openness_score-entry {% if openness_score != None and openness_score < 4 %}fail{% endif %}">★★★★☆ Linkable data - served at URIs (e.g. RDF)</div> #}
+<div class="star-openness_score-entry {% if openness_score != None and openness_score < 4 %}fail{% endif %}">★★★★☆ Povezljivi podatki - servirani na URI-jih (na primer RDF)</div>
-<div class="star-openness_score-entry {% if openness_score != None and openness_score < 3 %}fail{% endif %}">★★★☆☆ Structured data in open format (e.g. CSV)</div>
+{# <div class="star-openness_score-entry {% if openness_score != None and openness_score < 3 %}fail{% endif %}">★★★☆☆ Structured data in open format (e.g. CSV)</div> #}
+<div class="star-openness_score-entry {% if openness_score != None and openness_score < 3 %}fail{% endif %}">★★★☆☆ Strukturirani podatki ter odprt format (na primer CSV)</div>
-<div class="star-openness_score-entry {% if openness_score != None and openness_score < 2 %}fail{% endif %}fail">★★☆☆☆ Structured data but proprietry format (e.g. Excel)</div>
-
-<div class="star-openness_score-entry {% if openness_score != None and openness_score < 1 %}fail{% endif %}">★☆☆☆☆ Unstructured data (e.g. PDF)</div>
+{# <div class="star-openness_score-entry {% if openness_score != None and openness_score < 2 %}fail{% endif %}fail">★★☆☆☆ Structured data but proprietry format (e.g. Excel)</div> #}
+<div class="star-openness_score-entry {% if openness_score != None and openness_score < 2 %}fail{% endif %}fail">★★☆☆☆ Strukturirani podatki, a lastniški format (na primer Excel)</div>
+{# <div class="star-openness_score-entry {% if openness_score != None and openness_score < 1 %}fail{% endif %}">★☆☆☆☆ Unstructured data (e.g. PDF)</div> #}
+<div class="star-openness_score-entry {% if openness_score != None and openness_score < 1 %}fail{% endif %}">★☆☆☆☆ Nestrukturirani podatki (na primer PDF)</div>
diff --git a/ckanext/qa/templates/report/openness.html b/ckanext/qa/templates/report/openness.html
index 7381997..55cb23d 100644
--- a/ckanext/qa/templates/report/openness.html
+++ b/ckanext/qa/templates/report/openness.html
@@ -1,13 +1,13 @@
{% if c.options['organization'] == None %}
<ul>
- <li>Datasets given a score: {{ c.data['num_packages_scored'] }} / {{ c.data['num_packages'] }}</li>
- <li>Totals
+ <li>Ocenjene podatkovne zbirke: {{ c.data['num_packages_scored'] }} / {{ c.data['num_packages'] }}</li>
+ <li>Skupno
<table class="table table-striped table-bordered table-condensed">
<thead>
<tr>
- <th>Score TBC</th>
+ <th>Ocene TBC</th>
{% for n in range(6) %}
- <th>Score {{n}}</th>
+ <th>Ocena {{n}}</th>
{% endfor %}
</tr>
</thead>
@@ -25,13 +25,13 @@
<table class="table table-striped table-bordered table-condensed tablesorter" id="report-table">
<thead>
<tr>
- <th>Publisher</th>
- <th>Score TBC</th>
+ <th>Izdajatelj</th>
+ <th>Ocene TBC</th>
{% for n in range(6) %}
- <th>Score {{n}}</th>
+ <th>Ocena {{n}}</th>
{% endfor %}
- <th>Total stars</th>
- <th>Average stars</th>
+ <th>Skupaj zvezd</th>
+ <th>Skupna ocena</th>
</tr>
</thead>
<tbody>
@@ -52,16 +52,16 @@
{% else %}
<ul>
- <li>Average score: {{ c.data['average_stars'] }}</li>
- <li>Total stars: {{ c.data['total_stars'] }}</li>
- <li>Datasets given a score: {{ c.data['num_packages_scored'] }} / {{ c.data['num_packages'] }}</li>
- <li>Score frequencies:
+ <li>Povprečje: {{ c.data['average_stars'] }}</li>
+ <li>Skupna ocena: {{ c.data['total_stars'] }}</li>
+ <li>Ocenjene podatkovne zbirke: {{ c.data['num_packages_scored'] }} / {{ c.data['num_packages'] }}</li>
+ <li>Frekvence ocen:
<table class="table table-striped table-bordered table-condensed">
<thead>
<tr>
- <th>Score TBC</th>
+ <th>Ocene TBC</th>
{% for n in range(6) %}
- <th>Score {{n}}</th>
+ <th>Ocena {{n}}</th>
{% endfor %}
</tr>
</thead>
@@ -79,13 +79,13 @@
<table class="table table-striped table-bordered table-condensed tablesorter" id="report-table">
<thead>
<tr>
- <th>Dataset</th>
- <th>Notes</th>
+ <th>Podatkovna zbirka</th>
+ <th>Zapiski</th>
{% if c.options['include_sub_organizations'] %}
- <th>Publisher</th>
+ <th>Izdajatelj</th>
{% endif %}
- <th>Score</th>
- <th>Score reason</th>
+ <th>Ocena</th>
+ <th>Razlog ocene</th>
</tr>
</thead>
<tbody>
--
GitLab