Commit 8119428a authored by Marko Kuder's avatar Marko Kuder
Browse files

Merge branch 'development' into feature/downloadall

parents 2d3978f2 f50b2f5e
......@@ -330,6 +330,11 @@ def opsi_user_list(context, data_dict):
return {'success': True}
def opsi_user_delete(context, data_dict):
# Sysadmins only
return {'success': False, 'msg': _('Only sysadmins can delete publishers')}
def dgu_organization_delete(context, data_dict):
# Sysadmins only
return { 'success': False, 'msg': _('Only sysadmins can delete publishers') }
......
import json
import logging
from ckan.lib.cli import CkanCommand
# No other CKAN imports allowed until _load_config is run,
# or logging is disabled
class RemoveRevisions(CkanCommand):
"""
Removing old revisions
"""
summary = __doc__.split('\n')[0]
usage = __doc__
max_args = 1
min_args = 1
attributes = ['resource_group_id', 'url', 'format', 'description', 'position', 'state', 'name', 'resource_type', 'url_type']
def __init__(self, name):
super(CkanCommand, self).__init__(name)
def date_from_string(self, date_str):
from datetime import datetime
try:
_date = datetime.strptime(date_str, '%Y-%m-%d')
return _date
except:
print('Wrong date format!')
return None
def command(self):
self._load_config()
self.log = logging.getLogger(__name__)
pck_name = self.args[0]
self.remove(pck_name)
def are_equal(self, old_revision, new_revision):
equal = True
for attr in self.attributes:
if attr == 'url' and getattr(old_revision, 'url_type') == 'upload' and getattr(new_revision, 'url_type') == 'upload':
#compare without protocol prefix and domain, because on uploaded file url generation,
#some plugins might change http into https or localhost might be stored instead of site url
old_url = getattr(old_revision, attr).split('//',1)[-1].split('/',1)[-1]
new_url = getattr(new_revision, attr).split('//',1)[-1].split('/',1)[-1]
if old_url != new_url:
equal = False
break
elif getattr(old_revision, attr) != getattr(new_revision, attr):
equal = False
break
return equal
def remove(self, pck_name):
from ckan import model
import ckan.lib.dictization as d
def chunks(l, n):
'''Yield successive n-sized chunks from l.'''
for i in xrange(0, len(l), n):
yield l[i:i + n]
def delete(res_revs, res):
sql = ['''
ALTER TABLE package_tag DROP CONSTRAINT package_tag_revision_id_fkey;
ALTER TABLE package_extra DROP CONSTRAINT package_extra_revision_id_fkey;
ALTER TABLE resource DROP CONSTRAINT resource_revision_id_fkey;
''']
for res_rev in res_revs:
sql.append("DELETE from resource_revision where id='%s' and revision_id='%s';\n" % (res.id, res_rev.revision_id))
# a revision created (e.g. over the API) can be connect to other
# resources or a dataset, so only delete the revision if only
# connected to this one.
if model.Session.query(model.ResourceRevision). \
filter_by(revision_id=res_rev.revision_id). \
count() == 1 and \
model.Session.query(model.PackageRevision). \
filter_by(revision_id=res_rev.revision_id).count() == 0 and \
model.Session.query(model.PackageExtraRevision).filter_by(revision_id=res_rev.revision_id).count() == 0:
sql.append("DELETE from revision where id='%s';\n" % res_rev.revision_id)
# sql.append("UPDATE resource SET revision_id='%s' WHERE id='%s';\n" % \
# (latest_good_res_rev.revision_id, res.id))
sql.append('''
ALTER TABLE package_tag ADD CONSTRAINT package_tag_revision_id_fkey FOREIGN KEY (revision_id) REFERENCES revision(id);
ALTER TABLE package_extra ADD CONSTRAINT package_extra_revision_id_fkey FOREIGN KEY (revision_id) REFERENCES revision(id);
ALTER TABLE resource ADD CONSTRAINT resource_revision_id_fkey FOREIGN KEY (revision_id) REFERENCES revision(id);
''')
print("Executing sql....")
model.Session.execute(''.join(sql))
print("Committing sql....")
model.Session.commit()
print("Sql committed.")
model.Session.remove()
resources = model.Session.query(model.Resource) \
.filter_by(state='active') \
.join(model.ResourceGroup) \
.join(model.Package)\
.filter(model.Package.name == pck_name) \
.order_by(model.Resource.position).all()
for resource in resources:
resource = model.Resource.get(resource.id)
print(resource.id)
revisions = model.Session.query(model.ResourceRevision)\
.filter_by(id=resource.id) \
.order_by(model.ResourceRevision.revision_timestamp)\
.all()
print("Resource: %s" % resource.url)
print("Num revisions: %d" % len(revisions))
revisions_dict = {rev.revision_id: rev for rev in revisions}
grouped_revisions = {}
current_group = 0
prev = None
for revision in revisions:
if prev is None:
prev = revision
grouped_revisions[current_group] = [revision.revision_id]
continue
if not self.are_equal(prev, revision):
current_group += 1
group = grouped_revisions.get(current_group, [])
group.append(revision.revision_id)
grouped_revisions[current_group] = group
prev = revision
revisions_to_remove = []
for key, grouped in grouped_revisions.items():
if len(grouped) <= 2:
continue
_tmp = grouped[1:-1]
for revision_id in _tmp:
revision = revisions_dict.get(revision_id, None)
if revision is not None:
revisions_to_remove.append(revision)
print("Num revisions to remove: %d" % len(revisions_to_remove))
print("Revisions to remove: %s" % ','.join([x.revision_id for x in revisions_to_remove]))
for chunk_of_res_revs in chunks(revisions_to_remove, 1000):
print('Preparing sql')
delete(chunk_of_res_revs, resource)
print('-------------------------------------------------------------------------')
model.Session.commit()
model.Session.remove()
return
# -*- encoding: utf-8 -*-
import exceptions
import re
import urllib2
from urllib import urlencode
......@@ -266,6 +267,8 @@ class PackageController(ckan.controllers.package.PackageController):
if title.find('\n') > 0:
title = title[0:title.find('\n')].strip()
return '{"name": "'+ title + '" , "url": "' + (PISRS_URL + id)+'"}'
except exceptions.UnicodeError:
return '{"name": "Vnos ni v pravilnem formatu." , "url":""}'
except URLError:
return '{"name": "Napaka pri poizvedbi SOP" , "url":' + (PISRS_URL + id) + '"}'
return '{"name": "Predpis s tem SOP ni bil najden" , "url": "' + (PISRS_URL + id) + '"}'
......@@ -469,10 +472,10 @@ class PackageController(ckan.controllers.package.PackageController):
except HTTPError:
errors['captcha'] = [u'Napaka pri preverjanju captcha']
error_summary['captcha'] = u'Prišlo je do napake pri preverjanju ReCAPTCHA. Prosimo, poskusite ponovno.'
if data_dict["message"].strip() == '':
if data_dict.get('message', '').strip() == '':
errors['message'] = [u'Manjkajoča vrednost']
error_summary['message'] = u'Komentar je obvezen.'
elif data_dict.get('email', '') == '':
elif data_dict.get('email', '').strip() == '':
errors['email'] = [u'Manjkajoča vrednost']
error_summary['email'] = u'Email je obvezen.'
elif not re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)").match(data_dict.get('email', '')):
......@@ -527,12 +530,12 @@ class PackageController(ckan.controllers.package.PackageController):
except logic.NotAuthorized:
base.abort(401, _('Not authorized to see this page'))
if data_dict["message"].strip() == '' and (not 'publish' in request.params):
if data_dict.get('message', '').strip() == '') and (not 'publish' in request.params):
errors['message'] = [u'Če zbirka ne bo objavljena, je komentar obvezen.']
error_summary['message'] = u'Manjkajoča vrednost'
vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
return p.toolkit.render('package/change_status.html', extra_vars=vars)
if data_dict["message"].strip() != '':
if data_dict.get('message', '').strip() != '':
er, er_sum = self._send_message_to_owner(pkg_dict, data_dict)
errors.update(er)
error_summary.update(er_sum)
......
# -*- coding: utf-8 -*-
import ckan.plugins.toolkit as t
import ckan.lib.helpers as h
import ckan.logic as logic
......@@ -6,6 +8,8 @@ import ckan.new_authz as new_authz
from ckan.lib.base import BaseController, abort, render
from ckan.controllers.user import UserController as CoreUserController
from ckan.common import _, request
from ckanext.dgu.lib.user_helpers import merge_users_on_delete, move_user
from ckanext.dgu.lib.reports import refresh_opsi_reports_async
c = t.c
get_action = logic.get_action
......@@ -13,6 +17,7 @@ check_access = logic.check_access
NotFound = logic.NotFound
NotAuthorized = logic.NotAuthorized
class UserController(CoreUserController):
def me(self, locale=None):
if not c.user:
......@@ -69,3 +74,67 @@ class UserController(CoreUserController):
c.users = users_list
return render('user/list.html')
def delete(self, id):
errors = {}
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj,
'for_view': True}
data_dict = {'id': id,
'user_obj': c.userobj}
context['with_related'] = False
try:
check_access('user_delete', context, data_dict)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
c.is_sysadmin = new_authz.is_sysadmin(c.user)
try:
user_dict = get_action('user_show')(context, data_dict)
except NotFound:
abort(404, _('User not found'))
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
c.user_dict = user_dict
c.is_myself = user_dict['name'] == c.user
if request.POST:
if 'cancel' in request.params:
h.redirect_to(h.url_for(controller='ckanext.dgu.controllers.user:UserController', action='read', id=id))
return
else:
if c.is_myself:
errors['user'] = "Uporabnika ni dovoljeno izbrisati."
new_user_id = request.params.get('creator_user_id', None)
if c.user_dict['number_administered_packages'] > 0:
if not new_user_id:
errors[_('Editor')] = _("This field is required")
if new_user_id == c.user_dict['id'] or new_user_id == c.user_dict['name']:
errors[_('Editor')] = u"Napačen izbor urednika (%s)" % c.user_dict['display_name']
if not errors:
if c.user_dict['number_administered_packages'] > 0:
merged_users = merge_users_on_delete(c.user_dict['id'], new_user_id, context)
_error = merged_users.get('error', None)
else:
_error = None
if _error:
errors[_('Editor')] = _error
else:
try:
get_action('user_delete')(context, data_dict)
user_index = h.url_for(controller='ckanext.dgu.controllers.user:UserController', action='index')
h.flash_success(_('%s has been deleted.') % _('User'))
h.redirect_to(user_index)
except NotAuthorized:
msg = _('Unauthorized to delete user with id "{user_id}".')
abort(401, msg.format(user_id=id))
c.errors = errors
return render('user/delete.html')
......@@ -79,4 +79,5 @@ def _update_search_index(package_id, log):
package = p.toolkit.get_action('package_show')(
context_, {'id': package_id})
package_index.index_package(package, defer_commit=False)
model.Session.commit()
log.info('Search indexed %s', package['name'])
......@@ -24,7 +24,7 @@ def refresh_opsi_reports():
import time
registry = ReportRegistry.instance()
timings = {}
report_list = ['unpublished']
report_list = ['unpublished', 'draft']
for report_name in report_list:
s = time.time()
registry.get_report(report_name).refresh_cache_for_all_options()
......@@ -381,6 +381,43 @@ unpublished_report_info = {
'template': 'report/unpublished.html',
}
def draft():
pkgs = model.Session.query(model.Package)\
.filter(model.Package.state=='draft')\
.join(model.PackageExtra)\
.filter_by(key='unpublished')\
.filter_by(value='false')\
.all()
pkg_dicts = []
for pkg in pkgs:
org = pkg.get_organization()
podrocje = pkg.extras.get('podrocje', u'')
author = dgu_helpers.get_user_name(pkg.creator_user_id)
pkg_dict = {
'id': pkg.id,
'name': pkg.name,
'title': pkg.title,
'organization title': org.title if org else u"",
'organization name': org.name if org else u"",
'notes': pkg.notes,
'author': author,
'state': pkg.state,
'podrocje': podrocje
}
pkg_dicts.append(pkg_dict)
return {'table': pkg_dicts}
draft_report_info = {
'name': 'draft',
'title': u'Zbirke v osnutku',
'description': u'Seznam osnutkov, ki so jih avtorji poslali glavnim urednikom in čakajo na potrditev za objavo.',
'option_defaults': None,
'option_combinations': None,
'generate': draft,
'template': 'report/draft.html',
}
def last_resource_deleted(pkg):
resource_revisions = model.Session.query(model.ResourceRevision) \
......
import logging
from helpers import user_get_groups
from helpers import user_get_roles
import ckan.logic as logic
from ckan import model
from ckan.common import _
from ckanext.dgu.model.publisher_request import PublisherRequest
from ckanext.dgu.model import publisher_request
log = logging.getLogger(__name__)
get_action = logic.get_action
# a couple of users have different mails on multiple accounts
# provide proper exceptions for these by specifying mappings
# from their email hashes to the ID of matching duplicate user
......@@ -192,3 +196,26 @@ def move_user (old_id, new_id):
model.Session.add(role_obj)
log.debug('Merged user ' + old_id + ' to ' + new_id)
def merge_users_on_delete(user_id, new_user_id, context):
result = {}
new_user = model.User.get(new_user_id)
if not isinstance(new_user, model.User):
log.error('Could not migrate data from ' + str(user_id) + ' to ' + str(new_user_id) + ', new user not found!')
result['error'] = '%s (%s)' % (_('User not found'), str(new_user_id))
return result
packages = model.Session.query(model.Package) \
.filter(model.Package.creator_user_id == user_id)
for pkg in packages:
role_show_dict = {'domain_object': pkg.id, 'roles': ['admin']}
existing_roles = logic.get_action('roles_show')(context, role_show_dict)
role_update_dict_list = [{'user': new_user.id, 'roles': ['admin']}]
for role in existing_roles.get('roles', []):
if role.get('role', '') == 'admin' and role.get('user_id', '') != new_user:
role_update_dict_list.append({'user': role.get('user_id'), 'roles': []})
updated_roles = logic.get_action('user_role_bulk_update')(context, {'domain_object': pkg.id, 'user_roles': role_update_dict_list})
return result
......@@ -69,12 +69,6 @@ def opsi_action_resource_update(context, data_dict):
'datastore_active' not in data_dict):
data_dict['datastore_active'] = resource.extras['datastore_active']
#OPSI edit: check access appends package and its id to data_dict which may cause incorrect generation of resource filenames, we remove it
if data_dict.get('package', ''):
del data_dict['package']
if data_dict.get('id', ''):
del data_dict['id']
#remove date if it is empty (should be saved as individual resource)
if data_dict.get('date', 'not existing') == '':
del data_dict['date']
......
......@@ -18,7 +18,8 @@ from ckanext.dgu.authorize import (
opsi_extra_fields_editable,
opsi_dataset_delete, opsi_user_list, opsi_user_show,
dgu_organization_delete, dgu_group_change_state,
opsi_bulk_publish
opsi_bulk_publish,
opsi_user_delete
)
from ckanext.report.interfaces import IReport
from ckan.lib.helpers import url_for
......@@ -203,6 +204,7 @@ class ThemePlugin(p.SingletonPlugin):
# Remap the /user/me to the DGU version of the User controller
with SubMapper(map, controller=user_controller) as m:
m.connect('/data/user/me', action='me')
m.connect('user_delete', '/data/user/delete/{id}', action='delete', conditions=dict(method=['GET', 'POST']))
m.connect('user_datasets', '/data/user/{id:.*}', action='read',
ckan_icon='sitemap')
m.connect('user_index', '/data/user', action='index')
......@@ -267,7 +269,8 @@ class AuthApiPlugin(p.SingletonPlugin):
'organization_delete': dgu_organization_delete,
'organization_update': opsi_organization_update,
'group_change_state': dgu_group_change_state,
'bulk_publish': opsi_bulk_publish
'bulk_publish': opsi_bulk_publish,
'user_delete': opsi_user_delete
}
......@@ -402,6 +405,7 @@ class PublisherPlugin(p.SingletonPlugin):
reports.publisher_activity_report_info,
reports.publisher_resources_info,
reports.unpublished_report_info,
reports.draft_report_info,
reports.datasets_without_resources_info,
#reports.app_dataset_theme_report_info,
#reports.app_dataset_report_info,
......
......@@ -42,8 +42,9 @@
</thead>
<tbody>
{% for u in c.page.items %}
{% if u.state != 'deleted' or h.is_sysadmin() %}
<tr>
<td><a href="/data/user/{{u.name}}">{{u.display_name}}</a></td>
<td><a href="/data/user/{{u.name}}">{{u.display_name}}{% if u.state == 'deleted'%} (izbrisan){% endif %}</a></td>
<td>{{u.email}}</td>
<td>{{u.number_administered_packages()}}</td>
<td>{{u.number_of_edits()}}</td>
......@@ -57,6 +58,7 @@
</td>
<td>{{h.render_datetime(u.created, date_format="%d-%m-%Y")}}</td>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
......@@ -67,4 +69,4 @@
</div>
{% endblock %}
\ No newline at end of file
{% endblock %}
{% extends "report/unpublished.html" %}
{% block packages_count %}
<p>
Število zbirk v osnutku: {{data['table']|length }}
</p>
{% endblock %}
{% block table_list %}
<table class="table table-bordered table-condensed tablesorter" id="report-table" style="width: 100%;table-layout:fixed; margin-top: 8px;">
<thead>
<tr>
<th style="width: 30px"><input id="check_all" class="inline" type="checkbox" checked="checked" autocomplete="off"/></th>
<th style="width: 80px">Zbirka</th>
<th style="width: 80px">Organizacija</th>
<th style="width: 200px">Opis</th>
<th style="width: 80px">Predlagatelj</th>
<th style="display: none">Področje</th>
</tr>
</thead>
<tbody>
{% for dataset in data['table'] %}
<tr>
{% set podrocje = dataset.get('podrocje') %}
<td>
<input type="hidden" id="pck_name_{{ dataset['id']}}" value="{{dataset['name']}}"/>
<input id="checkbox_{{ dataset['id']}}" class="inline" type="checkbox" autocomplete="off"
{% if podrocje and podrocje != '' %}checked="checked"{% endif %}
{% if not podrocje or podrocje == '' %}disabled="disabled"{% endif %}
/>
</td>
<td><a href="/dataset/{{dataset['name']}}">{{dataset['title']}}</a></td>
<td><a href="/publisher/{{dataset['organization name']}}">{{dataset['organization title']}}</a></td>
<td>{{dataset['notes']}}</td>
<td style="overflow-wrap: anywhere">{{dataset['author']}}</td>
<td style="display: none">
<select class="form-control" id="podrocje_{{ dataset['id'] }}" name="podrocje" autocomplete="off">
<option value="" {% if not podrocje or podrocje == '' %}selected="selected"{% endif %} >Izberite področje...</option>
{% for podrocje_option in podrocje_choices %}
<option value="{{podrocje_option}}" {% if podrocje_option == podrocje %}selected="selected" {% endif %} >{{podrocje_option}}</option>
{% endfor %}
</select>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}
{% set podrocje_choices = h.podrocje_choices(data)%}
<div class="widget-container">
{% block packages_count %}
<p>
Število zbirk v potrjevanju: {{data['table']|length }}
</p>
<table class="table table-bordered table-condensed tablesorter" id="report-table" style="width: 100%;table-layout:fixed; margin-top: 8px;">
<thead>
<tr>
<th style="width: 30px"><input id="check_all" class="inline" type="checkbox" checked="checked" autocomplete="off"/></th>
<th style="width: 80px">Zbirka</th>
<th style="width: 80px">Organizacija</th>
<th style="width: 200px">Opis</th>
<th style="width: 80px">Predlagatelj</th>
<th style="width: 270px">Področje</th>
</tr>
</thead>
<tbody>
{% for dataset in data['table'] %}
<tr>
{% set podrocje = dataset.get('podrocje') %}
<td>
<input type="hidden" id="pck_name_{{ dataset['id']}}" value="{{dataset['name']}}"/>
<input id="checkbox_{{ dataset['id']}}" class="inline" type="checkbox" autocomplete="off"
{% if podrocje and podrocje != '' %}checked="checked"{% endif %}
{% if not podrocje or podrocje == '' %}disabled="disabled"{% endif %}
/>
</td>
<td><a href="/dataset/{{dataset['name']}}">{{dataset['title']}}</a></td>
<td><a href="/publisher/{{dataset['organization name']}}">{{dataset['organization title']}}</a></td>
<td>{{dataset['notes']}}</td>
<td>{{dataset['author']}}</td>
<td>
<select class="form-control" id="podrocje_{{ dataset['id'] }}" name="podrocje" autocomplete="off">
<option value="" {% if not podrocje or podrocje == '' %}selected="selected"{% endif %} >Izberite področje...</option>
{% for podrocje_option in podrocje_choices %}
<option value="{{podrocje_option}}" {% if podrocje_option == podrocje %}selected="selected" {% endif %} >{{podrocje_option}}</option>
{% endfor %}
</select>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}
{% block table_list %}
<table class="table table-bordered table-condensed tablesorter" id="report-table" style="width: 100%;table-layout:fixed; margin-top: 8px;">
<thead>
<tr>
<th style="width: 30px"><input id="check_all" class="inline" type="checkbox" checked="checked" autocomplete="off"/></th>
<th style="width: 80px">Zbirka</th>
<th style="width: 80px">Organizacija</th>
<th style="width: 200px">Opis</th>
<th style="width: 80px">Predlagatelj</th>
<th style="width: 270px">Področje</th>
</tr>
</thead>
<tbody>
{% for dataset in data['table'] %}
<tr>
{% set podrocje = dataset.get('podrocje') %}
<td>
<input type="hidden" id="pck_name_{{ dataset['id'] }}" value="{{ dataset['name'] }}"/>
<input id="checkbox_{{ dataset['id'] }}" class="inline" type="checkbox" autocomplete="off"
{% if podrocje and podrocje != '' %}checked="checked"{% endif %}
{% if not podrocje or podrocje == '' %}disabled="disabled"{% endif %}
/>
</td>
<td><a href="/dataset/{{ dataset['name'] }}">{{ dataset['title'] }}</a></td>
<td><a href="/publisher/{{ dataset['organization name'] }}">{{ dataset['organization title'] }}</a></td>
<td>{{ dataset['notes'] }}</td>
<td style="overflow-wrap: anywhere">{{ dataset['author'] }}</td>
<td>