Commit c7ed50de authored by David Read's avatar David Read
Browse files

Take account of deleted resources/datasets and also files already removed.

parent 11c34d1f
...@@ -568,26 +568,31 @@ class Archiver(CkanCommand): ...@@ -568,26 +568,31 @@ class Archiver(CkanCommand):
previous_bin = (0, '') previous_bin = (0, '')
counts = [] counts = []
total_sizes = [] total_sizes = []
#example_resources = []
print '{:>15}{:>10}{:>20}'.format( print '{:>15}{:>10}{:>20}'.format(
'file size', 'no. files', 'files size (bytes)') 'file size', 'no. files', 'files size (bytes)')
for size_bin in size_bins: for size_bin in size_bins:
q = model.Session.query(Archival) \ q = model.Session.query(Archival) \
.filter(Archival.size > previous_bin[0]) \ .filter(Archival.size > previous_bin[0]) \
.filter(Archival.size <= size_bin[0]) .filter(Archival.size <= size_bin[0]) \
.filter(Archival.cache_filepath != '') \
.join(model.Resource,
Archival.resource_id == model.Resource.id) \
.filter(model.Resource.state != 'deleted') \
.join(model.Package,
Archival.package_id == model.Package.id) \
.filter(model.Package.state != 'deleted')
count = q.count() count = q.count()
counts.append(count) counts.append(count)
#q = model.Session.query(Archival, model.Resource) \
# .filter(Archival.size > previous_bin[0]) \
# .filter(Archival.size <= size_bin[0]) \
# .join(model.Resource,
# Archival.resource_id == model.Resource.id) \
# .first()
#example_res = q.Resource if q else None
#example_resources.append(example_res)
total_size = model.Session.query(func.sum(Archival.size)) \ total_size = model.Session.query(func.sum(Archival.size)) \
.filter(Archival.size > previous_bin[0]) \ .filter(Archival.size > previous_bin[0]) \
.filter(Archival.size <= size_bin[0]) \ .filter(Archival.size <= size_bin[0]) \
.filter(Archival.cache_filepath != '') \
.join(model.Resource,
Archival.resource_id == model.Resource.id) \
.filter(model.Resource.state != 'deleted') \
.join(model.Package,
Archival.package_id == model.Package.id) \
.filter(model.Package.state != 'deleted') \
.all()[0][0] .all()[0][0]
total_size = int(total_size or 0) total_size = int(total_size or 0)
total_sizes.append(total_size) total_sizes.append(total_size)
...@@ -602,8 +607,7 @@ class Archiver(CkanCommand): ...@@ -602,8 +607,7 @@ class Archiver(CkanCommand):
max_size = settings.MAX_CONTENT_LENGTH max_size = settings.MAX_CONTENT_LENGTH
archivals = model.Session.query(Archival) \ archivals = model.Session.query(Archival) \
.filter(Archival.size > max_size) \ .filter(Archival.size > max_size) \
.join(model.Resource, .filter(Archival.cache_filepath != '') \
Archival.resource_id == model.Resource.id) \
.all() .all()
total_size = int(model.Session.query(func.sum(Archival.size)) \ total_size = int(model.Session.query(func.sum(Archival.size)) \
.filter(Archival.size > max_size) \ .filter(Archival.size > max_size) \
...@@ -613,7 +617,24 @@ class Archiver(CkanCommand): ...@@ -613,7 +617,24 @@ class Archiver(CkanCommand):
raw_input('Press Enter to DELETE them') raw_input('Press Enter to DELETE them')
for archival in archivals: for archival in archivals:
print 'Deleting %r' % archival print 'Deleting %r' % archival
resource = model.Resource.get(archival.resource_id)
if resource.state == 'deleted':
print 'Nothing to delete - Resource is deleted - deleting archival'
model.Session.delete(archival)
model.Session.commit()
model.Session.flush()
continue
pkg = model.Package.get(archival.package_id)
if pkg.state == 'deleted':
print 'Nothing to delete - Dataset is deleted - deleting archival'
model.Session.delete(archival)
model.Session.commit()
model.Session.flush()
continue
filepath = archival.cache_filepath filepath = archival.cache_filepath
if not os.path.exists(filepath):
print 'Skipping - file not on disk'
continue
try: try:
os.unlink(filepath) os.unlink(filepath)
except OSError: except OSError:
...@@ -622,4 +643,4 @@ class Archiver(CkanCommand): ...@@ -622,4 +643,4 @@ class Archiver(CkanCommand):
archival.cache_filepath = None archival.cache_filepath = None
model.Session.commit() model.Session.commit()
model.Session.flush() model.Session.flush()
print '..deleted %s' % filepath.decode('utf8') print '..deleted %s' % filepath.decode('utf8')
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment