diff --git a/Dockerfile b/Dockerfile index 8b07583..2438e1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,3 +4,7 @@ COPY ./ /app/ WORKDIR /app/ RUN pip install -r reqs.pip + +RUN adduser --disabled-password --gecos '' gedgo +RUN chown -R gedgo:gedgo /app +USER gedgo diff --git a/docker-compose.yml b/docker-compose.yml index ea76fdb..74f946b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,8 @@ services: MYSQL_DATABASE: 'gedgo' MYSQL_USER: 'gedgo' MYSQL_PASSWORD: 'gedgo' + volumes: + - './tmp:/gedgo_tmp' logging: driver: 'none' redis: @@ -49,7 +51,7 @@ services: worker: container_name: 'gedgo_worker' image: 'gedgo_app' - command: ['python', 'manage.py', 'celeryd', '-c', '1', '--loglevel=info'] + command: ['celery', '-A', 'gedgo.tasks', 'worker', '--loglevel=debug'] volumes: - './:/app' links: diff --git a/gedgo-web.conf b/gedgo-web.conf index 410c799..958f235 100644 --- a/gedgo-web.conf +++ b/gedgo-web.conf @@ -12,6 +12,7 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $http_host; proxy_redirect off; + client_max_body_size 10M; if (!-f $request_filename) { proxy_pass http://app_server; diff --git a/gedgo/admin.py b/gedgo/admin.py index 4189009..3f92f8a 100644 --- a/gedgo/admin.py +++ b/gedgo/admin.py @@ -1,9 +1,6 @@ from gedgo.models import Gedcom, BlogPost, Document, Documentary from django.contrib import admin -from djcelery.models import TaskState, WorkerState, \ - PeriodicTask, IntervalSchedule, CrontabSchedule - class GedcomAdmin(admin.ModelAdmin): exclude = ('key_people',) @@ -31,9 +28,3 @@ class DocumentaryAdmin(admin.ModelAdmin): admin.site.register(BlogPost, BlogPostAdmin) admin.site.register(Document, DocumentAdmin) admin.site.register(Documentary, DocumentaryAdmin) - -admin.site.unregister(TaskState) -admin.site.unregister(WorkerState) -admin.site.unregister(IntervalSchedule) -admin.site.unregister(CrontabSchedule) -admin.site.unregister(PeriodicTask) diff --git a/gedgo/gedcom_update.py b/gedgo/gedcom_update.py index 1f4a017..a6ab291 100644 --- a/gedgo/gedcom_update.py +++ b/gedgo/gedcom_update.py @@ -5,14 +5,11 @@ from django.db import transaction from django.utils.datetime_safe import date from django.utils import timezone -from datetime import datetime +from datetime import datetime, date from re import findall from os import path -from cStringIO import StringIO -from PIL import Image - -from gedgo.storages import gedcom_storage +from gedgo.storages import gedcom_storage, resize_thumb @transaction.atomic @@ -27,7 +24,7 @@ def update(g, file_name, verbose=True): title=__child_value_by_tags(parsed.header, 'TITL', default=''), last_updated=datetime(1920, 1, 1) # TODO: Fix. ) - print g.id + print 'Gedcom id=%s' % g.id if verbose: print 'Importing entries to models' @@ -71,7 +68,7 @@ def __process_all_relations(gedcom, parsed, verbose=True): # Process Person objects for index, person in enumerate(gedcom.person_set.iterator()): entry = parsed.entries.get(person.pointer) - print index + if entry is not None: __process_person_relations(gedcom, person, entry) else: @@ -139,13 +136,18 @@ def __process_family_relations(gedcom, family, entry): # --- Import Constructors def __process_Person(entry, g): - if __check_unchanged(entry, g): - return - p, _ = Person.objects.get_or_create( pointer=entry['pointer'], gedcom=g) + # No changes recorded in the gedcom, skip it + if __check_unchanged(entry, p): + return None + + p.last_changed = __parse_gen_date( + __child_value_by_tags(entry, ['CHAN', 'DATE']) + )[0] + # Name name_value = __child_value_by_tags(entry, 'NAME', default='') name = findall(r'^([^/]*) /([^/]+)/$', name_value) @@ -162,6 +164,8 @@ def __process_Person(entry, g): p.education = __child_value_by_tags(entry, 'EDUC') p.religion = __child_value_by_tags(entry, 'RELI') + p.save() + # Media document_entries = [ c for c in entry.get('children', []) @@ -172,17 +176,20 @@ def __process_Person(entry, g): if (d is not None) and (__child_value_by_tags(m, 'PRIM') == 'Y'): p.profile.add(d) - p.save() def __process_Family(entry, g): - if __check_unchanged(entry, g): - return - f, _ = Family.objects.get_or_create( pointer=entry['pointer'], gedcom=g) + if __check_unchanged(entry, f): + return None + + f.last_changed = __parse_gen_date( + __child_value_by_tags(entry, ['CHAN', 'DATE']) + )[0] + for k in ['MARR', 'DPAR']: f.joined = __create_Event(__child_by_tag(entry, k), g, f.joined) if f.joined: @@ -225,6 +232,7 @@ def __create_Event(entry, g, e): e.date_approxQ = date_approxQ e.save() + return e @@ -243,35 +251,35 @@ def __process_Note(entry, g): n.text = n.text.strip('\n') n.save() + return n def __process_Document(entry, obj, g): - name = __valid_document_entry(entry) - if not name: - return None + full_name = __child_value_by_tags(entry, 'FILE') + name = path.basename(full_name).decode('utf-8').strip() + known = Document.objects.filter(docfile=name).exists() - file_name = 'gedcom/%s' % name - known = Document.objects.filter(docfile=file_name) + if not known and not gedcom_storage.exists(name): + return None - if len(known) > 0: - m = known[0] + kind = __child_value_by_tags(entry, 'TYPE') + if known: + m = Document.objects.filter(docfile=name).first() else: - kind = __child_value_by_tags(entry, 'TYPE') m = Document(gedcom=g, kind=kind) - m.docfile.name = file_name - if kind == 'PHOTO': - try: - make_thumbnail(name, __child_value_by_tags(entry, 'CROP')) - thumb = path.join('default/thumbs', name) - except: - print ' Warning: failed to make or find thumbnail: %s' % name - return None # Bail on document creation if thumb fails - else: - thumb = None - if thumb is not None: - m.thumb.name = thumb - m.save() + m.docfile.name = name + + if kind == 'PHOTO': + try: + make_thumbnail(name, 'w128h128') + make_thumbnail(name, 'w640h480') + except Exception as e: + print e + print ' Warning: failed to make or find thumbnail: %s' % name + return None # Bail on document creation if thumb fails + + m.save() if isinstance(obj, Person) and \ not m.tagged_people.filter(pointer=obj.pointer).exists(): @@ -284,11 +292,12 @@ def __process_Document(entry, obj, g): # --- Helper Functions -def __check_unchanged(entry, g): +def __check_unchanged(entry, existing): changed = __parse_gen_date( __child_value_by_tags(entry, ['CHAN', 'DATE']) )[0] - return changed and g.last_updated and (changed <= g.last_updated) + return isinstance(existing.last_changed, date) and \ + changed == existing.last_changed DATE_FORMATS = [ @@ -361,37 +370,16 @@ def __child_by_tag(entry, tag): return child -def __valid_document_entry(e): - full_name = __child_value_by_tags(e, 'FILE') - name = path.basename(full_name).decode('utf-8').strip() - if gedcom_storage.exists(name): - return name - - -def make_thumbnail(name, crop): +def make_thumbnail(name, size): """ Copies an image from gedcom_storage, converts it to a thumbnail, and saves - it to default_storage for fast access + it to default_storage for fast access. This also gets done on the fly, + but it's better to pre-build """ - thumb_name = path.join('thumbs', name) + thumb_name = path.join('preview-cache', 'gedcom', size, name) if default_storage.exists(thumb_name): return thumb_name - im = Image.open(gedcom_storage.open(name)) - width, height = im.size - - # TODO: Use crop argument - if width > height: - offset = (width - height) / 2 - box = (offset, 0, offset + height, height) - else: - offset = ((height - width) * 3) / 10 - box = (0, offset, width, offset + width) - cropped = im.crop(box) - - size = 150, 150 - cropped.thumbnail(size, Image.ANTIALIAS) - output = StringIO() - cropped.save(output, 'JPEG') - return default_storage.save(thumb_name, output) + resized = resize_thumb(gedcom_storage.open(name), size=size) + return default_storage.save(thumb_name, resized) diff --git a/gedgo/management/commands/update_gedcom.py b/gedgo/management/commands/update_gedcom.py index d16d409..c608076 100644 --- a/gedgo/management/commands/update_gedcom.py +++ b/gedgo/management/commands/update_gedcom.py @@ -45,7 +45,7 @@ def handle(self, *args, **options): # Check file time against gedcom last_update time. file_time = datetime.fromtimestamp(path.getmtime(file_name)) last_update_time = g.last_updated.replace(tzinfo=None) - if (options['force'] or (file_time > last_update_time)): + if (options['force'] or True or (file_time > last_update_time)): start = datetime.now() errstr = '' @@ -55,6 +55,7 @@ def handle(self, *args, **options): e = exc_info()[0] errstr = 'There was an error: %s\n%s' % ( e, traceback.format_exc()) + print errstr end = datetime.now() diff --git a/gedgo/migrations/0002_auto_20180120_1030.py b/gedgo/migrations/0002_auto_20180120_1030.py new file mode 100644 index 0000000..53936d8 --- /dev/null +++ b/gedgo/migrations/0002_auto_20180120_1030.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.2 on 2018-01-20 15:30 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gedgo', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='document', + name='thumb', + ), + migrations.AddField( + model_name='family', + name='last_changed', + field=models.DateField(blank=True, null=True), + ), + migrations.AddField( + model_name='person', + name='last_changed', + field=models.DateField(blank=True, null=True), + ), + migrations.AlterField( + model_name='document', + name='docfile', + field=models.FileField(upload_to=b'gedcom'), + ), + ] diff --git a/gedgo/models/document.py b/gedgo/models/document.py index 82e33fc..302ffa6 100644 --- a/gedgo/models/document.py +++ b/gedgo/models/document.py @@ -11,7 +11,6 @@ class Meta: docfile = models.FileField(upload_to='gedcom') last_updated = models.DateTimeField(auto_now_add=True) gedcom = models.ForeignKey('Gedcom', null=True, blank=True) - thumb = models.FileField(upload_to='thumbs', null=True, blank=True) kind = models.CharField( max_length=5, diff --git a/gedgo/models/event.py b/gedgo/models/event.py index 1374f7b..dc4b018 100644 --- a/gedgo/models/event.py +++ b/gedgo/models/event.py @@ -5,6 +5,8 @@ class Event(models.Model): class Meta: app_label = 'gedgo' + gedcom = models.ForeignKey('Gedcom') + # Can't use DateFields because sometimes only a Year is known, and # we don't want to show those as January 01, , and datetime # doesn't allow missing values. @@ -12,7 +14,6 @@ class Meta: year_range_end = models.IntegerField(null=True) date_format = models.CharField(null=True, max_length=10) date_approxQ = models.BooleanField('Date is approximate') - gedcom = models.ForeignKey('Gedcom') place = models.CharField(max_length=50) # Breaks strict MVC conventions. diff --git a/gedgo/models/family.py b/gedgo/models/family.py index 9e9c9c0..4bf90ff 100644 --- a/gedgo/models/family.py +++ b/gedgo/models/family.py @@ -9,6 +9,8 @@ class Meta: app_label = 'gedgo' pointer = models.CharField(max_length=10, primary_key=True) gedcom = models.ForeignKey('Gedcom') + last_changed = models.DateField(null=True, blank=True) + husbands = models.ManyToManyField('Person', related_name='family_husbands') wives = models.ManyToManyField('Person', related_name='family_wives') children = models.ManyToManyField('Person', related_name='family_children') diff --git a/gedgo/models/gedcom.py b/gedgo/models/gedcom.py index d46dde1..fb61e7f 100644 --- a/gedgo/models/gedcom.py +++ b/gedgo/models/gedcom.py @@ -1,7 +1,7 @@ from django.db import models import random -from document import Document +from person import Person class Gedcom(models.Model): @@ -31,5 +31,12 @@ def __unicode__(self): @property def photo_sample(self): - photos = Document.objects.filter(gedcom=self, kind='PHOTO') - return random.sample(photos, min(24, len(photos))) + people = Person.objects.filter(gedcom=self).order_by('?') + + sample = [] + for person in people.iterator(): + if person.key_photo: + sample.append(person.key_photo) + if len(sample) == 24: + break + return sample diff --git a/gedgo/models/note.py b/gedgo/models/note.py index 940cad4..0055392 100644 --- a/gedgo/models/note.py +++ b/gedgo/models/note.py @@ -5,7 +5,6 @@ class Note(models.Model): class Meta: app_label = 'gedgo' pointer = models.CharField(max_length=10, primary_key=True) - text = models.TextField() gedcom = models.ForeignKey('Gedcom') diff --git a/gedgo/models/person.py b/gedgo/models/person.py index c3a18d7..1395e45 100644 --- a/gedgo/models/person.py +++ b/gedgo/models/person.py @@ -11,6 +11,7 @@ class Meta: verbose_name_plural = 'People' pointer = models.CharField(max_length=10, primary_key=True) gedcom = models.ForeignKey('Gedcom') + last_changed = models.DateField(null=True, blank=True) # Name first_name = models.CharField(max_length=255) diff --git a/gedgo/static/img/question.jpg b/gedgo/static/img/question.jpg new file mode 100644 index 0000000..aa0d4bd Binary files /dev/null and b/gedgo/static/img/question.jpg differ diff --git a/gedgo/static/styles/style-default.css b/gedgo/static/styles/style-default.css index 99f4683..1164e3f 100644 --- a/gedgo/static/styles/style-default.css +++ b/gedgo/static/styles/style-default.css @@ -115,6 +115,10 @@ path { stroke-width: 2; } +.word-break { + word-break: break-all; +} + #worker-section { min-height: 75px; diff --git a/gedgo/storages.py b/gedgo/storages.py index 3f63df3..413337a 100644 --- a/gedgo/storages.py +++ b/gedgo/storages.py @@ -3,14 +3,16 @@ from django.conf import settings from django.utils.module_loading import import_string -from cStringIO import StringIO +import re import os +from PIL import Image +from cStringIO import StringIO from dropbox.dropbox import Dropbox from dropbox.files import FileMetadata, FolderMetadata, ThumbnailFormat, \ ThumbnailSize -class DropboxStorage(Storage): +class DropBoxSearchableStorage(Storage): def __init__(self, *args, **kwargs): self.client = Dropbox(settings.DROPBOX_ACCESS_TOKEN) self.location = kwargs.get('location', settings.MEDIA_ROOT) @@ -61,15 +63,16 @@ def search(self, query, name='', start=0): # Ignore directories for now return (directories, files) - def preview(self, name, size='w64h64'): - return StringIO(self.client.files_get_thumbnail( + def preview(self, name, size='w128h128'): + file_ = StringIO(self.client.files_get_thumbnail( self.path(name), format=ThumbnailFormat('jpeg', None), size=ThumbnailSize(size, None) )[1].content) + return resize_thumb(file_, size) -class ResearchFileSystemStorage(FileSystemStorage): +class FileSystemSearchableStorage(FileSystemStorage): def search(self, query): terms = [term for term in query.lower().split()] directories, files = [], [] @@ -80,6 +83,31 @@ def search(self, query): files.append(os.path.join(r, f)) return directories, files + def preview(self, name, size='w128h128'): + return resize_thumb(self.open(name), size) + + +def resize_thumb(file_, size='w128h128', crop=None): + im = Image.open(file_) + width, height = im.size + + if size in ('w64h64', 'w128h128'): + if width > height: + offset = (width - height) / 2 + box = (offset, 0, offset + height, height) + else: + offset = ((height - width) * 3) / 10 + box = (0, offset, width, offset + width) + im = im.crop(box) + + m = re.match('w(\d+)h(\d+)', size) + new_size = [int(d) for d in m.groups()] + + im.thumbnail(new_size, Image.ANTIALIAS) + output = StringIO() + im.save(output, 'JPEG') + return output + research_storage = import_string(settings.GEDGO_RESEARCH_FILE_STORAGE)( location=settings.GEDGO_RESEARCH_FILE_ROOT) diff --git a/gedgo/tasks.py b/gedgo/tasks.py index 523be1a..e135944 100644 --- a/gedgo/tasks.py +++ b/gedgo/tasks.py @@ -1,10 +1,12 @@ - from __future__ import absolute_import +import os +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') +import django +django.setup() from gedgo.gedcom_update import update from gedgo import redis from gedgo.models import Gedcom -import os from celery import Celery from django.conf import settings from datetime import datetime @@ -14,10 +16,7 @@ import requests import json - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') - -app = Celery() +app = Celery('gedgo') app.config_from_object(settings) diff --git a/gedgo/templates/default/basic-information.html b/gedgo/templates/default/basic-information.html new file mode 100644 index 0000000..edad476 --- /dev/null +++ b/gedgo/templates/default/basic-information.html @@ -0,0 +1,37 @@ +{% block basic_information %} +

({{ person.year_range }})

+ {% if person.birth or person.death or person.education or person.religion %} + + {% if person.birth.date %} + + + + {% endif %} + {% if person.birth.place %} + + + + {% endif %} + {% if person.death.date %} + + + + {% endif %} + {% if person.death.place %} + + + + {% endif %} + {% if person.education %} + + + + {% endif %} + {% if person.religion %} + + + + {% endif %} +
Born:{{ person.birth.date_string }} {% if person.birth.date_approxQ %}(approximate){% endif %}
Birthplace:{{ person.birth.place }}
Died:{{ person.death.date_string }} {% if person.death.date_approxQ %}(approximate){% endif %}
Deathplace:{{ person.death.place }}
Education:{{ person.education_delimited|linebreaksbr }}
Religion:{{ person.religion }}
+ {% endif %} +{% endblock %} diff --git a/gedgo/templates/default/dashboard.html b/gedgo/templates/default/dashboard.html index 3b954e3..573a40e 100644 --- a/gedgo/templates/default/dashboard.html +++ b/gedgo/templates/default/dashboard.html @@ -1,15 +1,17 @@ {% extends "base.html" %} -{% block content %} -
+{% block leftsidebar %} +

Worker Status

+{% endblock %} +{% block content %} +

Update a Gedcom

-

Update a Gedcom

{% csrf_token %}