Get worker layer functioning again

Dockerfile

@@ -4,3 +4,7 @@ COPY ./ /app/
 
 WORKDIR /app/
 RUN pip install -r reqs.pip
+
+RUN adduser --disabled-password --gecos '' gedgo
+RUN chown -R gedgo:gedgo /app
+USER gedgo

docker-compose.yml

@@ -17,6 +17,8 @@ services:
             MYSQL_DATABASE: 'gedgo'
             MYSQL_USER: 'gedgo'
             MYSQL_PASSWORD: 'gedgo'
+        volumes:
+            - './tmp:/gedgo_tmp'
         logging:
             driver: 'none'
     redis:
@@ -49,7 +51,7 @@ services:
     worker:
         container_name: 'gedgo_worker'
         image: 'gedgo_app'
-        command: ['python', 'manage.py', 'celeryd', '-c', '1', '--loglevel=info']
+        command: ['celery', '-A', 'gedgo.tasks', 'worker', '--loglevel=debug']
         volumes:
             - './:/app'
         links:

gedgo-web.conf

@@ -12,6 +12,7 @@ server {
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header Host $http_host;
         proxy_redirect off;
+        client_max_body_size 10M;
 
         if (!-f $request_filename) {
             proxy_pass http://app_server;

gedgo/admin.py

@@ -1,9 +1,6 @@
 from gedgo.models import Gedcom, BlogPost, Document, Documentary
 from django.contrib import admin
 
-from djcelery.models import TaskState, WorkerState, \
-    PeriodicTask, IntervalSchedule, CrontabSchedule
-
 
 class GedcomAdmin(admin.ModelAdmin):
     exclude = ('key_people',)
@@ -31,9 +28,3 @@ class DocumentaryAdmin(admin.ModelAdmin):
 admin.site.register(BlogPost, BlogPostAdmin)
 admin.site.register(Document, DocumentAdmin)
 admin.site.register(Documentary, DocumentaryAdmin)
-
-admin.site.unregister(TaskState)
-admin.site.unregister(WorkerState)
-admin.site.unregister(IntervalSchedule)
-admin.site.unregister(CrontabSchedule)
-admin.site.unregister(PeriodicTask)

gedgo/gedcom_update.py

@@ -5,14 +5,11 @@
 from django.db import transaction
 from django.utils.datetime_safe import date
 from django.utils import timezone
-from datetime import datetime
+from datetime import datetime, date
 from re import findall
 from os import path
-from cStringIO import StringIO
 
-from PIL import Image
-
-from gedgo.storages import gedcom_storage
+from gedgo.storages import gedcom_storage, resize_thumb
 
 
 @transaction.atomic
@@ -27,7 +24,7 @@ def update(g, file_name, verbose=True):
             title=__child_value_by_tags(parsed.header, 'TITL', default=''),
             last_updated=datetime(1920, 1, 1)  # TODO: Fix.
         )
-    print g.id
+    print 'Gedcom id=%s' % g.id
 
     if verbose:
         print 'Importing entries to models'
@@ -71,7 +68,7 @@ def __process_all_relations(gedcom, parsed, verbose=True):
     # Process Person objects
     for index, person in enumerate(gedcom.person_set.iterator()):
         entry = parsed.entries.get(person.pointer)
-        print index
+
         if entry is not None:
             __process_person_relations(gedcom, person, entry)
         else:
@@ -139,13 +136,18 @@ def __process_family_relations(gedcom, family, entry):
 
 # --- Import Constructors
 def __process_Person(entry, g):
-    if __check_unchanged(entry, g):
-        return
-
     p, _ = Person.objects.get_or_create(
         pointer=entry['pointer'],
         gedcom=g)
 
+    # No changes recorded in the gedcom, skip it
+    if __check_unchanged(entry, p):
+        return None
+
+    p.last_changed = __parse_gen_date(
+        __child_value_by_tags(entry, ['CHAN', 'DATE'])
+    )[0]
+
     # Name
     name_value = __child_value_by_tags(entry, 'NAME', default='')
     name = findall(r'^([^/]*) /([^/]+)/$', name_value)
@@ -162,6 +164,8 @@ def __process_Person(entry, g):
     p.education = __child_value_by_tags(entry, 'EDUC')
     p.religion = __child_value_by_tags(entry, 'RELI')
 
+    p.save()
+
     # Media
     document_entries = [
         c for c in entry.get('children', [])
@@ -172,17 +176,20 @@ def __process_Person(entry, g):
         if (d is not None) and (__child_value_by_tags(m, 'PRIM') == 'Y'):
             p.profile.add(d)
 
-    p.save()
 
 
 def __process_Family(entry, g):
-    if __check_unchanged(entry, g):
-        return
-
     f, _ = Family.objects.get_or_create(
         pointer=entry['pointer'],
         gedcom=g)
 
+    if __check_unchanged(entry, f):
+        return None
+
+    f.last_changed = __parse_gen_date(
+        __child_value_by_tags(entry, ['CHAN', 'DATE'])
+    )[0]
+
     for k in ['MARR', 'DPAR']:
         f.joined = __create_Event(__child_by_tag(entry, k), g, f.joined)
         if f.joined:
@@ -225,6 +232,7 @@ def __create_Event(entry, g, e):
     e.date_approxQ = date_approxQ
 
     e.save()
+
     return e
 
 
@@ -243,35 +251,35 @@ def __process_Note(entry, g):
     n.text = n.text.strip('\n')
 
     n.save()
+
     return n
 
 
 def __process_Document(entry, obj, g):
-    name = __valid_document_entry(entry)
-    if not name:
-        return None
+    full_name = __child_value_by_tags(entry, 'FILE')
+    name = path.basename(full_name).decode('utf-8').strip()
+    known = Document.objects.filter(docfile=name).exists()
 
-    file_name = 'gedcom/%s' % name
-    known = Document.objects.filter(docfile=file_name)
+    if not known and not gedcom_storage.exists(name):
+        return None
 
-    if len(known) > 0:
-        m = known[0]
+    kind = __child_value_by_tags(entry, 'TYPE')
+    if known:
+        m = Document.objects.filter(docfile=name).first()
     else:
-        kind = __child_value_by_tags(entry, 'TYPE')
         m = Document(gedcom=g, kind=kind)
-        m.docfile.name = file_name
-        if kind == 'PHOTO':
-            try:
-                make_thumbnail(name, __child_value_by_tags(entry, 'CROP'))
-                thumb = path.join('default/thumbs', name)
-            except:
-                print '  Warning: failed to make or find thumbnail: %s' % name
-                return None  # Bail on document creation if thumb fails
-        else:
-            thumb = None
-        if thumb is not None:
-            m.thumb.name = thumb
-        m.save()
+        m.docfile.name = name
+
+    if kind == 'PHOTO':
+        try:
+            make_thumbnail(name, 'w128h128')
+            make_thumbnail(name, 'w640h480')
+        except Exception as e:
+            print e
+            print '  Warning: failed to make or find thumbnail: %s' % name
+            return None  # Bail on document creation if thumb fails
+
+    m.save()
 
     if isinstance(obj, Person) and \
             not m.tagged_people.filter(pointer=obj.pointer).exists():
@@ -284,11 +292,12 @@ def __process_Document(entry, obj, g):
 
 
 # --- Helper Functions
-def __check_unchanged(entry, g):
+def __check_unchanged(entry, existing):
     changed = __parse_gen_date(
         __child_value_by_tags(entry, ['CHAN', 'DATE'])
     )[0]
-    return changed and g.last_updated and (changed <= g.last_updated)
+    return isinstance(existing.last_changed, date) and \
+           changed == existing.last_changed
 
 
 DATE_FORMATS = [
@@ -361,37 +370,16 @@ def __child_by_tag(entry, tag):
             return child
 
 
-def __valid_document_entry(e):
-    full_name = __child_value_by_tags(e, 'FILE')
-    name = path.basename(full_name).decode('utf-8').strip()
-    if gedcom_storage.exists(name):
-        return name
-
-
-def make_thumbnail(name, crop):
+def make_thumbnail(name, size):
     """
     Copies an image from gedcom_storage, converts it to a thumbnail, and saves
-    it to default_storage for fast access
+    it to default_storage for fast access.  This also gets done on the fly,
+    but it's better to pre-build
     """
-    thumb_name = path.join('thumbs', name)
+    thumb_name = path.join('preview-cache', 'gedcom', size, name)
 
     if default_storage.exists(thumb_name):
         return thumb_name
 
-    im = Image.open(gedcom_storage.open(name))
-    width, height = im.size
-
-    # TODO: Use crop argument
-    if width > height:
-        offset = (width - height) / 2
-        box = (offset, 0, offset + height, height)
-    else:
-        offset = ((height - width) * 3) / 10
-        box = (0, offset, width, offset + width)
-    cropped = im.crop(box)
-
-    size = 150, 150
-    cropped.thumbnail(size, Image.ANTIALIAS)
-    output = StringIO()
-    cropped.save(output, 'JPEG')
-    return default_storage.save(thumb_name, output)
+    resized = resize_thumb(gedcom_storage.open(name), size=size)
+    return default_storage.save(thumb_name, resized)

gedgo/management/commands/update_gedcom.py

@@ -45,7 +45,7 @@ def handle(self, *args, **options):
         # Check file time against gedcom last_update time.
         file_time = datetime.fromtimestamp(path.getmtime(file_name))
         last_update_time = g.last_updated.replace(tzinfo=None)
-        if (options['force'] or (file_time > last_update_time)):
+        if (options['force'] or True or (file_time > last_update_time)):
             start = datetime.now()
 
             errstr = ''
@@ -55,6 +55,7 @@ def handle(self, *args, **options):
                 e = exc_info()[0]
                 errstr = 'There was an error: %s\n%s' % (
                     e, traceback.format_exc())
+                print errstr
 
             end = datetime.now()
 

gedgo/migrations/0002_auto_20180120_1030.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.2 on 2018-01-20 15:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gedgo', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='document',
+            name='thumb',
+        ),
+        migrations.AddField(
+            model_name='family',
+            name='last_changed',
+            field=models.DateField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='person',
+            name='last_changed',
+            field=models.DateField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='docfile',
+            field=models.FileField(upload_to=b'gedcom'),
+        ),
+    ]

gedgo/models/document.py

@@ -11,7 +11,6 @@ class Meta:
     docfile = models.FileField(upload_to='gedcom')
     last_updated = models.DateTimeField(auto_now_add=True)
     gedcom = models.ForeignKey('Gedcom', null=True, blank=True)
-    thumb = models.FileField(upload_to='thumbs', null=True, blank=True)
 
     kind = models.CharField(
         max_length=5,

gedgo/models/event.py

@@ -5,14 +5,15 @@
 class Event(models.Model):
     class Meta:
         app_label = 'gedgo'
+    gedcom = models.ForeignKey('Gedcom')
+
     # Can't use DateFields because sometimes only a Year is known, and
     # we don't want to show those as January 01, <year>, and datetime
     # doesn't allow missing values.
     date = models.DateField(null=True)
     year_range_end = models.IntegerField(null=True)
     date_format = models.CharField(null=True, max_length=10)
     date_approxQ = models.BooleanField('Date is approximate')
-    gedcom = models.ForeignKey('Gedcom')
     place = models.CharField(max_length=50)
 
     # Breaks strict MVC conventions.

gedgo/models/family.py

@@ -9,6 +9,8 @@ class Meta:
         app_label = 'gedgo'
     pointer = models.CharField(max_length=10, primary_key=True)
     gedcom = models.ForeignKey('Gedcom')
+    last_changed = models.DateField(null=True, blank=True)
+
     husbands = models.ManyToManyField('Person', related_name='family_husbands')
     wives = models.ManyToManyField('Person', related_name='family_wives')
     children = models.ManyToManyField('Person', related_name='family_children')

gedgo/models/gedcom.py

@@ -1,7 +1,7 @@
 from django.db import models
 import random
 
-from document import Document
+from person import Person
 
 
 class Gedcom(models.Model):
@@ -31,5 +31,12 @@ def __unicode__(self):
 
     @property
     def photo_sample(self):
-        photos = Document.objects.filter(gedcom=self, kind='PHOTO')
-        return random.sample(photos, min(24, len(photos)))
+        people = Person.objects.filter(gedcom=self).order_by('?')
+
+        sample = []
+        for person in people.iterator():
+            if person.key_photo:
+                sample.append(person.key_photo)
+            if len(sample) == 24:
+                break
+        return sample

gedgo/models/note.py

@@ -5,7 +5,6 @@ class Note(models.Model):
     class Meta:
         app_label = 'gedgo'
     pointer = models.CharField(max_length=10, primary_key=True)
-
     text = models.TextField()
     gedcom = models.ForeignKey('Gedcom')
 

gedgo/models/person.py

@@ -11,6 +11,7 @@ class Meta:
         verbose_name_plural = 'People'
     pointer = models.CharField(max_length=10, primary_key=True)
     gedcom = models.ForeignKey('Gedcom')
+    last_changed = models.DateField(null=True, blank=True)
 
     # Name
     first_name = models.CharField(max_length=255)