Update python/django and strip out celery

.dockerignore

@@ -3,6 +3,8 @@
 settings.pyc
 settings_local.py
 *.ged
-tmp/
-files/
-data/
+.env
+.tmp/
+.files/
+.data/
+.tmp/

.gitignore

@@ -1,6 +1,10 @@
 .DS_Store
 *.py[co]
 *.zip
+.env
+.tmp/
+.data/
+.files/
 
 # Packages
 *.egg

Dockerfile

@@ -1,12 +1,20 @@
-FROM python:2.7-alpine3.7
+FROM python:3.7-alpine3.7
 
 WORKDIR /app/
-COPY ./reqs.frozen.pip /app/
+COPY ./reqs.pip /app/
 ENV LIBRARY_PATH=/lib:/usr/lib
-RUN apk --update add jpeg-dev zlib-dev build-base mariadb-dev && \
-    pip install -r reqs.frozen.pip && \
-    apk add mariadb-client-libs && \
-    apk del build-base mariadb-dev
+RUN apk --update add jpeg-dev zlib-dev build-base && \
+    pip install -r reqs.pip && \
+    apk del build-base
+
+# Create a non-root user
+RUN addgroup -S appgroup && adduser -S app -G appgroup
 
 COPY ./ /app/
-RUN mkdir -p /static && python manage.py collectstatic -c --noinput
+RUN mkdir -p /static && \
+    chown app /static /app && \
+    python manage.py collectstatic -c --noinput
+
+USER app
+
+CMD uvicorn --host=0.0.0.0 asgi:application

asgi.py

@@ -0,0 +1,8 @@
+import os
+from django.core.asgi import get_asgi_application
+from django_simple_task import django_simple_task_middlware
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+
+app = get_asgi_application()
+application = django_simple_task_middlware(app)

docker-compose.yml

@@ -1,63 +1,24 @@
 version: '2.2'
 services:
-    # Dev services
-    avahi:
-        container_name: 'avahi'
-        image: 'enernoclabs/avahi:latest'
-        network_mode: 'host'
-        logging:
-            driver: 'none'
-    db:
-        container_name: 'db'
-        image: 'mysql:5.7'
-        ports:
-            - '3306:3306'
-        environment:
-            MYSQL_ROOT_PASSWORD: 'docker'
-            MYSQL_DATABASE: 'gedgo'
-            MYSQL_USER: 'gedgo'
-            MYSQL_PASSWORD: 'gedgo'
-        volumes:
-            - './tmp:/gedgo_tmp'
-        logging:
-            driver: 'none'
-    redis:
-        container_name: 'redis'
-        image: 'redis'
-        ports:
-            - '6379'
-        logging:
-            driver: 'none'
-    # Application
     app:
         build: '.'
         image: 'gedgo_app'
+        env_file: '.env'
         container_name: 'gedgo_app'
-        command: ['python', 'manage.py', 'runserver', '0.0.0.0:8000']
+        # command: ['python', 'manage.py', 'runserver', '0.0.0.0:8000']
+        command: ['uvicorn', '--host=0.0.0.0', '--reload', 'asgi:application']
+        ports:
+            - '8000:8000'
         volumes:
             - './:/app'
-        links:
-            - 'db'
-            - 'redis'
+            - '.data/:/data'
     web:
         image: 'nginx:alpine'
         command: 'nginx -g "daemon off;"'
         volumes:
             - './gedgo-web.conf:/etc/nginx/conf.d/default.conf:ro'
-            - './files:/src/files:ro'
+            - './.files:/src/files:ro'
         ports:
             - '80:80'
         links:
             - 'app'
-    worker:
-        container_name: 'gedgo_worker'
-        image: 'gedgo_app'
-        command: ['celery', '-A', 'gedgo.tasks', 'worker', '--loglevel=debug']
-        volumes:
-            - './:/app'
-        links:
-            - 'app'
-            - 'db'
-            - 'redis'
-        depends_on:
-            - 'app'

gedgo/__init__.py

@@ -1,10 +0,0 @@
-from django.conf import settings
-import redis as Redis
-
-redis = None
-if hasattr(settings, 'GEDGO_REDIS_SERVER'):
-    try:
-        redis = Redis.StrictRedis(host=settings.GEDGO_REDIS_SERVER)
-        redis.ping()
-    except Exception as e:
-        print e

gedgo/gedcom_parser.py

@@ -17,7 +17,7 @@ class GedcomParser(object):
     )
 
     def __init__(self, file_name_or_stream):
-        if isinstance(file_name_or_stream, basestring):
+        if isinstance(file_name_or_stream, str):
             self.file = open(file_name_or_stream, 'rU')
         else:
             self.file = file_name_or_stream
@@ -70,9 +70,7 @@ def __parse_element(self, line):
                 break
 
         # Keep the entry trimmed down
-        for key in entry.keys():
-            if not entry[key]:
-                del entry[key]
+        entry = dict((key, entry[key]) for key in entry.keys() if entry[key])
 
         return tag, entry
 

gedgo/gedcom_update.py

@@ -1,5 +1,5 @@
-from gedcom_parser import GedcomParser
-from models import Gedcom, Person, Family, Note, Document, Event
+from gedgo.gedcom_parser import GedcomParser
+from gedgo.models import Gedcom, Person, Family, Note, Document, Event
 
 from django.core.files.storage import default_storage
 from django.db import transaction
@@ -8,27 +8,31 @@
 from datetime import datetime
 from re import findall
 from os import path
+import sys
 
 from gedgo.storages import gedcom_storage, resize_thumb
 
+# For logging
+sys.stdout = sys.stderr
+
 
 @transaction.atomic
 def update(g, file_name, verbose=True):
     # Prevent circular dependencies
     if verbose:
-        print 'Parsing content'
+        print('Parsing content')
     parsed = GedcomParser(file_name)
 
     if g is None:
         g = Gedcom.objects.create(
             title=__child_value_by_tags(parsed.header, 'TITL', default=''),
             last_updated=datetime(1920, 1, 1)  # TODO: Fix.
         )
-    if verbose:
-        print 'Gedcom id=%s' % g.id
 
     if verbose:
-        print 'Importing entries to models'
+        print('Gedcom id=%s' % g.id)
+        print('Importing entries to models')
+
     person_counter = family_counter = note_counter = 0
     entries = parsed.entries.values()
     for index, entry in enumerate(entries):
@@ -45,26 +49,29 @@ def update(g, file_name, verbose=True):
             note_counter += 1
 
         if verbose and (index + 1) % 100 == 0:
-            print ' ... %d / %d' % (index + 1, len(entries))
+            print(' ... %d / %d' % (index + 1, len(entries)))
 
     if verbose:
-        print 'Found %d people, %d families, %d notes, and %d documents' % (
+        print('Found %d people, %d families, %d notes, and %d documents' % (
             person_counter, family_counter, note_counter,
-            Document.objects.count())
+            Document.objects.count()))
 
     if verbose:
-        print 'Creating ForeignKey links'
+        print('Creating ForeignKey links')
 
     __process_all_relations(g, parsed, verbose)
 
     g.last_updated = timezone.now()
     g.save()
 
+    if verbose:
+        print('Done updating gedcom')
+
 
 # --- Second Level script functions
 def __process_all_relations(gedcom, parsed, verbose=True):
     if verbose:
-        print '  Starting Person objects.'
+        print('  Starting Person objects.')
 
     # Process Person objects
     for index, person in enumerate(gedcom.person_set.iterator()):
@@ -75,7 +82,7 @@ def __process_all_relations(gedcom, parsed, verbose=True):
         else:
             person.delete()
     if verbose:
-        print '  Finished Person objects, starting Family objects.'
+        print('  Finished Person objects, starting Family objects.')
 
     # Process Family objects
     for family in gedcom.family_set.iterator():
@@ -86,15 +93,15 @@ def __process_all_relations(gedcom, parsed, verbose=True):
         else:
             family.delete()
     if verbose:
-        print '  Finished Family objects.'
+        print('  Finished Family objects.')
 
 
 def __process_person_relations(gedcom, person, entry):
     families = gedcom.family_set
     notes = gedcom.note_set
 
     # "FAMS"
-    person.spousal_families = []
+    person.spousal_families.clear()
     person.spousal_families.add(
         *__objects_from_entry_tag(families, entry, 'FAMS')
     )
@@ -106,7 +113,7 @@ def __process_person_relations(gedcom, person, entry):
         person.child_family = child_family[0]
 
     # "NOTE"
-    person.notes = []
+    person.notes.clear()
     person.notes.add(*__objects_from_entry_tag(notes, entry, 'NOTE'))
 
     person.save()
@@ -117,19 +124,19 @@ def __process_family_relations(gedcom, family, entry):
     notes = gedcom.note_set
 
     # "HUSB"
-    family.husbands = []
+    family.husbands.clear()
     family.husbands.add(*__objects_from_entry_tag(people, entry, 'HUSB'))
 
     # "WIFE"
-    family.wives = []
+    family.wives.clear()
     family.wives.add(*__objects_from_entry_tag(people, entry, 'WIFE'))
 
     # "CHIL"
-    family.children = []
+    family.children.clear()
     family.children.add(*__objects_from_entry_tag(people, entry, 'CHIL'))
 
     # "NOTE"
-    family.notes = []
+    family.notes.clear()
     family.notes.add(*__objects_from_entry_tag(notes, entry, 'NOTE'))
 
     family.save()
@@ -257,7 +264,7 @@ def __process_Note(entry, g):
 
 def __process_Document(entry, obj, g):
     full_name = __child_value_by_tags(entry, 'FILE')
-    name = path.basename(full_name).decode('utf-8').strip()
+    name = path.basename(full_name).strip()
     known = Document.objects.filter(docfile=name).exists()
 
     if not known and not gedcom_storage.exists(name):
@@ -275,8 +282,8 @@ def __process_Document(entry, obj, g):
             make_thumbnail(name, 'w128h128')
             make_thumbnail(name, 'w640h480')
         except Exception as e:
-            print e
-            print '  Warning: failed to make or find thumbnail: %s' % name
+            print(e)
+            print('  Warning: failed to make or find thumbnail: %s' % name)
             return None  # Bail on document creation if thumb fails
 
     m.save()
@@ -352,7 +359,7 @@ def __objects_from_entry_tag(qset, entry, tag):
 
 
 def __child_value_by_tags(entry, tags, default=None):
-    if isinstance(tags, basestring):
+    if isinstance(tags, str):
         tags = [tags]
     tags.reverse()
     next = entry

gedgo/middleware.py

@@ -1,4 +1,3 @@
-from gedgo import redis
 import json
 import time
 import re
@@ -16,40 +15,5 @@ class SimpleTrackerMiddleware(object):
     """
 
     def process_response(self, request, response):
-        # Don't process if redis isn't configured or non-200 response
-        if redis is None or response.status_code != 200:
-            return response
-
-        # Only track non-superuser visitors
-        if request.user is None or request.user.is_superuser \
-                or not request.user.username:
-            return response
-
-        for pattern in IGNORE_PATTERNS:
-            if pattern.match(request.path_info):
-                return response
-
-        # Increment counters and record pageview.
-        # This is pretty fast, but could be done in a celery task to reduce
-        # per-page overhead.
-        id_ = request.user.id
-        _increment_key('gedgo_page_view_count')
-        _increment_key('gedgo_user_%d_page_view_count' % id_)
-
-        page_view = {
-            'ip': request.META['REMOTE_ADDR'],
-            'path': request.path_info,
-            'time': int(time.time())
-        }
-        redis.lpush('gedgo_user_%d_page_views' % id_, json.dumps(page_view))
-        redis.ltrim('gedgo_user_%d_page_views' % id_, 0, 100)
-
+        # TODO: Add user tracking
         return response
-
-
-def _increment_key(key_name):
-    try:
-        pvc = int(redis.get(key_name))
-    except TypeError:
-        pvc = 0
-    redis.set(key_name, pvc + 1)