Multi-storage approach for media

.gitignore

@@ -31,6 +31,6 @@ pip-log.txt
 *bootstrap*.css
 *d3.*.js
 
-.files/
+/files/
 *.ged
 settings_local.py

gedgo/gedcom_update.py

@@ -1,20 +1,26 @@
 from gedcom_parser import GedcomParser
 from models import Gedcom, Person, Family, Note, Document, Event
 
+from django.core.files.storage import default_storage
 from django.db import transaction
 from django.utils.datetime_safe import date
 from django.utils import timezone
-from django.conf import settings
 from datetime import datetime
-from dropbox.dropbox import Dropbox
 from re import findall
-from os import path, mkdir
+from os import path
+from cStringIO import StringIO
 
 from PIL import Image
 
+gedcom_storage = None
+
 
 @transaction.atomic
 def update(g, file_name, verbose=True):
+    # Prevent circular dependencies
+    global gedcom_storage
+    from gedgo.views.util import gedcom_storage as gs
+    gedcom_storage = gs
     if verbose:
         print 'Parsing content'
     parsed = GedcomParser(file_name)
@@ -238,30 +244,28 @@ def __process_Note(entry, g):
 
 
 def __process_Document(entry, obj, g):
-    if not __valid_document_entry(entry):
+    name = __valid_document_entry(entry)
+    if not name:
         return None
 
-    file_name = __strip_files_directories(entry)
-    kind = __child_value_by_tags(entry, 'TYPE')
-
-    if kind == 'PHOTO':
-        try:
-            make_thumbnail(path.join(settings.MEDIA_ROOT, file_name))
-            thumb = path.join('thumbs', file_name)
-        except:
-            print '  Warning: failed to make or find thumbnail: ' + file_name
-            return None  # Bail on document creation if thumb fails
-
-    else:
-        thumb = None
-
-    known = Document.objects.filter(docfile=file_name.decode('utf-8').strip())
+    file_name = 'gedcom/%s' % name
+    known = Document.objects.filter(docfile=file_name)
 
     if len(known) > 0:
         m = known[0]
     else:
+        kind = __child_value_by_tags(entry, 'TYPE')
         m = Document(gedcom=g, kind=kind)
         m.docfile.name = file_name
+        if kind == 'PHOTO':
+            try:
+                make_thumbnail(name)
+                thumb = path.join('default/thumbs', name)
+            except:
+                print '  Warning: failed to make or find thumbnail: %s' % name
+                return None  # Bail on document creation if thumb fails
+        else:
+            thumb = None
         if thumb is not None:
             m.thumb.name = thumb
         m.save()
@@ -354,67 +358,37 @@ def __child_by_tag(entry, tag):
             return child
 
 
-# TODO: Proper Storage class approach to this code
 def __valid_document_entry(e):
-    file_name = __strip_files_directories(e)
-    media_file = path.join(settings.MEDIA_ROOT, file_name)
-
-    # The file exists already
-    if isinstance(file_name, basestring) and file_name and \
-            path.exists(media_file):
-        return True
-
-    if __get_from_dropbox(file_name, media_file, ):
-        return True
-
-    return False
-
+    full_name = __child_value_by_tags(e, 'FILE')
+    name = path.basename(full_name).decode('utf-8').strip()
+    if gedcom_storage.exists(name):
+        return name
 
-def __strip_files_directories(e):
-    file_name = __child_value_by_tags(e, 'FILE')
-    return path.basename(file_name)
 
+def make_thumbnail(name):
+    """
+    Copies an image from gedcom_storage, converts it to a thumbnail, and saves
+    it to default_storage for fast access
+    """
+    thumb_name = path.join('thumbs', name)
 
-def __get_from_dropbox(file_name, media_file_name):
-    if not getattr(settings, 'DROPBOX_ACCESS_TOKEN', None):
-        return False
-    d = Dropbox(settings.DROPBOX_ACCESS_TOKEN)
-    resource_path = path.join(settings.DROPBOX_BASE_PATH, file_name)
-    print '  .. %s' % resource_path
-    try:
-        d.files_download_to_file(media_file_name, resource_path)
-        return True
-    except:
-        return False
+    if default_storage.exists(thumb_name):
+        return thumb_name
 
-
-def make_thumbnail(file_name):
-    base_name = path.basename(file_name)
-    dir_name = path.dirname(file_name)
-
-    thumb_path = path.join(dir_name, 'thumbs')
-    thumb_file = path.join(thumb_path, base_name)
-
-    if not path.exists(thumb_path):
-        mkdir(thumb_path)  # Worry about permissions?
-
-    size = 150, 150
-
-    if path.exists(thumb_file):
-        return thumb_file
-
-    im = Image.open(file_name)
+    im = Image.open(gedcom_storage.open(name))
     width, height = im.size
 
+    # TODO: Use gedcom _CROP attribute to set the box
     if width > height:
         offset = (width - height) / 2
         box = (offset, 0, offset + height, height)
     else:
         offset = ((height - width) * 3) / 10
         box = (0, offset, width, offset + width)
-
     cropped = im.crop(box)
-    cropped.thumbnail(size, Image.ANTIALIAS)
-    cropped.save(thumb_file)
 
-    return thumb_file
+    size = 150, 150
+    cropped.thumbnail(size, Image.ANTIALIAS)
+    output = StringIO()
+    cropped.save(output, 'JPEG')
+    return default_storage.save(thumb_name, output)

gedgo/models/document.py

@@ -8,11 +8,10 @@ class Meta:
 
     title = models.CharField(max_length=40, null=True, blank=True)
     description = models.TextField(null=True, blank=True)
-    docfile = models.FileField(upload_to='uploaded')
+    docfile = models.FileField(upload_to='gedcom')
     last_updated = models.DateTimeField(auto_now_add=True)
     gedcom = models.ForeignKey('Gedcom', null=True, blank=True)
-    thumb = models.FileField(upload_to='uploaded/thumbs',
-                             null=True, blank=True)
+    thumb = models.FileField(upload_to='thumbs', null=True, blank=True)
 
     kind = models.CharField(
         max_length=5,

gedgo/storages.py

@@ -18,7 +18,10 @@ def created_time(self, name):
         raise NotImplementedError
 
     def exists(self, name):
-        raise NotImplementedError
+        try:
+            return isinstance(self.client.metadata(self.path(name)), dict)
+        except:
+            return False
 
     def get_available_name(self, name):
         raise NotImplementedError
@@ -37,20 +40,17 @@ def listdir(self, path):
                 files.append(name)
         return (directories, files)
 
-    def preview(self, name):
-        return self.client.thumbnail(self.path(name)).read()
-
     def modified_time(self, name):
         raise NotImplementedError
 
     def open(self, name, mode='rb'):
-        raise NotImplementedError
+        return self.client.get_file(self.path(name))
 
     def save(self, name, content, max_length=None):
         raise NotImplementedError
 
     def size(self, name):
-        raise NotImplementedError
+        return self.client.metadata(self.path(name)).bytes
 
     def url(self, name):
         return self.client.media(self.path(name))['url']

gedgo/templates/default/person-card.html

@@ -1,6 +1,6 @@
 <div class="card row">
   <div class="col-xs-5">
-    <a href="/gedgo/{{ somebody.gedcom.id }}/{{ somebody.pointer }}/"><img src="{% if somebody.photos %}{{ MEDIA_URL }}thumbs/{{ somebody.key_photo }}{% else %}{{ STATIC_URL }}img/generic_person.gif{% endif %}" class="thumb img-rounded"></a>
+    <a href="/gedgo/{{ somebody.gedcom.id }}/{{ somebody.pointer }}/"><img src="{% if somebody.photos %}{{ MEDIA_URL }}default/thumbs/{{ somebody.key_photo }}{% else %}{{ STATIC_URL }}img/generic_person.gif{% endif %}" class="thumb img-rounded"></a>
   </div>
   <div class="card-content col-xs-7">
     <h4 class="card-title"><a href="/gedgo/{{ somebody.gedcom.id }}/{{ somebody.pointer }}/">{{ somebody.full_name }}</a></h4>

gedgo/templates/default/person.html

@@ -16,8 +16,8 @@
 {% block content %}
 <div class="row">
     <div class="col-xs-5">
-      <a href="{% if person.photos %}{{ MEDIA_URL }}{{ person.key_photo }}{% else %}#{% endif %}">
-          <img src="{% if person.photos %}{{ MEDIA_URL }}thumbs/{{ person.key_photo }}{% else %}{{ STATIC_URL }}img/generic_person.gif{% endif %}" class="thumb img-rounded">
+      <a href="{% if person.photos %}{{ MEDIA_URL }}gedcom/{{ person.key_photo }}{% else %}#{% endif %}">
+          <img src="{% if person.photos %}{{ MEDIA_URL }}default/thumbs/{{ person.key_photo }}{% else %}{{ STATIC_URL }}img/generic_person.gif{% endif %}" class="thumb img-rounded">
       </a>
     </div>
     <div class="col-xs-7">

gedgo/urls.py

@@ -48,8 +48,8 @@
             'template_name': 'auth/password_reset_confirm.html'
         }),
 
-    # Backup media fileserve view
-    url(r'^media/(?P<file_base_name>.*)$', views.media),
+    # Authenticated media fileserve view
+    url(r'^media/(?P<storage_name>\w+)/(?P<file_base_name>.*)$', views.media),
 
     url(r'^$', lambda r: redirect('/gedgo/1/')),
 ]

gedgo/views/research.py

@@ -1,18 +1,10 @@
-from django.conf import settings
-from django.http import Http404, HttpResponseRedirect
+from django.http import Http404
 from django.contrib.auth.decorators import login_required
-from django.utils.module_loading import import_string
 
 from os import path
 import mimetypes
 
-from gedgo.views.util import render
-
-
-storage = None
-if getattr(settings, 'GEDGO_RESEARCH_FILE_STORAGE', None):
-    storage = import_string(settings.GEDGO_RESEARCH_FILE_STORAGE)(
-        location=settings.GEDGO_RESEARCH_FILE_ROOT)
+from gedgo.views.util import serve_content, render, research_storage as storage
 
 
 @login_required
@@ -25,7 +17,7 @@ def research(request, pathname):
     # Serve the content through xsendfile or directly.
     try:
         if '.' in name:
-            return HttpResponseRedirect(storage.url(name))
+            return serve_content(storage, name)
         else:
             directories, files = storage.listdir(name)
             directories = [__process(name, d, True) for d in directories]

gedgo/views/util.py

@@ -9,23 +9,39 @@
 from django.contrib import messages
 from django.shortcuts import render_to_response
 from django.template import RequestContext
+from django.utils.module_loading import import_string
 
 from gedgo.models import BlogPost, Documentary
 from gedgo.forms import CommentForm
 
 from os import path
 import mimetypes
 
+research_storage = None
+if getattr(settings, 'GEDGO_RESEARCH_FILE_STORAGE', None):
+    research_storage = import_string(settings.GEDGO_RESEARCH_FILE_STORAGE)(
+        location=settings.GEDGO_RESEARCH_FILE_ROOT)
+
+gedcom_storage = None
+if getattr(settings, 'GEDGO_GEDCOM_FILE_STORAGE', None):
+    gedcom_storage = import_string(settings.GEDGO_GEDCOM_FILE_STORAGE)(
+        location=settings.GEDGO_GEDCOM_FILE_ROOT)
+
+
+STORAGES = {
+    'research': research_storage,
+    'gedcom': gedcom_storage
+}
+
 
 @login_required
-def media(request, file_base_name):
+def media(request, storage_name, file_base_name):
     """
-    Authenticated view to serve media content if necessary,
-    it's much better to have a webserver handle this through
-    an authenticated proxy
+    Authenticated view to serve media content and toggle storage classes
     """
     filename = file_base_name.strip('/')
-    return serve_content(request, filename)
+    storage = STORAGES.get(storage_name, default_storage)
+    return serve_content(storage, filename)
 
 
 def process_comments(request, noun):
@@ -100,40 +116,33 @@ def site_context(request):
     }
 
 
-def serve_content(request, filename):
+def serve_content(storage, name):
     """
     Generate a response to server protected content.
-    http://djangosnippets.org/snippets/365/
-    http://www.chicagodjango.com/blog/permission-based-file-serving/
     """
-    storage_class = default_storage.__class__.__name__
-    if storage_class == 'FileSystemStorage':
-        file_path = default_storage.path(filename)
-        if not default_storage.exists(filename):
-            raise Http404
-        if settings.DEBUG:
-            # Serve it ourselves in debug mode only
-            wrapper = FileWrapper(file(file_path))
-            response = HttpResponse(wrapper)
-        else:
-            # Set sendfile headers
-            response['X-Sendfile'] = file_path  # apache
-            response['X-Accel-Redirect'] = file_path  # nginx
-            response = HttpResponse()
-        file_type = mimetypes.guess_type(filename)[0]
-        response['Content-Type'] = file_type
-        response['Content-Length'] = path.getsize(file_path)
-        if file_type is None:
-            response['Content-Disposition'] = "attachment; filename=%s;" % (
-                path.basename(filename))
-        return response
+    if not storage.exists(name):
+        raise Http404
+
+    # Non-filesystem storages should re-direct to a temporary URL
+    if not storage.__class__.__name__ == 'FileSystemStorage':
+        return HttpResponseRedirect(storage.url(name))
+
+    # If behind a real server, use send-file
+    if settings.GEDGO_SENDFILE_HEADER:
+        response = HttpResponse()
+        response[settings.GEDGO_SENDFILE_HEADER] = default_storage.path(name)
+    # Otherwise, serve it ourselves, which should only happen in DEBUG mode
     else:
-        # Other storages we'll use the url attribute
-        try:
-            url = default_storage.url(filename)
-            return HttpResponseRedirect(url)
-        except Exception as e:
-            raise e  # Http404
+        wrapper = FileWrapper(storage.open(name))
+        response = HttpResponse(wrapper)
+
+    # Set various file headers and return
+    base = path.basename(name)
+    response['Content-Type'] = mimetypes.guess_type(base)[0]
+    response['Content-Length'] = storage.size(name)
+    if response['Content-Type'] is None:
+        response['Content-Disposition'] = "attachment; filename=%s;" % (base)
+    return response
 
 
 def logout_view(request):