Merge pull request #51 from geoadmin/develop

New Release v1.4.1 - #patch
geoadmin · Apr 21, 2022 · 61d942b · 61d942b
2 parents 77bcfe3 + 8d80613
commit 61d942b
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 13 deletions.
diff --git a/app/helpers/utils.py b/app/helpers/utils.py
@@ -79,14 +79,40 @@ def wrapped(*args, **kwargs):
     return inner_decorator
 
 
-def validate_content_length(file_content, max_length):
+def validate_content_length():
+
+    def inner_decorator(func):
+
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            # NOTE: the multipart/form-data has an unknown overhead (boundary string is set by the
+            # client), but with a 1KB overhead we are good. The goal of this check is to avoid
+            # uploading huge file which would starve the memory and rejecting them later by
+            # validate_file_length()
+            multipart_overhead = 1024
+            max_length = KML_MAX_SIZE + multipart_overhead
+            if request.content_length > max_length:
+                logger.error(
+                    'Payload too large: payload=%s MB, max_allowed=%s MB',
+                    bytes_conversion(request.content_length, 'MB'),
+                    bytes_conversion(max_length, 'MB'),
+                )
+                abort(413, f"Payload too large, max allowed={bytes_conversion(max_length, 'MB')}MB")
+            return func(*args, **kwargs)
+
+        return wrapped
+
+    return inner_decorator
+
+
+def validate_file_length(file_content, max_length):
     if len(file_content) > max_length:
         logger.error(
-            'Payload too large: payload=%s MB, max_allowed=%s MB',
+            'KML file too large: payload=%s MB, max_allowed=%s MB',
             bytes_conversion(request.content_length, 'MB'),
             bytes_conversion(max_length, 'MB'),
         )
-        abort(413, "KML file too large")
+        abort(413, f"KML file too large, max allowed={bytes_conversion(max_length, 'MB')}MB")
 
 
 def validate_kml_string(kml_string):
@@ -139,7 +165,7 @@ def validate_kml_file():
         )
         abort(415, "Unsupported KML media type")
     file_content = file.read()
-    validate_content_length(file_content, KML_MAX_SIZE)
+    validate_file_length(file_content, KML_MAX_SIZE)
     file_content = decompress_if_gzipped(file_content)
     try:
         if 'charset' in file.mimetype_params:

diff --git a/app/routes.py b/app/routes.py
@@ -14,6 +14,7 @@
 from app.helpers.dynamodb import get_db
 from app.helpers.s3 import get_storage
 from app.helpers.utils import get_kml_file_link
+from app.helpers.utils import validate_content_length
 from app.helpers.utils import validate_content_type
 from app.helpers.utils import validate_kml_file
 from app.helpers.utils import validate_permissions
@@ -32,6 +33,7 @@ def checker():
 
 
 @app.route('/admin', methods=['POST'])
+@validate_content_length()
 @validate_content_type("multipart/form-data")
 def create_kml():
     # Get the kml file data
@@ -120,6 +122,7 @@ def get_kml_metadata(kml_id):
 
 
 @app.route('/admin/<kml_id>', methods=['PUT'])
+@validate_content_length()
 @validate_content_type("multipart/form-data")
 def update_kml(kml_id):
     db = get_db()

diff --git a/buildspec.yml b/buildspec.yml
@@ -24,22 +24,34 @@ phases:
     commands:
       - echo "export of the image tag for build and push purposes"
       - echo "CODEBUILD_WEBHOOK_HEAD_REF=${CODEBUILD_WEBHOOK_HEAD_REF} CODEBUILD_WEBHOOK_BASE_REF=${CODEBUILD_WEBHOOK_BASE_REF}"
-      - export GITHUB_BRANCH="${CODEBUILD_WEBHOOK_HEAD_REF#refs/heads/}"
+      - |
+        if [[ -n "${CODEBUILD_WEBHOOK_HEAD_REF}" ]]; then
+          export GITHUB_BRANCH="${CODEBUILD_WEBHOOK_HEAD_REF#refs/heads/}"
+        else
+          # NOTE: For manual build trigger, CODEBUILD_WEBHOOK_HEAD_REF is not set therefore get
+          # the branch name from git command. This is a bit hacky but did not find any other solution
+          export GITHUB_BRANCH=$(git show-ref --heads | grep $(git --no-pager show --format=%H) | head -1 | awk '{gsub("refs/heads/", ""); print $2}')
+        fi
       - export GITHUB_COMMIT=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7)
       - export GITHUB_TAG="$(git describe --tags 2>/dev/null)"
-      - echo "GITHUB_BRANCH=${GITHUB_BRANCH} GITHUB_COMMIT=${GITHUB_COMMIT} GITHUB_TAG=${GITHUB_TAG} DOCKER_IMG_TAG=${DOCKER_IMG_TAG}"
+      - |
+        if [[ -n "${GITHUB_TAG}" ]]; then
+          export DOCKER_IMG_TAG=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_TAG}
+        else
+          export DOCKER_IMG_TAG=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_BRANCH//\//_}.${GITHUB_COMMIT}
+          export GITHUB_TAG=${GITHUB_COMMIT}
+        fi
+      - export DOCKER_IMG_TAG_LATEST=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_BRANCH//\//_}.latest
+      - echo "GITHUB_BRANCH=${GITHUB_BRANCH}"
+      - echo "GITHUB_COMMIT=${GITHUB_COMMIT}"
+      - echo "GITHUB_TAG=${GITHUB_TAG}"
+      - echo "DOCKER_IMG_TAG=${DOCKER_IMG_TAG}"
+      - echo "DOCKER_IMG_TAG_LATEST=${DOCKER_IMG_TAG_LATEST}"
       - echo "creating a clean environment"
       - make ci
   build:
     commands:
       - echo Build started on $(date)
-      - export DOCKER_IMG_TAG=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_TAG}
-      - export DOCKER_IMG_TAG_LATEST=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_BRANCH//\//_}.latest
-      - |-
-        if [ "${GITHUB_TAG}" = "" ] ; then
-          export DOCKER_IMG_TAG=${REGISTRY}/${IMAGE_BASE_NAME}:${GITHUB_BRANCH//\//_}.${GITHUB_COMMIT}
-          export GITHUB_TAG=${GITHUB_COMMIT}
-        fi
       - echo "Building docker image with tags ${DOCKER_IMG_TAG} and ${DOCKER_IMG_TAG_LATEST}"
       - >
         docker build