Kai.wang/new baseon latest aws airflow

README.md

@@ -54,7 +54,7 @@ VERSION
 ## Get started
 
 ```bash
-git clone https://github.com/aws/aws-mwaa-local-runner.git
+compass repo get aws-mwaa-local-runner
 cd aws-mwaa-local-runner
 ```
 

docker/config/webserver_config.py

@@ -24,6 +24,8 @@
 
 basedir = os.path.abspath(os.path.dirname(__file__))
 
+AUTH_ROLE_PUBLIC = "Admin"
+
 # The SQLAlchemy connection string.
 SQLALCHEMY_DATABASE_URI = conf.get('core', 'SQL_ALCHEMY_CONN')
 

docker/docker-compose-local.yml

@@ -1,4 +1,6 @@
 version: '3.7'
+volumes:
+    postgres_data: {}
 services:
     postgres:
         image: postgres:10-alpine
@@ -11,7 +13,7 @@ services:
                 max-size: 10m
                 max-file: "3"
         volumes:
-            - "${PWD}/db-data:/var/lib/postgresql/data"
+            - "postgres_data:/var/lib/postgresql/data"
 
     local-runner:
         image: amazon/mwaa-local:2_5

docker/docker-compose-resetdb.yml

@@ -11,7 +11,7 @@ services:
                 max-size: 10m
                 max-file: "3"
         volumes:
-            - "${PWD}/db-data:/var/lib/postgresql/data"
+            - "postgres_data:/var/lib/postgresql/data"
 
     resetdb:
         image: amazon/mwaa-local:2_5

docker/script/entrypoint.sh

@@ -31,7 +31,7 @@ install_requirements() {
           fi
       fi    
         echo "Installing requirements.txt"
-        pip3 install --user -r "$AIRFLOW_HOME/$REQUIREMENTS_FILE"
+        pip3 install --user -r "$AIRFLOW_HOME/$REQUIREMENTS_FILE" --force-reinstall
     fi
 }
 
@@ -131,13 +131,17 @@ case "$1" in
     export AIRFLOW__CORE__LOAD_EXAMPLES="False"
 
     install_requirements
+    # Forward ports for App Domain. 192.168.5.2 is the host system IP. This should be host.lima.internal, but that DNS
+    # entry is currently broken. The IP should be static however.
+    socat TCP4-LISTEN:31337,fork,reuseaddr TCP4:192.168.5.2:31337 &
+    socat TCP4-LISTEN:31338,fork,reuseaddr TCP4:192.168.5.2:31338 &
     airflow db init
     if [ "$AIRFLOW__CORE__EXECUTOR" = "LocalExecutor" ] || [ "$AIRFLOW__CORE__EXECUTOR" = "SequentialExecutor" ]; then
       # With the "Local" and "Sequential" executors it should all run in one container.
       airflow scheduler &
       sleep 2
     fi
-    airflow users create -r Admin -u admin -e [email protected] -f admin -l user -p $DEFAULT_PASSWORD
+
     exec airflow webserver
     ;;
   resetdb)

docker/script/systemlibs.sh

@@ -29,6 +29,9 @@ yum install -y nc
 # Needed for generating fernet key for local runner
 yum install -y python2-cryptography
 
+# Needed to forward App Domain related ports: 31337, 31338
+yum install -y socat
+
 # Install additional system library dependencies. Provided as a string of libraries separated by space
 if [ -n "${SYSTEM_DEPS}" ]; then yum install -y "${SYSTEM_DEPS}"; fi
 

requirements/requirements.txt

@@ -1,4 +1,3 @@
---constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-3.10.txt"
+# --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-3.10.txt"
 
-apache-airflow-providers-snowflake==4.0.2
-apache-airflow-providers-mysql==4.0.0
+/usr/local/airflow/plugins/demo_airflow-0.0.0-py3-none-any.whl