ci: Add default image build step to checks.yml (#1659)

Signed-off-by: Kevin Su <[email protected]>
flyteorg · Apr 24, 2024 · 13eadcc · 13eadcc
1 parent c302ec5
commit 13eadcc
Show file tree

Hide file tree

Showing 41 changed files with 95 additions and 80 deletions.
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -99,12 +99,28 @@ jobs:
           registry: ghcr.io
           username: ${{ secrets.FLYTE_BOT_USERNAME }}
           password: ${{ secrets.FLYTE_BOT_PAT }}
+      - name: Build and push default image
+        working-directory: examples/${{ matrix.example }}
+        run: |
+          if [ -f Dockerfile ]; then
+              tag1=ghcr.io/flyteorg/flytecookbook:${{ matrix.example }}-${{ github.sha }}
+              tag2=ghcr.io/flyteorg/flytecookbook:latest
+              docker build -t tag1 -t tag2 .
+          fi
+      - name: Push default image
+        if: ${{ github.event_name != 'pull_request' }}
+        run: |
+          if [ -f Dockerfile ]; then
+              docker push ghcr.io/flyteorg/flytecookbook --all-tags
+          fi
       - name: Pyflyte package
         working-directory: examples/${{ matrix.example }}
         run: |
+          default_image=ghcr.io/flyteorg/flytecookbook:${{ matrix.example }}-${{ github.sha }}
           source .venv/bin/activate
           pyflyte \
           --pkgs ${{ matrix.example }} package \
+          --image $default_image \
           --image mindmeld="ghcr.io/flyteorg/flytecookbook:core-latest" \
           --image borebuster="ghcr.io/flyteorg/flytekit:py3.9-latest" \
           --output flyte-package.tgz \

diff --git a/_example_template/Dockerfile b/_example_template/Dockerfile
@@ -18,8 +18,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 RUN pip freeze
 
 # Copy the actual code

diff --git a/examples/athena_plugin/Dockerfile b/examples/athena_plugin/Dockerfile
@@ -25,8 +25,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/aws_batch_plugin/Dockerfile b/examples/aws_batch_plugin/Dockerfile
@@ -24,8 +24,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install --no-deps -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install --no-deps -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/bigquery_agent/Dockerfile b/examples/bigquery_agent/Dockerfile
@@ -21,8 +21,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/bigquery_plugin/Dockerfile b/examples/bigquery_plugin/Dockerfile
@@ -21,8 +21,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/blast/Dockerfile b/examples/blast/Dockerfile
@@ -47,8 +47,8 @@ RUN wget ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.13.0/ncbi-blast-2
 WORKDIR /root
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN ${VENV}/bin/pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN ${VENV}/bin/pip install -r /root/requirements.in
 
 # Copy data
 # COPY blast/kitasatospora /root/kitasatospora

diff --git a/examples/customizing_dependencies/Dockerfile b/examples/customizing_dependencies/Dockerfile
@@ -18,8 +18,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 RUN pip freeze
 
 # Copy the actual code

diff --git a/examples/databricks_agent/Dockerfile b/examples/databricks_agent/Dockerfile
@@ -11,8 +11,8 @@ USER 0
 RUN sudo apt-get update && sudo apt-get install -y make build-essential libssl-dev git
 
 # Install Python dependencies
-COPY ./requirements.txt /databricks/driver/requirements.txt
-RUN /databricks/python3/bin/pip install -r /databricks/driver/requirements.txt
+COPY ./requirements.in /databricks/driver/requirements.in
+RUN /databricks/python3/bin/pip install -r /databricks/driver/requirements.in
 
 WORKDIR /databricks/driver
 

diff --git a/examples/databricks_plugin/Dockerfile b/examples/databricks_plugin/Dockerfile
@@ -11,8 +11,8 @@ USER 0
 RUN sudo apt-get update && sudo apt-get install -y make build-essential libssl-dev git
 
 # Install Python dependencies
-COPY ./requirements.txt /databricks/driver/requirements.txt
-RUN /databricks/python3/bin/pip install -r /databricks/driver/requirements.txt
+COPY ./requirements.in /databricks/driver/requirements.in
+RUN /databricks/python3/bin/pip install -r /databricks/driver/requirements.in
 
 WORKDIR /databricks/driver
 

diff --git a/examples/dbt_plugin/Dockerfile b/examples/dbt_plugin/Dockerfile
@@ -17,8 +17,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root/
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root/
+RUN pip install -r /root/requirements.in
 # psycopg2-binary is a dependency of the dbt-postgres adapter, but that doesn't work on mac M1s.
 # As per https://github.com/psycopg/psycopg2/issues/1360, we install psycopg to circumvent this.
 RUN pip uninstall -y psycopg2-binary && pip install psycopg2

diff --git a/examples/development_lifecycle/Dockerfile b/examples/development_lifecycle/Dockerfile
@@ -20,8 +20,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root

diff --git a/examples/dolt_plugin/Dockerfile b/examples/dolt_plugin/Dockerfile
@@ -29,8 +29,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root/.
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root/.
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/duckdb_plugin/Dockerfile b/examples/duckdb_plugin/Dockerfile
@@ -24,8 +24,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root/
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root/
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/exploratory_data_analysis/Dockerfile b/examples/exploratory_data_analysis/Dockerfile
@@ -33,8 +33,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN ${VENV}/bin/pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN ${VENV}/bin/pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/feast_integration/Dockerfile b/examples/feast_integration/Dockerfile
@@ -24,8 +24,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root/.
-RUN ${VENV}/bin/pip install -r /root/requirements.txt
+COPY requirements.in /root/.
+RUN ${VENV}/bin/pip install -r /root/requirements.in
 
 # Copy the actual co
 COPY . /root/

diff --git a/examples/flyteinteractive_plugin/Dockerfile b/examples/flyteinteractive_plugin/Dockerfile
@@ -14,8 +14,8 @@ RUN apt-get update && apt-get install build-essential -y \
     && :
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 RUN pip freeze
 
 # Copy the actual code

diff --git a/examples/forecasting_sales/Dockerfile b/examples/forecasting_sales/Dockerfile
@@ -83,8 +83,8 @@ RUN HOROVOD_WITH_MPI=1 HOROVOD_WITH_TENSORFLOW=1 pip install --no-cache-dir horo
 # ENV HOROVOD_PROGRAM /opt/venv/bin/flytekit_mpi_runner.py
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # SPARK
 RUN flytekit_install_spark3.sh

diff --git a/examples/greatexpectations_plugin/Dockerfile b/examples/greatexpectations_plugin/Dockerfile
@@ -24,8 +24,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root/.
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root/.
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/house_price_prediction/Dockerfile b/examples/house_price_prediction/Dockerfile
@@ -33,8 +33,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN ${VENV}/bin/pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN ${VENV}/bin/pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/k8s_dask_plugin/Dockerfile b/examples/k8s_dask_plugin/Dockerfile
@@ -34,8 +34,8 @@ RUN python3.11 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/k8s_pod_plugin/Dockerfile b/examples/k8s_pod_plugin/Dockerfile
@@ -25,8 +25,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root/.
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root/.
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/k8s_spark_plugin/Dockerfile b/examples/k8s_spark_plugin/Dockerfile
@@ -23,8 +23,8 @@ ENV PATH="${VENV}/bin:$PATH"
 RUN pip3 install wheel
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 RUN wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.4/hadoop-aws-3.2.4.jar -P /opt/spark/jars && \
     wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.901/aws-java-sdk-bundle-1.11.901.jar -P /opt/spark/jars

diff --git a/examples/kfmpi_plugin/Dockerfile b/examples/kfmpi_plugin/Dockerfile
@@ -53,8 +53,8 @@ RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_confi
     mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # Install TensorFlow
 # In case you encounter the "The TensorFlow library was compiled to use AVX instructions, which are not present on your machine" error,

diff --git a/examples/kfpytorch_plugin/Dockerfile b/examples/kfpytorch_plugin/Dockerfile
@@ -24,8 +24,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 RUN pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
 RUN pip install tensorboardX
 

diff --git a/examples/kftensorflow_plugin/Dockerfile b/examples/kftensorflow_plugin/Dockerfile
@@ -38,8 +38,8 @@ ENV PATH="${VENV}/bin:$PATH"
 RUN pip3 install wheel
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # Install TensorFlow
 RUN pip install tensorflow

diff --git a/examples/mlflow_plugin/Dockerfile b/examples/mlflow_plugin/Dockerfile
@@ -24,8 +24,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/mmcloud_agent/Dockerfile b/examples/mmcloud_agent/Dockerfile
@@ -15,8 +15,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root

diff --git a/examples/mmcloud_plugin/Dockerfile b/examples/mmcloud_plugin/Dockerfile
@@ -15,8 +15,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root

diff --git a/examples/mnist_classifier/Dockerfile b/examples/mnist_classifier/Dockerfile
@@ -22,8 +22,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/modin_plugin/Dockerfile b/examples/modin_plugin/Dockerfile
@@ -24,8 +24,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root/.
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root/.
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/nlp_processing/Dockerfile b/examples/nlp_processing/Dockerfile
@@ -33,8 +33,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root
-RUN ${VENV}/bin/pip install -r /root/requirements.txt
+COPY requirements.in /root
+RUN ${VENV}/bin/pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/pandera_plugin/Dockerfile b/examples/pandera_plugin/Dockerfile
@@ -24,8 +24,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root/.
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root/.
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/

diff --git a/examples/papermill_plugin/Dockerfile b/examples/papermill_plugin/Dockerfile
@@ -24,8 +24,8 @@ RUN python3 -m venv ${VENV}
 ENV PATH="${VENV}/bin:$PATH"
 
 # Install Python dependencies
-COPY requirements.txt /root/.
-RUN pip install -r /root/requirements.txt
+COPY requirements.in /root/.
+RUN pip install -r /root/requirements.in
 
 # Copy the actual code
 COPY . /root/