CI Improvements (#3150)

* chore(ci): add glibc version check * chore(ci): dockerfile cleanup * chore(ci): move duplicated logic to a reusable workflow * chore(ci): Add reusable prod build workflow and assert glibc version * chore(ui): use moonbeam-release-medium when preparing binary draft * chore(ci): fix GLIBC version check * chore(ci): update .github/workflows/publish-docker.yml * chore(ci): update publish-binary.yml and remove moonbeam-release.Dockerfile * chore(ci): move docker image publish to a reusable workflow * chore(ci): fix docker publishing
moonbeam-foundation · Jan 28, 2025 · d73775c · d73775c
1 parent a902cf6
commit d73775c
Show file tree

Hide file tree

Showing 15 changed files with 276 additions and 344 deletions.
diff --git a/.github/workflow-templates/build-prod-binary/action.yml b/.github/workflow-templates/build-prod-binary/action.yml
@@ -0,0 +1,53 @@
+name: Build Production Binary
+description: |
+  Builds production a moonbeam binary for a given CPU target
+
+inputs:
+  target:
+    description: The CPU target for the binary
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build production moonbeam
+      shell: bash
+      run: |
+        # Build moonbeam
+        # (we don't use volumes because of ownership/permissions issues)
+        docker build \
+          --tag prod --no-cache \
+          --build-arg="COMMIT=${{ github.event.inputs.sha }}" \
+          --build-arg="RUSTFLAGS=-C target-cpu=${{ inputs.target }}" \
+          - < docker/moonbeam-production.Dockerfile
+
+        # Copy moonbeam binary
+        docker rm -f dummy 2> /dev/null | true
+        docker create -ti --name dummy prod bash
+        docker cp dummy:/moonbeam/moonbeam moonbeam
+        docker rm -f dummy
+
+        GLIBC_VERSION="$(objdump -T moonbeam | grep "GLIBC_" | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -1)"
+
+        if [[ $GLIBC_VERSION == "2.34" ]]; then
+          echo "✅ Using expected GLIBC version: ${GLIBC_VERSION}";
+        else
+          echo "❌ Unexpected GLIBC version: ${GLIBC_VERSION}";
+          exit 1;
+        fi
+
+        # Cleanup
+        docker rmi prod
+
+    - name: Save parachain binary
+      shell: bash
+      run: |
+        mkdir -p build
+        cp moonbeam build/moonbeam-${{ inputs.target }}
+    - name: Upload binary
+      uses: actions/upload-artifact@v4
+      with:
+        name: binaries-${{inputs.target}}
+        path: build/moonbeam-${{inputs.target}}
diff --git a/.github/workflow-templates/cargo-build/action.yml b/.github/workflow-templates/cargo-build/action.yml
@@ -17,6 +17,7 @@ runs:
       shell: bash
       run: |
         echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
+        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
         echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
         echo "SCCACHE_CACHE_SIZE=100GB" >> $GITHUB_ENV
         # Set RUSTFLAGS if not already set
@@ -51,7 +52,10 @@ runs:
         cargo build $params
     - name: Display binary comments
       shell: bash
-      run: readelf -p .comment ./target/release/moonbeam
+      run: |
+        readelf -p .comment ./target/release/moonbeam
+        GLIBC_VERSION="$(objdump -T ./target/release/moonbeam | grep "GLIBC_" | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -1)"
+        echo "GLIBC Version: $GLIBC_VERSION"
     - name: Display sccache stats
       shell: bash
       run: ${SCCACHE_PATH} --show-stats
@@ -73,4 +77,4 @@ runs:
       shell: bash
       run: |
         mkdir -p build
-        cp target/release/moonbeam build/moonbeam;
+        cp target/release/moonbeam build/moonbeam;
diff --git a/.github/workflow-templates/publish-docker/action.yml b/.github/workflow-templates/publish-docker/action.yml
@@ -0,0 +1,70 @@
+name: Publish docker image
+description: |
+  Publish docker image tags to dockerhub
+
+inputs:
+  dockerhub_username:
+    description: "Dockerhub username"
+    required: true
+  dockerhub_password:
+    description: "Dockerhub password"
+    required: true
+  image_tags:
+    description: "Image tags"
+    required: true
+  image_title:
+    description: "Image title"
+    required: true
+  image_description:
+    description: "Image description"
+    required: true
+  image_url:
+    description: "Image url"
+    required: true
+  image_source:
+    description: "Image source"
+    required: true
+  image_created:
+    description: "Image creation timestamp"
+    required: true
+  image_revision:
+    description: "Image revision"
+    required: true
+  image_licenses:
+    description: "Image licenses"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+    - name: Set up Docker Buildx
+      uses: docker/[email protected]
+      with:
+        version: latest
+        driver-opts: |
+          image=moby/buildkit:master
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ inputs.dockerhub_username }}
+        password: ${{ inputs.dockerhub_password }}
+    - name: Build and push moonbeam
+      id: docker_build
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ./docker/moonbeam.Dockerfile
+        platforms: linux/amd64
+        push: true
+        tags: ${{ inputs.image_tags }}
+        labels: |
+          org.opencontainers.image.title=${{ inputs.image_title }}
+          org.opencontainers.image.description=${{ inputs.image_title }}
+          org.opencontainers.image.url=${{ inputs.image_url }}
+          org.opencontainers.image.source=${{ inputs.image_source }}
+          org.opencontainers.image.created=${{ inputs.image_created }}
+          org.opencontainers.image.revision=${{ inputs.image_revision }}
+          org.opencontainers.image.licenses=${{ inputs.image_licenses }}
+          
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -486,6 +486,7 @@ jobs:
       RUSTC_WRAPPER: "sccache"
       CARGO_INCREMENTAL: "0"
       SCCACHE_CACHE_SIZE: "100GB"
+      SCCACHE_GHA_ENABLED: true
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -678,59 +679,6 @@ jobs:
           pnpm compile-solidity
           pnpm moonwall test dev_moonbase_tracing
 
-  docker-moonbeam:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-    needs: ["set-tags", "build"]
-    if: ${{ needs.set-tags.outputs.image_exists == 'false' && !github.event.pull_request.head.repo.fork }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.set-tags.outputs.git_ref }}
-      - uses: actions/download-artifact@v4
-        with:
-          name: moonbeam
-          path: build
-      - name: Prepare
-        id: prep
-        run: |
-          DOCKER_IMAGE=moonbeamfoundation/moonbeam
-          TAGS="${DOCKER_IMAGE}:sha-${{ needs.set-tags.outputs.sha8 }}"
-          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
-          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/[email protected]
-        with:
-          version: latest
-          driver-opts: |
-            image=moby/buildkit:master
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.MBF_DOCKERHUB_USERNAME }}
-          password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }}
-      - name: Build and push moonbeam
-        id: docker_build
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: ./docker/moonbeam.Dockerfile
-          platforms: linux/amd64
-          push: true
-          tags: ${{ steps.prep.outputs.tags }}
-          labels: |
-            org.opencontainers.image.title=${{ github.event.repository.name }}
-            org.opencontainers.image.description=${{ github.event.repository.description }}
-            org.opencontainers.image.url=${{ github.event.repository.html_url }}
-            org.opencontainers.image.source=${{ github.event.repository.clone_url }}
-            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
-            org.opencontainers.image.revision=${{ github.sha }}
-            org.opencontainers.image.licenses=${{ github.event.repository.license.spdx_id }}
-
   lazy-loading-tests:
     runs-on:
       labels: bare-metal
@@ -879,13 +827,9 @@ jobs:
           path: target/release
       - name: Retrieve moonbeam binary from docker (for plainSpec generation)
         run: |
-          MOONBEAM_COMMIT=${{ needs.set-tags.outputs.latest_rt_sha8 }}
-          DOCKER_TAG="moonbeamfoundation/moonbeam:sha-$MOONBEAM_COMMIT"
-
-          docker rm -f moonbeam_container 2> /dev/null | true
-          docker create --name moonbeam_container $DOCKER_TAG bash
-          docker cp moonbeam_container:moonbeam/moonbeam test/tmp/moonbeam_rt
-          docker rm -f moonbeam_container
+          LATEST_CLIENT=$(curl -s https://api.github.com/repos/moonbeam-foundation/moonbeam/releases | jq -r '.[] | select(.name | test("v";"i")) | .tag_name' | sort -rs | head -n 1 | tr -d '[:blank:]')
+          echo "Latest client: $LATEST_CLIENT"
+          wget https://github.com/moonbeam-foundation/moonbeam/releases/download/$LATEST_CLIENT/moonbeam -O test/tmp/moonbeam_rt
       - name: Prepare Chainspecs
         run: |
           cd test

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -100,6 +100,7 @@ jobs:
       RUSTC_WRAPPER: "sccache"
       CARGO_INCREMENTAL: "0"
       SCCACHE_CACHE_SIZE: "100GB"
+      SCCACHE_GHA_ENABLED: true
       CARGO_TERM_COLOR: always
     steps:
       - name: Checkout
@@ -227,4 +228,4 @@ jobs:
           comment-id: ${{ steps.fc.outputs.comment-id }}
           issue-number: ${{ github.event.pull_request.number }}
           body-path: coverage_report.md
-          edit-mode: replace
+          edit-mode: replace
diff --git a/.github/workflows/prepare-binary.yml b/.github/workflows/prepare-binary.yml
@@ -15,7 +15,7 @@ jobs:
   ####### Building binaries #######
 
   build-binary:
-    runs-on: ubuntu-latest
+    runs-on: moonbeam-release-medium
     permissions:
       contents: read
     strategy:
@@ -24,34 +24,10 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Build production moonbeam
-        run: |
-          # Build moonbeam
-          # (we don't use volumes because of ownership/permissions issues)
-          docker build \
-            --tag prod --no-cache \
-            --build-arg="COMMIT=${{ github.event.inputs.sha }}" \
-            --build-arg="RUSTFLAGS=-C target-cpu=${{ matrix.cpu }}" \
-            - < docker/moonbeam-production.Dockerfile
-
-          # Copy moonbeam binary
-          docker rm -f dummy 2> /dev/null | true
-          docker create -ti --name dummy prod bash
-          docker cp dummy:/moonbeam/moonbeam moonbeam
-          docker rm -f dummy
-
-          # Cleanup
-          docker rmi prod
-
-      - name: Save parachain binary
-        run: |
-          mkdir -p build
-          cp moonbeam build/moonbeam-${{matrix.cpu}}
-      - name: Upload binary
-        uses: actions/upload-artifact@v4
+      - name: Cargo build
+        uses: ./.github/workflow-templates/build-prod-binary
         with:
-          name: binaries-${{matrix.cpu}}
-          path: build/moonbeam-${{matrix.cpu}}
+          target: ${{ matrix.cpu }}
 
   ####### Prepare the release draft #######
   docker-release-candidate:
@@ -79,33 +55,16 @@ jobs:
           TAGS="${TAG_SHA},${TAG_VER}"
           echo "tags=${TAGS}" >> $GITHUB_OUTPUT
           echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/[email protected]
-        with:
-          version: latest
-          driver-opts: |
-            image=moby/buildkit:master
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.MBF_DOCKERHUB_USERNAME }}
-          password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }}
-      - name: Build and push moonbeam
-        id: docker_build
-        uses: docker/build-push-action@v6
+      - name: Cargo build
+        uses: ./.github/workflow-templates/publish-docker
         with:
-          context: .
-          file: ./docker/moonbeam.Dockerfile
-          platforms: linux/amd64
-          push: true
-          tags: ${{ steps.prep.outputs.tags }}
-          labels: |
-            org.opencontainers.image.title=${{ github.event.repository.name }}
-            org.opencontainers.image.description=${{ github.event.repository.description }}
-            org.opencontainers.image.url=${{ github.event.repository.html_url }}
-            org.opencontainers.image.source=${{ github.event.repository.clone_url }}
-            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
-            org.opencontainers.image.revision=${{ github.sha }}
-            org.opencontainers.image.licenses=${{ github.event.repository.license.spdx_id }}
+          dockerhub_username: ${{ secrets.MBF_DOCKERHUB_USERNAME }}
+          dockerhub_password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }}
+          image_tags: ${{ steps.prep.outputs.tags }}
+          image_title: ${{ github.event.repository.name }}
+          image_description: ${{ github.event.repository.description }}
+          image_url: ${{ github.event.repository.html_url }}
+          image_source: ${{ github.event.repository.clone_url }}
+          image_created: ${{ steps.prep.outputs.created }}
+          image_revision: ${{ github.sha }}
+          image_licenses: ${{ github.event.repository.license.spdx_id }}