From d73775c84de77cd838cc951ff06ec84dbd18ecdb Mon Sep 17 00:00:00 2001 From: Rodrigo Quelhas <22591718+RomarQ@users.noreply.github.com> Date: Tue, 28 Jan 2025 14:03:09 +0000 Subject: [PATCH] CI Improvements (#3150) * chore(ci): add glibc version check * chore(ci): dockerfile cleanup * chore(ci): move duplicated logic to a reusable workflow * chore(ci): Add reusable prod build workflow and assert glibc version * chore(ui): use moonbeam-release-medium when preparing binary draft * chore(ci): fix GLIBC version check * chore(ci): update .github/workflows/publish-docker.yml * chore(ci): update publish-binary.yml and remove moonbeam-release.Dockerfile * chore(ci): move docker image publish to a reusable workflow * chore(ci): fix docker publishing --- .../build-prod-binary/action.yml | 53 ++++++++++++++ .../workflow-templates/cargo-build/action.yml | 8 +- .../publish-docker/action.yml | 70 ++++++++++++++++++ .github/workflows/build.yml | 64 +--------------- .github/workflows/coverage.yml | 3 +- .github/workflows/prepare-binary.yml | 73 ++++--------------- .github/workflows/publish-binary.yml | 72 ++++++------------ .github/workflows/publish-docker-runtime.yml | 57 +++++++++++---- .github/workflows/publish-docker.yml | 51 +++++++------ .github/workflows/subxt-diff.yml | 65 ++++++++--------- docker/moonbase-parachain.Dockerfile | 33 --------- docker/moonbeam-production.Dockerfile | 6 +- docker/moonbeam-release.Dockerfile | 9 --- docker/moonbeam.Dockerfile | 5 +- docker/polkadot-relay.Dockerfile | 51 ------------- 15 files changed, 276 insertions(+), 344 deletions(-) create mode 100644 .github/workflow-templates/build-prod-binary/action.yml create mode 100644 .github/workflow-templates/publish-docker/action.yml delete mode 100644 docker/moonbase-parachain.Dockerfile delete mode 100644 docker/moonbeam-release.Dockerfile delete mode 100644 docker/polkadot-relay.Dockerfile diff --git a/.github/workflow-templates/build-prod-binary/action.yml b/.github/workflow-templates/build-prod-binary/action.yml new file mode 100644 index 0000000000..2ba95397f4 --- /dev/null +++ b/.github/workflow-templates/build-prod-binary/action.yml @@ -0,0 +1,53 @@ +name: Build Production Binary +description: | + Builds production a moonbeam binary for a given CPU target + +inputs: + target: + description: The CPU target for the binary + required: true + +runs: + using: "composite" + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build production moonbeam + shell: bash + run: | + # Build moonbeam + # (we don't use volumes because of ownership/permissions issues) + docker build \ + --tag prod --no-cache \ + --build-arg="COMMIT=${{ github.event.inputs.sha }}" \ + --build-arg="RUSTFLAGS=-C target-cpu=${{ inputs.target }}" \ + - < docker/moonbeam-production.Dockerfile + + # Copy moonbeam binary + docker rm -f dummy 2> /dev/null | true + docker create -ti --name dummy prod bash + docker cp dummy:/moonbeam/moonbeam moonbeam + docker rm -f dummy + + GLIBC_VERSION="$(objdump -T moonbeam | grep "GLIBC_" | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -1)" + + if [[ $GLIBC_VERSION == "2.34" ]]; then + echo "✅ Using expected GLIBC version: ${GLIBC_VERSION}"; + else + echo "❌ Unexpected GLIBC version: ${GLIBC_VERSION}"; + exit 1; + fi + + # Cleanup + docker rmi prod + + - name: Save parachain binary + shell: bash + run: | + mkdir -p build + cp moonbeam build/moonbeam-${{ inputs.target }} + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: binaries-${{inputs.target}} + path: build/moonbeam-${{inputs.target}} diff --git a/.github/workflow-templates/cargo-build/action.yml b/.github/workflow-templates/cargo-build/action.yml index dd9cf28ca0..64380ad2f9 100644 --- a/.github/workflow-templates/cargo-build/action.yml +++ b/.github/workflow-templates/cargo-build/action.yml @@ -17,6 +17,7 @@ runs: shell: bash run: | echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV echo "SCCACHE_CACHE_SIZE=100GB" >> $GITHUB_ENV # Set RUSTFLAGS if not already set @@ -51,7 +52,10 @@ runs: cargo build $params - name: Display binary comments shell: bash - run: readelf -p .comment ./target/release/moonbeam + run: | + readelf -p .comment ./target/release/moonbeam + GLIBC_VERSION="$(objdump -T ./target/release/moonbeam | grep "GLIBC_" | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu | tail -1)" + echo "GLIBC Version: $GLIBC_VERSION" - name: Display sccache stats shell: bash run: ${SCCACHE_PATH} --show-stats @@ -73,4 +77,4 @@ runs: shell: bash run: | mkdir -p build - cp target/release/moonbeam build/moonbeam; + cp target/release/moonbeam build/moonbeam; \ No newline at end of file diff --git a/.github/workflow-templates/publish-docker/action.yml b/.github/workflow-templates/publish-docker/action.yml new file mode 100644 index 0000000000..1c0397b47a --- /dev/null +++ b/.github/workflow-templates/publish-docker/action.yml @@ -0,0 +1,70 @@ +name: Publish docker image +description: | + Publish docker image tags to dockerhub + +inputs: + dockerhub_username: + description: "Dockerhub username" + required: true + dockerhub_password: + description: "Dockerhub password" + required: true + image_tags: + description: "Image tags" + required: true + image_title: + description: "Image title" + required: true + image_description: + description: "Image description" + required: true + image_url: + description: "Image url" + required: true + image_source: + description: "Image source" + required: true + image_created: + description: "Image creation timestamp" + required: true + image_revision: + description: "Image revision" + required: true + image_licenses: + description: "Image licenses" + required: true + +runs: + using: "composite" + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.8.0 + with: + version: latest + driver-opts: | + image=moby/buildkit:master + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ inputs.dockerhub_username }} + password: ${{ inputs.dockerhub_password }} + - name: Build and push moonbeam + id: docker_build + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/moonbeam.Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ inputs.image_tags }} + labels: | + org.opencontainers.image.title=${{ inputs.image_title }} + org.opencontainers.image.description=${{ inputs.image_title }} + org.opencontainers.image.url=${{ inputs.image_url }} + org.opencontainers.image.source=${{ inputs.image_source }} + org.opencontainers.image.created=${{ inputs.image_created }} + org.opencontainers.image.revision=${{ inputs.image_revision }} + org.opencontainers.image.licenses=${{ inputs.image_licenses }} + \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd41936b8c..0d067b27d4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -486,6 +486,7 @@ jobs: RUSTC_WRAPPER: "sccache" CARGO_INCREMENTAL: "0" SCCACHE_CACHE_SIZE: "100GB" + SCCACHE_GHA_ENABLED: true steps: - name: Checkout uses: actions/checkout@v4 @@ -678,59 +679,6 @@ jobs: pnpm compile-solidity pnpm moonwall test dev_moonbase_tracing - docker-moonbeam: - runs-on: ubuntu-latest - permissions: - contents: read - needs: ["set-tags", "build"] - if: ${{ needs.set-tags.outputs.image_exists == 'false' && !github.event.pull_request.head.repo.fork }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ needs.set-tags.outputs.git_ref }} - - uses: actions/download-artifact@v4 - with: - name: moonbeam - path: build - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=moonbeamfoundation/moonbeam - TAGS="${DOCKER_IMAGE}:sha-${{ needs.set-tags.outputs.sha8 }}" - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.8.0 - with: - version: latest - driver-opts: | - image=moby/buildkit:master - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.MBF_DOCKERHUB_USERNAME }} - password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }} - - name: Build and push moonbeam - id: docker_build - uses: docker/build-push-action@v6 - with: - context: . - file: ./docker/moonbeam.Dockerfile - platforms: linux/amd64 - push: true - tags: ${{ steps.prep.outputs.tags }} - labels: | - org.opencontainers.image.title=${{ github.event.repository.name }} - org.opencontainers.image.description=${{ github.event.repository.description }} - org.opencontainers.image.url=${{ github.event.repository.html_url }} - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.licenses=${{ github.event.repository.license.spdx_id }} - lazy-loading-tests: runs-on: labels: bare-metal @@ -879,13 +827,9 @@ jobs: path: target/release - name: Retrieve moonbeam binary from docker (for plainSpec generation) run: | - MOONBEAM_COMMIT=${{ needs.set-tags.outputs.latest_rt_sha8 }} - DOCKER_TAG="moonbeamfoundation/moonbeam:sha-$MOONBEAM_COMMIT" - - docker rm -f moonbeam_container 2> /dev/null | true - docker create --name moonbeam_container $DOCKER_TAG bash - docker cp moonbeam_container:moonbeam/moonbeam test/tmp/moonbeam_rt - docker rm -f moonbeam_container + LATEST_CLIENT=$(curl -s https://api.github.com/repos/moonbeam-foundation/moonbeam/releases | jq -r '.[] | select(.name | test("v";"i")) | .tag_name' | sort -rs | head -n 1 | tr -d '[:blank:]') + echo "Latest client: $LATEST_CLIENT" + wget https://github.com/moonbeam-foundation/moonbeam/releases/download/$LATEST_CLIENT/moonbeam -O test/tmp/moonbeam_rt - name: Prepare Chainspecs run: | cd test diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index db6fa033ee..f8999293c2 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -100,6 +100,7 @@ jobs: RUSTC_WRAPPER: "sccache" CARGO_INCREMENTAL: "0" SCCACHE_CACHE_SIZE: "100GB" + SCCACHE_GHA_ENABLED: true CARGO_TERM_COLOR: always steps: - name: Checkout @@ -227,4 +228,4 @@ jobs: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} body-path: coverage_report.md - edit-mode: replace + edit-mode: replace \ No newline at end of file diff --git a/.github/workflows/prepare-binary.yml b/.github/workflows/prepare-binary.yml index 449e5c37ce..dd87393f6b 100644 --- a/.github/workflows/prepare-binary.yml +++ b/.github/workflows/prepare-binary.yml @@ -15,7 +15,7 @@ jobs: ####### Building binaries ####### build-binary: - runs-on: ubuntu-latest + runs-on: moonbeam-release-medium permissions: contents: read strategy: @@ -24,34 +24,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Build production moonbeam - run: | - # Build moonbeam - # (we don't use volumes because of ownership/permissions issues) - docker build \ - --tag prod --no-cache \ - --build-arg="COMMIT=${{ github.event.inputs.sha }}" \ - --build-arg="RUSTFLAGS=-C target-cpu=${{ matrix.cpu }}" \ - - < docker/moonbeam-production.Dockerfile - - # Copy moonbeam binary - docker rm -f dummy 2> /dev/null | true - docker create -ti --name dummy prod bash - docker cp dummy:/moonbeam/moonbeam moonbeam - docker rm -f dummy - - # Cleanup - docker rmi prod - - - name: Save parachain binary - run: | - mkdir -p build - cp moonbeam build/moonbeam-${{matrix.cpu}} - - name: Upload binary - uses: actions/upload-artifact@v4 + - name: Cargo build + uses: ./.github/workflow-templates/build-prod-binary with: - name: binaries-${{matrix.cpu}} - path: build/moonbeam-${{matrix.cpu}} + target: ${{ matrix.cpu }} ####### Prepare the release draft ####### docker-release-candidate: @@ -79,33 +55,16 @@ jobs: TAGS="${TAG_SHA},${TAG_VER}" echo "tags=${TAGS}" >> $GITHUB_OUTPUT echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.8.0 - with: - version: latest - driver-opts: | - image=moby/buildkit:master - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.MBF_DOCKERHUB_USERNAME }} - password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }} - - name: Build and push moonbeam - id: docker_build - uses: docker/build-push-action@v6 + - name: Cargo build + uses: ./.github/workflow-templates/publish-docker with: - context: . - file: ./docker/moonbeam.Dockerfile - platforms: linux/amd64 - push: true - tags: ${{ steps.prep.outputs.tags }} - labels: | - org.opencontainers.image.title=${{ github.event.repository.name }} - org.opencontainers.image.description=${{ github.event.repository.description }} - org.opencontainers.image.url=${{ github.event.repository.html_url }} - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.licenses=${{ github.event.repository.license.spdx_id }} + dockerhub_username: ${{ secrets.MBF_DOCKERHUB_USERNAME }} + dockerhub_password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }} + image_tags: ${{ steps.prep.outputs.tags }} + image_title: ${{ github.event.repository.name }} + image_description: ${{ github.event.repository.description }} + image_url: ${{ github.event.repository.html_url }} + image_source: ${{ github.event.repository.clone_url }} + image_created: ${{ steps.prep.outputs.created }} + image_revision: ${{ github.sha }} + image_licenses: ${{ github.event.repository.license.spdx_id }} diff --git a/.github/workflows/publish-binary.yml b/.github/workflows/publish-binary.yml index f68f7997d5..69aa00badb 100644 --- a/.github/workflows/publish-binary.yml +++ b/.github/workflows/publish-binary.yml @@ -38,34 +38,10 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ github.event.inputs.to }} - - name: Build production moonbeam - run: | - # Build moonbeam - # (we don't use volumes because of ownership/permissions issues) - docker build \ - --tag prod --no-cache \ - --build-arg="COMMIT=${{ github.event.inputs.to }}" \ - --build-arg="RUSTFLAGS=-C target-cpu=${{ matrix.cpu }}" \ - - < docker/moonbeam-production.Dockerfile - - # Copy moonbeam binary - docker rm -f dummy 2> /dev/null | true - docker create -ti --name dummy prod bash - docker cp dummy:/moonbeam/moonbeam moonbeam - docker rm -f dummy - - # Cleanup - docker rmi prod - - - name: Save parachain binary - run: | - mkdir -p build - cp moonbeam build/moonbeam-${{matrix.cpu}} - - name: Upload binary - uses: actions/upload-artifact@v4 + - name: Cargo build + uses: ./.github/workflow-templates/build-prod-binary with: - name: binaries-${{matrix.cpu}} - path: build/moonbeam-${{matrix.cpu}} + target: ${{ matrix.cpu }} ####### Prepare the release draft ####### @@ -168,27 +144,25 @@ jobs: pattern: binaries-* merge-multiple: true path: build - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.MBF_DOCKERHUB_USERNAME }} - password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }} - - run: | - mv build/moonbeam-x86-64 build/moonbeam - + - name: Prepare + id: prep + run: | DOCKER_IMAGE=moonbeamfoundation/moonbeam - COMMIT=`git rev-list -n 1 '${{ github.event.inputs.to }}'` - SHA=sha-${COMMIT::8} - echo using "${DOCKER_IMAGE}:${SHA} as base image" + VERSION="${{ github.event.inputs.to }}" + TAG="${VERSION}-rc" - TAG="${{ github.event.inputs.to }}-rc" - - echo building "${DOCKER_IMAGE}:${TAG}" - docker build \ - --build-arg DOCKER_IMAGE="$DOCKER_IMAGE" \ - --build-arg SHA="$SHA" \ - -f docker/moonbeam-release.Dockerfile \ - -t "${DOCKER_IMAGE}:${TAG}" \ - . - - docker push "${DOCKER_IMAGE}:${TAG}" + echo "tags=${DOCKER_IMAGE}:${TAG}" >> $GITHUB_OUTPUT + echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + - name: Cargo build + uses: ./.github/workflow-templates/publish-docker + with: + dockerhub_username: ${{ secrets.MBF_DOCKERHUB_USERNAME }} + dockerhub_password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }} + image_tags: ${{ steps.prep.outputs.tags }} + image_title: ${{ github.event.repository.name }} + image_description: ${{ github.event.repository.description }} + image_url: ${{ github.event.repository.html_url }} + image_source: ${{ github.event.repository.clone_url }} + image_created: ${{ steps.prep.outputs.created }} + image_revision: ${{ github.sha }} + image_licenses: ${{ github.event.repository.license.spdx_id }} diff --git a/.github/workflows/publish-docker-runtime.yml b/.github/workflows/publish-docker-runtime.yml index 48dd01457e..8163b5109a 100644 --- a/.github/workflows/publish-docker-runtime.yml +++ b/.github/workflows/publish-docker-runtime.yml @@ -4,31 +4,56 @@ on: workflow_dispatch: inputs: tag: - description: runtime tag (ex. runtime-2200) to publish on docker + description: runtime tag (ex. runtime-3400) to publish on docker required: true jobs: - tag-docker: - runs-on: ubuntu-latest + ####### Building binaries ####### + + build-binary: + runs-on: moonbeam-release-medium permissions: contents: read steps: - name: Checkout uses: actions/checkout@v4 + - name: Cargo build + uses: ./.github/workflow-templates/build-prod-binary with: - fetch-depth: 0 - - name: Login to DockerHub - uses: docker/login-action@v3 + target: "x86-64" + + publish-docker: + runs-on: ubuntu-latest + permissions: + contents: read + needs: ["build-binary"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: - username: ${{ secrets.MBF_DOCKERHUB_USERNAME }} - password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }} - - name: Publish runtime docker image + pattern: binaries-* + merge-multiple: true + path: build + - name: Prepare + id: prep run: | + echo "rename default binary" + mv build/moonbeam-x86-64 build/moonbeam DOCKER_IMAGE=moonbeamfoundation/moonbeam - DOCKER_TAG="${{ github.event.inputs.tag }}" - COMMIT=`git rev-list -n 1 '${{ github.event.inputs.tag }}'` - SHA=sha-${COMMIT::8} - echo tagging "${DOCKER_IMAGE}:${SHA}" - docker pull "${DOCKER_IMAGE}:${SHA}" - docker tag "${DOCKER_IMAGE}:${SHA}" "${DOCKER_IMAGE}:${DOCKER_TAG}" - docker push "${DOCKER_IMAGE}:${DOCKER_TAG}" + TAGS="${DOCKER_IMAGE}:${{ github.event.inputs.tag }}" + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + - name: Cargo build + uses: ./.github/workflow-templates/publish-docker + with: + dockerhub_username: ${{ secrets.MBF_DOCKERHUB_USERNAME }} + dockerhub_password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }} + image_tags: ${{ steps.prep.outputs.tags }} + image_title: ${{ github.event.repository.name }} + image_description: ${{ github.event.repository.description }} + image_url: ${{ github.event.repository.html_url }} + image_source: ${{ github.event.repository.clone_url }} + image_created: ${{ steps.prep.outputs.created }} + image_revision: ${{ github.sha }} + image_licenses: ${{ github.event.repository.license.spdx_id }} \ No newline at end of file diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 0bc5e48174..33ebbc8f53 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -17,45 +17,48 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.MBF_DOCKERHUB_USERNAME }} - password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }} - - run: | + - name: Prepare + id: prep + run: | DOCKER_IMAGE=moonbeamfoundation/moonbeam VERSION="${{ github.event.inputs.tag }}" COMMIT=`git rev-list -n 1 '${{ github.event.inputs.tag }}'` - SHA=sha-${COMMIT::8} - echo using "${DOCKER_IMAGE}:${SHA} as base image" + COMMIT_SHA8=sha-${COMMIT::8} - mkdir -p build + mkdir -p build wget https://github.com/moonbeam-foundation/moonbeam/releases/download/$VERSION/moonbeam -O build/moonbeam wget https://github.com/moonbeam-foundation/moonbeam/releases/download/$VERSION/moonbeam-skylake -O build/moonbeam-skylake wget https://github.com/moonbeam-foundation/moonbeam/releases/download/$VERSION/moonbeam-znver3 -O build/moonbeam-znver3 - echo building "${DOCKER_IMAGE}:${VERSION}" - docker build \ - --build-arg DOCKER_IMAGE="$DOCKER_IMAGE" \ - --build-arg SHA="$SHA" \ - -f docker/moonbeam-release.Dockerfile \ - -t "${DOCKER_IMAGE}:${VERSION}" \ - . - - docker push "${DOCKER_IMAGE}:${VERSION}" + TAG_SHA="${DOCKER_IMAGE}:sha-${COMMIT_SHA8}" + TAG_VER="${DOCKER_IMAGE}:${VERSION}" + TAGS="${TAG_SHA},${TAG_VER}" if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then MINOR=${VERSION%.*} echo tagging "${DOCKER_IMAGE}:${MINOR}" - docker tag "${DOCKER_IMAGE}:${VERSION}" "${DOCKER_IMAGE}:${MINOR}" - docker push "${DOCKER_IMAGE}:${MINOR}" + TAGS="${TAGS},${DOCKER_IMAGE}:${MINOR}" MAJOR=${MINOR%.*} echo tagging "${DOCKER_IMAGE}:${MAJOR}" - docker tag "${DOCKER_IMAGE}:${VERSION}" "${DOCKER_IMAGE}:${MAJOR}" - docker push "${DOCKER_IMAGE}:${MAJOR}" + TAGS="${TAGS},${DOCKER_IMAGE}:${MAJOR}" echo tagging "${DOCKER_IMAGE}:latest" - docker tag "${DOCKER_IMAGE}:${VERSION}" "${DOCKER_IMAGE}:latest" - docker push "${DOCKER_IMAGE}:latest" + TAGS="${TAGS},${DOCKER_IMAGE}:latest" fi + + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + - name: Cargo build + uses: ./.github/workflow-templates/publish-docker + with: + dockerhub_username: ${{ secrets.MBF_DOCKERHUB_USERNAME }} + dockerhub_password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }} + image_tags: ${{ steps.prep.outputs.tags }} + image_title: ${{ github.event.repository.name }} + image_description: ${{ github.event.repository.description }} + image_url: ${{ github.event.repository.html_url }} + image_source: ${{ github.event.repository.clone_url }} + image_created: ${{ steps.prep.outputs.created }} + image_revision: ${{ github.sha }} + image_licenses: ${{ github.event.repository.license.spdx_id }} \ No newline at end of file diff --git a/.github/workflows/subxt-diff.yml b/.github/workflows/subxt-diff.yml index d229ca4ac4..3834a39ccf 100644 --- a/.github/workflows/subxt-diff.yml +++ b/.github/workflows/subxt-diff.yml @@ -10,50 +10,47 @@ on: required: true jobs: - build: - runs-on: - labels: bare-metal - permissions: - contents: read - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: "runtime-${{ github.event.inputs.spec_version }}" - - name: Local build new Node - uses: ./.github/workflow-templates/cargo-build - - name: Upload Node - uses: actions/upload-artifact@v4 - with: - name: moonbeam - path: build - local-diff: - needs: build - runs-on: moonbeam-release-medium + runs-on: ubuntu-latest permissions: + actions: read contents: read strategy: matrix: runtime: [moonbeam, moonbase, moonriver] + env: + GH_TOKEN: ${{ github.token }} steps: - name: Checkout uses: actions/checkout@v4 - - name: Download Node - uses: actions/download-artifact@v4 with: - name: moonbeam - path: build + fetch-depth: 0 - name: Install Subxt-cli run: | rustup override unset rustup show cargo install subxt-cli@0.37.0 --locked echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - name: Run last_release node via Docker + - name: Lookup previous runtime release build + run: | + COMMIT=`git rev-list -n 1 'runtim-${{ github.event.inputs.last_spec_version }}'` + PREVIOUS_RUNTIME_BUILD=$(gh run -R moonbeam-foundation/moonbeam list -w Build --limit=100 --json databaseId,status,conclusion,headSha --jq ".[] | select(.headSha == \"$COMMIT\" and .status == \"completed\" and .conclusion == \"success\") | .databaseId" | head -n 1) + echo "Latest build for runtim-${{ github.event.inputs.last_spec_version }}: $PREVIOUS_RUNTIME_BUILD" + echo "PREVIOUS_RUNTIME_BUILD=$PREVIOUS_RUNTIME_BUILD" >> $GITHUB_OUTPUT + - name: Lookup previous runtime release build + run: | + COMMIT=`git rev-list -n 1 'runtim-${{ github.event.inputs.spec_version }}'` + RUNTIME_BUILD=$(gh run -R moonbeam-foundation/moonbeam list -w Build --limit=100 --json databaseId,status,conclusion,headSha --jq ".[] | select(.headSha == \"$COMMIT\" and .status == \"completed\" and .conclusion == \"success\") | .databaseId" | head -n 1) + echo "Latest build for runtim-${{ github.event.inputs.spec_version }}: $RUNTIME_BUILD" + echo "NEXT_RUNTIME_BUILD=$RUNTIME_BUILD" >> $GITHUB_OUTPUT + - name: Download binaries for each runtime + run: | + gh run -R moonbeam-foundation/moonbeam download $PREVIOUS_RUNTIME_BUILD -n moonbeam --dir previous-runtime-release + gh run -R moonbeam-foundation/moonbeam download $NEXT_RUNTIME_BUILD -n moonbeam --dir next-runtime-release + - name: Run previous runtime run: | - docker pull moonbeamfoundation/moonbeam:runtime-${{ github.event.inputs.last_spec_version }} - docker run -d --name moonbeam_container -p 9911:9911 moonbeamfoundation/moonbeam:runtime-${{ github.event.inputs.last_spec_version }} \ + chmod +x previous-runtime-release/moonbeam + nohup sh -c './previous-runtime-release/moonbeam \ --chain=${{ matrix.runtime }}-dev \ --no-hardware-benchmarks \ --no-telemetry \ @@ -65,15 +62,13 @@ jobs: --no-prometheus \ --unsafe-rpc-external \ --alice \ - --unsafe-force-node-key-generation \ --rpc-port=9911 \ - --tmp - - sleep 2 - - name: Run Local Node + --unsafe-force-node-key-generation \ + --tmp & sleep 2' & + - name: Run current runtime run: | - chmod +x build/moonbeam - nohup sh -c './build/moonbeam \ + chmod +x next-runtime-release/moonbeam + nohup sh -c './next-runtime-release/moonbeam \ --chain=${{ matrix.runtime }}-dev \ --no-hardware-benchmarks \ --no-telemetry \ @@ -96,8 +91,6 @@ jobs: - name: Stop Node Services if: always() run: | - docker stop moonbeam_container - docker rm moonbeam_container pkill moonbeam - name: Upload diff artifacts diff --git a/docker/moonbase-parachain.Dockerfile b/docker/moonbase-parachain.Dockerfile deleted file mode 100644 index c822300184..0000000000 --- a/docker/moonbase-parachain.Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# Node for Moonbase Parachains. -# -# Requires to run from repository root and to copy the binary in the build folder (part of the release workflow) - -FROM phusion/baseimage:0.11 -LABEL maintainer="alan@moonsonglabs.com" -LABEL description="Moonbeam network node. Supports Alphanet/Stagenet. Will support Moonriver and Moonbeam mainnet." -ARG PROFILE=release - -RUN mv /usr/share/ca* /tmp && \ - rm -rf /usr/share/* && \ - mv /tmp/ca-certificates /usr/share/ && \ - rm -rf /usr/lib/python* && \ - useradd -m -u 1000 -U -s /bin/sh -d /moonbase-parachain moonbeam && \ - mkdir -p /moonbase-parachain/.local/share/moonbase-parachain && \ - chown -R moonbeam:moonbeam /moonbase-parachain && \ - ln -s /moonbase-parachain/.local/share/moonbase-parachain /data && \ - rm -rf /usr/bin /usr/sbin - -USER moonbeam - -COPY --chown=moonbeam build /moonbase-parachain -RUN chmod uog+x /moonbase-parachain/moonbeam - -# 30333 for parachain p2p -# 30334 for relaychain p2p -# 9944 for Websocket and RPC call -# 9615 for Prometheus (metrics) -EXPOSE 30333 30334 9944 9615 - -VOLUME ["/data"] - -CMD ["/moonbase-parachain/moonbeam"] diff --git a/docker/moonbeam-production.Dockerfile b/docker/moonbeam-production.Dockerfile index 558c65f31f..e73c4127e6 100644 --- a/docker/moonbeam-production.Dockerfile +++ b/docker/moonbeam-production.Dockerfile @@ -1,8 +1,8 @@ -# Node for Moonbeam +# Production Node for Moonbeam # # Requires to run from repository root and to copy the binary in the build folder (part of the release workflow) -FROM debian:stable AS builder +FROM docker.io/library/ubuntu:22.04 AS builder # Branch or tag to build moonbeam from ARG COMMIT="master" @@ -50,7 +50,7 @@ RUN cargo build --profile=production --all FROM debian:stable-slim LABEL maintainer="alan@moonsonglabs.com" -LABEL description="Binary for Moonbeam Nodes" +LABEL description="Production Binary for Moonbeam Nodes" RUN useradd -m -u 1000 -U -s /bin/sh -d /moonbeam moonbeam && \ mkdir -p /moonbeam/.local/share && \ diff --git a/docker/moonbeam-release.Dockerfile b/docker/moonbeam-release.Dockerfile deleted file mode 100644 index f99a84b062..0000000000 --- a/docker/moonbeam-release.Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -# Node for Moonbeam networks - -ARG DOCKER_IMAGE -ARG SHA -FROM "$DOCKER_IMAGE:$SHA" -USER moonbeam - -COPY --chown=moonbeam build/* /moonbeam/ -RUN chmod uog+x /moonbeam/moonbeam* diff --git a/docker/moonbeam.Dockerfile b/docker/moonbeam.Dockerfile index 746243e88c..af44f52c9e 100644 --- a/docker/moonbeam.Dockerfile +++ b/docker/moonbeam.Dockerfile @@ -1,4 +1,4 @@ -# Node for Moonbase Alphanet. +# Moonbeam Binary # # Requires to run from repository root and to copy the binary in the build folder (part of the release workflow) @@ -8,8 +8,7 @@ RUN apt-get update && apt-get install -y ca-certificates && update-ca-certificat FROM debian:stable-slim LABEL maintainer="alan@moonsonglabs.com" -LABEL description="Binary for Moonbeam Collator" - +LABEL description="Moonbeam Binary" RUN useradd -m -u 1000 -U -s /bin/sh -d /moonbeam moonbeam && \ mkdir -p /moonbeam/.local/share && \ diff --git a/docker/polkadot-relay.Dockerfile b/docker/polkadot-relay.Dockerfile deleted file mode 100644 index 52d68de5b6..0000000000 --- a/docker/polkadot-relay.Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -# Inspired by Polkadot Dockerfile - -FROM docker.io/paritytech/ci-linux:production as builder -LABEL maintainer="alan@moonsonglabs.com" -LABEL description="This is the build stage for Polkadot. Here we create the binary." - -ARG POLKADOT_COMMIT=master -ARG POLKADOT_REPO=https://github.com/paritytech/polkadot-sdk -RUN echo "Using polkadot ${POLKADOT_COMMIT}" -WORKDIR / - -# Grab the Polkadot Code -# TODO how to grab the correct commit from the lock file? -RUN git clone --depth 1 ${POLKADOT_REPO} -WORKDIR /polkadot-sdk -RUN git checkout ${POLKADOT_COMMIT} - -# RUN sed -i 's/pub const EPOCH_DURATION_IN_SLOTS: BlockNumber = 1 \* HOURS/pub const EPOCH_DURATION_IN_SLOTS: BlockNumber = 2 \* MINUTES/' runtime/*/src/constants.rs -# Download rust dependencies and build the rust binary -RUN cargo build --profile release --locked - -# ===== SECOND STAGE ====== - -FROM debian:stable-slim -LABEL maintainer="alan@moonsonglabs.com" -LABEL description="Polkadot for Moonbeam Relay Chains" -COPY --from=builder /polkadot-sdk/target/release/polkadot /usr/local/bin -COPY --from=builder /polkadot-sdk/target/release/polkadot-execute-worker /usr/local/bin -COPY --from=builder /polkadot-sdk/target/release/polkadot-prepare-worker /usr/local/bin - -RUN useradd -m -u 1000 -U -s /bin/sh -d /moonbase-alphanet moonbeam && \ - mkdir -p /moonbase-alphanet/.local/share/moonbase-alphanet && \ - chown -R moonbeam:moonbeam /moonbase-alphanet && \ - ln -s /moonbase-alphanet/.local/share/moonbase-alphanet /data && \ - rm -rf /usr/bin /usr/sbin - -USER moonbeam - -COPY --chown=moonbeam specs/alphanet/westend-embedded-specs-v8.json /moonbase-alphanet/alphanet-relay-raw-specs.json -RUN grep -v '/p2p/' /moonbase-alphanet/alphanet-relay-raw-specs.json > \ - /moonbase-alphanet/alphanet-relay-raw-specs-no-bootnodes.json - -# 30333 for p2p traffic -# 9933 for RPC call -# 9944 for Websocket -# 9615 for Prometheus (metrics) -EXPOSE 30333 9933 9944 9615 - -VOLUME ["/data"] - -CMD ["/usr/local/bin/polkadot"]