From b594a3c74967002c1ea847bdde521f9b6d375734 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Sat, 27 Apr 2024 00:04:12 -0400 Subject: [PATCH] libsql: add codspeed benchmarking in CI --- .github/workflows/benchmark.yml | 31 ++ .github/workflows/brew-test.yml | 27 -- .github/workflows/c-bindings.yml | 78 ----- .github/workflows/extensions-test.yml | 84 ------ .github/workflows/golang-bindings.yml | 81 ----- .github/workflows/libsql-server-release.yml | 312 -------------------- .github/workflows/nemesis.yml | 87 ------ .github/workflows/publish-crsqlite.yml | 43 --- .github/workflows/publish-server.yml | 244 --------------- .github/workflows/release-drafter.yml | 41 --- .github/workflows/release-libsql.yml | 207 ------------- .github/workflows/rust.yml | 199 ------------- .github/workflows/server-pr-images.yml | 88 ------ .github/workflows/sqlite3.yml | 50 ---- .github/workflows/storage-server-images.yml | 64 ---- Cargo.lock | 36 ++- libsql/Cargo.toml | 6 +- 17 files changed, 71 insertions(+), 1607 deletions(-) create mode 100644 .github/workflows/benchmark.yml delete mode 100644 .github/workflows/brew-test.yml delete mode 100644 .github/workflows/c-bindings.yml delete mode 100644 .github/workflows/extensions-test.yml delete mode 100644 .github/workflows/golang-bindings.yml delete mode 100644 .github/workflows/libsql-server-release.yml delete mode 100644 .github/workflows/nemesis.yml delete mode 100644 .github/workflows/publish-crsqlite.yml delete mode 100644 .github/workflows/publish-server.yml delete mode 100644 .github/workflows/release-drafter.yml delete mode 100644 .github/workflows/release-libsql.yml delete mode 100644 .github/workflows/rust.yml delete mode 100644 .github/workflows/server-pr-images.yml delete mode 100644 .github/workflows/sqlite3.yml delete mode 100644 .github/workflows/storage-server-images.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000000..0156ce04b3c --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,31 @@ +name: codspeed-benchmarks + +on: + # Run on pushes to the main branch + push: + branches: + - "main" + # Run on pull requests + pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +jobs: + benchmarks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup rust toolchain, cache and cargo-codspeed binary + uses: moonrepo/setup-rust@v0 + with: + cache-target: release + bins: cargo-codspeed + + - name: Build the benchmark target(s) + run: cargo codspeed build -p libsql + + - name: Run the benchmarks + uses: CodSpeedHQ/action@v2 + with: + run: cargo codspeed run -p libsql diff --git a/.github/workflows/brew-test.yml b/.github/workflows/brew-test.yml deleted file mode 100644 index 8abe5dbbbb9..00000000000 --- a/.github/workflows/brew-test.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Workflow to test our brew installations - -on: - workflow_dispatch: - -jobs: - install: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-latest, macos-14 ] - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Set up Homebrew - id: set-up-homebrew - uses: Homebrew/actions/setup-homebrew@master - - - name: Brew install tap - run: | - brew tap libsql/sqld - brew install libsql-server - - - name: Verify CLI installation - run: sqld --version diff --git a/.github/workflows/c-bindings.yml b/.github/workflows/c-bindings.yml deleted file mode 100644 index c96b13b41d5..00000000000 --- a/.github/workflows/c-bindings.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: C bindings tests CI - -on: - push: - branches: [ "main" ] - pull_request: - merge_group: - branches: [ "main" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - c-bundle-validate: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - name: Set up cargo cache - uses: actions/cache@v3 - continue-on-error: false - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- - - - name: Generate C bundle - run: cargo xtask build-bundled - - - name: Check that C bundle is up to date - run: git diff --quiet || (echo "found uncommited changes in the bundle; please run 'cargo xtask build-bundled' and commit the result" && exit 1) - - c-bindings: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - - name: Install Protoc - uses: arduino/setup-protoc@v2 - - - name: Set up cargo cache - uses: actions/cache@v3 - continue-on-error: false - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- - - - name: Build crates - run: cargo build --release - - - name: copy lib - working-directory: bindings/c - run: cp ../../target/release/libsql_experimental.a . - - - name: clean rust - run: cargo clean - - - name: Build C bindings example - working-directory: bindings/c - env: - LIBSQL_EXPERIMENTAL_PATH: libsql_experimental.a - run: make diff --git a/.github/workflows/extensions-test.yml b/.github/workflows/extensions-test.yml deleted file mode 100644 index bdc7b5d4ce4..00000000000 --- a/.github/workflows/extensions-test.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Extensions Tests - -on: - push: - branches: [ "main" ] - pull_request: - merge_group: - branches: [ "main" ] - -env: - CARGO_TERM_COLOR: always - PROTOC_VERSION: 3.23.4 - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - c-tests: - runs-on: ubuntu-latest - defaults: - run: - working-directory: libsql-sqlite3 - name: CR SQLite C Tests - - steps: - - uses: hecrj/setup-rust-action@v2 - - name: Checkout repository - uses: actions/checkout@v3 - - name: build libsql - run: | - ./configure - make libsql - - name: build - run: | - cd ext/crr - make loadable - - name: test - run: | - cd ext/crr - make test - - rs-tests: - runs-on: ubuntu-latest - defaults: - run: - working-directory: libsql-sqlite3 - name: CR SQLite Rust Tests - - steps: - - uses: hecrj/setup-rust-action@v2 - - name: Checkout repository - uses: actions/checkout@v3 - - name: build libsql - run: | - ./configure - make libsql - - name: test - run: | - cd ext/crr/rs/core - cargo test --features=loadable_extension - - extensions-tests: - runs-on: ubuntu-latest - name: Extensions Tests - - steps: - - uses: hecrj/setup-rust-action@v2 - - name: Checkout repository - uses: actions/checkout@v3 - - name: build libsql and ffi bindings - run: | - cargo xtask build-bundled - - name: download extensions - run: | - cd libsql-sqlite3/test/rust_suite - export VSS_VERSION="v0.1.2" - wget https://github.com/asg017/sqlite-vss/releases/download/$VSS_VERSION/sqlite-vss-$VSS_VERSION-loadable-linux-x86_64.tar.gz - tar -xvf sqlite-vss-$VSS_VERSION-loadable-linux-x86_64.tar.gz -C src - - name: test - run: | - cd libsql-sqlite3/test/rust_suite - # Only execute extensions tests - cargo test --features=extensions extensions diff --git a/.github/workflows/golang-bindings.yml b/.github/workflows/golang-bindings.yml deleted file mode 100644 index 54027d8f18f..00000000000 --- a/.github/workflows/golang-bindings.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Go bindings tests CI - -on: - push: - branches: [ "main" ] - pull_request: - merge_group: - branches: [ "main" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - golang-bindings: - - runs-on: ubuntu-latest - - services: - sqld: - image: ghcr.io/tursodatabase/libsql-server:latest - ports: - - 8080:8080 - - steps: - - uses: actions/checkout@v3 - - # needed because we run out of disk space during tests - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # when set to "true" but frees about 6 GB - tool-cache: true - - - uses: dtolnay/rust-toolchain@stable - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: '>=1.21' - - name: get TCL - run: sudo apt-get install -y tcl8.6-dev - - - name: Install Protoc - uses: arduino/setup-protoc@v2 - - - name: Set up cargo cache - uses: actions/cache@v3 - continue-on-error: false - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- - - - name: Build sql-experimental - run: cargo b -j16 --release -p sql-experimental - - - name: go-libsql checkout - uses: actions/checkout@v3 - with: - repository: tursodatabase/go-libsql - ref: "main" - path: "go-libsql" - - - name: Copy sql-experimental library to go-libsql - run: cp target/release/libsql_experimental.a go-libsql/lib/darwin_amd64/ - && cp target/release/libsql_experimental.a go-libsql/lib/darwin_arm64/ - && cp target/release/libsql_experimental.a go-libsql/lib/linux_amd64/ - && cp target/release/libsql_experimental.a go-libsql/lib/linux_arm64/ - && cp bindings/c/include/libsql.h go-libsql/lib/include/ - - - name: Run go-libsql tests - working-directory: go-libsql - run: go test -v -count=1 ./... - env: - LIBSQL_PRIMARY_URL: "http://127.0.0.1:8080" diff --git a/.github/workflows/libsql-server-release.yml b/.github/workflows/libsql-server-release.yml deleted file mode 100644 index 919d6fa179c..00000000000 --- a/.github/workflows/libsql-server-release.yml +++ /dev/null @@ -1,312 +0,0 @@ -# Copyright 2022-2024, axodotdev -# SPDX-License-Identifier: MIT or Apache-2.0 -# -# CI that: -# -# * checks for a Git Tag that looks like a release -# * builds artifacts with cargo-dist (archives, installers, hashes) -# * uploads those artifacts to temporary workflow zip -# * on success, uploads the artifacts to a GitHub Release -# -# Note that the GitHub Release will be created with a generated -# title/body based on your changelogs. - -name: Release - -permissions: - contents: write - -# This task will run whenever you push a git tag that looks like a version -# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. -# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where -# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION -# must be a Cargo-style SemVer Version (must have at least major.minor.patch). -# -# If PACKAGE_NAME is specified, then the announcement will be for that -# package (erroring out if it doesn't have the given version or isn't cargo-dist-able). -# -# If PACKAGE_NAME isn't specified, then the announcement will be for all -# (cargo-dist-able) packages in the workspace with that version (this mode is -# intended for workspaces with only one dist-able package, or with all dist-able -# packages versioned/released in lockstep). -# -# If you push multiple tags at once, separate instances of this workflow will -# spin up, creating an independent announcement for each one. However, GitHub -# will hard limit this to 3 tags per commit, as it will assume more tags is a -# mistake. -# -# If there's a prerelease-style suffix to the version, then the release(s) -# will be marked as a prerelease. -on: - push: - tags: - - 'libsql-server**[0-9]+.[0-9]+.[0-9]+*' - pull_request: - -jobs: - # Run 'cargo dist plan' (or host) to determine what tasks we need to do - plan: - runs-on: ubuntu-latest - outputs: - val: ${{ steps.plan.outputs.manifest }} - tag: ${{ !github.event.pull_request && github.ref_name || '' }} - tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} - publishing: ${{ !github.event.pull_request }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Install cargo-dist - # we specify bash to get pipefail; it guards against the `curl` command - # failing. otherwise `sh` won't catch that `curl` returned non-0 - shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.14.1/cargo-dist-installer.sh | sh" - # sure would be cool if github gave us proper conditionals... - # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible - # functionality based on whether this is a pull_request, and whether it's from a fork. - # (PRs run on the *source* but secrets are usually on the *target* -- that's *good* - # but also really annoying to build CI around when it needs secrets to work right.) - - id: plan - run: | - cargo dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json - echo "cargo dist ran successfully" - cat plan-dist-manifest.json - echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" - - name: "Upload dist-manifest.json" - uses: actions/upload-artifact@v4 - with: - name: artifacts-plan-dist-manifest - path: plan-dist-manifest.json - - # Build and packages all the platform-specific things - build-local-artifacts: - name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) - # Let the initial task tell us to not run (currently very blunt) - needs: - - plan - if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} - strategy: - fail-fast: false - # Target platforms/runners are computed by cargo-dist in create-release. - # Each member of the matrix has the following arguments: - # - # - runner: the github runner - # - dist-args: cli flags to pass to cargo dist - # - install-dist: expression to run to install cargo-dist on the runner - # - # Typically there will be: - # - 1 "global" task that builds universal installers - # - N "local" tasks that build each platform's binaries and platform-specific installers - matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} - runs-on: ${{ matrix.runner }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json - steps: - - name: enable windows longpaths - run: | - git config --global core.longpaths true - - uses: actions/checkout@v4 - with: - submodules: recursive - - uses: swatinem/rust-cache@v2 - with: - key: ${{ join(matrix.targets, '-') }} - - name: Install cargo-dist - run: ${{ matrix.install_dist }} - # Get the dist-manifest - - name: Fetch local artifacts - uses: actions/download-artifact@v4 - with: - pattern: artifacts-* - path: target/distrib/ - merge-multiple: true - - name: Install dependencies - run: | - ${{ matrix.packages_install }} - - name: Build artifacts - run: | - # Actually do builds and make zips and whatnot - cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json - echo "cargo dist ran successfully" - - id: cargo-dist - name: Post-build - # We force bash here just because github makes it really hard to get values up - # to "real" actions without writing to env-vars, and writing to env-vars has - # inconsistent syntax between shell and powershell. - shell: bash - run: | - # Parse out what we just built and upload it to scratch storage - echo "paths<> "$GITHUB_OUTPUT" - jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" - echo "EOF" >> "$GITHUB_OUTPUT" - - cp dist-manifest.json "$BUILD_MANIFEST_NAME" - - name: "Upload artifacts" - uses: actions/upload-artifact@v4 - with: - name: artifacts-build-local-${{ join(matrix.targets, '_') }} - path: | - ${{ steps.cargo-dist.outputs.paths }} - ${{ env.BUILD_MANIFEST_NAME }} - - # Build and package all the platform-agnostic(ish) things - build-global-artifacts: - needs: - - plan - - build-local-artifacts - runs-on: "ubuntu-20.04" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Install cargo-dist - shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.14.1/cargo-dist-installer.sh | sh" - # Get all the local artifacts for the global tasks to use (for e.g. checksums) - - name: Fetch local artifacts - uses: actions/download-artifact@v4 - with: - pattern: artifacts-* - path: target/distrib/ - merge-multiple: true - - id: cargo-dist - shell: bash - run: | - cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json - echo "cargo dist ran successfully" - - # Parse out what we just built and upload it to scratch storage - echo "paths<> "$GITHUB_OUTPUT" - jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" - echo "EOF" >> "$GITHUB_OUTPUT" - - cp dist-manifest.json "$BUILD_MANIFEST_NAME" - - name: "Upload artifacts" - uses: actions/upload-artifact@v4 - with: - name: artifacts-build-global - path: | - ${{ steps.cargo-dist.outputs.paths }} - ${{ env.BUILD_MANIFEST_NAME }} - # Determines if we should publish/announce - host: - needs: - - plan - - build-local-artifacts - - build-global-artifacts - # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine) - if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - runs-on: "ubuntu-20.04" - outputs: - val: ${{ steps.host.outputs.manifest }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Install cargo-dist - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.14.1/cargo-dist-installer.sh | sh" - # Fetch artifacts from scratch-storage - - name: Fetch artifacts - uses: actions/download-artifact@v4 - with: - pattern: artifacts-* - path: target/distrib/ - merge-multiple: true - # This is a harmless no-op for GitHub Releases, hosting for that happens in "announce" - - id: host - shell: bash - run: | - cargo dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json - echo "artifacts uploaded and released successfully" - cat dist-manifest.json - echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" - - name: "Upload dist-manifest.json" - uses: actions/upload-artifact@v4 - with: - # Overwrite the previous copy - name: artifacts-dist-manifest - path: dist-manifest.json - - publish-homebrew-formula: - needs: - - plan - - host - runs-on: "ubuntu-20.04" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PLAN: ${{ needs.plan.outputs.val }} - GITHUB_USER: "axo bot" - GITHUB_EMAIL: "admin+bot@axo.dev" - if: ${{ !fromJson(needs.plan.outputs.val).announcement_is_prerelease || fromJson(needs.plan.outputs.val).publish_prereleases }} - steps: - - uses: actions/checkout@v4 - with: - repository: "libsql/homebrew-sqld" - token: ${{ secrets.HOMEBREW_TAP_TOKEN }} - # So we have access to the formula - - name: Fetch homebrew formulae - uses: actions/download-artifact@v4 - with: - pattern: artifacts-* - path: Formula/ - merge-multiple: true - # This is extra complex because you can make your Formula name not match your app name - # so we need to find releases with a *.rb file, and publish with that filename. - - name: Commit formula files - run: | - git config --global user.name "${GITHUB_USER}" - git config --global user.email "${GITHUB_EMAIL}" - - for release in $(echo "$PLAN" | jq --compact-output '.releases[] | select([.artifacts[] | endswith(".rb")] | any)'); do - filename=$(echo "$release" | jq '.artifacts[] | select(endswith(".rb"))' --raw-output) - name=$(echo "$filename" | sed "s/\.rb$//") - version=$(echo "$release" | jq .app_version --raw-output) - - git add "Formula/${filename}" - git commit -m "${name} ${version}" - done - git push - - # Create a GitHub Release while uploading all files to it - announce: - needs: - - plan - - host - - publish-homebrew-formula - # use "always() && ..." to allow us to wait for all publish jobs while - # still allowing individual publish jobs to skip themselves (for prereleases). - # "host" however must run to completion, no skipping allowed! - if: ${{ always() && needs.host.result == 'success' && (needs.publish-homebrew-formula.result == 'skipped' || needs.publish-homebrew-formula.result == 'success') }} - runs-on: "ubuntu-20.04" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: "Download GitHub Artifacts" - uses: actions/download-artifact@v4 - with: - pattern: artifacts-* - path: artifacts - merge-multiple: true - - name: Cleanup - run: | - # Remove the granular manifests - rm -f artifacts/*-dist-manifest.json - - name: Create GitHub Release - uses: ncipollo/release-action@v1 - with: - tag: ${{ needs.plan.outputs.tag }} - name: ${{ fromJson(needs.host.outputs.val).announcement_title }} - body: ${{ fromJson(needs.host.outputs.val).announcement_github_body }} - prerelease: ${{ fromJson(needs.host.outputs.val).announcement_is_prerelease }} - artifacts: "artifacts/*" diff --git a/.github/workflows/nemesis.yml b/.github/workflows/nemesis.yml deleted file mode 100644 index 9cfdbc39b06..00000000000 --- a/.github/workflows/nemesis.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: Nemesis Tests - -on: - schedule: - - cron: '* */4 * * *' - workflow_dispatch: - -env: - CARGO_TERM_COLOR: always - PROTOC_VERSION: 3.23.4 - # FIXME: There are some warnings depending on certain feature flags that - # we need to fix before we can enable this. - # RUSTFLAGS: "-D warnings" - -jobs: - test-nemesis: - runs-on: ubuntu-latest - if: github.repository == 'tursodatabase/libsql' - name: Run Nemesis Tests - env: - RUSTFLAGS: -D warnings - steps: - - uses: hecrj/setup-rust-action@v2 - - # needed because we run out of disk space during tests - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # when set to "true" but frees about 6 GB - tool-cache: true - - - name: Install deps - run: sudo apt update && sudo apt install -y libclang-dev - - - name: Install protoc - uses: taiki-e/install-action@v2 - with: - tool: protoc@${{ env.PROTOC_VERSION }} - - - uses: actions/checkout@v3 - - - name: Set up cargo cache - uses: actions/cache@v3 - continue-on-error: false - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- - - - name: Cargo build - run: | - cargo build - mv ./target/debug/sqld ~/.cargo/bin - sqld --version - - - name: Download MinIO binary - run: | - wget -q https://dl.min.io/server/minio/release/linux-amd64/minio -O minio - chmod +x minio - mv minio ~/.cargo/bin - minio --version - - - name: Nemesis tests checkout - uses: actions/checkout@v3 - with: - repository: tursodatabase/tursotest - ref: "main" - path: "nemesis-tests" - token: ${{ secrets.ACCESS_TOKEN_TURSO_TEST }} - - - name: Build nemesis test - uses: actions/setup-go@v4 - with: - go-version: '>=1.21.4' - - run: | - cd nemesis-tests - go build -o ../tursotests cmd/tursotest/main.go - - - name: Run nemesis tests - run: | - ./tursotests nemesis local diff --git a/.github/workflows/publish-crsqlite.yml b/.github/workflows/publish-crsqlite.yml deleted file mode 100644 index 52e643d49e5..00000000000 --- a/.github/workflows/publish-crsqlite.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: publish-crsqlite - -on: - push: - tags: - - "v*" - - "prebuild-test.*" - -jobs: - prebuild: - name: Prebuild for ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-latest - library_name: crsqlite.so - artifact_name: crsqlite.zip - asset_name: crsqlite-linux-x86_64.zip - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - - name: Load .env file - uses: xom9ikk/dotenv@v2 - with: - path: ./ - - - name: Build - run: | - cd libsql-sqlite3/ext/crr - make loadable - cd dist; zip crsqlite.zip ${{ matrix.library_name }} - - - name: Upload binaries to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: libsql-sqlite3/ext/crr/dist/${{ matrix.artifact_name }} - asset_name: ${{ matrix.asset_name }} - tag: ${{ github.ref }} diff --git a/.github/workflows/publish-server.yml b/.github/workflows/publish-server.yml deleted file mode 100644 index 10820457b87..00000000000 --- a/.github/workflows/publish-server.yml +++ /dev/null @@ -1,244 +0,0 @@ -name: ci - -on: - push: - branches: - - "main" - tags: - - libsql-server-v*.*.* - -env: - REGISTRY: ghcr.io - IMAGE_NAME: tursodatabase/libsql-server - -jobs: - build-amd64: - permissions: write-all - runs-on: ubuntu-latest - env: - platform: "linux/amd64" - steps: - - - name: Prepare - run: | - platform=${{ env.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout - uses: actions/checkout@v4 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=match,pattern=libsql-server-(.*),group=1 - type=raw,value=latest - type=sha,prefix=,format=short - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push by digest - id: build - uses: docker/build-push-action@v5 - with: - context: . - platforms: ${{ env.platform }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - build-amd64-debug: - permissions: write-all - runs-on: ubuntu-latest - env: - platform: "linux/amd64" - steps: - - - name: Prepare - run: | - platform=${{ env.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout - uses: actions/checkout@v4 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-debug - tags: | - type=match,pattern=libsql-server-(.*),group=1 - type=raw,value=debug - type=sha,prefix=,format=short - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push by digest - id: build - uses: docker/build-push-action@v5 - with: - context: . - platforms: ${{ env.platform }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - build-args: | - BUILD_DEBUG=true - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-debug-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - build-arm64: - permissions: write-all - runs-on: buildjet-8vcpu-ubuntu-2204-arm - env: - platform: "linux/arm64" - steps: - - - name: Prepare - run: | - platform=${{ env.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout - uses: actions/checkout@v4 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=match,pattern=libsql-server-(.*),group=1 - type=raw,value=latest - type=sha,prefix=,format=short - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push by digest - id: build - uses: docker/build-push-action@v5 - with: - context: . - platforms: ${{ env.platform }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - - - name: Export digestmatrix - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - - merge: - permissions: write-all - runs-on: ubuntu-latest - needs: - - build-amd64 - - build-arm64 - steps: - - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digests-* - merge-multiple: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=match,pattern=libsql-server-(.*),group=1 - type=raw,value=latest - type=sha,prefix=,format=short - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml deleted file mode 100644 index 82e66d26174..00000000000 --- a/.github/workflows/release-drafter.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Release Drafter - -on: - push: - # branches to consider in the event; optional, defaults to all - branches: - - main - # pull_request event is required only for autolabeler - pull_request: - # Only following types are handled by the action, but one can default to all as well - types: [opened, reopened, synchronize] - # pull_request_target event is required for autolabeler to support PRs from forks - # pull_request_target: - # types: [opened, reopened, synchronize] - -permissions: - contents: read - -jobs: - update_release_draft: - permissions: - # write permission is required to create a github release - contents: write - # write permission is required for autolabeler - # otherwise, read permission is required at least - pull-requests: write - runs-on: ubuntu-latest - steps: - # (Optional) GitHub Enterprise requires GHE_HOST variable set - #- name: Set GHE_HOST - # run: | - # echo "GHE_HOST=${GITHUB_SERVER_URL##https:\/\/}" >> $GITHUB_ENV - - # Drafts your next Release notes as Pull Requests are merged into "master" - - uses: release-drafter/release-drafter@v5 - # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml - # with: - # config-name: my-config.yml - # disable-autolabeler: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-libsql.yml b/.github/workflows/release-libsql.yml deleted file mode 100644 index 1d9e3e1e55b..00000000000 --- a/.github/workflows/release-libsql.yml +++ /dev/null @@ -1,207 +0,0 @@ -name: Create and publish libsql binaries - -on: - push: - #branches: ['main'] - tags: - - libsql-sqlite3-v*.*.* - -jobs: - - start-runner: - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Start EC2 runner - id: start-ec2-runner - uses: machulav/ec2-github-runner@v2 - with: - mode: start - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - ec2-image-id: ami-0e811aad4d084edb9 - ec2-instance-type: t4g.2xlarge - subnet-id: subnet-05010d45bfb86f5fb - security-group-id: sg-01bd3845621443669 - - build_binaries: - needs: start-runner - strategy: - matrix: - os: - - ubuntu-latest - - macos-14 - - ${{ needs.start-runner.outputs.label }} - variant: - - 'wasm-udf-dynamic' - - 'wasm-udf-wasmedge' - - 'wasm-udf' - - '' - - runs-on: ${{ matrix.os }} - steps: - #Debug output + special rustup config for the ubuntu/aarch64 runner - - name: show step matrix details - run: | - env - set -ex - uname -a - echo "os: ${{ matrix.os }}" - echo "variant: ${{ matrix.variant }}" - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > ./rustup.sh - chmod a+x rustup.sh - ./rustup.sh -y - if [[ "${{ matrix.os}}" == "${{ needs.start-runner.outputs.label }}" ]]; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH - echo "PATH=$PATH:/root/.cargo/bin" >> $GITHUB_PATH - fi - pwd - whoami - - #for windows only, use correct newlines - - name: Set git to use LF - if: matrix.os == 'windows-latest' - run: | - git config --global core.autocrlf false - git config --global core.eol lf - - - uses: actions/checkout@v3 - - - uses: dtolnay/rust-toolchain@stable - - - name: setup msys2 for windows - if: matrix.os == 'windows-latest' - uses: msys2/setup-msys2@v2 - with: - update: true - install: mingw-w64-x86_64-toolchain mingw-w64-x86_64-sqlite3 gcc make base-devel mingw-w64-x86_64-rust zip - msystem: mingw64 - path-type: inherit - - - name: run configure - working-directory: libsql-sqlite3 - run: | - set -ex - mkdir build - pushd build - case "${{ matrix.variant }}" in - "wasm-udf-dynamic") - ../configure --enable-wasm-runtime-dynamic - ;; - "wasm-udf-wasmedge") - # Fix for https://github.com/WasmEdge/WasmEdge/issues/2622 - if [[ "${{ matrix.os}}" == "${{ needs.start-runner.outputs.label }}" ]]; then - curl -SLO https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh - chmod a+x install.sh - ./install.sh --dist manylinux2014 -D - else - curl -sSf https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh | bash - fi - source ~/.wasmedge/env - ../configure --enable-wasm-runtime-wasmedge - ;; - "wasm-udf") - ../configure --enable-wasm-runtime - ;; - *) - ../configure - ;; - esac - popd - - - name: build - working-directory: libsql-sqlite3 - run: | - set -ex - if [[ "${{ matrix.variant }}" == "wasm-udf-wasmedge" ]]; then - source ~/.wasmedge/env || echo "no wasmedge installed" - fi - pushd build - make - make sqlite3.c - ls -la - popd - - - name: package artefacts (for named variants) - working-directory: libsql-sqlite3 - if: matrix.variant != '' - run: | - set -ex - uname -a - OS=$(uname -s) - ARCH=$(uname -m) - cd build - tar cvzf libsql-${{ github.ref_name }}-${{ matrix.variant }}-$OS-$ARCH.tar.gz .libs libsql sqlite3 - tar cvzf libsql-amalgamation-${{ github.ref_name }}-${{ matrix.variant }}-$OS-$ARCH.tar.gz sqlite3.c sqlite3.h - ls -la - cd .. - - - name: package artefacts - working-directory: libsql-sqlite3 - if: matrix.variant == '' - run: | - set -ex - uname -a - OS=$(uname -s) - ARCH=$(uname -m) - cd build - tar cvzf libsql-${{ github.ref_name }}-$OS-$ARCH.tar.gz .libs libsql sqlite3 - tar cvzf libsql-amalgamation-${{ github.ref_name }}-$OS-$ARCH.tar.gz sqlite3.c sqlite3.h - ls -la - cd .. - - - name: Push artefacts into release draft - working-directory: libsql-sqlite3 - uses: softprops/action-gh-release@v1 - if: ${{startsWith(github.ref, 'refs/tags/') }} - with: - fail_on_unmatched_files: false - files: | - build/libsql-${{ github.ref_name }}-${{ matrix.variant }}-Linux-x86_64.tar.gz - build/libsql-amalgamation-${{ github.ref_name }}-${{ matrix.variant }}-Linux-x86_64.tar.gz - build/libsql-${{ github.ref_name }}-Linux-x86_64.tar.gz - build/libsql-amalgamation-${{ github.ref_name }}-Linux-x86_64.tar.gz - build/libsql-${{ github.ref_name }}-${{ matrix.variant }}-Linux-aarch64.tar.gz - build/libsql-amalgamation-${{ github.ref_name }}-${{ matrix.variant }}-Linux-aarch64.tar.gz - build/libsql-${{ github.ref_name }}-Linux-aarch64.tar.gz - build/libsql-amalgamation-${{ github.ref_name }}-Linux-aarch64.tar.gz - build/libsql-${{ github.ref_name }}-${{ matrix.variant }}-Darwin-x86_64.tar.gz - build/libsql-amalgamation-${{ github.ref_name }}-${{ matrix.variant }}-Darwin-x86_64.tar.gz - build/libsql-${{ github.ref_name }}-Darwin-x86_64.tar.gz - build/libsql-amalgamation-${{ github.ref_name }}-Darwin-x86_64.tar.gz - build/libsql-${{ github.ref_name }}-${{ matrix.variant }}-Darwin-arm64.tar.gz - build/libsql-amalgamation-${{ github.ref_name }}-${{ matrix.variant }}-Darwin-arm64.tar.gz - build/libsql-${{ github.ref_name }}-Darwin-arm64.tar.gz - build/libsql-amalgamation-${{ github.ref_name }}-Darwin-arm64.tar.gz - - stop-runner: - name: Stop self-hosted EC2 runner - needs: - - start-runner # required to get output from the start-runner job - - build_binaries # required to wait when the main job is done - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@v2 - with: - mode: stop - github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - label: ${{ needs.start-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index 26eaba46cfa..00000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,199 +0,0 @@ -name: Rust - -on: - push: - branches: [ "main" ] - pull_request: - merge_group: - branches: [ "main" ] - -env: - CARGO_TERM_COLOR: always - PROTOC_VERSION: 3.23.4 - # FIXME: There are some warnings depending on certain feature flags that - # we need to fix before we can enable this. - # RUSTFLAGS: "-D warnings" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - checks: - runs-on: ubuntu-latest - name: Run Checks - env: - RUSTFLAGS: -D warnings - steps: - - uses: hecrj/setup-rust-action@v2 - - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Set up cargo cache - uses: actions/cache@v3 - continue-on-error: false - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- - - - name: Cargo fmt - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --check - - - name: Cargo check - uses: actions-rs/cargo@v1 - with: - command: check - args: --all-targets --all-features - - - name: Test openssl - shell: bash {0} - run: cargo tree -p libsql-server -i openssl; [ $? = 101 ] - - # Disabled because we vendor rusqlite which does not use - # clippy so it warns. - # - name: Clippy - # uses: actions-rs/clippy-check@v1 - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - # args: --all-targets --all-features -- -D warnings - - - features: - name: Check features and unused dependencies - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@nightly - - uses: taiki-e/install-action@cargo-hack - - uses: taiki-e/install-action@cargo-udeps - - uses: Swatinem/rust-cache@v2 - - run: cargo +nightly hack udeps -p libsql --each-feature - - run: RUSTFLAGS="-D warnings" cargo check -p libsql --no-default-features --features core - - run: RUSTFLAGS="-D warnings" cargo check -p libsql --no-default-features --features replication - - run: RUSTFLAGS="-D warnings" cargo check -p libsql --no-default-features --features remote - - test: - runs-on: ubuntu-latest - name: Run Tests - env: - RUSTFLAGS: -D warnings - steps: - - uses: hecrj/setup-rust-action@v2 - - # needed because we run out of disk space during tests - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - # this might remove tools that are actually needed, - # when set to "true" but frees about 6 GB - tool-cache: true - - - name: Install deps - run: sudo apt update && sudo apt install -y libclang-dev - - - name: Install protoc - uses: taiki-e/install-action@v2 - with: - tool: protoc@${{ env.PROTOC_VERSION }} - - - uses: taiki-e/install-action@v2 - with: - tool: nextest - - - uses: actions/checkout@v3 - - - name: Set up cargo cache - uses: actions/cache@v3 - continue-on-error: false - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- - - - name: Run tests - run: cargo nextest run - - - name: Run tests with custom pager - env: - LIBSQL_EXPERIMENTAL_PAGER: 1000 - run: cargo nextest run - - name: clean build dir - run: rm -rf libsql-ffi/bundled/SQLite3MultipleCiphers/build - - name: embedded replica encryption tests - run: cargo test -F test-encryption --package libsql-server --test tests embedded_replica - windows: - runs-on: windows-latest - name: Windows checks - steps: - - uses: hecrj/setup-rust-action@v2 - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Set up cargo cache - uses: actions/cache@v3 - continue-on-error: false - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: ${{ runner.os }}-cargo- - - name: check libsql remote - run: cargo check -p libsql --no-default-features -F remote - - # test-rust-wasm: - # runs-on: ubuntu-latest - # name: Run Rust Wasm Tests - # env: - # RUSTFLAGS: -D warnings - # steps: - # - uses: hecrj/setup-rust-action@v2 - # with: - # targets: [wasm32-unknown-unknown] - - # # needed because we run out of disk space during tests - # - name: Free Disk Space (Ubuntu) - # uses: jlumbroso/free-disk-space@main - # with: - # # this might remove tools that are actually needed, - # # when set to "true" but frees about 6 GB - # tool-cache: true - - # - uses: actions/checkout@v3 - - # - name: Set up cargo cache - # uses: actions/cache@v3 - # continue-on-error: false - # with: - # path: | - # ~/.cargo/bin/ - # ~/.cargo/registry/index/ - # ~/.cargo/registry/cache/ - # ~/.cargo/git/db/ - # target/ - # key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - # restore-keys: ${{ runner.os }}-cargo- - - # - name: Run check - # run: cargo check --verbose -p libsql --target wasm32-unknown-unknown --no-default-features --features cloudflare diff --git a/.github/workflows/server-pr-images.yml b/.github/workflows/server-pr-images.yml deleted file mode 100644 index 46cc2ec5931..00000000000 --- a/.github/workflows/server-pr-images.yml +++ /dev/null @@ -1,88 +0,0 @@ -# This workflow builds and pushes the pull request Docker image. Note -# that this workflow is only triggered for internal users and the pull -# request body must contain the string `+build`. -name: libsql server pull request Docker image - -on: - workflow_dispatch: - pull_request: - branches: [ "main" ] - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }}-server-devel - -jobs: - # docker image build and upload to ghcr - build-and-push-image: - # run this job if either: - # 1. The pull request is from the same repository and contains '+build' in the body - # 2. The workflow is manually triggered (workflow_dispatch event) - if: > - ( - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository && - github.event.pull_request.body && - contains(github.event.pull_request.body, '+build') - ) || - github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Get short SHA - id: get-short-sha - run: | - SHA="$(echo ${GITHUB_SHA::7})" - echo "sha=$SHA" >> "$GITHUB_OUTPUT" - - - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }}-${{ steps.get-short-sha.outputs.sha }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build and push Docker image with durable-wal - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }}-durable-wal-${{ steps.get-short-sha.outputs.sha }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - BUILD_DEBUG=true - ENABLE_FEATURES=durable-wal - - - name: Echo image name - run: | - echo "Pushed: ${{ steps.meta.outputs.tags }}-${{ steps.get-short-sha.outputs.sha }}" - echo "Pushed: ${{ steps.meta.outputs.tags }}-durable-wal-${{ steps.get-short-sha.outputs.sha }}" diff --git a/.github/workflows/sqlite3.yml b/.github/workflows/sqlite3.yml deleted file mode 100644 index 2610a698967..00000000000 --- a/.github/workflows/sqlite3.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Makefile CI with Wasm enabled - -on: - push: - branches: [ "main" ] - paths: - - "libsql-sqlite3/**" - pull_request: - paths: - - "libsql-sqlite3/**" - merge_group: - branches: [ "main" ] - paths: - - "libsql-sqlite3/**" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - make-sqlite3: - runs-on: ubuntu-latest - defaults: - run: - working-directory: libsql-sqlite3 - - steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@stable - - - name: get TCL - run: sudo apt-get install -y tcl8.6-dev - - - name: get WasmEdge - run: > - wget -O- https://github.com/WasmEdge/WasmEdge/releases/download/0.11.2/WasmEdge-slim-runtime-0.11.2-manylinux2014_x86_64.tar.gz | tar zxv - && sudo cp -r WasmEdge-0.11.2-Linux/include/* /usr/include/ - && sudo cp -r WasmEdge-0.11.2-Linux/lib64/* /usr/lib/ - - - name: configure with Wasm - run: ./configure --enable-wasm-runtime-dynamic - - - name: make the library generally available - run: mkdir tmp_lib && DESTDIR=$(pwd)/tmp_lib make liblibsql_install && sudo cp -r tmp_lib/usr/local/lib/* /usr/lib/ - - - name: Run tests - run: make test - - - name: Run Rust tests with Wasm - run: make rusttestwasm diff --git a/.github/workflows/storage-server-images.yml b/.github/workflows/storage-server-images.yml deleted file mode 100644 index bb54ab8ebff..00000000000 --- a/.github/workflows/storage-server-images.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Storage server Docker image - -on: - workflow_dispatch: - push: - branches: [ "main" ] - paths: - - "libsql-storage-server/**" - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }}-storage-server-devel - -jobs: - build-and-push-image: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Get short SHA - id: get-short-sha - run: | - SHA="$(echo ${GITHUB_SHA::7})" - echo "sha=$SHA" >> "$GITHUB_OUTPUT" - - - name: Build and push Docker image - uses: docker/build-push-action@v6 - with: - context: . - push: true - tags: | - ${{ steps.meta.outputs.tags }}-${{ steps.get-short-sha.outputs.sha }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - file: "./libsql-storage-server/Dockerfile" - - - name: Echo image name - run: | - echo "Pushed: ${{ steps.meta.outputs.tags }}-${{ steps.get-short-sha.outputs.sha }}" diff --git a/Cargo.lock b/Cargo.lock index 04ae7280657..2fdc1952b3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1671,12 +1671,46 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +[[package]] +name = "codspeed" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a104ac948e0188b921eb3fcbdd55dcf62e542df4c7ab7e660623f6288302089" +dependencies = [ + "colored", + "libc", + "serde_json", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "722c36bdc62d9436d027256ce2627af81ac7a596dfc7d13d849d0d212448d7fe" +dependencies = [ + "codspeed", + "colored", + "criterion", + "futures", + "tokio", +] + [[package]] name = "colorchoice" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "combine" version = "4.6.7" @@ -3439,7 +3473,7 @@ dependencies = [ "bincode", "bitflags 2.6.0", "bytes", - "criterion", + "codspeed-criterion-compat", "fallible-iterator 0.3.0", "futures", "http 0.2.12", diff --git a/libsql/Cargo.toml b/libsql/Cargo.toml index fa89cc68add..f5c0ed1e914 100644 --- a/libsql/Cargo.toml +++ b/libsql/Cargo.toml @@ -44,7 +44,11 @@ libsql_replication = { version = "0.4", path = "../libsql-replication", optional async-stream = { version = "0.3.5", optional = true } [dev-dependencies] -criterion = { version = "0.5", features = ["html_reports", "async", "async_futures", "async_tokio"] } +criterion = { package = "codspeed-criterion-compat", version = "2.6.0", features = [ + "async", + "async_futures", + "async_tokio", +] } pprof = { version = "0.12.1", features = ["criterion", "flamegraph"] } tokio = { version = "1.29.1", features = ["full"] } tokio-test = "0.4"