From e0803c459891325e83bd81eeb91a1620ca601425 Mon Sep 17 00:00:00 2001
From: Vincent Sarago <vincent.sarago@gmail.com>
Date: Thu, 27 Jul 2023 15:24:49 +0200
Subject: [PATCH] add benchmark comparison (#676)

* add benchmark comparison

* fix
---
 .github/workflows/benchmark.yml | 35 ++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 7afca7757..b92311196 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -19,7 +19,9 @@ jobs:
       - name: install siege
         run: |
           sudo apt update
-          sudo apt install --yes siege
+          sudo apt install --yes siege jq
+          siege -C
+
       - name: Start containers
         run: docker-compose -f "docker-compose.yml" up -d --build benchmark
 
@@ -30,12 +32,39 @@ jobs:
 
       - name: Run siege (WebMercator TMS)
         run: |
-          siege --file .github/data/urls.txt -b -c 1 -r 100 --quiet --json-output
+          siege --file .github/data/urls.txt -b -c 1 -r 100 --json-output 2>&1 | jq -c > results.json
+          echo "Benchmark Results"
+          cat results.json | jq
+          echo "Parse Results"
+          cat results.json | jq '{"name": "WebMercator elapsed_time", "unit": "s", "value": .elapsed_time}, {"name": "WebMercator data_transferred", "unit": "Megabytes", "value": .data_transferred}, {"name": "WebMercator response_time", "unit": "s", "value": .response_time}, {"name": "WebMercator longest_transaction", "unit": "s", "value": .longest_transaction}' > output.json
 
       - name: Run siege (WGS1984Quad TMS)
         run: |
-          siege --file .github/data/urls_wgs84.txt -b -c 1 -r 100 --quiet --json-output
+          siege --file .github/data/urls_wgs84.txt -b -c 1 -r 100 --json-output 2>&1 | jq -c > results.json
+          echo "Benchmark Results"
+          cat results.json | jq
+          echo "Parse Results"
+          cat results.json | jq '{"name": "WGS1984Quad elapsed_time", "unit": "s", "value": .elapsed_time}, {"name": "WGS1984Quad data_transferred", "unit": "Megabytes", "value": .data_transferred}, {"name": "WGS1984Quad response_time", "unit": "s", "value": .response_time}, {"name": "WGS1984Quad longest_transaction", "unit": "s", "value": .longest_transaction}' >> output.json
 
       - name: Stop containers
         if: always()
         run: docker-compose -f "docker-compose.yml" down
+
+      - name: Merge Outputs
+        run: |
+          cat output.json | jq '[inputs]' > benchmark.json
+
+      - name: Check and Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: TiTiler performance Benchmarks
+          tool: 'customSmallerIsBetter'
+          output-file-path: benchmark.json
+          alert-threshold: '130%'
+          comment-on-alert: true
+          fail-on-alert: false
+          # GitHub API token to make a commit comment
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          gh-pages-branch: 'gh-benchmarks'
+          # Make a commit only if main
+          auto-push: ${{ github.ref == 'refs/heads/main' }}