canary-multi-turn.yml

name: Canary for multi-turn

on: [workflow_dispatch, push, pull_request]

jobs:

  canary-multi-turn:
    runs-on: ubuntu-22.04
    timeout-minutes: 3
    strategy:
      max-parallel: 3
      fail-fast: false
      matrix:
        model:
          - meta-llama/llama-3.2-3b-instruct             # $0.0300/$0.0300 [ 128K]
          - qwen/qwen-2-7b-instruct                      # $0.0540/$0.0540 [  32K]
          - mistralai/mistral-7b-instruct-v0.3           # $0.0550/$0.0550 [  32K]
          - meta-llama/llama-3.1-8b-instruct             # $0.0550/$0.0550 [ 128K]
          - microsoft/phi-3.5-mini-128k-instruct         # $0.1000/$0.1000 [ 128K]
          - mistralai/mistral-nemo                       # $0.1300/$0.1300 [ 128K]
          - mistralai/mixtral-8x7b-instruct              # $0.2400/$0.2400 [  32K]
          - meta-llama/llama-3.1-70b-instruct            # $0.3000/$0.3000 [ 128K]
          - google/gemini-flash-1.5                      # $0.0750/$0.3000 [1000K]
          - ai21/jamba-1-5-mini                          # $0.2000/$0.4000 [ 256K]
          - openai/gpt-4o-mini                           # $0.1500/$0.6000 [ 128K]
    steps:
      - uses: actions/checkout@v4

      - run: ./query-llm.js tests/canary-single-turn.txt
        env:
          LLM_API_BASE_URL: ${{ secrets.LLM_API_BASE_URL }}
          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
          LLM_CHAT_MODEL: ${{ matrix.model }}

      - run: ./query-llm.js tests/canary-multi-turn.txt
        env:
          LLM_API_BASE_URL: ${{ secrets.LLM_API_BASE_URL }}
          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
          LLM_CHAT_MODEL: ${{ matrix.model }}