From 1b23da63eae422a7e371203e935c78bbf02f94ee Mon Sep 17 00:00:00 2001
From: Wei-Zao <80608670+Wei-Zao@users.noreply.github.com>
Date: Fri, 5 Jan 2024 16:40:51 +0800
Subject: [PATCH] Add batch_evaluate.sh

---
 taskbench/batch_evaluate.sh | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 taskbench/batch_evaluate.sh

diff --git a/taskbench/batch_evaluate.sh b/taskbench/batch_evaluate.sh
new file mode 100644
index 0000000..60b6932
--- /dev/null
+++ b/taskbench/batch_evaluate.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+set -x
+set -e
+
+export CUDA_VISIBLE_DEVICES=0
+
+data_dir=$1
+prediction_dir=$2
+
+if [[ $data_dir == *"dailylifeapis"* ]]; then
+    dependency_type="temporal"
+else
+    dependency_type="resource"
+fi
+
+for file in $data_dir/$prediction_dir/*.json
+do
+    llm=$(basename $file .json)
+    #  replace prediction_dir's "predictions" with "metrics"
+    metrics=$(echo $prediction_dir | sed 's/predictions/metrics/g')
+    if [ -f $data_dir/$metrics/${llm}_splits_all_tools_all_metric_all.json ] && [ -s $data_dir/$metrics/${llm}_splits_all_tools_all_metric_all.json ];
+    then
+        continue
+    fi
+    echo $llm
+    python evaluate.py --data_dir $data_dir --prediction_dir $prediction_dir --llm $llm --splits all --n_tools all --mode add --dependency_type $dependency_type -m all 
+done
\ No newline at end of file