notebooks/run_all_benchmark.ipynb

{
    "cells": [
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Run all TCAB benchmark"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {},
            "outputs": [],
            "source": [
                "%load_ext autoreload\n",
                "%autoreload 2"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 2,
            "metadata": {},
            "outputs": [],
            "source": [
                "import time\n",
                "from functools import reduce\n",
                "from itertools import product\n",
                "\n",
                "from tqdm.auto import tqdm\n",
                "\n",
                "from scripts import (\n",
                "    distribute_experiments,\n",
                "    make_experiment,\n",
                "    make_official_dataset_splits,\n",
                "    run_experiment,\n",
                ")\n",
                "\n",
                "# from scripts import encode_main\n",
                "\n",
                "t = time.time()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 3,
            "metadata": {},
            "outputs": [],
            "source": [
                "def process_args(args_grid):\n",
                "    liste1 = [dict(zip(args_grid.keys(), v)) for v in product(*args_grid.values())]\n",
                "    liste2 = [list(reduce(lambda x, y: x + y, dico.items())) for dico in liste1]\n",
                "    liste3 = [list(filter(None, liste)) for liste in liste2]\n",
                "\n",
                "    print(f\"Number of parameters combinations: {len(liste3)}\")\n",
                "    for elem in liste3:\n",
                "        print(elem)\n",
                "\n",
                "    return liste3"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 4,
            "metadata": {},
            "outputs": [],
            "source": [
                "def call_main_args(main, processed_args_grid):\n",
                "    for args in tqdm(processed_args_grid):\n",
                "        print(f\"\\n### Running main with {args}\\n\")\n",
                "        main(args)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 1. scripts/encode_main.py"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 5,
            "metadata": {},
            "outputs": [],
            "source": [
                "# encode_main.main()"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 2. scripts/make_official_dataset_splits.py"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 6,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- reading data\n",
                        "--- dropping unsupported datasets\n",
                        "--- dropping duplicates\n",
                        "--- making splits across all datasets ---\n",
                        "allocine\n",
                        "--- filtering for dataset  (28913, 17)\n"
                    ]
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "0286e49ba2bf4a4e84757a9e8b04ea1e",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/20000 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "07fd741286bd4a35b72390853e6cdf54",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/8010 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- Train DF stats ---\n",
                        "total_instances: 17348, \n",
                        "attack_name: {'input_reduction': 1187, 'clean': 11990, 'deepwordbug': 976, 'bae': 751, 'textfooler': 987, 'textbugger': 1004, 'pwws': 453}, \n",
                        "target_dataset: {'allocine': 17348}, \n",
                        "target_model: {'distilcamembert': 17348}, \n",
                        "status: {'success': 5358, 'clean': 11990}, \n",
                        "attack_toolchain: {'textattack': 5358, 'none': 11990}, \n",
                        "scenario: {'sentiment': 17348}, \n",
                        "\n",
                        "--- Test DF stats ---\n",
                        "total_instances: 5783, \n",
                        "attack_name: {'input_reduction': 402, 'clean': 4006, 'deepwordbug': 326, 'textfooler': 333, 'pwws': 147, 'bae': 238, 'textbugger': 331}, \n",
                        "target_dataset: {'allocine': 5783}, \n",
                        "target_model: {'distilcamembert': 5783}, \n",
                        "status: {'success': 1777, 'clean': 4006}, \n",
                        "attack_toolchain: {'textattack': 1777, 'none': 4006}, \n",
                        "scenario: {'sentiment': 5783}, \n",
                        "\n",
                        "--- Val DF stats ---\n",
                        "total_instances: 5782, \n",
                        "attack_name: {'clean': 4004, 'deepwordbug': 334, 'textfooler': 336, 'bae': 228, 'input_reduction': 411, 'textbugger': 328, 'pwws': 141}, \n",
                        "target_dataset: {'allocine': 5782}, \n",
                        "target_model: {'distilcamembert': 5782}, \n",
                        "status: {'clean': 4004, 'success': 1778}, \n",
                        "attack_toolchain: {'none': 4004, 'textattack': 1778}, \n",
                        "scenario: {'sentiment': 5782}, \n",
                        "\n",
                        "--- filtering for target model \n",
                        "--- Train DF stats ---\n",
                        "total_instances: 17348, \n",
                        "attack_name: {'input_reduction': 1187, 'clean': 11990, 'deepwordbug': 976, 'bae': 751, 'textfooler': 987, 'textbugger': 1004, 'pwws': 453}, \n",
                        "target_dataset: {'allocine': 17348}, \n",
                        "target_model: {'distilcamembert': 17348}, \n",
                        "status: {'success': 5358, 'clean': 11990}, \n",
                        "attack_toolchain: {'textattack': 5358, 'none': 11990}, \n",
                        "scenario: {'sentiment': 17348}, \n",
                        "\n",
                        "--- Test DF stats ---\n",
                        "total_instances: 5783, \n",
                        "attack_name: {'input_reduction': 402, 'clean': 4006, 'deepwordbug': 326, 'textfooler': 333, 'pwws': 147, 'bae': 238, 'textbugger': 331}, \n",
                        "target_dataset: {'allocine': 5783}, \n",
                        "target_model: {'distilcamembert': 5783}, \n",
                        "status: {'success': 1777, 'clean': 4006}, \n",
                        "attack_toolchain: {'textattack': 1777, 'none': 4006}, \n",
                        "scenario: {'sentiment': 5783}, \n",
                        "\n",
                        "--- Val DF stats ---\n",
                        "total_instances: 5782, \n",
                        "attack_name: {'clean': 4004, 'deepwordbug': 334, 'textfooler': 336, 'bae': 228, 'input_reduction': 411, 'textbugger': 328, 'pwws': 141}, \n",
                        "target_dataset: {'allocine': 5782}, \n",
                        "target_model: {'distilcamembert': 5782}, \n",
                        "status: {'clean': 4004, 'success': 1778}, \n",
                        "attack_toolchain: {'none': 4004, 'textattack': 1778}, \n",
                        "scenario: {'sentiment': 5782}, \n",
                        "\n",
                        "--- allocine distilcamembert --- DONE.\n",
                        "--- allocine --- DONE.\n",
                        "--- combining splits across all datasets ---\n",
                        "--- DONE.\n"
                    ]
                }
            ],
            "source": [
                "make_official_dataset_splits.main()"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 3. scripts/distribute_experiments.py"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 7,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Number of parameters combinations: 4\n",
                        "['--embeddings_name', 'default', '--experiment_setting', 'clean_vs_all']\n",
                        "['--embeddings_name', 'default', '--experiment_setting', 'multiclass_with_clean']\n",
                        "['--embeddings_name', 'fr+small', '--experiment_setting', 'clean_vs_all']\n",
                        "['--embeddings_name', 'fr+small', '--experiment_setting', 'multiclass_with_clean']\n"
                    ]
                }
            ],
            "source": [
                "args_grid = {\n",
                "    \"--embeddings_name\": [\"default\", \"fr+small\"],\n",
                "    \"--experiment_setting\": [\"clean_vs_all\", \"multiclass_with_clean\"],\n",
                "}\n",
                "args_grid = process_args(args_grid)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 8,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "6767204c2ca64798beaba1c93574997a",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/4 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "\n",
                        "### Running main with ['--embeddings_name', 'default', '--experiment_setting', 'clean_vs_all']\n",
                        "\n",
                        "Reading train.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\train.csv\n",
                        "Reading val.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\val.csv\n",
                        "Reading test.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\test.csv\n",
                        "Dropping attacks by count, if less than < 10\n"
                    ]
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "49dc6526caaa4896924f89bd8c6fb3da",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/4006 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        " *** clean_vs_all saving to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "\n",
                        "### Running main with ['--embeddings_name', 'default', '--experiment_setting', 'multiclass_with_clean']\n",
                        "\n",
                        "Reading train.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\train.csv\n",
                        "Reading val.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\val.csv\n",
                        "Reading test.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\test.csv\n",
                        "Dropping attacks by count, if less than < 10\n"
                    ]
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "7feeefb7e347446f82ea7ea0d83e80b8",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/4006 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        " *** multiclass_with_clean saving to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "\n",
                        "### Running main with ['--embeddings_name', 'fr+small', '--experiment_setting', 'clean_vs_all']\n",
                        "\n",
                        "Reading train.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\train.csv\n",
                        "Reading val.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\val.csv\n",
                        "Reading test.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\test.csv\n",
                        "Dropping attacks by count, if less than < 10\n"
                    ]
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "337120c850ad4399a73bb4d578000a87",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/4006 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        " *** clean_vs_all saving to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "\n",
                        "### Running main with ['--embeddings_name', 'fr+small', '--experiment_setting', 'multiclass_with_clean']\n",
                        "\n",
                        "Reading train.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\train.csv\n",
                        "Reading val.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\val.csv\n",
                        "Reading test.csv from  data_tcab\\official_TCAB_splits\\splits_by_dataset_and_tm\\allocine\\distilcamembert\\test.csv\n",
                        "Dropping attacks by count, if less than < 10\n"
                    ]
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "aba6e40d5e84402e84df44f168f67c9e",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/4006 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        " *** multiclass_with_clean saving to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n"
                    ]
                }
            ],
            "source": [
                "call_main_args(distribute_experiments.main, args_grid)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 4. scripts/make_experiments.py"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 9,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Number of parameters combinations: 4\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean']\n"
                    ]
                }
            ],
            "source": [
                "embeddings_list = [\"default\", \"fr+small\"]\n",
                "experiment_list = [\"clean_vs_all\", \"multiclass_with_clean\"]\n",
                "experiment_dir_list = [\n",
                "    f\"data_tcab/detection-experiments/allocine/distilcamembert/{emb}/{exp}\"\n",
                "    for emb, exp in product(embeddings_list, experiment_list)\n",
                "]\n",
                "args_grid = {\"--experiment_dir\": experiment_dir_list}\n",
                "args_grid = process_args(args_grid)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 10,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "448c790596904aec8c211ae9cd1ebd2a",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/4 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all']\n",
                        "\n",
                        "making exp into...  data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all\n",
                        "\n",
                        "--- loading known instances from  data_tcab/embeddings/\n",
                        "--- target_model: distilcamembert, target_dataset: allocine\n",
                        "--- No. joblib files of varied size to read: 1\n",
                        "[0] data_tcab\\embeddings\\default_distilcamembert_allocine_ALL_ALL.joblib\n",
                        "\n",
                        "done, no. unique instances with extracted features: 28,913\n",
                        "\n",
                        "--- creating train.joblib\n",
                        "    cannot find embeddings for 0 / 10,716 instances\n",
                        "\n",
                        "--- creating val.joblib\n",
                        "    cannot find embeddings for 0 / 3,556 instances\n",
                        "\n",
                        "--- creating test_release.joblib\n",
                        "    cannot find embeddings for 0 / 2,891 instances\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean']\n",
                        "\n",
                        "making exp into...  data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean\n",
                        "\n",
                        "--- loading known instances from  data_tcab/embeddings/\n",
                        "--- target_model: distilcamembert, target_dataset: allocine\n",
                        "--- No. joblib files of varied size to read: 1\n",
                        "[0] data_tcab\\embeddings\\default_distilcamembert_allocine_ALL_ALL.joblib\n",
                        "\n",
                        "done, no. unique instances with extracted features: 28,913\n",
                        "\n",
                        "--- creating train.joblib\n",
                        "    cannot find embeddings for 0 / 6,545 instances\n",
                        "\n",
                        "--- creating val.joblib\n",
                        "    cannot find embeddings for 0 / 2,189 instances\n",
                        "\n",
                        "--- creating test_release.joblib\n",
                        "    cannot find embeddings for 0 / 2,891 instances\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all']\n",
                        "\n",
                        "making exp into...  data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all\n",
                        "\n",
                        "--- loading known instances from  data_tcab/embeddings/\n",
                        "--- target_model: distilcamembert, target_dataset: allocine\n",
                        "--- No. joblib files of varied size to read: 1\n",
                        "[0] data_tcab\\embeddings\\fr+small_distilcamembert_allocine_ALL_ALL.joblib\n",
                        "\n",
                        "done, no. unique instances with extracted features: 28,913\n",
                        "\n",
                        "--- creating train.joblib\n",
                        "    cannot find embeddings for 0 / 10,716 instances\n",
                        "\n",
                        "--- creating val.joblib\n",
                        "    cannot find embeddings for 0 / 3,556 instances\n",
                        "\n",
                        "--- creating test_release.joblib\n",
                        "    cannot find embeddings for 0 / 2,891 instances\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean']\n",
                        "\n",
                        "making exp into...  data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean\n",
                        "\n",
                        "--- loading known instances from  data_tcab/embeddings/\n",
                        "--- target_model: distilcamembert, target_dataset: allocine\n",
                        "--- No. joblib files of varied size to read: 1\n",
                        "[0] data_tcab\\embeddings\\fr+small_distilcamembert_allocine_ALL_ALL.joblib\n",
                        "\n",
                        "done, no. unique instances with extracted features: 28,913\n",
                        "\n",
                        "--- creating train.joblib\n",
                        "    cannot find embeddings for 0 / 6,545 instances\n",
                        "\n",
                        "--- creating val.joblib\n",
                        "    cannot find embeddings for 0 / 2,189 instances\n",
                        "\n",
                        "--- creating test_release.joblib\n",
                        "    cannot find embeddings for 0 / 2,891 instances\n"
                    ]
                }
            ],
            "source": [
                "call_main_args(make_experiment.main, args_grid)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 5. scripts/run_experiment.py"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 11,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Number of parameters combinations: 48\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'all', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'all', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'all', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'all', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'all', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'all', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'all', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'all', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'all', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'all', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'all', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'all', '--model', 'LGB', '--model_n_jobs', '10']\n"
                    ]
                }
            ],
            "source": [
                "args_grid = {\n",
                "    \"--experiment_dir\": experiment_dir_list,\n",
                "    \"--feature_setting\": [\"bert\", \"bert+tp\", \"bert+tp+lm\", \"all\"],\n",
                "    \"--model\": [\"LR\", \"RF\", \"LGB\"],\n",
                "    \"--model_n_jobs\": [\"10\"],\n",
                "}\n",
                "args_grid = process_args(args_grid)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 12,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "42a8b5341d6b4a33b859866b9539405f",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/48 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LR\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.716 total time=   2.4s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.723 total time=   1.8s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.714 total time=   1.6s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.713 total time=   1.6s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.721 total time=   1.7s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.717 total time=   1.7s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:13\n",
                        "{'train_accuracy': 0.7692235908921239, 'train_balanced_accuracy': 0.7692235908921239, 'train_confusion_matrix': [[4245, 1113], [1360, 3998]], 'train_recall': 0.746173945502053, 'train_precision': 0.7822343963999218, 'train_f1_score': 0.7637787754322285, 'train_roc_auc': 0.7692235908921239}\n",
                        "{'validation_accuracy': 0.6991001124859393, 'validation_balanced_accuracy': 0.6991001124859393, 'validation_confusion_matrix': [[1368, 410], [660, 1118]], 'validation_recall': 0.6287964004499438, 'validation_precision': 0.731675392670157, 'validation_f1_score': 0.6763460375075621, 'validation_roc_auc': 0.6991001124859393}\n",
                        "{'test_accuracy': 0.7343479764787271, 'test_balanced_accuracy': 0.7090039267739616, 'test_confusion_matrix': [[1555, 454], [314, 568]], 'test_recall': 0.6439909297052154, 'test_precision': 0.5557729941291585, 'test_f1_score': 0.5966386554621849, 'test_roc_auc': 0.7090039267739616}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LR\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\RF\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.610 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.592 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.595 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.615 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.596 total time=   1.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.604 total time=   1.2s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.610 total time=   0.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.592 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.595 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.615 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.595 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.604 total time=   1.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.624 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.613 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.624 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.631 total time=   1.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.619 total time=   1.5s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.629 total time=   1.5s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.624 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.619 total time=   1.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.624 total time=   1.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.631 total time=   1.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.620 total time=   1.5s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.630 total time=   1.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:32\n",
                        "{'train_accuracy': 0.7051138484509145, 'train_balanced_accuracy': 0.7051138484509145, 'train_confusion_matrix': [[4129, 1229], [1931, 3427]], 'train_recall': 0.6396043299738708, 'train_precision': 0.7360395189003437, 'train_f1_score': 0.6844417815058917, 'train_roc_auc': 0.7051138484509145}\n",
                        "{'validation_accuracy': 0.6220472440944882, 'validation_balanced_accuracy': 0.6220472440944882, 'validation_confusion_matrix': [[1248, 530], [814, 964]], 'validation_recall': 0.5421822272215973, 'validation_precision': 0.6452476572958501, 'validation_f1_score': 0.589242053789731, 'validation_roc_auc': 0.6220472440944882}\n",
                        "{'test_accuracy': 0.6703562781044621, 'test_balanced_accuracy': 0.6387921022067363, 'test_confusion_matrix': [[1446, 563], [390, 492]], 'test_recall': 0.5578231292517006, 'test_precision': 0.4663507109004739, 'test_f1_score': 0.5080020650490449, 'test_roc_auc': 0.6387921022067363}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\RF\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LGB\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.617 total time=   0.4s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.627 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.620 total time=   0.4s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.653 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.660 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.653 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.624 total time=   0.4s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.637 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.636 total time=   0.4s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.664 total time=   0.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.667 total time=   0.7s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.669 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.617 total time=   0.4s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.627 total time=   0.4s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.620 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.660 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.669 total time=   0.7s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.672 total time=   0.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.624 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.637 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.636 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.671 total time=   1.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.688 total time=   1.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.694 total time=   1.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:20\n",
                        "{'train_accuracy': 0.8608622620380739, 'train_balanced_accuracy': 0.8608622620380739, 'train_confusion_matrix': [[4569, 789], [702, 4656]], 'train_recall': 0.8689809630459127, 'train_precision': 0.8550964187327824, 'train_f1_score': 0.8619827825603998, 'train_roc_auc': 0.8608622620380739}\n",
                        "{'validation_accuracy': 0.6605736782902137, 'validation_balanced_accuracy': 0.6605736782902136, 'validation_confusion_matrix': [[1372, 406], [801, 977]], 'validation_recall': 0.5494938132733408, 'validation_precision': 0.7064352856109906, 'validation_f1_score': 0.6181588105030053, 'validation_roc_auc': 0.6605736782902136}\n",
                        "{'test_accuracy': 0.6952611553095814, 'test_balanced_accuracy': 0.6541673580001106, 'test_confusion_matrix': [[1526, 483], [398, 484]], 'test_recall': 0.5487528344671202, 'test_precision': 0.500517063081696, 'test_f1_score': 0.523526230394808, 'test_roc_auc': 0.6541673580001106}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LGB\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LR\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.740 total time=   2.2s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.721 total time=   2.3s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.744 total time=   2.2s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.734 total time=   2.2s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.716 total time=   2.3s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.739 total time=   2.2s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:16\n",
                        "{'train_accuracy': 0.7892870474057484, 'train_balanced_accuracy': 0.7892870474057484, 'train_confusion_matrix': [[4387, 971], [1287, 4071]], 'train_recall': 0.7597984322508399, 'train_precision': 0.8074176913923047, 'train_f1_score': 0.7828846153846154, 'train_roc_auc': 0.7892870474057484}\n",
                        "{'validation_accuracy': 0.7275028121484814, 'validation_balanced_accuracy': 0.7275028121484814, 'validation_confusion_matrix': [[1421, 357], [612, 1166]], 'validation_recall': 0.655793025871766, 'validation_precision': 0.7655942219304005, 'validation_f1_score': 0.7064525901242048, 'validation_roc_auc': 0.7275028121484816}\n",
                        "{'test_accuracy': 0.7512971290211, 'test_balanced_accuracy': 0.729149383330568, 'test_confusion_matrix': [[1579, 430], [289, 593]], 'test_recall': 0.6723356009070295, 'test_precision': 0.5796676441837733, 'test_f1_score': 0.6225721784776903, 'test_roc_auc': 0.7291493833305679}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LR\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\RF\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.649 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.636 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.617 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.648 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.635 total time=   1.3s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.624 total time=   1.2s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.648 total time=   0.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.634 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.617 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.645 total time=   1.3s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.634 total time=   1.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.623 total time=   1.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.669 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.652 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.667 total time=   1.0s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.667 total time=   1.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.657 total time=   1.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.665 total time=   1.6s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.669 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.659 total time=   1.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.658 total time=   1.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.665 total time=   1.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.661 total time=   1.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.663 total time=   1.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:32\n",
                        "{'train_accuracy': 0.7317095931317655, 'train_balanced_accuracy': 0.7317095931317656, 'train_confusion_matrix': [[4358, 1000], [1875, 3483]], 'train_recall': 0.6500559910414334, 'train_precision': 0.7769350881106402, 'train_f1_score': 0.7078548927954478, 'train_roc_auc': 0.7317095931317656}\n",
                        "{'validation_accuracy': 0.6690101237345332, 'validation_balanced_accuracy': 0.6690101237345332, 'validation_confusion_matrix': [[1378, 400], [777, 1001]], 'validation_recall': 0.562992125984252, 'validation_precision': 0.7144896502498216, 'validation_f1_score': 0.6297577854671281, 'validation_roc_auc': 0.6690101237345332}\n",
                        "{'test_accuracy': 0.7097890003459011, 'test_balanced_accuracy': 0.6751147613516951, 'test_confusion_matrix': [[1535, 474], [365, 517]], 'test_recall': 0.5861678004535147, 'test_precision': 0.5216952573158425, 'test_f1_score': 0.5520555258942872, 'test_roc_auc': 0.6751147613516951}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\RF\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LGB\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.667 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.668 total time=   0.4s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.664 total time=   0.4s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.701 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.695 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.695 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.684 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.682 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.671 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.719 total time=   0.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.712 total time=   0.7s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.710 total time=   0.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.667 total time=   0.4s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.668 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.664 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.709 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.710 total time=   0.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.712 total time=   0.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.684 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.682 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.671 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.730 total time=   1.1s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.734 total time=   1.2s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.726 total time=   1.2s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:19\n",
                        "{'train_accuracy': 0.860675625233296, 'train_balanced_accuracy': 0.860675625233296, 'train_confusion_matrix': [[4722, 636], [857, 4501]], 'train_recall': 0.8400522583053378, 'train_precision': 0.8761923301537863, 'train_f1_score': 0.8577417818008576, 'train_roc_auc': 0.860675625233296}\n",
                        "{'validation_accuracy': 0.7255343082114736, 'validation_balanced_accuracy': 0.7255343082114736, 'validation_confusion_matrix': [[1484, 294], [682, 1096]], 'validation_recall': 0.6164229471316085, 'validation_precision': 0.7884892086330936, 'validation_f1_score': 0.691919191919192, 'validation_roc_auc': 0.7255343082114736}\n",
                        "{'test_accuracy': 0.7540643375994466, 'test_balanced_accuracy': 0.7177838615120845, 'test_confusion_matrix': [[1629, 380], [331, 551]], 'test_recall': 0.6247165532879818, 'test_precision': 0.5918367346938775, 'test_f1_score': 0.6078323221180363, 'test_roc_auc': 0.7177838615120844}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LGB\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LR\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.781 total time=   2.4s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.780 total time=   2.0s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.774 total time=   2.3s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.769 total time=   2.4s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.778 total time=   2.2s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.772 total time=   2.1s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:17\n",
                        "{'train_accuracy': 0.8260544979469951, 'train_balanced_accuracy': 0.8260544979469951, 'train_confusion_matrix': [[4586, 772], [1092, 4266]], 'train_recall': 0.7961926091825308, 'train_precision': 0.8467645891226677, 'train_f1_score': 0.8207002693343594, 'train_roc_auc': 0.8260544979469951}\n",
                        "{'validation_accuracy': 0.7674353205849269, 'validation_balanced_accuracy': 0.7674353205849269, 'validation_confusion_matrix': [[1486, 292], [535, 1243]], 'validation_recall': 0.6991001124859393, 'validation_precision': 0.8097719869706841, 'validation_f1_score': 0.7503773015393903, 'validation_roc_auc': 0.7674353205849269}\n",
                        "{'test_accuracy': 0.7848495330335524, 'test_balanced_accuracy': 0.7574249211879873, 'test_confusion_matrix': [[1663, 346], [276, 606]], 'test_recall': 0.6870748299319728, 'test_precision': 0.6365546218487395, 'test_f1_score': 0.6608505997818974, 'test_roc_auc': 0.7574249211879874}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LR\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\RF\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.751 total time=   0.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.745 total time=   1.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.756 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.753 total time=   1.3s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.747 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.756 total time=   1.3s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.750 total time=   0.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.745 total time=   0.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.756 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.753 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.747 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.758 total time=   1.2s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.757 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.756 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.768 total time=   1.0s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.757 total time=   1.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.758 total time=   1.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.772 total time=   1.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.758 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.754 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.764 total time=   1.0s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.759 total time=   1.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.756 total time=   1.5s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.771 total time=   1.6s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:32\n",
                        "{'train_accuracy': 0.7837812616648003, 'train_balanced_accuracy': 0.7837812616648003, 'train_confusion_matrix': [[4464, 894], [1423, 3935]], 'train_recall': 0.7344158268010451, 'train_precision': 0.8148685027956099, 'train_f1_score': 0.7725532541474428, 'train_roc_auc': 0.7837812616648003}\n",
                        "{'validation_accuracy': 0.7696850393700787, 'validation_balanced_accuracy': 0.7696850393700787, 'validation_confusion_matrix': [[1473, 305], [514, 1264]], 'validation_recall': 0.7109111361079865, 'validation_precision': 0.8056086679413639, 'validation_f1_score': 0.7553032566477442, 'validation_roc_auc': 0.7696850393700788}\n",
                        "{'test_accuracy': 0.788654444828779, 'test_balanced_accuracy': 0.7668408826945412, 'test_confusion_matrix': [[1653, 356], [255, 627]], 'test_recall': 0.7108843537414966, 'test_precision': 0.6378433367243134, 'test_f1_score': 0.6723860589812333, 'test_roc_auc': 0.7668408826945412}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\RF\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LGB\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.759 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.745 total time=  12.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.765 total time=   1.4s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.779 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.765 total time=   0.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.789 total time=   0.7s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.768 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.754 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.774 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.784 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.775 total time=   0.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.796 total time=   1.4s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.759 total time=   0.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.745 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.765 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.787 total time=   0.9s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.779 total time=   0.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.796 total time=   0.9s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.768 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.754 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.774 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.797 total time=   1.3s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.789 total time=   1.3s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.803 total time=   1.3s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:35\n",
                        "{'train_accuracy': 0.8878312803284808, 'train_balanced_accuracy': 0.8878312803284808, 'train_confusion_matrix': [[4899, 459], [743, 4615]], 'train_recall': 0.8613288540500187, 'train_precision': 0.9095388253843122, 'train_f1_score': 0.8847776073619632, 'train_roc_auc': 0.8878312803284807}\n",
                        "{'validation_accuracy': 0.7879640044994376, 'validation_balanced_accuracy': 0.7879640044994376, 'validation_confusion_matrix': [[1533, 245], [509, 1269]], 'validation_recall': 0.7137232845894264, 'validation_precision': 0.8381770145310435, 'validation_f1_score': 0.7709599027946535, 'validation_roc_auc': 0.7879640044994376}\n",
                        "{'test_accuracy': 0.8177101349014182, 'test_balanced_accuracy': 0.7880648194237044, 'test_confusion_matrix': [[1736, 273], [254, 628]], 'test_recall': 0.7120181405895691, 'test_precision': 0.6970033296337403, 'test_f1_score': 0.7044307347167695, 'test_roc_auc': 0.7880648194237044}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LGB\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'all', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LR\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.954 total time=   6.8s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.955 total time=   7.3s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.962 total time=   7.0s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.953 total time=   7.0s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.951 total time=   6.8s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.960 total time=   7.1s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:52\n",
                        "{'train_accuracy': 0.9993467711832773, 'train_balanced_accuracy': 0.9993467711832773, 'train_confusion_matrix': [[5353, 5], [2, 5356]], 'train_recall': 0.9996267263904441, 'train_precision': 0.9990673381831748, 'train_f1_score': 0.9993469540069037, 'train_roc_auc': 0.9993467711832773}\n",
                        "{'validation_accuracy': 0.9513498312710911, 'validation_balanced_accuracy': 0.9513498312710911, 'validation_confusion_matrix': [[1715, 63], [110, 1668]], 'validation_recall': 0.9381327334083239, 'validation_precision': 0.9636048526863085, 'validation_f1_score': 0.9506982046166998, 'validation_roc_auc': 0.9513498312710911}\n",
                        "{'test_accuracy': 0.9564164648910412, 'test_balanced_accuracy': 0.9540124993086665, 'test_confusion_matrix': [[1929, 80], [46, 836]], 'test_recall': 0.9478458049886621, 'test_precision': 0.9126637554585153, 'test_f1_score': 0.9299221357063403, 'test_roc_auc': 0.9540124993086665}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LR\\all\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'all', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\RF\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.926 total time=   2.3s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.932 total time=   2.3s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.922 total time=   2.3s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.930 total time=   3.3s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.933 total time=   3.5s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.926 total time=   3.3s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.926 total time=   2.4s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.932 total time=   3.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.922 total time=   2.5s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.930 total time=   3.3s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.933 total time=   3.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.925 total time=   3.5s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.950 total time=   3.3s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.952 total time=   3.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.947 total time=   2.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.950 total time=   4.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.951 total time=   4.3s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.948 total time=   4.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.950 total time=   2.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.952 total time=   2.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.948 total time=   2.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.951 total time=   4.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.952 total time=   4.5s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.950 total time=   4.3s\n",
                        "--- finished training\n",
                        "--- training time: 00:01:27\n",
                        "{'train_accuracy': 0.9582866741321389, 'train_balanced_accuracy': 0.9582866741321389, 'train_confusion_matrix': [[5060, 298], [149, 5209]], 'train_recall': 0.9721911160880926, 'train_precision': 0.9458870528418377, 'train_f1_score': 0.9588587206626783, 'train_roc_auc': 0.9582866741321387}\n",
                        "{'validation_accuracy': 0.953880764904387, 'validation_balanced_accuracy': 0.953880764904387, 'validation_confusion_matrix': [[1668, 110], [54, 1724]], 'validation_recall': 0.96962879640045, 'validation_precision': 0.9400218102508179, 'validation_f1_score': 0.9545957918050941, 'validation_roc_auc': 0.9538807649043869}\n",
                        "{'test_accuracy': 0.9436181252161882, 'test_balanced_accuracy': 0.9476660582932359, 'test_confusion_matrix': [[1883, 126], [37, 845]], 'test_recall': 0.9580498866213152, 'test_precision': 0.870236869207003, 'test_f1_score': 0.9120345385860766, 'test_roc_auc': 0.947666058293236}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\RF\\all\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/clean_vs_all', '--feature_setting', 'all', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LGB\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.946 total time=   4.1s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.932 total time=   4.4s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.937 total time=   4.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.960 total time=   5.4s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.955 total time=   5.0s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.956 total time=   5.1s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.951 total time=   4.4s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.942 total time=   4.4s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.943 total time=   4.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.964 total time=   6.1s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.962 total time=   6.0s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.957 total time=   6.2s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.946 total time=   4.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.932 total time=   4.7s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.937 total time=   4.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.963 total time=   7.1s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.957 total time=   7.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.959 total time=   7.9s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.951 total time=   4.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.942 total time=   4.7s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.943 total time=   4.7s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.966 total time=  10.3s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.966 total time=  11.1s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.962 total time=  12.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:02:40\n",
                        "{'train_accuracy': 0.9990668159761105, 'train_balanced_accuracy': 0.9990668159761105, 'train_confusion_matrix': [[5352, 6], [4, 5354]], 'train_recall': 0.9992534527808884, 'train_precision': 0.9988805970149254, 'train_f1_score': 0.9990669901100951, 'train_roc_auc': 0.9990668159761104}\n",
                        "{'validation_accuracy': 0.9637232845894264, 'validation_balanced_accuracy': 0.9637232845894264, 'validation_confusion_matrix': [[1698, 80], [49, 1729]], 'validation_recall': 0.9724409448818898, 'validation_precision': 0.9557766721945826, 'validation_f1_score': 0.9640367995539448, 'validation_roc_auc': 0.9637232845894264}\n",
                        "{'test_accuracy': 0.9588377723970944, 'test_balanced_accuracy': 0.9608428737348598, 'test_confusion_matrix': [[1920, 89], [30, 852]], 'test_recall': 0.9659863945578231, 'test_precision': 0.9054197662061636, 'test_f1_score': 0.9347229840921558, 'test_roc_auc': 0.9608428737348598}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\clean_vs_all\\LGB\\all\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LR\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.469 total time=   2.9s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.484 total time=   2.8s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.463 total time=   2.8s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.458 total time=   2.7s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.477 total time=   2.7s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.450 total time=   2.8s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:21\n",
                        "{'train_accuracy': 0.6849503437738732, 'train_balanced_accuracy': 0.6905706888795135, 'train_confusion_matrix': [[463, 128, 40, 7, 49, 26, 38], [260, 596, 87, 18, 77, 39, 110], [85, 87, 674, 6, 37, 50, 37], [11, 12, 7, 1133, 9, 8, 7], [23, 18, 10, 0, 371, 8, 23], [86, 71, 93, 5, 42, 647, 60], [110, 118, 55, 4, 67, 34, 599]]}\n",
                        "{'validation_accuracy': 0.5322064869803563, 'validation_balanced_accuracy': 0.5113104384554835, 'validation_confusion_matrix': [[75, 55, 18, 15, 19, 18, 28], [56, 155, 45, 31, 31, 30, 63], [25, 37, 182, 14, 16, 24, 36], [10, 12, 11, 354, 5, 9, 10], [18, 23, 3, 4, 65, 6, 22], [32, 48, 32, 3, 14, 163, 36], [32, 55, 27, 10, 24, 17, 171]]}\n",
                        "{'test_accuracy': 0.4230370114147354, 'test_balanced_accuracy': 0.4957168764809185, 'test_confusion_matrix': [[35, 26, 12, 7, 12, 9, 15], [359, 742, 241, 109, 168, 128, 262], [10, 18, 81, 8, 13, 15, 18], [11, 6, 5, 173, 4, 2, 3], [3, 7, 5, 1, 36, 7, 13], [11, 17, 24, 4, 15, 80, 14], [12, 22, 19, 8, 18, 7, 76]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LR\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\RF\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.265 total time=   0.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.259 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.293 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.264 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.258 total time=   0.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.287 total time=   1.0s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.263 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.259 total time=   0.5s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.295 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.265 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.258 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.286 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.279 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.263 total time=   0.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.286 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.277 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.270 total time=   1.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.297 total time=   1.2s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.276 total time=   0.9s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.264 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.276 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.280 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.270 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.297 total time=   1.2s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:24\n",
                        "{'train_accuracy': 0.3951107715813598, 'train_balanced_accuracy': 0.4147294304494604, 'train_confusion_matrix': [[283, 55, 38, 112, 151, 63, 49], [238, 233, 83, 192, 236, 105, 100], [122, 89, 195, 196, 200, 104, 70], [17, 33, 19, 954, 70, 47, 47], [29, 23, 7, 30, 334, 17, 13], [110, 80, 74, 126, 219, 324, 71], [120, 79, 62, 144, 231, 88, 263]]}\n",
                        "{'validation_accuracy': 0.3033348560986752, 'validation_balanced_accuracy': 0.3080343356771852, 'validation_confusion_matrix': [[55, 33, 4, 42, 54, 12, 28], [81, 61, 16, 86, 90, 36, 41], [39, 37, 31, 100, 69, 24, 34], [5, 19, 13, 314, 25, 23, 12], [17, 16, 3, 11, 73, 7, 14], [46, 31, 17, 53, 81, 71, 29], [48, 37, 14, 56, 99, 23, 59]]}\n",
                        "{'test_accuracy': 0.2224143894846074, 'test_balanced_accuracy': 0.3296362287779924, 'test_confusion_matrix': [[21, 14, 1, 17, 40, 15, 8], [361, 329, 105, 416, 452, 180, 166], [17, 14, 19, 39, 45, 17, 12], [2, 4, 6, 154, 23, 7, 8], [4, 4, 4, 7, 46, 5, 2], [12, 14, 7, 21, 52, 40, 19], [15, 10, 7, 20, 54, 22, 34]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\RF\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LGB\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.319 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.301 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.323 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.323 total time=   1.1s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.315 total time=   1.2s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.313 total time=   1.2s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.327 total time=   0.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.327 total time=   0.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.336 total time=   0.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.321 total time=   2.0s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.312 total time=   2.1s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.326 total time=   2.0s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.319 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.301 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.323 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.314 total time=   2.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.310 total time=   2.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.308 total time=   2.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.327 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.327 total time=   0.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.336 total time=   0.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.305 total time=   4.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.308 total time=   4.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.303 total time=   4.7s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:43\n",
                        "{'train_accuracy': 0.40718105423987777, 'train_balanced_accuracy': 0.41267033953188814, 'train_confusion_matrix': [[224, 116, 65, 104, 120, 54, 68], [221, 293, 103, 164, 177, 108, 121], [96, 92, 321, 131, 162, 89, 85], [33, 42, 61, 892, 58, 38, 63], [36, 27, 17, 36, 270, 36, 31], [105, 80, 99, 104, 152, 383, 81], [113, 101, 97, 132, 169, 93, 282]]}\n",
                        "{'validation_accuracy': 0.3563270899954317, 'validation_balanced_accuracy': 0.34359629713067574, 'validation_confusion_matrix': [[53, 48, 20, 37, 26, 13, 31], [73, 89, 40, 71, 55, 30, 53], [36, 39, 89, 65, 46, 25, 34], [6, 14, 27, 311, 24, 11, 18], [17, 16, 8, 12, 53, 12, 23], [39, 40, 42, 36, 44, 90, 37], [42, 41, 39, 39, 51, 29, 95]]}\n",
                        "{'test_accuracy': 0.28294707713593914, 'test_balanced_accuracy': 0.36743290981647075, 'test_confusion_matrix': [[24, 23, 6, 14, 24, 15, 10], [387, 464, 175, 317, 324, 144, 198], [14, 20, 33, 30, 27, 18, 21], [4, 9, 12, 148, 11, 9, 11], [2, 7, 4, 7, 38, 8, 6], [8, 20, 16, 16, 27, 60, 18], [14, 17, 13, 17, 29, 21, 51]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LGB\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LR\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.551 total time=   2.9s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.548 total time=   2.8s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.542 total time=   2.8s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.545 total time=   3.0s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.537 total time=   2.8s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.529 total time=   2.8s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:21\n",
                        "{'train_accuracy': 0.7527883880825057, 'train_balanced_accuracy': 0.7607789759153187, 'train_confusion_matrix': [[539, 107, 26, 4, 16, 22, 37], [214, 690, 53, 11, 46, 63, 110], [48, 71, 772, 0, 16, 39, 30], [4, 4, 1, 1176, 0, 1, 1], [11, 11, 4, 1, 405, 5, 16], [60, 97, 55, 0, 31, 701, 60], [88, 144, 28, 4, 37, 42, 644]]}\n",
                        "{'validation_accuracy': 0.5897670169026953, 'validation_balanced_accuracy': 0.5775776248482484, 'validation_confusion_matrix': [[91, 43, 13, 15, 21, 20, 25], [65, 172, 29, 19, 32, 37, 57], [23, 31, 223, 8, 11, 17, 21], [10, 3, 4, 377, 6, 4, 7], [15, 11, 7, 6, 86, 5, 11], [29, 50, 22, 5, 16, 175, 31], [35, 63, 18, 5, 19, 29, 167]]}\n",
                        "{'test_accuracy': 0.46316153580076097, 'test_balanced_accuracy': 0.5637791844436408, 'test_confusion_matrix': [[42, 26, 9, 6, 10, 8, 15], [359, 794, 176, 83, 160, 174, 263], [10, 17, 101, 3, 8, 11, 13], [8, 3, 0, 190, 2, 0, 1], [5, 5, 4, 2, 44, 5, 7], [12, 17, 14, 4, 10, 89, 19], [18, 28, 12, 6, 12, 7, 79]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LR\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\RF\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.366 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.387 total time=   0.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.351 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.380 total time=   0.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.392 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.372 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.369 total time=   0.5s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.386 total time=   0.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.351 total time=   0.6s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.383 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.394 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.371 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.395 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.401 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.384 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.417 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.423 total time=   1.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.402 total time=   1.2s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.405 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.409 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.388 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.416 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.422 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.410 total time=   1.0s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:24\n",
                        "{'train_accuracy': 0.4909090909090909, 'train_balanced_accuracy': 0.5017037515919717, 'train_confusion_matrix': [[284, 56, 69, 122, 94, 51, 75], [192, 218, 123, 165, 155, 133, 201], [68, 44, 577, 109, 75, 54, 49], [24, 2, 7, 1115, 22, 2, 15], [29, 15, 15, 31, 329, 13, 21], [96, 66, 118, 112, 140, 337, 135], [96, 75, 94, 124, 151, 94, 353]]}\n",
                        "{'validation_accuracy': 0.4458656920968479, 'validation_balanced_accuracy': 0.44288280667132923, 'validation_confusion_matrix': [[66, 13, 19, 47, 35, 19, 29], [46, 66, 37, 74, 60, 41, 87], [16, 6, 201, 55, 29, 10, 17], [11, 2, 7, 376, 8, 3, 4], [11, 11, 5, 19, 81, 6, 8], [42, 17, 35, 42, 52, 75, 65], [30, 28, 33, 51, 56, 27, 111]]}\n",
                        "{'test_accuracy': 0.2608094085091664, 'test_balanced_accuracy': 0.424021796703146, 'test_confusion_matrix': [[27, 8, 17, 20, 23, 10, 11], [276, 324, 202, 357, 298, 195, 357], [14, 6, 84, 23, 20, 9, 7], [7, 0, 4, 187, 4, 0, 2], [3, 0, 10, 9, 43, 3, 4], [17, 4, 26, 19, 32, 40, 27], [18, 7, 22, 23, 31, 12, 49]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\RF\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LGB\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.476 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.467 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.481 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.484 total time=   1.2s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.466 total time=   1.3s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.481 total time=   1.2s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.494 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.476 total time=   1.0s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.501 total time=   0.9s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.472 total time=   2.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.456 total time=   2.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.483 total time=   2.4s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.476 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.467 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.481 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.456 total time=   2.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.450 total time=   2.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.458 total time=   2.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.494 total time=   0.9s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.476 total time=   0.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.501 total time=   0.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.442 total time=   5.0s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.440 total time=   5.3s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.443 total time=   5.0s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:46\n",
                        "{'train_accuracy': 0.5475935828877005, 'train_balanced_accuracy': 0.5393168062369941, 'train_confusion_matrix': [[293, 145, 84, 74, 47, 54, 54], [159, 460, 137, 61, 72, 135, 163], [46, 92, 651, 45, 37, 49, 56], [36, 11, 18, 1098, 3, 6, 15], [36, 32, 33, 27, 265, 30, 30], [94, 147, 131, 43, 64, 422, 103], [90, 154, 115, 48, 66, 119, 395]]}\n",
                        "{'validation_accuracy': 0.5194152581087255, 'validation_balanced_accuracy': 0.5027478302510561, 'validation_confusion_matrix': [[75, 44, 25, 33, 18, 9, 24], [49, 143, 41, 27, 33, 42, 76], [8, 29, 230, 24, 16, 12, 15], [13, 4, 10, 371, 4, 2, 7], [10, 21, 10, 8, 72, 7, 13], [27, 56, 33, 13, 31, 114, 54], [30, 58, 35, 16, 27, 38, 132]]}\n",
                        "{'test_accuracy': 0.42269111034244206, 'test_balanced_accuracy': 0.5009459719989295, 'test_confusion_matrix': [[40, 23, 17, 10, 9, 7, 10], [266, 731, 235, 121, 117, 199, 340], [9, 13, 105, 10, 8, 10, 8], [7, 2, 6, 187, 1, 1, 0], [4, 5, 10, 8, 34, 2, 9], [18, 17, 24, 7, 14, 60, 25], [20, 18, 21, 7, 11, 20, 65]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LGB\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LR\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.560 total time=   3.1s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.584 total time=   3.1s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.564 total time=   2.5s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.537 total time=   2.5s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.557 total time=   2.6s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.557 total time=   2.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:20\n",
                        "{'train_accuracy': 0.7779984721161192, 'train_balanced_accuracy': 0.783623447084792, 'train_confusion_matrix': [[543, 125, 19, 2, 13, 18, 31], [199, 787, 36, 8, 33, 36, 88], [40, 60, 789, 0, 14, 41, 32], [4, 3, 0, 1177, 0, 1, 2], [9, 11, 5, 0, 412, 5, 11], [55, 67, 57, 0, 32, 722, 71], [83, 125, 28, 4, 45, 40, 662]]}\n",
                        "{'validation_accuracy': 0.6176336226587483, 'validation_balanced_accuracy': 0.5993478489864171, 'validation_confusion_matrix': [[87, 57, 11, 16, 15, 18, 24], [58, 221, 24, 16, 18, 34, 40], [20, 24, 228, 9, 13, 19, 21], [13, 4, 5, 372, 5, 7, 5], [10, 13, 4, 3, 86, 7, 18], [22, 50, 26, 3, 13, 179, 35], [25, 62, 21, 4, 20, 25, 179]]}\n",
                        "{'test_accuracy': 0.5309581459702525, 'test_balanced_accuracy': 0.5697699347153408, 'test_confusion_matrix': [[40, 33, 10, 5, 9, 4, 15], [349, 993, 143, 74, 121, 124, 205], [13, 20, 100, 4, 5, 9, 12], [5, 0, 3, 192, 1, 0, 3], [3, 10, 4, 5, 40, 4, 6], [10, 17, 12, 3, 10, 92, 21], [15, 27, 16, 3, 17, 6, 78]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LR\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\RF\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.439 total time=   0.5s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.434 total time=   0.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.443 total time=   0.6s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.453 total time=   0.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.444 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.457 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.440 total time=   0.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.434 total time=   0.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.442 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.453 total time=   0.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.443 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.456 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.470 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.472 total time=   0.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.467 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.487 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.483 total time=   1.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.482 total time=   1.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.474 total time=   0.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.467 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.477 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.488 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.484 total time=   1.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.491 total time=   1.1s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:23\n",
                        "{'train_accuracy': 0.5411764705882353, 'train_balanced_accuracy': 0.5279810043926737, 'train_confusion_matrix': [[273, 267, 14, 94, 42, 41, 20], [100, 833, 13, 106, 42, 61, 32], [40, 122, 530, 114, 76, 73, 21], [15, 2, 6, 1155, 8, 1, 0], [22, 58, 14, 34, 300, 14, 11], [69, 243, 84, 138, 138, 273, 59], [58, 261, 65, 141, 179, 105, 178]]}\n",
                        "{'validation_accuracy': 0.5038830516217451, 'validation_balanced_accuracy': 0.47406525708719055, 'validation_confusion_matrix': [[62, 89, 3, 41, 18, 11, 4], [34, 293, 4, 48, 13, 11, 8], [13, 43, 172, 65, 25, 9, 7], [1, 0, 5, 401, 1, 3, 0], [10, 21, 3, 19, 77, 7, 4], [30, 97, 17, 51, 45, 72, 16], [22, 95, 28, 52, 80, 33, 26]]}\n",
                        "{'test_accuracy': 0.6077481840193705, 'test_balanced_accuracy': 0.4517321854541622, 'test_confusion_matrix': [[27, 38, 8, 19, 10, 11, 3], [182, 1368, 34, 224, 73, 78, 50], [5, 23, 78, 24, 11, 16, 6], [4, 1, 2, 196, 1, 0, 0], [4, 10, 6, 12, 35, 4, 1], [13, 35, 13, 25, 31, 37, 11], [12, 33, 14, 26, 43, 18, 16]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\RF\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LGB\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.544 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.545 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.533 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.544 total time=   1.2s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.558 total time=   1.2s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.550 total time=   1.3s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.556 total time=   0.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.562 total time=   0.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.559 total time=   0.9s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.538 total time=   2.3s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.556 total time=   2.1s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.549 total time=   2.3s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.544 total time=   0.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.545 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.533 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.527 total time=   2.9s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.542 total time=   2.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.545 total time=   2.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.556 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.562 total time=   0.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.559 total time=   0.9s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.513 total time=   4.9s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.538 total time=   5.1s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.533 total time=   5.3s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:46\n",
                        "{'train_accuracy': 0.5998472116119175, 'train_balanced_accuracy': 0.5883478514682807, 'train_confusion_matrix': [[338, 198, 60, 54, 24, 41, 36], [173, 715, 68, 44, 43, 41, 103], [52, 74, 654, 44, 32, 63, 57], [24, 9, 20, 1100, 5, 10, 19], [31, 35, 22, 27, 287, 17, 34], [113, 144, 113, 40, 63, 425, 106], [90, 184, 91, 48, 72, 95, 407]]}\n",
                        "{'validation_accuracy': 0.5774326176336226, 'validation_balanced_accuracy': 0.5495356693576628, 'validation_confusion_matrix': [[77, 75, 16, 30, 13, 5, 12], [64, 245, 20, 20, 17, 19, 26], [21, 27, 230, 19, 10, 14, 13], [5, 3, 5, 384, 2, 7, 5], [19, 13, 8, 8, 74, 7, 12], [32, 65, 27, 15, 24, 127, 38], [28, 73, 29, 16, 28, 35, 127]]}\n",
                        "{'test_accuracy': 0.5627810446212383, 'test_balanced_accuracy': 0.5220273467652453, 'test_confusion_matrix': [[32, 37, 13, 8, 6, 7, 13], [344, 1143, 123, 95, 80, 76, 148], [9, 15, 108, 9, 4, 7, 11], [5, 1, 4, 189, 2, 2, 1], [4, 7, 9, 7, 35, 2, 8], [19, 19, 23, 8, 13, 58, 25], [16, 25, 21, 7, 13, 18, 62]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LGB\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'all', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LR\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.681 total time=   9.1s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.685 total time=   8.9s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.682 total time=   8.7s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.675 total time=   8.8s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.690 total time=   8.8s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.681 total time=   9.1s\n",
                        "--- finished training\n",
                        "--- training time: 00:01:07\n",
                        "{'train_accuracy': 0.9920550038197097, 'train_balanced_accuracy': 0.992582046300145, 'train_confusion_matrix': [[749, 0, 0, 0, 2, 0, 0], [3, 1181, 0, 3, 0, 0, 0], [0, 1, 974, 0, 0, 1, 0], [0, 0, 0, 1187, 0, 0, 0], [0, 0, 0, 0, 453, 0, 0], [2, 0, 6, 0, 0, 981, 15], [6, 0, 0, 0, 2, 11, 968]]}\n",
                        "{'validation_accuracy': 0.7418912745545911, 'validation_balanced_accuracy': 0.7068818820458319, 'validation_confusion_matrix': [[124, 9, 13, 11, 10, 25, 36], [9, 374, 6, 3, 6, 4, 9], [13, 3, 244, 4, 14, 39, 17], [6, 2, 0, 401, 0, 1, 1], [12, 2, 11, 1, 83, 9, 23], [26, 2, 36, 4, 8, 206, 46], [41, 3, 30, 5, 25, 40, 192]]}\n",
                        "{'test_accuracy': 0.8339674852992044, 'test_balanced_accuracy': 0.6934667802302907, 'test_confusion_matrix': [[60, 3, 8, 4, 6, 12, 23], [77, 1800, 31, 21, 22, 25, 33], [14, 2, 112, 2, 5, 15, 13], [3, 0, 0, 199, 2, 0, 0], [5, 1, 6, 1, 40, 7, 12], [13, 2, 12, 2, 15, 104, 17], [22, 2, 9, 3, 17, 13, 96]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LR\\all\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'all', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\RF\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.548 total time=   1.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.525 total time=   1.4s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.527 total time=   1.6s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.548 total time=   2.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.538 total time=   2.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.537 total time=   2.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.548 total time=   1.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.524 total time=   1.5s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.527 total time=   1.6s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.548 total time=   2.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.538 total time=   2.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.537 total time=   2.0s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.577 total time=   1.9s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.597 total time=   1.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.582 total time=   1.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.589 total time=   2.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.600 total time=   2.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.589 total time=   2.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.592 total time=   1.9s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.583 total time=   1.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.590 total time=   1.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.601 total time=   2.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.590 total time=   2.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.594 total time=   2.8s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:55\n",
                        "{'train_accuracy': 0.6331550802139038, 'train_balanced_accuracy': 0.622817155431451, 'train_confusion_matrix': [[573, 19, 15, 47, 48, 36, 13], [49, 1097, 6, 29, 4, 2, 0], [179, 36, 512, 32, 135, 65, 17], [10, 3, 5, 1165, 0, 1, 3], [100, 8, 20, 8, 305, 5, 7], [296, 31, 174, 18, 148, 307, 30], [332, 27, 116, 35, 212, 80, 185]]}\n",
                        "{'validation_accuracy': 0.6094106898126999, 'validation_balanced_accuracy': 0.5887600466477236, 'validation_confusion_matrix': [[153, 15, 1, 23, 15, 14, 7], [13, 382, 2, 13, 1, 0, 0], [53, 15, 177, 21, 43, 18, 7], [1, 3, 3, 399, 1, 0, 4], [35, 4, 6, 1, 85, 6, 4], [89, 10, 63, 10, 55, 90, 11], [109, 6, 54, 13, 78, 28, 48]]}\n",
                        "{'test_accuracy': 0.7941888619854721, 'test_balanced_accuracy': 0.5565321055848403, 'test_confusion_matrix': [[70, 8, 5, 6, 9, 12, 6], [74, 1848, 9, 65, 8, 4, 1], [30, 12, 78, 10, 20, 10, 3], [0, 3, 1, 200, 0, 0, 0], [15, 1, 11, 2, 39, 1, 3], [46, 8, 31, 5, 26, 42, 7], [56, 4, 25, 6, 39, 13, 19]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\RF\\all\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/default/multiclass_with_clean', '--feature_setting', 'all', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LGB\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.665 total time=   4.2s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.673 total time=   4.1s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.673 total time=   4.2s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.694 total time=   9.9s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.715 total time=   9.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.709 total time=  11.1s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.686 total time=   6.0s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.700 total time=   5.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.698 total time=   6.9s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.693 total time=  18.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.708 total time=  18.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.705 total time=  17.1s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.665 total time=   4.0s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.673 total time=   3.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.673 total time=   4.3s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.689 total time=  21.2s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.700 total time=  21.0s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.701 total time=  20.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.686 total time=   6.1s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.700 total time=   6.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.698 total time=   5.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.684 total time=  40.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.702 total time=  38.1s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.703 total time=  36.9s\n",
                        "--- finished training\n",
                        "--- training time: 00:05:40\n",
                        "{'train_accuracy': 0.86768525592055, 'train_balanced_accuracy': 0.8660404612010414, 'train_confusion_matrix': [[692, 5, 13, 5, 5, 11, 20], [27, 1145, 6, 2, 0, 1, 6], [74, 10, 784, 3, 16, 63, 26], [0, 2, 0, 1185, 0, 0, 0], [23, 1, 3, 0, 410, 7, 9], [114, 8, 45, 1, 22, 741, 73], [150, 10, 23, 2, 37, 43, 722]]}\n",
                        "{'validation_accuracy': 0.7400639561443582, 'validation_balanced_accuracy': 0.705331349491856, 'validation_confusion_matrix': [[154, 8, 10, 11, 8, 16, 21], [14, 385, 3, 4, 2, 2, 1], [25, 7, 245, 2, 7, 28, 20], [1, 4, 1, 402, 1, 1, 1], [22, 2, 9, 1, 75, 10, 22], [48, 6, 28, 4, 16, 181, 45], [63, 3, 23, 6, 28, 35, 178]]}\n",
                        "{'test_accuracy': 0.8540297474922173, 'test_balanced_accuracy': 0.6951405075857122, 'test_confusion_matrix': [[72, 5, 5, 5, 3, 9, 17], [83, 1864, 13, 17, 10, 14, 8], [13, 2, 112, 2, 2, 21, 11], [2, 2, 1, 198, 0, 0, 1], [12, 1, 5, 1, 38, 7, 8], [24, 2, 12, 3, 11, 89, 24], [20, 2, 10, 3, 18, 13, 96]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\default\\multiclass_with_clean\\LGB\\all\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LR\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.850 total time=   2.6s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.838 total time=   1.8s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.847 total time=   1.7s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.843 total time=   1.7s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.833 total time=   1.8s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.845 total time=   1.8s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:14\n",
                        "{'train_accuracy': 0.8784994400895857, 'train_balanced_accuracy': 0.8784994400895856, 'train_confusion_matrix': [[4813, 545], [757, 4601]], 'train_recall': 0.858715938783128, 'train_precision': 0.8940924990283715, 'train_f1_score': 0.8760472201066261, 'train_roc_auc': 0.8784994400895856}\n",
                        "{'validation_accuracy': 0.8481439820022497, 'validation_balanced_accuracy': 0.8481439820022497, 'validation_confusion_matrix': [[1556, 222], [318, 1460]], 'validation_recall': 0.8211473565804275, 'validation_precision': 0.8680142687277052, 'validation_f1_score': 0.8439306358381503, 'validation_roc_auc': 0.8481439820022497}\n",
                        "{'test_accuracy': 0.8498789346246973, 'test_balanced_accuracy': 0.8334716000221227, 'test_confusion_matrix': [[1759, 250], [184, 698]], 'test_recall': 0.7913832199546486, 'test_precision': 0.7362869198312236, 'test_f1_score': 0.7628415300546448, 'test_roc_auc': 0.8334716000221226}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LR\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\RF\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.739 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.733 total time=   0.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.741 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.742 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.737 total time=   1.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.744 total time=   1.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.739 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.733 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.741 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.742 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.737 total time=   1.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.744 total time=   1.2s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.748 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.749 total time=   1.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.749 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.749 total time=   1.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.747 total time=   1.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.753 total time=   1.6s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.748 total time=   0.9s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.751 total time=   1.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.750 total time=   1.0s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.748 total time=   1.4s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.746 total time=   1.5s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.756 total time=   1.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:32\n",
                        "{'train_accuracy': 0.7899402762224711, 'train_balanced_accuracy': 0.7899402762224711, 'train_confusion_matrix': [[4363, 995], [1256, 4102]], 'train_recall': 0.7655841731989549, 'train_precision': 0.8047871296841279, 'train_f1_score': 0.7846963175514108, 'train_roc_auc': 0.789940276222471}\n",
                        "{'validation_accuracy': 0.7556242969628797, 'validation_balanced_accuracy': 0.7556242969628797, 'validation_confusion_matrix': [[1399, 379], [490, 1288]], 'validation_recall': 0.7244094488188977, 'validation_precision': 0.7726454709058188, 'validation_f1_score': 0.7477503628447025, 'validation_roc_auc': 0.7556242969628797}\n",
                        "{'test_accuracy': 0.7741265997924593, 'test_balanced_accuracy': 0.7573419611747139, 'test_confusion_matrix': [[1608, 401], [252, 630]], 'test_recall': 0.7142857142857143, 'test_precision': 0.6110572259941804, 'test_f1_score': 0.6586513329848406, 'test_roc_auc': 0.7573419611747139}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\RF\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LGB\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.749 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.750 total time=   0.4s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.756 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.784 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.787 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.779 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.762 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.762 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.769 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.801 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.803 total time=   0.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.799 total time=   0.7s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.749 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.750 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.756 total time=   0.4s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.796 total time=   0.9s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.801 total time=   0.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.794 total time=   1.0s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.762 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.762 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.769 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.814 total time=   1.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.819 total time=   1.2s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.816 total time=   1.3s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:21\n",
                        "{'train_accuracy': 0.9228256812243374, 'train_balanced_accuracy': 0.9228256812243374, 'train_confusion_matrix': [[4963, 395], [432, 4926]], 'train_recall': 0.9193729003359462, 'train_precision': 0.9257658334899455, 'train_f1_score': 0.922558291974904, 'train_roc_auc': 0.9228256812243375}\n",
                        "{'validation_accuracy': 0.797806524184477, 'validation_balanced_accuracy': 0.797806524184477, 'validation_confusion_matrix': [[1544, 234], [485, 1293]], 'validation_recall': 0.7272215973003374, 'validation_precision': 0.8467583497053045, 'validation_f1_score': 0.7824508320726172, 'validation_roc_auc': 0.7978065241844767}\n",
                        "{'test_accuracy': 0.814251124178485, 'test_balanced_accuracy': 0.7811238316464797, 'test_confusion_matrix': [[1740, 269], [268, 614]], 'test_recall': 0.6961451247165533, 'test_precision': 0.695356738391846, 'test_f1_score': 0.6957507082152975, 'test_roc_auc': 0.7811238316464798}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LGB\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LR\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.849 total time=   2.1s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.838 total time=   2.3s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.845 total time=   2.1s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.847 total time=   2.1s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.838 total time=   2.2s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.842 total time=   2.2s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:16\n",
                        "{'train_accuracy': 0.8818589025755879, 'train_balanced_accuracy': 0.8818589025755879, 'train_confusion_matrix': [[4818, 540], [726, 4632]], 'train_recall': 0.864501679731243, 'train_precision': 0.8955916473317865, 'train_f1_score': 0.8797720797720798, 'train_roc_auc': 0.8818589025755879}\n",
                        "{'validation_accuracy': 0.8498312710911136, 'validation_balanced_accuracy': 0.8498312710911136, 'validation_confusion_matrix': [[1556, 222], [312, 1466]], 'validation_recall': 0.8245219347581553, 'validation_precision': 0.8684834123222749, 'validation_f1_score': 0.845931909982689, 'validation_roc_auc': 0.8498312710911136}\n",
                        "{'test_accuracy': 0.8471117260463508, 'test_balanced_accuracy': 0.8317985730877717, 'test_confusion_matrix': [[1750, 259], [183, 699]], 'test_recall': 0.7925170068027211, 'test_precision': 0.7296450939457203, 'test_f1_score': 0.7597826086956522, 'test_roc_auc': 0.8317985730877717}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LR\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\RF\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.729 total time=   0.9s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.734 total time=   0.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.748 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.728 total time=   1.4s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.737 total time=   1.4s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.746 total time=   1.2s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.729 total time=   0.9s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.734 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.748 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.728 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.737 total time=   1.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.746 total time=   1.4s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.738 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.755 total time=   1.3s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.765 total time=   1.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.741 total time=   1.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.751 total time=   1.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.763 total time=   1.5s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.742 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.752 total time=   1.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.762 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.740 total time=   1.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.753 total time=   1.5s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.765 total time=   1.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:32\n",
                        "{'train_accuracy': 0.7897536394176932, 'train_balanced_accuracy': 0.7897536394176932, 'train_confusion_matrix': [[4390, 968], [1285, 4073]], 'train_recall': 0.7601717058603956, 'train_precision': 0.8079746082126562, 'train_f1_score': 0.7833445523608038, 'train_roc_auc': 0.7897536394176932}\n",
                        "{'validation_accuracy': 0.750843644544432, 'validation_balanced_accuracy': 0.750843644544432, 'validation_confusion_matrix': [[1399, 379], [507, 1271]], 'validation_recall': 0.7148481439820022, 'validation_precision': 0.7703030303030303, 'validation_f1_score': 0.7415402567094516, 'validation_roc_auc': 0.750843644544432}\n",
                        "{'test_accuracy': 0.7706675890695261, 'test_balanced_accuracy': 0.7529450804712129, 'test_confusion_matrix': [[1604, 405], [258, 624]], 'test_recall': 0.7074829931972789, 'test_precision': 0.6064139941690962, 'test_f1_score': 0.6530612244897961, 'test_roc_auc': 0.7529450804712129}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\RF\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LGB\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.747 total time=   0.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.757 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.751 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.785 total time=   0.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.784 total time=   0.7s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.795 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.766 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.762 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.765 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.807 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.800 total time=   0.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.807 total time=   1.0s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.747 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.757 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.751 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.803 total time=   1.0s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.798 total time=   0.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.802 total time=   0.9s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.766 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.762 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.765 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.830 total time=   1.2s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.824 total time=   1.4s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.818 total time=   1.3s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:23\n",
                        "{'train_accuracy': 0.9243187756625607, 'train_balanced_accuracy': 0.9243187756625606, 'train_confusion_matrix': [[4978, 380], [431, 4927]], 'train_recall': 0.9195595371407241, 'train_precision': 0.9283964575089504, 'train_f1_score': 0.9239568682606658, 'train_roc_auc': 0.9243187756625606}\n",
                        "{'validation_accuracy': 0.8076490438695163, 'validation_balanced_accuracy': 0.8076490438695163, 'validation_confusion_matrix': [[1557, 221], [463, 1315]], 'validation_recall': 0.7395950506186727, 'validation_precision': 0.8561197916666666, 'validation_f1_score': 0.7936028968014485, 'validation_roc_auc': 0.8076490438695164}\n",
                        "{'test_accuracy': 0.8256658595641646, 'test_balanced_accuracy': 0.7953791272606603, 'test_confusion_matrix': [[1754, 255], [249, 633]], 'test_recall': 0.717687074829932, 'test_precision': 0.7128378378378378, 'test_f1_score': 0.7152542372881355, 'test_roc_auc': 0.7953791272606604}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LGB\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LR\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.854 total time=   2.4s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.874 total time=   2.2s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.865 total time=   2.2s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.856 total time=   2.2s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.864 total time=   2.5s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.860 total time=   2.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:17\n",
                        "{'train_accuracy': 0.8994960806270996, 'train_balanced_accuracy': 0.8994960806270997, 'train_confusion_matrix': [[4911, 447], [630, 4728]], 'train_recall': 0.8824188129899216, 'train_precision': 0.9136231884057971, 'train_f1_score': 0.8977499287952151, 'train_roc_auc': 0.8994960806270996}\n",
                        "{'validation_accuracy': 0.8624859392575928, 'validation_balanced_accuracy': 0.8624859392575928, 'validation_confusion_matrix': [[1584, 194], [295, 1483]], 'validation_recall': 0.8340832395950506, 'validation_precision': 0.8843172331544424, 'validation_f1_score': 0.858465991316932, 'validation_roc_auc': 0.8624859392575928}\n",
                        "{'test_accuracy': 0.863369076444137, 'test_balanced_accuracy': 0.8460400420330734, 'test_confusion_matrix': [[1789, 220], [175, 707]], 'test_recall': 0.8015873015873016, 'test_precision': 0.7626752966558792, 'test_f1_score': 0.7816473189607518, 'test_roc_auc': 0.8460400420330734}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LR\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\RF\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.798 total time=   0.9s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.794 total time=   0.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.782 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.796 total time=   1.3s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.800 total time=   1.3s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.785 total time=   1.3s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.798 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.794 total time=   0.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.782 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.796 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.800 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.785 total time=   1.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.815 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.813 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.802 total time=   1.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.816 total time=   1.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.817 total time=   1.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.806 total time=   1.6s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.812 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.813 total time=   1.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.804 total time=   1.2s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.820 total time=   1.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.817 total time=   1.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.806 total time=   1.6s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:32\n",
                        "{'train_accuracy': 0.8400522583053378, 'train_balanced_accuracy': 0.8400522583053378, 'train_confusion_matrix': [[4856, 502], [1212, 4146]], 'train_recall': 0.7737961926091825, 'train_precision': 0.8919965576592083, 'train_f1_score': 0.8287027783330003, 'train_roc_auc': 0.8400522583053378}\n",
                        "{'validation_accuracy': 0.8062429696287964, 'validation_balanced_accuracy': 0.8062429696287965, 'validation_confusion_matrix': [[1571, 207], [482, 1296]], 'validation_recall': 0.7289088863892014, 'validation_precision': 0.8622754491017964, 'validation_f1_score': 0.790003047851265, 'validation_roc_auc': 0.8062429696287964}\n",
                        "{'test_accuracy': 0.8446904185402975, 'test_balanced_accuracy': 0.8138377302140369, 'test_confusion_matrix': [[1794, 215], [234, 648]], 'test_recall': 0.7346938775510204, 'test_precision': 0.7508690614136733, 'test_f1_score': 0.7426934097421203, 'test_roc_auc': 0.8138377302140368}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\RF\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'bert+tp+lm', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LGB\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.795 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.808 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.809 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.828 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.851 total time=   0.7s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.844 total time=   0.7s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.811 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.823 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.821 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.850 total time=   1.0s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.869 total time=   0.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.866 total time=   1.0s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.795 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.808 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.809 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.849 total time=   0.9s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.865 total time=   0.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.859 total time=   0.9s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.811 total time=   0.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.823 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.821 total time=   0.7s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.876 total time=   1.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.889 total time=   1.7s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.882 total time=   2.0s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:24\n",
                        "{'train_accuracy': 0.9511011571481897, 'train_balanced_accuracy': 0.9511011571481895, 'train_confusion_matrix': [[5139, 219], [305, 5053]], 'train_recall': 0.9430757745427398, 'train_precision': 0.9584597875569044, 'train_f1_score': 0.9507055503292569, 'train_roc_auc': 0.9511011571481895}\n",
                        "{'validation_accuracy': 0.858548931383577, 'validation_balanced_accuracy': 0.858548931383577, 'validation_confusion_matrix': [[1625, 153], [350, 1428]], 'validation_recall': 0.8031496062992126, 'validation_precision': 0.9032258064516129, 'validation_f1_score': 0.8502530515034237, 'validation_roc_auc': 0.858548931383577}\n",
                        "{'test_accuracy': 0.8834313386371497, 'test_balanced_accuracy': 0.8604750843426802, 'test_confusion_matrix': [[1847, 162], [175, 707]], 'test_recall': 0.8015873015873016, 'test_precision': 0.8135788262370541, 'test_f1_score': 0.8075385494003426, 'test_roc_auc': 0.8604750843426802}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LGB\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'all', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LR\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.939 total time=   8.2s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.944 total time=   8.3s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.947 total time=   8.0s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.934 total time=   8.5s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.941 total time=   8.1s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.947 total time=   8.1s\n",
                        "--- finished training\n",
                        "--- training time: 00:01:01\n",
                        "{'train_accuracy': 0.99822695035461, 'train_balanced_accuracy': 0.99822695035461, 'train_confusion_matrix': [[5345, 13], [6, 5352]], 'train_recall': 0.9988801791713325, 'train_precision': 0.9975768872320596, 'train_f1_score': 0.9982281078056514, 'train_roc_auc': 0.99822695035461}\n",
                        "{'validation_accuracy': 0.9336332958380202, 'validation_balanced_accuracy': 0.9336332958380202, 'validation_confusion_matrix': [[1683, 95], [141, 1637]], 'validation_recall': 0.9206974128233971, 'validation_precision': 0.9451501154734411, 'validation_f1_score': 0.9327635327635327, 'validation_roc_auc': 0.9336332958380203}\n",
                        "{'test_accuracy': 0.9318574887582152, 'test_balanced_accuracy': 0.9264836015707096, 'test_confusion_matrix': [[1889, 120], [77, 805]], 'test_recall': 0.9126984126984127, 'test_precision': 0.8702702702702703, 'test_f1_score': 0.8909795240730494, 'test_roc_auc': 0.9264836015707095}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LR\\all\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'all', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\RF\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.889 total time=   2.5s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.907 total time=   2.4s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.903 total time=   2.4s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.898 total time=   3.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.914 total time=   3.5s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.909 total time=   3.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.889 total time=   2.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.907 total time=   2.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.903 total time=   2.5s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.898 total time=   3.5s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.914 total time=   3.5s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.909 total time=   3.3s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.919 total time=   3.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.925 total time=   3.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.925 total time=   3.0s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.920 total time=   4.3s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.926 total time=   4.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.928 total time=   4.3s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.917 total time=   2.9s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.926 total time=   3.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.924 total time=   2.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.918 total time=   4.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.926 total time=   4.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.926 total time=   4.6s\n",
                        "--- finished training\n",
                        "--- training time: 00:01:29\n",
                        "{'train_accuracy': 0.9327174318775663, 'train_balanced_accuracy': 0.9327174318775662, 'train_confusion_matrix': [[4781, 577], [144, 5214]], 'train_recall': 0.973124300111982, 'train_precision': 0.9003626316698325, 'train_f1_score': 0.9353305229168536, 'train_roc_auc': 0.9327174318775663}\n",
                        "{'validation_accuracy': 0.9260404949381328, 'validation_balanced_accuracy': 0.9260404949381327, 'validation_confusion_matrix': [[1582, 196], [67, 1711]], 'validation_recall': 0.9623172103487064, 'validation_precision': 0.8972207656004195, 'validation_f1_score': 0.928629579375848, 'validation_roc_auc': 0.9260404949381328}\n",
                        "{'test_accuracy': 0.9090280179868557, 'test_balanced_accuracy': 0.9237320944638018, 'test_confusion_matrix': [[1780, 229], [34, 848]], 'test_recall': 0.9614512471655329, 'test_precision': 0.787372330547818, 'test_f1_score': 0.8657478305257785, 'test_roc_auc': 0.9237320944638017}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\RF\\all\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/clean_vs_all', '--feature_setting', 'all', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LGB\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\train.joblib\n",
                        "num of instances in the file:  10716\n",
                        "loaded, labels:  Counter({'perturbed': 5358, 'clean': 5358})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\val.joblib\n",
                        "num of instances in the file:  3556\n",
                        "loaded, labels:  Counter({'perturbed': 1778, 'clean': 1778})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'perturbed': 882})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 10,716\n",
                        "no. val: 3,556\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.919 total time=   5.0s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.926 total time=   5.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.917 total time=   5.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.943 total time=   6.3s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.942 total time=   6.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.932 total time=   6.3s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.933 total time=   5.4s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.934 total time=   6.0s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.927 total time=   5.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.945 total time=   7.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.948 total time=   7.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.942 total time=   8.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.919 total time=   5.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.926 total time=   5.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.917 total time=   5.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.945 total time=   8.1s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.944 total time=   7.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.942 total time=   8.0s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.933 total time=   5.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.934 total time=   6.1s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.927 total time=   5.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.950 total time=  11.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.951 total time=  11.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.952 total time=  11.1s\n",
                        "--- finished training\n",
                        "--- training time: 00:03:04\n",
                        "{'train_accuracy': 0.9952407614781635, 'train_balanced_accuracy': 0.9952407614781635, 'train_confusion_matrix': [[5323, 35], [16, 5342]], 'train_recall': 0.9970138111235536, 'train_precision': 0.993490794123117, 'train_f1_score': 0.9952491849091756, 'train_roc_auc': 0.9952407614781634}\n",
                        "{'validation_accuracy': 0.9457255343082115, 'validation_balanced_accuracy': 0.9457255343082115, 'validation_confusion_matrix': [[1658, 120], [73, 1705]], 'validation_recall': 0.9589426321709786, 'validation_precision': 0.9342465753424658, 'validation_f1_score': 0.9464335276158757, 'validation_roc_auc': 0.9457255343082115}\n",
                        "{'test_accuracy': 0.9363542026980284, 'test_balanced_accuracy': 0.9395774569990598, 'test_confusion_matrix': [[1871, 138], [46, 836]], 'test_recall': 0.9478458049886621, 'test_precision': 0.8583162217659137, 'test_f1_score': 0.9008620689655171, 'test_roc_auc': 0.9395774569990597}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\clean_vs_all\\LGB\\all\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LR\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.590 total time=   2.5s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.602 total time=   2.6s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.595 total time=   2.7s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.573 total time=   2.8s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.578 total time=   2.3s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.584 total time=   2.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:19\n",
                        "{'train_accuracy': 0.7827349121466769, 'train_balanced_accuracy': 0.7814776573930755, 'train_confusion_matrix': [[567, 70, 12, 0, 30, 29, 43], [108, 915, 30, 1, 52, 35, 46], [53, 47, 742, 0, 54, 47, 33], [0, 2, 0, 1183, 0, 0, 2], [18, 16, 7, 0, 386, 3, 23], [64, 56, 61, 1, 46, 699, 77], [103, 86, 34, 2, 78, 53, 631]]}\n",
                        "{'validation_accuracy': 0.650525354042942, 'validation_balanced_accuracy': 0.6249844165513723, 'validation_confusion_matrix': [[108, 29, 9, 10, 19, 19, 34], [56, 258, 17, 4, 26, 17, 33], [19, 22, 216, 3, 14, 34, 26], [5, 1, 0, 401, 0, 0, 4], [13, 10, 5, 0, 79, 11, 23], [28, 22, 35, 1, 23, 185, 34], [37, 32, 21, 5, 36, 28, 177]]}\n",
                        "{'test_accuracy': 0.6281563472846766, 'test_balanced_accuracy': 0.5925974666530399, 'test_confusion_matrix': [[50, 20, 7, 5, 8, 7, 19], [259, 1270, 94, 16, 137, 97, 136], [6, 15, 95, 0, 16, 17, 14], [4, 0, 0, 196, 1, 0, 3], [5, 10, 6, 0, 37, 4, 10], [14, 17, 17, 2, 13, 85, 17], [16, 19, 10, 4, 22, 8, 83]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LR\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\RF\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.377 total time=   0.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.370 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.370 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.378 total time=   1.0s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.383 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.358 total time=   1.0s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.376 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.371 total time=   0.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.370 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.378 total time=   0.9s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.382 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.357 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.399 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.396 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.398 total time=   0.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.403 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.402 total time=   1.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.398 total time=   1.0s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.401 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.389 total time=   0.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.397 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.401 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.397 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.402 total time=   1.1s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:24\n",
                        "{'train_accuracy': 0.5126050420168067, 'train_balanced_accuracy': 0.5021621824728876, 'train_confusion_matrix': [[264, 197, 33, 79, 106, 61, 11], [115, 815, 21, 44, 123, 56, 13], [65, 173, 410, 82, 134, 104, 8], [3, 18, 3, 1143, 12, 4, 4], [28, 64, 18, 22, 307, 14, 0], [97, 209, 140, 69, 172, 303, 14], [121, 223, 97, 138, 204, 91, 113]]}\n",
                        "{'validation_accuracy': 0.429419826404751, 'validation_balanced_accuracy': 0.39208073651552805, 'validation_confusion_matrix': [[38, 70, 11, 43, 38, 19, 9], [27, 267, 12, 30, 48, 22, 5], [16, 62, 111, 52, 49, 38, 6], [4, 1, 2, 388, 11, 1, 4], [9, 30, 15, 11, 59, 12, 5], [34, 81, 45, 34, 66, 59, 9], [27, 71, 38, 74, 78, 30, 18]]}\n",
                        "{'test_accuracy': 0.5617433414043583, 'test_balanced_accuracy': 0.37505878289384514, 'test_confusion_matrix': [[11, 48, 7, 15, 20, 10, 5], [176, 1302, 43, 112, 257, 92, 27], [10, 37, 41, 23, 27, 24, 1], [2, 1, 1, 195, 3, 1, 1], [7, 15, 5, 10, 28, 5, 2], [11, 40, 25, 14, 37, 35, 3], [12, 44, 21, 25, 33, 15, 12]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\RF\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_bert']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LGB\\bert\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 768\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.428 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.435 total time=   0.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.431 total time=   0.6s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.459 total time=   1.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.458 total time=   1.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.455 total time=   1.2s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.453 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.465 total time=   0.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.456 total time=   1.0s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.466 total time=   2.4s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.464 total time=   2.1s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.466 total time=   2.1s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.428 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.435 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.431 total time=   0.4s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.446 total time=   2.3s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.450 total time=   2.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.448 total time=   2.7s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.453 total time=   0.9s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.465 total time=   0.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.456 total time=   0.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.445 total time=   5.0s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.456 total time=   5.2s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.458 total time=   5.6s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:48\n",
                        "{'train_accuracy': 0.8502673796791443, 'train_balanced_accuracy': 0.8560614876319509, 'train_confusion_matrix': [[651, 39, 9, 0, 21, 16, 15], [72, 1001, 19, 4, 35, 29, 27], [58, 43, 787, 0, 34, 31, 23], [0, 2, 1, 1184, 0, 0, 0], [7, 5, 0, 0, 439, 0, 2], [62, 53, 39, 2, 33, 783, 32], [70, 86, 20, 4, 43, 44, 720]]}\n",
                        "{'validation_accuracy': 0.5641845591594336, 'validation_balanced_accuracy': 0.5275536296244946, 'validation_confusion_matrix': [[95, 52, 9, 13, 15, 23, 21], [57, 274, 11, 5, 17, 28, 19], [33, 39, 171, 6, 20, 46, 19], [13, 2, 1, 381, 2, 1, 11], [19, 19, 9, 3, 55, 16, 20], [37, 53, 39, 6, 20, 133, 40], [55, 47, 31, 18, 21, 38, 126]]}\n",
                        "{'test_accuracy': 0.6153580076098236, 'test_balanced_accuracy': 0.49233321636354965, 'test_confusion_matrix': [[37, 34, 9, 6, 8, 10, 12], [267, 1335, 80, 28, 90, 96, 113], [16, 20, 72, 5, 13, 19, 18], [5, 1, 1, 189, 0, 1, 7], [11, 10, 6, 2, 26, 5, 12], [20, 27, 26, 3, 12, 64, 13], [27, 28, 12, 7, 16, 16, 56]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LGB\\bert\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LR\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.619 total time=   2.4s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.627 total time=   2.4s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.624 total time=   2.4s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.600 total time=   2.6s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.621 total time=   2.7s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.598 total time=   2.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:18\n",
                        "{'train_accuracy': 0.8131398013750955, 'train_balanced_accuracy': 0.8138237163417517, 'train_confusion_matrix': [[589, 63, 16, 0, 17, 22, 44], [103, 926, 23, 2, 39, 37, 57], [34, 47, 789, 0, 28, 39, 39], [0, 1, 0, 1184, 0, 0, 2], [14, 12, 2, 0, 403, 1, 21], [41, 54, 45, 0, 32, 757, 75], [84, 95, 39, 1, 51, 43, 674]]}\n",
                        "{'validation_accuracy': 0.6701690269529466, 'validation_balanced_accuracy': 0.6488184208458498, 'validation_confusion_matrix': [[118, 28, 11, 8, 14, 16, 33], [54, 258, 21, 4, 24, 15, 35], [17, 19, 234, 3, 18, 20, 23], [5, 1, 0, 402, 2, 0, 1], [10, 6, 8, 0, 85, 10, 22], [27, 21, 30, 1, 28, 189, 32], [31, 34, 24, 4, 31, 31, 181]]}\n",
                        "{'test_accuracy': 0.6316153580076098, 'test_balanced_accuracy': 0.6125821421797385, 'test_confusion_matrix': [[52, 22, 8, 4, 11, 5, 14], [255, 1262, 107, 12, 128, 107, 138], [4, 11, 107, 0, 13, 13, 15], [5, 0, 0, 198, 0, 0, 1], [5, 8, 5, 0, 41, 2, 11], [14, 16, 16, 1, 12, 85, 21], [16, 20, 16, 3, 17, 9, 81]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LR\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\RF\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.468 total time=   0.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.471 total time=   0.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.455 total time=   0.6s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.478 total time=   0.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.467 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.453 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.468 total time=   0.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.472 total time=   0.6s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.455 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.479 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.468 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.453 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.483 total time=   0.9s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.493 total time=   0.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.472 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.496 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.497 total time=   1.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.475 total time=   1.2s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.499 total time=   0.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.491 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.474 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.500 total time=   1.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.493 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.470 total time=   1.1s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:23\n",
                        "{'train_accuracy': 0.5657754010695187, 'train_balanced_accuracy': 0.5456745975771227, 'train_confusion_matrix': [[265, 181, 57, 79, 56, 83, 30], [58, 771, 83, 40, 65, 127, 43], [44, 116, 583, 56, 46, 107, 24], [4, 3, 2, 1177, 0, 0, 1], [26, 61, 16, 24, 265, 40, 21], [72, 207, 121, 67, 80, 410, 47], [65, 224, 103, 108, 80, 175, 232]]}\n",
                        "{'validation_accuracy': 0.5212425765189584, 'validation_balanced_accuracy': 0.484753070952086, 'validation_confusion_matrix': [[49, 54, 16, 39, 26, 33, 11], [15, 247, 25, 28, 31, 42, 23], [9, 34, 203, 39, 17, 26, 6], [2, 0, 1, 405, 2, 0, 1], [11, 20, 11, 14, 66, 14, 5], [23, 73, 33, 33, 41, 99, 26], [20, 63, 34, 48, 40, 59, 72]]}\n",
                        "{'test_accuracy': 0.5569007263922519, 'test_balanced_accuracy': 0.43962801224413267, 'test_confusion_matrix': [[23, 34, 15, 18, 10, 12, 4], [118, 1212, 136, 116, 138, 213, 76], [8, 23, 89, 13, 11, 16, 3], [1, 0, 1, 201, 0, 0, 1], [6, 7, 11, 8, 29, 8, 3], [12, 44, 26, 16, 21, 35, 11], [14, 41, 22, 20, 20, 24, 21]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\RF\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LGB\\bert+tp\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 804\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.543 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.537 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.529 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.561 total time=   1.2s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.556 total time=   1.2s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.554 total time=   1.2s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.549 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.560 total time=   0.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.555 total time=   0.9s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.560 total time=   2.2s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.564 total time=   2.1s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.548 total time=   2.1s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.543 total time=   0.7s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.537 total time=   0.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.529 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.546 total time=   2.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.544 total time=   2.7s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.554 total time=   2.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.549 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.560 total time=   0.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.555 total time=   0.9s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.543 total time=   5.1s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.546 total time=   5.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.540 total time=   5.7s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:48\n",
                        "{'train_accuracy': 0.8701298701298701, 'train_balanced_accuracy': 0.8721255634089194, 'train_confusion_matrix': [[651, 41, 16, 0, 9, 10, 24], [54, 1011, 32, 0, 20, 31, 39], [29, 35, 844, 0, 16, 35, 17], [0, 4, 0, 1183, 0, 0, 0], [6, 6, 1, 0, 424, 2, 14], [52, 49, 31, 1, 18, 810, 43], [54, 83, 17, 1, 18, 42, 772]]}\n",
                        "{'validation_accuracy': 0.628140703517588, 'validation_balanced_accuracy': 0.5940720154600699, 'validation_confusion_matrix': [[107, 39, 17, 15, 11, 18, 21], [43, 266, 24, 5, 17, 23, 33], [12, 16, 237, 4, 11, 35, 19], [6, 2, 2, 396, 2, 1, 2], [18, 20, 7, 1, 63, 13, 19], [26, 51, 31, 3, 21, 157, 39], [37, 50, 29, 9, 22, 40, 149]]}\n",
                        "{'test_accuracy': 0.6236596333448634, 'test_balanced_accuracy': 0.5627579006286413, 'test_confusion_matrix': [[43, 28, 15, 5, 4, 13, 8], [240, 1281, 141, 32, 63, 105, 147], [13, 11, 112, 3, 4, 11, 9], [3, 0, 1, 199, 0, 0, 1], [10, 6, 10, 1, 31, 5, 9], [23, 18, 20, 3, 12, 70, 19], [17, 26, 14, 3, 16, 19, 67]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LGB\\bert+tp\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LR\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.626 total time=   2.4s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.634 total time=   2.6s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.641 total time=   2.6s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.614 total time=   2.6s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.618 total time=   2.6s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.624 total time=   2.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:19\n",
                        "{'train_accuracy': 0.8275019098548511, 'train_balanced_accuracy': 0.8272180864628614, 'train_confusion_matrix': [[603, 63, 13, 0, 15, 19, 38], [79, 969, 27, 2, 31, 39, 40], [30, 51, 803, 0, 27, 36, 29], [0, 1, 0, 1184, 0, 0, 2], [15, 9, 3, 0, 404, 4, 18], [46, 66, 48, 0, 24, 754, 66], [80, 87, 30, 1, 49, 41, 699]]}\n",
                        "{'validation_accuracy': 0.6834170854271356, 'validation_balanced_accuracy': 0.6619095127773978, 'validation_confusion_matrix': [[128, 23, 13, 9, 16, 14, 25], [49, 266, 22, 2, 29, 17, 26], [19, 21, 239, 2, 18, 21, 14], [4, 1, 0, 403, 2, 0, 1], [14, 8, 8, 0, 84, 10, 17], [26, 22, 30, 1, 25, 191, 33], [35, 25, 24, 4, 33, 30, 185]]}\n",
                        "{'test_accuracy': 0.6596333448633691, 'test_balanced_accuracy': 0.6322796446434948, 'test_confusion_matrix': [[54, 16, 11, 4, 10, 7, 14], [224, 1327, 103, 16, 115, 119, 105], [3, 13, 110, 1, 11, 14, 11], [4, 2, 0, 196, 0, 0, 2], [7, 9, 4, 1, 41, 3, 7], [17, 15, 14, 1, 14, 90, 14], [13, 19, 11, 3, 14, 13, 89]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LR\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\RF\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.473 total time=   0.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.465 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.483 total time=   0.6s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.491 total time=   0.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.489 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.497 total time=   0.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.473 total time=   0.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.465 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.483 total time=   0.6s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.491 total time=   0.9s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.489 total time=   0.9s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.497 total time=   1.0s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.512 total time=   0.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.503 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.514 total time=   0.7s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.522 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.523 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.533 total time=   1.1s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.509 total time=   0.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.511 total time=   0.7s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.520 total time=   0.6s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.524 total time=   1.2s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.518 total time=   1.1s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.537 total time=   1.0s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:24\n",
                        "{'train_accuracy': 0.5876241405653171, 'train_balanced_accuracy': 0.5593679030200143, 'train_confusion_matrix': [[294, 216, 36, 86, 47, 61, 11], [53, 994, 29, 41, 24, 40, 6], [47, 153, 533, 69, 36, 120, 18], [2, 0, 2, 1177, 2, 4, 0], [36, 70, 10, 44, 249, 31, 13], [69, 244, 80, 70, 71, 440, 30], [81, 235, 61, 167, 90, 194, 159]]}\n",
                        "{'validation_accuracy': 0.5417999086340795, 'validation_balanced_accuracy': 0.49476289761924824, 'validation_confusion_matrix': [[58, 78, 5, 34, 24, 29, 0], [26, 331, 4, 18, 10, 18, 4], [9, 58, 181, 40, 13, 30, 3], [0, 0, 2, 403, 2, 4, 0], [14, 30, 2, 14, 58, 19, 4], [15, 97, 18, 26, 40, 117, 15], [26, 78, 18, 65, 44, 67, 38]]}\n",
                        "{'test_accuracy': 0.7046004842615012, 'test_balanced_accuracy': 0.47001868710442013, 'test_confusion_matrix': [[25, 42, 13, 17, 7, 10, 2], [113, 1631, 46, 91, 41, 74, 13], [2, 32, 76, 21, 5, 24, 3], [0, 0, 2, 202, 0, 0, 0], [6, 16, 6, 10, 23, 11, 0], [5, 42, 18, 14, 17, 62, 7], [9, 43, 18, 33, 15, 26, 18]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\RF\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'bert+tp+lm', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LGB\\bert+tp+lm\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 848\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.584 total time=   0.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.572 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.561 total time=   0.7s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.610 total time=   1.3s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.601 total time=   1.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.600 total time=   1.2s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.602 total time=   0.8s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.594 total time=   0.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.591 total time=   0.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.614 total time=   2.3s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.607 total time=   2.2s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.608 total time=   2.4s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.584 total time=   0.6s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.572 total time=   0.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.561 total time=   0.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.610 total time=   2.9s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.597 total time=   2.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.597 total time=   2.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.602 total time=   0.9s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.594 total time=   0.8s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.591 total time=   0.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.602 total time=   5.0s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.595 total time=   5.3s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.597 total time=   5.1s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:48\n",
                        "{'train_accuracy': 0.8873949579831932, 'train_balanced_accuracy': 0.8869048633309335, 'train_confusion_matrix': [[669, 51, 6, 0, 5, 11, 9], [31, 1099, 15, 1, 12, 18, 11], [26, 36, 844, 1, 15, 36, 18], [0, 2, 0, 1185, 0, 0, 0], [10, 13, 1, 0, 423, 1, 5], [43, 61, 30, 0, 15, 822, 33], [72, 74, 20, 1, 26, 28, 766]]}\n",
                        "{'validation_accuracy': 0.6706258565555048, 'validation_balanced_accuracy': 0.635449024809977, 'validation_confusion_matrix': [[124, 35, 12, 12, 7, 16, 22], [42, 307, 14, 1, 18, 13, 16], [12, 20, 245, 4, 10, 34, 9], [2, 1, 2, 401, 2, 2, 1], [21, 18, 4, 1, 66, 10, 21], [25, 51, 25, 3, 21, 176, 27], [44, 38, 26, 9, 21, 49, 149]]}\n",
                        "{'test_accuracy': 0.7115185057073677, 'test_balanced_accuracy': 0.5941016132557168, 'test_confusion_matrix': [[50, 28, 13, 2, 5, 8, 10], [190, 1516, 103, 14, 50, 68, 68], [10, 14, 112, 2, 3, 13, 9], [1, 0, 1, 199, 1, 1, 1], [10, 7, 9, 0, 27, 8, 11], [17, 15, 20, 2, 9, 80, 22], [25, 17, 17, 2, 11, 17, 73]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LGB\\bert+tp+lm\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'all', '--model', 'LR', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LR\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 2 candidates, totalling 6 fits\n",
                        "[CV 1/3] END .........logisticregression__C=0.1;, score=0.666 total time=   9.6s\n",
                        "[CV 2/3] END .........logisticregression__C=0.1;, score=0.663 total time=   8.9s\n",
                        "[CV 3/3] END .........logisticregression__C=0.1;, score=0.665 total time=   9.2s\n",
                        "[CV 1/3] END .........logisticregression__C=1.0;, score=0.669 total time=   8.9s\n",
                        "[CV 2/3] END .........logisticregression__C=1.0;, score=0.674 total time=   8.9s\n",
                        "[CV 3/3] END .........logisticregression__C=1.0;, score=0.666 total time=   9.1s\n",
                        "--- finished training\n",
                        "--- training time: 00:01:08\n",
                        "{'train_accuracy': 0.9912910618792972, 'train_balanced_accuracy': 0.9919127021385593, 'train_confusion_matrix': [[751, 0, 0, 0, 0, 0, 0], [1, 1184, 1, 1, 0, 0, 0], [1, 0, 965, 0, 1, 7, 2], [0, 2, 0, 1185, 0, 0, 0], [0, 0, 0, 0, 453, 0, 0], [3, 0, 6, 0, 0, 980, 15], [4, 0, 1, 0, 0, 12, 970]]}\n",
                        "{'validation_accuracy': 0.7313841936957515, 'validation_balanced_accuracy': 0.695410142995069, 'validation_confusion_matrix': [[125, 5, 14, 11, 9, 24, 40], [19, 362, 3, 5, 5, 8, 9], [17, 5, 239, 3, 11, 40, 19], [3, 1, 0, 406, 1, 0, 0], [20, 2, 8, 1, 79, 14, 17], [37, 8, 30, 1, 18, 199, 35], [40, 11, 18, 6, 29, 41, 191]]}\n",
                        "{'test_accuracy': 0.8087167070217918, 'test_balanced_accuracy': 0.666552618768675, 'test_confusion_matrix': [[58, 5, 6, 6, 4, 17, 20], [86, 1747, 45, 15, 36, 37, 43], [15, 0, 109, 2, 3, 10, 24], [0, 1, 0, 202, 0, 0, 1], [9, 1, 5, 1, 36, 8, 12], [17, 1, 13, 2, 8, 96, 28], [26, 4, 8, 5, 9, 20, 90]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LR\\all\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'all', '--model', 'RF', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\RF\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.576 total time=   1.5s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.504 total time=   1.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.539 total time=   1.5s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.559 total time=   2.3s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.495 total time=   2.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.504 total time=   2.2s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.576 total time=   1.6s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.505 total time=   1.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.539 total time=   1.5s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.558 total time=   2.1s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.495 total time=   2.4s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=3, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.505 total time=   2.3s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.583 total time=   1.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.546 total time=   2.2s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=50;, score=0.574 total time=   1.9s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.575 total time=   2.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.543 total time=   2.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=2, randomforestclassifier__n_estimators=100;, score=0.563 total time=   2.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.582 total time=   1.7s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.549 total time=   1.8s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=50;, score=0.578 total time=   1.8s\n",
                        "[CV 1/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.569 total time=   2.8s\n",
                        "[CV 2/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.547 total time=   3.0s\n",
                        "[CV 3/3] END randomforestclassifier__max_depth=5, randomforestclassifier__min_samples_leaf=4, randomforestclassifier__n_estimators=100;, score=0.557 total time=   2.9s\n",
                        "--- finished training\n",
                        "--- training time: 00:00:56\n",
                        "{'train_accuracy': 0.6119174942704354, 'train_balanced_accuracy': 0.5979708839550514, 'train_confusion_matrix': [[511, 10, 13, 53, 104, 40, 20], [130, 979, 8, 15, 38, 7, 10], [187, 15, 488, 37, 132, 92, 25], [4, 0, 4, 1175, 0, 2, 2], [98, 3, 32, 10, 279, 18, 13], [275, 13, 130, 27, 174, 334, 51], [281, 19, 89, 39, 179, 141, 239]]}\n",
                        "{'validation_accuracy': 0.5975331201461854, 'validation_balanced_accuracy': 0.5747936127381875, 'validation_confusion_matrix': [[136, 4, 1, 23, 44, 14, 6], [32, 348, 0, 11, 15, 5, 0], [48, 4, 185, 18, 47, 26, 6], [0, 1, 4, 405, 1, 0, 0], [35, 2, 5, 2, 82, 10, 5], [84, 9, 42, 8, 65, 98, 22], [91, 11, 38, 14, 71, 57, 54]]}\n",
                        "{'test_accuracy': 0.7295053614666206, 'test_balanced_accuracy': 0.5402535254577695, 'test_confusion_matrix': [[60, 5, 4, 8, 21, 13, 5], [180, 1666, 9, 52, 66, 17, 19], [19, 5, 79, 7, 32, 18, 3], [0, 2, 1, 199, 0, 1, 1], [12, 2, 8, 2, 43, 2, 3], [35, 1, 28, 6, 42, 43, 10], [44, 2, 20, 8, 38, 31, 19]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\RF\\all\n",
                        "\n",
                        "### Running main with ['--experiment_dir', 'data_tcab/detection-experiments/allocine/distilcamembert/fr+small/multiclass_with_clean', '--feature_setting', 'all', '--model', 'LGB', '--model_n_jobs', '10']\n",
                        "\n",
                        "starting experiment: data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "CPU available : 16\n",
                        "--- searching exp data in data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\n",
                        "--- using these features:\n",
                        "['lm_perplexity', 'lm_proba_and_rank', 'tm_activation', 'tm_gradient', 'tm_posterior', 'tm_saliency', 'tp_avg_word_length', 'tp_bert', 'tp_is_first_word_lowercase', 'tp_num_alpha_chars', 'tp_num_cased_letters', 'tp_num_cased_word_switches', 'tp_num_chars', 'tp_num_digits', 'tp_num_lowercase_after_punctuation', 'tp_num_mixed_case_words', 'tp_num_multi_spaces', 'tp_num_non_ascii', 'tp_num_punctuation', 'tp_num_single_lowercase_letters', 'tp_num_words']\n",
                        "--- output to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LGB\\all\n"
                    ]
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "c:\\Users\\Baptiste\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\nlp-adversarial-attacks-ghBt6cj_-py3.10\\lib\\site-packages\\torch\\nn\\modules\\module.py:1117: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
                        "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\train.joblib\n",
                        "num of instances in the file:  6545\n",
                        "loaded, labels:  Counter({'clean': 1187, 'input_reduction': 1187, 'textbugger': 1004, 'textfooler': 987, 'deepwordbug': 976, 'bae': 751, 'pwws': 453})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\val.joblib\n",
                        "num of instances in the file:  2189\n",
                        "loaded, labels:  Counter({'clean': 411, 'input_reduction': 411, 'textfooler': 336, 'deepwordbug': 334, 'textbugger': 328, 'bae': 228, 'pwws': 141})\n",
                        "--- loading from data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\test_release.joblib\n",
                        "num of instances in the file:  2891\n",
                        "loaded, labels:  Counter({'clean': 2009, 'input_reduction': 204, 'textbugger': 165, 'deepwordbug': 163, 'textfooler': 162, 'bae': 116, 'pwws': 72})\n",
                        "--- training main starting\n",
                        "skip if done: False\n",
                        "\n",
                        "no. train: 6,545\n",
                        "no. val: 2,189\n",
                        "no. test: 2,891\n",
                        "no. features: 4,539\n",
                        "\n",
                        "--- starting training\n",
                        "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.651 total time=   4.5s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.663 total time=   4.7s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.633 total time=   4.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.689 total time=  10.2s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.675 total time=   9.9s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.666 total time=  10.2s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.681 total time=   6.3s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.675 total time=   6.7s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.653 total time=   7.1s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.697 total time=  16.0s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.673 total time=  18.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=3, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.669 total time=  18.1s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.651 total time=   4.3s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.663 total time=   4.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=2;, score=0.633 total time=   4.8s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.683 total time=  22.1s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.670 total time=  21.6s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=50, lgbmclassifier__num_leaves=15;, score=0.653 total time=  21.5s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.681 total time=   6.1s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.675 total time=   6.0s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=2;, score=0.653 total time=   6.7s\n",
                        "[CV 1/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.680 total time=  38.9s\n",
                        "[CV 2/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.666 total time=  38.5s\n",
                        "[CV 3/3] END lgbmclassifier__boosting_type=gbdt, lgbmclassifier__max_depth=5, lgbmclassifier__n_estimators=100, lgbmclassifier__num_leaves=15;, score=0.657 total time=  35.5s\n",
                        "--- finished training\n",
                        "--- training time: 00:05:49\n",
                        "{'train_accuracy': 0.9483575248281131, 'train_balanced_accuracy': 0.9504076637367082, 'train_confusion_matrix': [[739, 0, 2, 0, 0, 4, 6], [5, 1170, 3, 2, 2, 2, 3], [32, 2, 891, 0, 5, 36, 10], [0, 1, 0, 1186, 0, 0, 0], [2, 0, 0, 0, 449, 2, 0], [42, 6, 22, 0, 7, 899, 28], [56, 8, 7, 0, 19, 24, 873]]}\n",
                        "{'validation_accuracy': 0.7309273640931933, 'validation_balanced_accuracy': 0.6950234680982748, 'validation_confusion_matrix': [[146, 6, 9, 9, 14, 17, 27], [19, 375, 3, 3, 3, 4, 4], [15, 4, 249, 3, 9, 28, 26], [2, 2, 1, 400, 2, 1, 3], [26, 2, 11, 0, 73, 11, 18], [41, 6, 33, 3, 20, 185, 40], [57, 12, 25, 4, 26, 40, 172]]}\n",
                        "{'test_accuracy': 0.8253199584918713, 'test_balanced_accuracy': 0.6764344920936615, 'test_confusion_matrix': [[70, 6, 6, 3, 6, 9, 16], [103, 1793, 24, 12, 17, 25, 35], [13, 3, 119, 2, 5, 13, 8], [1, 1, 0, 199, 0, 2, 1], [12, 0, 7, 1, 36, 4, 12], [19, 0, 21, 2, 8, 86, 29], [31, 4, 15, 2, 14, 13, 83]]}\n",
                        "--- saving metrics to data_tcab\\detection-experiments\\allocine\\distilcamembert\\fr+small\\multiclass_with_clean\\LGB\\all\n"
                    ]
                }
            ],
            "source": [
                "call_main_args(run_experiment.main, args_grid)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 13,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Total time: 00:58:40\n"
                    ]
                }
            ],
            "source": [
                "print(f\"Total time: {time.strftime('%H:%M:%S', time.gmtime(time.time()-t))}\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": []
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "nlp-adversarial-attacks-ghBt6cj_-py3.10",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.10.9"
        },
        "orig_nbformat": 4
    },
    "nbformat": 4,
    "nbformat_minor": 2
}