exp02_growing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "10d6a1ac-ee33-4426-a03d-4058150940b3",
   "metadata": {},
   "source": [
    "## Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cdf69ba6-7679-4bf3-bd8e-5d296a347ec9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, subprocess, re, ast, pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "99c194b9-95a7-4992-a809-7557dc8855a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "kitti_pointpillars_model_names = [\n",
    "    \"pointpillars_kitti_eloss33\",\n",
    "    \"pointpillars_kitti_eloss66\",\n",
    "    \"pointpillars_kitti_eloss99\"\n",
    "]\n",
    "kitti_voxelnet_model_names = [\n",
    "    \"voxelnet_kitti_eloss50\",\n",
    "    \"voxelnet_kitti_eloss100\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "65fd4728-cc48-4889-983b-2d2434cc8684",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pointpillars_kitti_eloss33\n",
      "pointpillars_kitti_eloss66\n",
      "pointpillars_kitti_eloss99\n"
     ]
    }
   ],
   "source": [
    "kitti_pointpillars_eval_results = []\n",
    "for i, model_name in enumerate(kitti_pointpillars_model_names):\n",
    "    model_root = f\"work_dirs/{model_name}\"\n",
    "    config_file_name = f\"{model_name}.py\"\n",
    "    config_file_path = os.path.join(model_root, config_file_name)\n",
    "    \n",
    "    if os.path.exists(os.path.join(model_root, \"eval_result.pickle\")):\n",
    "        print(model_name)\n",
    "        with open(os.path.join(model_root, \"eval_result.pickle\"), \"rb\") as f:\n",
    "            eval_result = pickle.load(f)\n",
    "    else:\n",
    "        print(model_name)\n",
    "        eval_result = []\n",
    "        for epoch in range(81+5*i, 86+5*i):\n",
    "            checkpoint_path = os.path.join(model_root, f\"epoch_{epoch}.pth\")\n",
    "            command = f\"tools/dist_test.sh {config_file_path} {checkpoint_path} 8 --eval mAP\"\n",
    "            process = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE)\n",
    "            out = process.stdout.read().decode(\"gbk\")\n",
    "            pattern = re.compile(\"{'.*:.*}\")\n",
    "            dict_string = pattern.search(out).group()\n",
    "            eval_dict = ast.literal_eval(dict_string.replace(\"nan\", \"None\"))\n",
    "            eval_result.append(eval_dict)\n",
    "\n",
    "        with open(os.path.join(model_root, \"eval_result.pickle\"), \"wb\") as f:\n",
    "            pickle.dump(eval_result, f)\n",
    "            \n",
    "    kitti_pointpillars_eval_results.append(eval_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "023ec273-5435-40f8-8142-1949c006886b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "voxelnet_kitti_eloss50\n",
      "voxelnet_kitti_eloss100\n"
     ]
    }
   ],
   "source": [
    "kitti_voxelnet_eval_results = []\n",
    "for i, model_name in enumerate(kitti_voxelnet_model_names):\n",
    "    model_root = f\"work_dirs/{model_name}\"\n",
    "    config_file_name = f\"{model_name}.py\"\n",
    "    config_file_path = os.path.join(model_root, config_file_name)\n",
    "    \n",
    "    if os.path.exists(os.path.join(model_root, \"eval_result.pickle\")):\n",
    "        print(model_name)\n",
    "        with open(os.path.join(model_root, \"eval_result.pickle\"), \"rb\") as f:\n",
    "            eval_result = pickle.load(f)\n",
    "    else:\n",
    "        print(model_name)\n",
    "        with open(\"work_dirs/voxelnet_kitti_eloss50/20220722_105331.log.json\", \"r\") as f:\n",
    "        # with open(\"work_dirs/voxelnet_kitti_eloss100/20220722_114805.log.json\", \"r\") as f:\n",
    "            content = f.read()\n",
    "        pattern = re.compile(\"{\\\"mode.*:.*}\")\n",
    "        dict_string_iter = pattern.finditer(content)\n",
    "\n",
    "        eval_result = []\n",
    "        for dict_string in dict_string_iter:\n",
    "            eval_dict = ast.literal_eval(dict_string.group().replace(\"nan\", \"None\"))\n",
    "            if \"mode\" in eval_dict and eval_dict[\"mode\"] == \"val\":\n",
    "                eval_result.append(eval_dict)\n",
    "\n",
    "        with open(os.path.join(model_root, \"eval_result.pickle\"), \"wb\") as f:\n",
    "            pickle.dump(eval_result, f)\n",
    "            \n",
    "    kitti_voxelnet_eval_results.append(eval_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "a2701842-81a3-423c-a1dc-a0d89219e0f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re, ast, pandas#, seaborn\n",
    "# import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "e182994e-bb40-4120-80f4-a4713fb704ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "kitti_pointpillars_eval_dfs = list(map(lambda x : pandas.DataFrame.from_dict(x), kitti_pointpillars_eval_results))\n",
    "kitti_voxelnet_eval_dfs = list(map(lambda x : pandas.DataFrame.from_dict(x), kitti_voxelnet_eval_results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "d790af0f-d004-4648-a5d7-54ceeeb7891f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   KITTI/Car_3D_AP40_moderate_strict\n",
      "0                          75.816906\n",
      "1                          79.162018\n",
      "2                          78.992568\n",
      "3                          78.417320\n",
      "4                          79.545259\n",
      "   KITTI/Car_3D_AP40_moderate_strict\n",
      "0                          81.698426\n",
      "1                          79.071148\n",
      "2                          78.031822\n",
      "3                          80.003314\n",
      "4                          82.613027\n",
      "   KITTI/Car_3D_AP40_moderate_strict\n",
      "0                          73.109124\n",
      "1                          79.132848\n",
      "2                          79.301024\n",
      "3                          81.200176\n",
      "4                          77.562604\n"
     ]
    }
   ],
   "source": [
    "for eval_df in kitti_pointpillars_eval_dfs:\n",
    "    print(eval_df[[\"KITTI/Car_3D_AP40_moderate_strict\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "95ad9762-13d9-4baa-831c-ef13d841dfff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   epoch  KITTI/Car_3D_AP40_moderate_strict\n",
      "0     36                           66.32669\n",
      "1     37                           79.29933\n",
      "2     38                           76.46516\n",
      "3     39                           78.90461\n",
      "4     40                           77.95029\n",
      "5     41                           78.30299\n",
      "6     42                           78.63699\n",
      "7     43                           78.91905\n",
      "8     44                           80.78463\n",
      "9     45                           81.59947\n",
      "   epoch  KITTI/Car_3D_AP40_moderate_strict\n",
      "0     46                           80.27750\n",
      "1     47                           82.44411\n",
      "2     48                           82.33411\n",
      "3     49                           82.32945\n",
      "4     50                           79.90354\n"
     ]
    }
   ],
   "source": [
    "for eval_df in kitti_voxelnet_eval_dfs:\n",
    "    print(eval_df[[\"epoch\", \"KITTI/Car_3D_AP40_moderate_strict\"]])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a888f99-2ab2-4700-886b-898620b854c6",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "a6d7ce0e-c675-485b-aec5-53be864acb8e",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pointpillars_kitti_eloss33\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.8/site-packages/torch/distributed/launch.py:178: FutureWarning: The module torch.distributed.launch is deprecated\n",
      "and will be removed in future. Use torch.distributed.run.\n",
      "Note that --use_env is set by default in torch.distributed.run.\n",
      "If your script expects `--local_rank` argument to be set, please\n",
      "change it to read from `os.environ['LOCAL_RANK']` instead. See \n",
      "https://pytorch.org/docs/stable/distributed.html#launch-utility for \n",
      "further instructions\n",
      "\n",
      "  warnings.warn(\n",
      "WARNING:torch.distributed.run:*****************************************\n",
      "Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \n",
      "*****************************************\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss33/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss33/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss33/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss33/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss33/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss33/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss33/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss33/final.pth\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7520/7518, 379.9 task/s, elapsed: 20s, ETA:     0s\n",
      "writing results to results/kitti-car/pointpillars_kitti_eloss33/results_eval.pkl\n",
      "\n",
      "Converting prediction to KITTI format\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7518/7518, 669.7 task/s, elapsed: 11s, ETA:     0s\n",
      "Result is saved to results/kitti-car/pointpillars_kitti_eloss33/kitti_results.pkl.\n",
      "pointpillars_kitti_eloss66\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.8/site-packages/torch/distributed/launch.py:178: FutureWarning: The module torch.distributed.launch is deprecated\n",
      "and will be removed in future. Use torch.distributed.run.\n",
      "Note that --use_env is set by default in torch.distributed.run.\n",
      "If your script expects `--local_rank` argument to be set, please\n",
      "change it to read from `os.environ['LOCAL_RANK']` instead. See \n",
      "https://pytorch.org/docs/stable/distributed.html#launch-utility for \n",
      "further instructions\n",
      "\n",
      "  warnings.warn(\n",
      "WARNING:torch.distributed.run:*****************************************\n",
      "Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \n",
      "*****************************************\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss66/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss66/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss66/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss66/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss66/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss66/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss66/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss66/final.pth\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7520/7518, 397.3 task/s, elapsed: 19s, ETA:     0s\n",
      "writing results to results/kitti-car/pointpillars_kitti_eloss66/results_eval.pkl\n",
      "\n",
      "Converting prediction to KITTI format\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7518/7518, 660.4 task/s, elapsed: 11s, ETA:     0s\n",
      "Result is saved to results/kitti-car/pointpillars_kitti_eloss66/kitti_results.pkl.\n",
      "pointpillars_kitti_eloss99\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.8/site-packages/torch/distributed/launch.py:178: FutureWarning: The module torch.distributed.launch is deprecated\n",
      "and will be removed in future. Use torch.distributed.run.\n",
      "Note that --use_env is set by default in torch.distributed.run.\n",
      "If your script expects `--local_rank` argument to be set, please\n",
      "change it to read from `os.environ['LOCAL_RANK']` instead. See \n",
      "https://pytorch.org/docs/stable/distributed.html#launch-utility for \n",
      "further instructions\n",
      "\n",
      "  warnings.warn(\n",
      "WARNING:torch.distributed.run:*****************************************\n",
      "Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \n",
      "*****************************************\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss99/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss99/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss99/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss99/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss99/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss99/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss99/final.pth\n",
      "load checkpoint from local path: work_dirs/pointpillars_kitti_eloss99/final.pth\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7520/7518, 389.3 task/s, elapsed: 19s, ETA:     0s\n",
      "writing results to results/kitti-car/pointpillars_kitti_eloss99/results_eval.pkl\n",
      "\n",
      "Converting prediction to KITTI format\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7518/7518, 637.9 task/s, elapsed: 12s, ETA:     0s\n",
      "Result is saved to results/kitti-car/pointpillars_kitti_eloss99/kitti_results.pkl.\n",
      "voxelnet_kitti_eloss50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.8/site-packages/torch/distributed/launch.py:178: FutureWarning: The module torch.distributed.launch is deprecated\n",
      "and will be removed in future. Use torch.distributed.run.\n",
      "Note that --use_env is set by default in torch.distributed.run.\n",
      "If your script expects `--local_rank` argument to be set, please\n",
      "change it to read from `os.environ['LOCAL_RANK']` instead. See \n",
      "https://pytorch.org/docs/stable/distributed.html#launch-utility for \n",
      "further instructions\n",
      "\n",
      "  warnings.warn(\n",
      "WARNING:torch.distributed.run:*****************************************\n",
      "Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \n",
      "*****************************************\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss50/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss50/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss50/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss50/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss50/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss50/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss50/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss50/final.pth\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7520/7518, 219.5 task/s, elapsed: 34s, ETA:     0s\n",
      "writing results to results/kitti-car/voxelnet_kitti_eloss50/results_eval.pkl\n",
      "\n",
      "Converting prediction to KITTI format\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7518/7518, 660.9 task/s, elapsed: 11s, ETA:     0s\n",
      "Result is saved to results/kitti-car/voxelnet_kitti_eloss50/kitti_results.pkl.\n",
      "voxelnet_kitti_eloss100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.8/site-packages/torch/distributed/launch.py:178: FutureWarning: The module torch.distributed.launch is deprecated\n",
      "and will be removed in future. Use torch.distributed.run.\n",
      "Note that --use_env is set by default in torch.distributed.run.\n",
      "If your script expects `--local_rank` argument to be set, please\n",
      "change it to read from `os.environ['LOCAL_RANK']` instead. See \n",
      "https://pytorch.org/docs/stable/distributed.html#launch-utility for \n",
      "further instructions\n",
      "\n",
      "  warnings.warn(\n",
      "WARNING:torch.distributed.run:*****************************************\n",
      "Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \n",
      "*****************************************\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss100/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss100/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss100/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss100/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss100/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss100/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss100/final.pth\n",
      "load checkpoint from local path: work_dirs/voxelnet_kitti_eloss100/final.pth\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7520/7518, 229.2 task/s, elapsed: 33s, ETA:     0s\n",
      "writing results to results/kitti-car/voxelnet_kitti_eloss100/results_eval.pkl\n",
      "\n",
      "Converting prediction to KITTI format\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7518/7518, 689.9 task/s, elapsed: 11s, ETA:     0s\n",
      "Result is saved to results/kitti-car/voxelnet_kitti_eloss100/kitti_results.pkl.\n"
     ]
    }
   ],
   "source": [
    "for model_name in kitti_pointpillars_model_names+kitti_voxelnet_model_names:\n",
    "    model_root = f\"work_dirs/{model_name}\"\n",
    "    config_file_name = f\"{model_name}.py\"\n",
    "    print(model_name)\n",
    "        \n",
    "    config_file_path = os.path.join(model_root, config_file_name)\n",
    "    \n",
    "    checkpoint_path = os.path.join(model_root, f\"final.pth\")\n",
    "    \n",
    "    test_dir = f\"results/kitti-car/{model_name}\"\n",
    "    \n",
    "    # command_1 = f\"mkdir -p {test_dir}\"\n",
    "    command_2 = f\"./tools/dist_test.sh {config_file_path} {checkpoint_path} 8 --out {os.path.join(test_dir, 'results_eval.pkl')} --format-only --eval-options 'pklfile_prefix={os.path.join(test_dir, 'kitti_results')}' 'submission_prefix={os.path.join(test_dir, 'kitti_results')}'\"\n",
    "    \n",
    "    # os.system(command_1)\n",
    "    os.system(command_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "5badeac1-883b-475d-841b-a09a8706e6ad",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.8/site-packages/torch/distributed/launch.py:178: FutureWarning: The module torch.distributed.launch is deprecated\n",
      "and will be removed in future. Use torch.distributed.run.\n",
      "Note that --use_env is set by default in torch.distributed.run.\n",
      "If your script expects `--local_rank` argument to be set, please\n",
      "change it to read from `os.environ['LOCAL_RANK']` instead. See \n",
      "https://pytorch.org/docs/stable/distributed.html#launch-utility for \n",
      "further instructions\n",
      "\n",
      "  warnings.warn(\n",
      "WARNING:torch.distributed.run:*****************************************\n",
      "Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \n",
      "*****************************************\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load checkpoint from local path: checkpoints/pointpillars_kitti/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car_20220331_134606-d42d15ed.pth\n",
      "load checkpoint from local path: checkpoints/pointpillars_kitti/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car_20220331_134606-d42d15ed.pth\n",
      "load checkpoint from local path: checkpoints/pointpillars_kitti/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car_20220331_134606-d42d15ed.pth\n",
      "load checkpoint from local path: checkpoints/pointpillars_kitti/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car_20220331_134606-d42d15ed.pth\n",
      "load checkpoint from local path: checkpoints/pointpillars_kitti/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car_20220331_134606-d42d15ed.pth\n",
      "load checkpoint from local path: checkpoints/pointpillars_kitti/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car_20220331_134606-d42d15ed.pth\n",
      "load checkpoint from local path: checkpoints/pointpillars_kitti/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car_20220331_134606-d42d15ed.pth\n",
      "load checkpoint from local path: checkpoints/pointpillars_kitti/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car_20220331_134606-d42d15ed.pth\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7520/7518, 393.6 task/s, elapsed: 19s, ETA:     0s\n",
      "writing results to results/kitti-car/pointpillars_kitti_eloss0/results_eval.pkl\n",
      "\n",
      "Converting prediction to KITTI format\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7518/7518, 673.1 task/s, elapsed: 11s, ETA:     0s\n",
      "Result is saved to results/kitti-car/pointpillars_kitti_eloss0/kitti_results.pkl.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name = \"pointpillars_kitti_eloss33\"\n",
    "model_root = f\"work_dirs/{model_name}\"\n",
    "config_file_name = f\"{model_name}.py\"\n",
    "config_file_path = os.path.join(model_root, config_file_name)\n",
    "checkpoint_path = \"checkpoints/pointpillars_kitti/hv_pointpillars_secfpn_6x8_160e_kitti-3d-car_20220331_134606-d42d15ed.pth\"\n",
    "test_dir = f\"results/kitti-car/pointpillars_kitti_eloss0\"\n",
    "\n",
    "# command_1 = f\"mkdir -p {test_dir}\"\n",
    "command_2 = f\"./tools/dist_test.sh {config_file_path} {checkpoint_path} 8 --out {os.path.join(test_dir, 'results_eval.pkl')} --format-only --eval-options 'pklfile_prefix={os.path.join(test_dir, 'kitti_results')}' 'submission_prefix={os.path.join(test_dir, 'kitti_results')}'\"\n",
    "\n",
    "# os.system(command_1)\n",
    "os.system(command_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7ede164a-d2ec-4d7f-a6b2-8c57437dc576",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.8/site-packages/torch/distributed/launch.py:178: FutureWarning: The module torch.distributed.launch is deprecated\n",
      "and will be removed in future. Use torch.distributed.run.\n",
      "Note that --use_env is set by default in torch.distributed.run.\n",
      "If your script expects `--local_rank` argument to be set, please\n",
      "change it to read from `os.environ['LOCAL_RANK']` instead. See \n",
      "https://pytorch.org/docs/stable/distributed.html#launch-utility for \n",
      "further instructions\n",
      "\n",
      "  warnings.warn(\n",
      "WARNING:torch.distributed.run:*****************************************\n",
      "Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \n",
      "*****************************************\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/backbones/mink_resnet.py:9: UserWarning: Please follow `getting_started.md` to install MinkowskiEngine.`\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.8/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n",
      "/user-data/mmdetection3d/mmdet3d/models/dense_heads/anchor3d_head.py:84: UserWarning: dir_offset and dir_limit_offset will be depressed and be incorporated into box coder in the future\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load checkpoint from local path: checkpoints/second_kitti/hv_second_secfpn_6x8_80e_kitti-3d-car_20200620_230238-393f000c.pth\n",
      "load checkpoint from local path: checkpoints/second_kitti/hv_second_secfpn_6x8_80e_kitti-3d-car_20200620_230238-393f000c.pth\n",
      "load checkpoint from local path: checkpoints/second_kitti/hv_second_secfpn_6x8_80e_kitti-3d-car_20200620_230238-393f000c.pth\n",
      "load checkpoint from local path: checkpoints/second_kitti/hv_second_secfpn_6x8_80e_kitti-3d-car_20200620_230238-393f000c.pth\n",
      "load checkpoint from local path: checkpoints/second_kitti/hv_second_secfpn_6x8_80e_kitti-3d-car_20200620_230238-393f000c.pth\n",
      "load checkpoint from local path: checkpoints/second_kitti/hv_second_secfpn_6x8_80e_kitti-3d-car_20200620_230238-393f000c.pth\n",
      "load checkpoint from local path: checkpoints/second_kitti/hv_second_secfpn_6x8_80e_kitti-3d-car_20200620_230238-393f000c.pth\n",
      "load checkpoint from local path: checkpoints/second_kitti/hv_second_secfpn_6x8_80e_kitti-3d-car_20200620_230238-393f000c.pth\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7520/7518, 222.6 task/s, elapsed: 34s, ETA:     0s\n",
      "writing results to results/kitti-car/voxelnet_kitti_eloss0/results_eval.pkl\n",
      "\n",
      "Converting prediction to KITTI format\n",
      "[>>>>>>>>>>>>>>>>>>>>>>>>>>] 7518/7518, 656.3 task/s, elapsed: 11s, ETA:     0s\n",
      "Result is saved to results/kitti-car/voxelnet_kitti_eloss0/kitti_results.pkl.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name = \"voxelnet_kitti_eloss50\"\n",
    "model_root = f\"work_dirs/{model_name}\"\n",
    "config_file_name = f\"{model_name}.py\"\n",
    "config_file_path = os.path.join(model_root, config_file_name)\n",
    "checkpoint_path = \"checkpoints/second_kitti/hv_second_secfpn_6x8_80e_kitti-3d-car_20200620_230238-393f000c.pth\"\n",
    "test_dir = f\"results/kitti-car/voxelnet_kitti_eloss0\"\n",
    "\n",
    "command_1 = f\"mkdir -p {test_dir}\"\n",
    "command_2 = f\"./tools/dist_test.sh {config_file_path} {checkpoint_path} 8 --out {os.path.join(test_dir, 'results_eval.pkl')} --format-only --eval-options 'pklfile_prefix={os.path.join(test_dir, 'kitti_results')}' 'submission_prefix={os.path.join(test_dir, 'kitti_results')}'\"\n",
    "\n",
    "os.system(command_1)\n",
    "os.system(command_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a736b00-32cf-47c3-bb22-a235c8b88574",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}