TensorBoard導入によるパッケージリストの更新。捕まった逃走者がシーンリセット時に解放されない問題を修正

1on1FlagGame/1on1DoroK-MLTEST1/Assets/DorokAgent.cs

@@ -1,3 +1,4 @@
+using System.Collections.Generic;
 using UnityEngine;
 using Unity.MLAgents;
 using Unity.MLAgents.Actuators;
@@ -23,7 +24,6 @@ public class DorokAgent: Agent {
 
     //捕らわれているかどうか
     public bool isCaptured = false;
-    PrisonController prisonController;
 
     EnvironmentParameters m_ResetParams;
     EnvController envController;
@@ -62,8 +62,6 @@ public override void Initialize() {
         m_LateralSpeed = m_Settings.agentLateralSpeed;
         m_ResetParams = Academy.Instance.EnvironmentParameters;
 
-        //牢屋のインスタンスを取得
-        prisonController = gameObject.GetComponent<PrisonController>();    
     }
 
 
@@ -189,12 +187,11 @@ void OnCollisionEnter(Collision c) {
         } else {
             onCollisionCriminer(c);
         }
-        envController.onGameEnd(team);
+        //envController.onGameEnd(team);
         EndEpisode();
     }
 
     private void onCollisionPolice(Collision c) {
-        print("onCollisionPolice");
         // 警察エージェントが逃走役エージェントと接触した場合、捕まえたと判定し、報酬を与える
         if (c.gameObject.CompareTag("Criminer")) {
             print("onCollisionPolice: Catch Criminer");
@@ -203,7 +200,6 @@ private void onCollisionPolice(Collision c) {
     }
 
     private void onCollisionCriminer(Collision c) {
-        print("onCollisionCriminer");
         //捕まっている場合、牢屋の位置から動かないようにする
         if(isCaptured) {
             agentRb.velocity = Vector3.zero;
@@ -215,15 +211,16 @@ private void onCollisionCriminer(Collision c) {
             isCaptured = true;
             //エージェントを所定の牢屋位置に移動させた後、エージェントの動きを止める
             //FIXME:牢屋の位置へ移動しない、prisonController is null
-            transform.position = prisonController.prisonPos;
+            transform.position = m_Settings.prisonPos;
             agentRb.velocity = Vector3.zero;
-            m_EnvController.onCaught();
+            envController.onCaught(team);
         }
         // 逃走役エージェントが牢屋にいる他の逃走者と接触した場合、報酬を与え、接触した他の逃走者を解放する
         if (c.gameObject.CompareTag("Criminer") && c.gameObject.GetComponent<DorokAgent>().isCaptured) {
             AddReward(10.0f);
             c.gameObject.GetComponent<DorokAgent>().isCaptured = false;
-            prisonController.ReleaseCapturedAgents(team);
+            List<GameObject> capturedAgents = envController.GetCapturedAgents();
+            envController.ReleaseCapturedAgents(capturedAgents);
         }
     }
 

1on1FlagGame/1on1DoroK-MLTEST1/Assets/DorokSettings.cs

@@ -6,6 +6,7 @@ public class DorokSettings : MonoBehaviour {
     public float agentForwardSpeed;
     public float agentLateralSpeed;
     public float rewardConstant;
+    public Vector3 prisonPos;
 }
 
 public enum Team {

1on1FlagGame/1on1DoroK-MLTEST1/Assets/EnvController.cs

@@ -30,16 +30,12 @@ public class PlayerInfo {
 
     private int m_ResetTimer;
 
-    //牢屋オブジェクト
-    public PrisonController Prison;
-
     [Tooltip("Max Environment Steps")] public int MaxEnvironmentSteps = 25000;
 
     void Start() {
         var PoliceCount = 0;
         var CriminerCount = 0;
         m_DorokSettings = FindObjectOfType<DorokSettings>();
-        Prison = FindObjectOfType<PrisonController>();
         // Initialize TeamManager
         PoliceGroup = new SimpleMultiAgentGroup();
         CriminerGroup = new SimpleMultiAgentGroup();
@@ -64,6 +60,7 @@ void FixedUpdate() {
         if (m_ResetTimer >= MaxEnvironmentSteps && MaxEnvironmentSteps > 0) {
             PoliceGroup.GroupEpisodeInterrupted();
             CriminerGroup.GroupEpisodeInterrupted();
+            onTimeUp();
             ResetScene();
         }
     }
@@ -74,16 +71,64 @@ void FixedUpdate() {
     public void onCaught(Team team) {
         if (team == Team.Police) {
             //牢屋にとらえている犯人役の人数分だけ報酬を与える
-            int count = Prison.GetCapturedAgents().Count;
+            int count = GetCapturedAgents().Count;
             PoliceGroup.AddGroupReward(count);
         } else {
             //牢屋にとらえている犯人役の人数分だけ報酬を減らす
-            int count = Prison.GetCapturedAgents().Count;
+            int count = GetCapturedAgents().Count;
             CriminerGroup.AddGroupReward(-count);
         }
         PoliceGroup.EndGroupEpisode();
         CriminerGroup.EndGroupEpisode();
-        ResetScene();
+        //生き残っている逃走者の数がいない場合、シーンをリセットする
+        List<GameObject> survivers = GetFreeCriminers();
+        if(survivers.Count == 0) {
+            ResetScene();
+        }
+    }
+
+
+    public void onTimeUp() {
+        //捕まっていない逃走者の分だけ負の報酬を与える
+        int count = GetFreeCriminers().Count;
+        PoliceGroup.AddGroupReward(-count);
+        CriminerGroup.AddGroupReward(count);
+    }
+
+
+    /**
+    * 捕らわれている逃走役エージェントを取得
+    */
+    public List<GameObject> GetCapturedAgents() {
+        List<GameObject> capturedAgents = new List<GameObject>();
+        foreach (GameObject agent in GameObject.FindGameObjectsWithTag("Criminer")) {
+            if (agent.GetComponent<DorokAgent>().isCaptured) {
+                capturedAgents.Add(agent);
+            }
+        }
+        print("PrisonController: capturedAgents.Count = " + capturedAgents.Count);
+        return capturedAgents;
+    }
+
+
+    public List<GameObject> GetFreeCriminers() {
+        List<GameObject> freeCriminers = new List<GameObject>();
+        foreach (GameObject agent in GameObject.FindGameObjectsWithTag("Criminer")) {
+            if (agent.GetComponent<DorokAgent>().isCaptured == false) {
+                freeCriminers.Add(agent);
+            }
+        }
+        return freeCriminers;
+    }
+
+    /**
+    * 捕らわれている逃走役エージェントを牢屋からランダムにフィールドに開放する
+    */
+    public void ReleaseCapturedAgents(List<GameObject> capturedAgents) {
+        foreach (GameObject agent in capturedAgents) {
+            agent.GetComponent<DorokAgent>().isCaptured = false;
+            agent.transform.position = new Vector3(Random.Range(-5.0f, 5.0f), 0.5f, Random.Range(-5.0f, 5.0f));
+        }
     }
 
 
@@ -100,5 +145,11 @@ public void ResetScene() {
             item.Rb.velocity = Vector3.zero;
             item.Rb.angularVelocity = Vector3.zero;
         }
+        //捕まっているフラグを戻す
+        foreach (GameObject agent in GameObject.FindGameObjectsWithTag("Criminer")) {
+            if (agent.GetComponent<DorokAgent>().isCaptured) {
+                agent.GetComponent<DorokAgent>().isCaptured = false;
+            }
+        }
     }
 }

1on1FlagGame/1on1DoroK-MLTEST1/Assets/ML-Agents/Timers/Dorok_timers.json

@@ -1 +1 @@
-{"count":1,"self":47.00152,"total":218.3845523,"children":{"InitializeActuators":{"count":2,"self":0.0020014,"total":0.0020014,"children":null},"InitializeSensors":{"count":2,"self":0.0040003,"total":0.0040003,"children":null},"AgentSendState":{"count":43601,"self":1.5696522,"total":2.2772867999999997,"children":{"CollectObservations":{"count":13082,"self":0.0089896,"total":0.0089896,"children":null},"WriteActionMask":{"count":13082,"self":0.011998199999999999,"total":0.011998199999999999,"children":null},"RequestDecision":{"count":13082,"self":0.06988939999999999,"total":0.6866468,"children":{"AgentInfo.ToProto":{"count":13082,"self":0.0899504,"total":0.6167574,"children":{"GenerateSensorData":{"count":13082,"self":0.47079649999999995,"total":0.526807,"children":{"RayPerceptionSensor.Perceive":{"count":13082,"self":0.0560105,"total":0.0560105,"children":null}}}}}}}}},"DecideAction":{"count":43601,"self":165.36213759999998,"total":165.3621325,"children":null},"AgentAct":{"count":43601,"self":3.1667266,"total":3.7366116,"children":{"AgentInfo.ToProto":{"count":13082,"self":0.0830617,"total":0.569885,"children":{"GenerateSensorData":{"count":13082,"self":0.41378499999999996,"total":0.48682329999999996,"children":{"RayPerceptionSensor.Perceive":{"count":13082,"self":0.0730383,"total":0.0730383,"children":null}}}}}}}},"gauges":{"Criminer.CumulativeReward":{"count":8721,"max":-0.100000024,"min":-0.200000048,"runningAverage":-0.199973956,"value":-0.200000048,"weightedAverage":-0.200000018},"Police.CumulativeReward":{"count":4361,"max":1.02780557,"min":-43.4392166,"runningAverage":-20.700201,"value":-24.8419113,"weightedAverage":-24.1771278}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1674962426","unity_version":"2021.3.14f1","command_line_arguments":"D:\\Program Files\\Unity\\2021.3.14f1\\Editor\\Unity.exe -projectpath D:\\ProdFolder\\GraduateReport2\\1on1FlagGame\\1on1DoroK-MLTEST1 -useHub -hubIPC -cloudEnvironment production","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.3.0-exp.3","scene_name":"Dorok","end_time_seconds":"1674962645"}}
+{"count":1,"self":511.23041279999995,"total":1297.982624,"children":{"InitializeActuators":{"count":2,"self":0.0029695,"total":0.0029695,"children":null},"InitializeSensors":{"count":2,"self":0.0029953,"total":0.0029953,"children":null},"AgentSendState":{"count":162551,"self":7.0138896,"total":10.1603309,"children":{"CollectObservations":{"count":48767,"self":0.055990599999999995,"total":0.055990599999999995,"children":null},"WriteActionMask":{"count":48767,"self":0.0639348,"total":0.0639348,"children":null},"RequestDecision":{"count":48767,"self":0.3389156,"total":3.0265161999999997,"children":{"AgentInfo.ToProto":{"count":48767,"self":0.38936509999999996,"total":2.6876005999999997,"children":{"GenerateSensorData":{"count":48767,"self":2.0333152,"total":2.2982355,"children":{"RayPerceptionSensor.Perceive":{"count":48767,"self":0.2649204,"total":0.2649204,"children":null}}}}}}}}},"DecideAction":{"count":162551,"self":760.7414784,"total":760.74149449999993,"children":null},"AgentAct":{"count":162551,"self":13.4759424,"total":15.841416899999999,"children":{"AgentInfo.ToProto":{"count":48767,"self":0.31282099999999996,"total":2.3654739,"children":{"GenerateSensorData":{"count":48767,"self":1.7751355999999998,"total":2.0526529,"children":{"RayPerceptionSensor.Perceive":{"count":48767,"self":0.27751739999999997,"total":0.27751739999999997,"children":null}}}}}}}},"gauges":{"Criminer.CumulativeReward":{"count":32511,"max":79.33128,"min":5.009826,"runningAverage":42.05008,"value":49.0591431,"weightedAverage":49.95115},"Police.CumulativeReward":{"count":16256,"max":0.985105,"min":-73.33128,"runningAverage":-36.098423,"value":-43.0591431,"weightedAverage":-44.29476}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1674973929","unity_version":"2021.3.14f1","command_line_arguments":"D:\\Program Files\\Unity\\2021.3.14f1\\Editor\\Unity.exe -projectpath D:\\ProdFolder\\GraduateReport2\\1on1FlagGame\\1on1DoroK-MLTEST1 -useHub -hubIPC -cloudEnvironment production","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.3.0-exp.3","scene_name":"Dorok","end_time_seconds":"1674975227"}}

1on1FlagGame/1on1DoroK-MLTEST1/Assets/Scenes/Dorok.unity

@@ -354,6 +354,7 @@ MonoBehaviour:
   agentForwardSpeed: 2
   agentLateralSpeed: 2
   rewardConstant: 1.42
+  prisonPos: {x: 48.48, y: -5.735, z: 43.47}
 --- !u!4 &899765140
 Transform:
   m_ObjectHideFlags: 0
@@ -483,7 +484,7 @@ PrefabInstance:
       objectReference: {fileID: 0}
     - target: {fileID: 3017807615762775812, guid: 85f239f4a25ddd441ba64358bc864f46, type: 3}
       propertyPath: m_LocalPosition.y
-      value: -5.63
+      value: -5.735
       objectReference: {fileID: 0}
     - target: {fileID: 3017807615762775812, guid: 85f239f4a25ddd441ba64358bc864f46, type: 3}
       propertyPath: m_LocalPosition.z
@@ -565,7 +566,6 @@ MonoBehaviour:
     StartingPos: {x: 0, y: 0, z: 0}
     StartingRot: {x: 0, y: 0, z: 0, w: 0}
     Rb: {fileID: 0}
-  Prison: {fileID: 1130999078}
   MaxEnvironmentSteps: 25000
 --- !u!4 &1021618441
 Transform:
@@ -624,7 +624,7 @@ MonoBehaviour:
   m_PrefabInstance: {fileID: 0}
   m_PrefabAsset: {fileID: 0}
   m_GameObject: {fileID: 1130999076}
-  m_Enabled: 1
+  m_Enabled: 0
   m_EditorHideFlags: 0
   m_Script: {fileID: 11500000, guid: dbcdc9c2aa3eaf84c9dfb4aae946b2bc, type: 3}
   m_Name: 
@@ -642,21 +642,25 @@ PrefabInstance:
       propertyPath: m_Name
       value: Criminer
       objectReference: {fileID: 0}
+    - target: {fileID: 5206574036134512812, guid: 5d5a9a68d509cae49a66ed0c03f8c758, type: 3}
+      propertyPath: m_BrainParameters.VectorObservationSize
+      value: 1
+      objectReference: {fileID: 0}
     - target: {fileID: 8850290593895417182, guid: 5d5a9a68d509cae49a66ed0c03f8c758, type: 3}
       propertyPath: m_RootOrder
       value: 6
       objectReference: {fileID: 0}
     - target: {fileID: 8850290593895417182, guid: 5d5a9a68d509cae49a66ed0c03f8c758, type: 3}
       propertyPath: m_LocalPosition.x
-      value: 29.08
+      value: 32.36
       objectReference: {fileID: 0}
     - target: {fileID: 8850290593895417182, guid: 5d5a9a68d509cae49a66ed0c03f8c758, type: 3}
       propertyPath: m_LocalPosition.y
       value: -5.08
       objectReference: {fileID: 0}
     - target: {fileID: 8850290593895417182, guid: 5d5a9a68d509cae49a66ed0c03f8c758, type: 3}
       propertyPath: m_LocalPosition.z
-      value: 17
+      value: 33.59
       objectReference: {fileID: 0}
     - target: {fileID: 8850290593895417182, guid: 5d5a9a68d509cae49a66ed0c03f8c758, type: 3}
       propertyPath: m_LocalRotation.w
@@ -1005,6 +1009,10 @@ PrefabInstance:
       propertyPath: m_Name
       value: Police
       objectReference: {fileID: 0}
+    - target: {fileID: 5530155795508512091, guid: c9f55c9c60104944d91806d300bbc031, type: 3}
+      propertyPath: m_BrainParameters.VectorObservationSize
+      value: 1
+      objectReference: {fileID: 0}
     - target: {fileID: 7002257196600256709, guid: c9f55c9c60104944d91806d300bbc031, type: 3}
       propertyPath: m_RootOrder
       value: 7