【マルチエージェント化】環境全体の操作とグループの報酬を担当するEnvControllerを追加。

1on1FlagGame/1on1DoroK-MLTEST1/Assets/DorokAgent.cs

@@ -2,6 +2,7 @@
 using Unity.MLAgents;
 using Unity.MLAgents.Actuators;
 using Unity.MLAgents.Policies;
+using Unity.MLAgents.Sensors;
 
 public enum Team {
     Police = 0,
@@ -11,7 +12,7 @@ public enum Team {
 
 public class DorokAgent: Agent {
 
-
+    EnvController envController;
     [HideInInspector]
     public Team team;
 
@@ -24,9 +25,12 @@ public class DorokAgent: Agent {
 
     EnvironmentParameters m_ResetParams;
 
-
+    /**
+    エージェントの初期化
+    */
     public override void Initialize() {
-        EnvCtrler envController = GetComponentInParent<EnvCtrler>();
+
+        envController = GetComponentInParent<EnvController>();
         m_BehaviorParameters = gameObject.GetComponent<BehaviorParameters>();
 
         if (m_BehaviorParameters.TeamId == (int)Team.Police) {
@@ -44,37 +48,77 @@ public override void Initialize() {
         agentRb.maxAngularVelocity = 500;
 
         m_ResetParams = Academy.Instance.EnvironmentParameters;
-    }
-
-
-    public void MoveAgent(ActionSegment<int> act) {
-        var dirToGo = Vector3.zero;
-        var rotateDir = Vector3.zero;
 
+        print("Initialized Agent For Team: " + team + "");
+    }
 
-        var forwardAxis = act[0];
-        var rightAxis = act[1];
-        var rotateAxis = act[2];
 
+    /**
+    環境の観測
+    */
+    public override void CollectObservations(VectorSensor sensor) {
+        // 自身の位置
+        sensor.AddObservation(agentRb.velocity.x);
+        sensor.AddObservation(agentRb.velocity.z);
+
+        // 敵エージェントの位置
+        var tagname = team == Team.Police ? "Criminer" : "Police";
+        GameObject[] enemys = GameObject.FindGameObjectsWithTag(tagname);
+
+        foreach(GameObject enemy in enemys) {
+            sensor.AddObservation(enemy.transform.position.x);
+            sensor.AddObservation(enemy.transform.position.z);
+        }
 
-        transform.Rotate(rotateDir, Time.deltaTime * 100f);
-        agentRb.AddForce(dirToGo * m_Settings.agentRunSpeed,ForceMode.VelocityChange);
     }
 
 
     /**
-    エージェントの行動による報酬の設定
+    エージェントの行動による報酬の設定(報酬設計)
     */
     public override void OnActionReceived(ActionBuffers actionBuffers) {
+        //Action size is 2
+        var actions = actionBuffers.ContinuousActions;
+        Vector3 controlSignal = Vector3.zero;
+        controlSignal.x = actions[0];
+        controlSignal.z = actions[1];
+        agentRb.AddForce(controlSignal * m_Settings.agentRunSpeed);
+
+        //敵エージェントとの距離を算出
+        var tagname = team == Team.Police ? "Criminer" : "Police";
+        GameObject[] enemys = GameObject.FindGameObjectsWithTag(tagname);
+        Vector3[] enemyPos = new Vector3[enemys.Length];
+        for (int i = 0; i < enemys.Length; i++) {
+            enemyPos[i] = enemys[i].transform.position;
+        }
+        var distanceToEnemy = Vector3.Distance(transform.position, enemyPos[0]);
+
+        //敵エージェントとの距離が近いほど報酬を与える
+        AddReward(1f / distanceToEnemy);
+        //敵エージェントとの距離が遠いほど報酬を与える
+        AddReward(-1f / distanceToEnemy);
+        //警官が逃走役を捕まえたら報酬を与え,逃走役は報酬を引く。
+        if (distanceToEnemy < 1f) {
+            if (team == Team.Police) {
+                AddReward(1f);
+            } else {
+                AddReward(-1f);
+                //逃走役を初期位置にもどす。
+                transform.position = initialPos;
+            }
+            EndEpisode();
+        }
 
-        MoveAgent(actionBuffers.DiscreteActions);
     }
 
     /**
-    シートの初期化
+    シーンの初期化
     */
     public override void OnEpisodeBegin() {
-
+        // エージェントの初期位置を設定
+        transform.position = initialPos;
+        agentRb.velocity = Vector3.zero;
+        agentRb.angularVelocity = Vector3.zero;
     }
 
 }

1on1FlagGame/1on1DoroK-MLTEST1/Assets/EnvController.cs

@@ -0,0 +1,103 @@
+using System.Collections.Generic;
+using UnityEngine;
+using Unity.MLAgents;
+using Unity.MLAgents.Sensors;
+using Unity.MLAgents.Actuators;
+
+public class EnvController : MonoBehaviour
+{
+    private DorokSettings m_DorokSettings;
+
+    [System.Serializable]
+    public class PlayerInfo {
+        public DorokAgent Agent;
+        [HideInInspector]
+        public Vector3 StartingPos;
+        [HideInInspector]
+        public Quaternion StartingRot;
+        [HideInInspector]
+        public Rigidbody Rb;
+    }
+
+
+    /*警察グループ*/
+    public SimpleMultiAgentGroup PoliceGroup;
+    /*逃走者グループ*/
+    public SimpleMultiAgentGroup CriminerGroup;
+
+
+    //List of Agents On Platform
+    public List<PlayerInfo> AgentsList = new List<PlayerInfo>();
+
+    private int m_ResetTimer;
+
+    [Tooltip("Max Environment Steps")] public int MaxEnvironmentSteps = 25000;
+
+    void Start() {
+        var PoliceCount = 0;
+        var CriminerCount = 0;
+        m_DorokSettings = FindObjectOfType<DorokSettings>();
+        // Initialize TeamManager
+        PoliceGroup = new SimpleMultiAgentGroup();
+        CriminerGroup = new SimpleMultiAgentGroup();
+        print("AgentsList Count: ");
+        print(AgentsList);
+        foreach (var item in AgentsList) {
+            if (item.Agent.team == Team.Police) {
+                PoliceGroup.RegisterAgent(item.Agent);
+                PoliceCount++;
+            } else {
+                CriminerGroup.RegisterAgent(item.Agent);
+                CriminerCount++;
+            }
+        }
+        print("TeamManager Initialized");
+        print("PoliceGroup: " + PoliceCount + " Agents");
+        print("CriminerGroup: " + CriminerCount + " Agents");
+    }
+
+
+    void FixedUpdate() {
+        m_ResetTimer += 1;
+        if (m_ResetTimer >= MaxEnvironmentSteps && MaxEnvironmentSteps > 0) {
+            PoliceGroup.GroupEpisodeInterrupted();
+            CriminerGroup.GroupEpisodeInterrupted();
+            ResetScene();
+        }
+    }
+
+    /**
+    1マッチ終了時の報酬処理
+    （逃走者が全員捕まるか、制限時間切れの場合）
+    */
+    public void MatchEnd(Team team) {
+        if (team == Team.Police) {
+            PoliceGroup.AddGroupReward(1);
+            CriminerGroup.AddGroupReward(-1);
+        } else {
+            PoliceGroup.AddGroupReward(-1);
+            CriminerGroup.AddGroupReward(1);
+        }
+
+        PoliceGroup.GroupEpisodeInterrupted();
+        CriminerGroup.GroupEpisodeInterrupted();
+        ResetScene();
+    }
+
+
+    public void ResetScene() {
+        m_ResetTimer = 0;
+
+        //Reset Agents
+        foreach (var item in AgentsList) {
+            var randomPosX = Random.Range(-5f, 5f);
+            var newStartPos = item.Agent.initialPos + new Vector3(randomPosX, 0f, 0f);
+            var rot = item.Agent.rotSign * Random.Range(80.0f, 100.0f);
+            var newRot = Quaternion.Euler(0, rot, 0);
+            item.Agent.transform.SetPositionAndRotation(newStartPos, newRot);
+
+            item.Rb.velocity = Vector3.zero;
+            item.Rb.angularVelocity = Vector3.zero;
+        }
+    }
+}

1on1FlagGame/1on1DoroK-MLTEST1/Assets/ML-Agents/Timers/Dorok_timers.json

@@ -1 +1 @@
-{"count":1,"self":118.0938496,"total":118.09881689999999,"children":{"InitializeActuators":{"count":1,"self":0.001967,"total":0.001967,"children":null},"InitializeSensors":{"count":1,"self":0.0019974999999999997,"total":0.0019974999999999997,"children":null}},"gauges":{},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1671984344","unity_version":"2021.3.14f1","command_line_arguments":"F:\\Program Files\\Unity\\2021.3.14f1\\Editor\\Unity.exe -projectpath F:\\Products\\GraduateReport2\\1on1FlagGame\\1on1DoroK-MLTEST1 -useHub -hubIPC -cloudEnvironment production","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.3.0-exp.3","scene_name":"Dorok","end_time_seconds":"1671984462"}}
+{"count":1,"self":27.4244192,"total":27.4354715,"children":{"InitializeActuators":{"count":2,"self":0.0009969,"total":0.0009969,"children":null},"InitializeSensors":{"count":2,"self":0.0019944,"total":0.0019944,"children":null},"AgentSendState":{"count":1,"self":0.0040199,"total":0.0060907,"children":{"CollectObservations":{"count":1,"self":0,"total":0,"children":null},"WriteActionMask":{"count":1,"self":0,"total":0,"children":null},"RequestDecision":{"count":1,"self":0.0020708,"total":0.0020708,"children":{"RayPerceptionSensor.Perceive":{"count":1,"self":0,"total":0,"children":null}}}}},"DecideAction":{"count":1,"self":0.00094409999999999991,"total":0.00094409999999999991,"children":null},"AgentAct":{"count":1,"self":0,"total":0,"children":null}},"gauges":{},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1672032004","unity_version":"2021.3.14f1","command_line_arguments":"F:\\Program Files\\Unity\\2021.3.14f1\\Editor\\Unity.exe -projectpath F:\\Products\\GraduateReport2\\1on1FlagGame\\1on1DoroK-MLTEST1 -useHub -hubIPC -cloudEnvironment production","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.3.0-exp.3","scene_name":"Dorok","end_time_seconds":"1672032031"}}

1on1FlagGame/1on1DoroK-MLTEST1/Assets/New Material 1.mat

@@ -75,6 +75,6 @@ Material:
     - _UVSec: 0
     - _ZWrite: 1
     m_Colors:
-    - _Color: {r: 1, g: 1, b: 1, a: 1}
+    - _Color: {r: 0.9811321, g: 0.5045568, b: 0.013883919, a: 1}
     - _EmissionColor: {r: 0, g: 0, b: 0, a: 1}
   m_BuildTextureStacks: []

1on1FlagGame/1on1DoroK-MLTEST1/Assets/New Material.mat

@@ -75,6 +75,6 @@ Material:
     - _UVSec: 0
     - _ZWrite: 1
     m_Colors:
-    - _Color: {r: 1, g: 1, b: 1, a: 1}
+    - _Color: {r: 0.124955505, g: 0.20940363, b: 0.9811321, a: 1}
     - _EmissionColor: {r: 0, g: 0, b: 0, a: 1}
   m_BuildTextureStacks: []

1on1FlagGame/1on1DoroK-MLTEST1/Assets/PoliceAgent.prefab → 1on1FlagGame/1on1DoroK-MLTEST1/Assets/Police.prefab

@@ -18,7 +18,7 @@ GameObject:
   - component: {fileID: 4093369590317185654}
   - component: {fileID: 4093369590317185657}
   m_Layer: 0
-  m_Name: PoliceAgent
+  m_Name: Police
   m_TagString: Police
   m_Icon: {fileID: 0}
   m_NavMeshLayer: 0