MultiAgentGroup Interface (#4923)

* add SimpleMultiAgentGroup

* add group reward field to agent and proto

Author: Ruo-Ping Dong

com.unity.ml-agents/Runtime/Agent.cs

@@ -34,6 +34,11 @@ internal struct AgentInfo
         /// </summary>
         public float reward;
 
+        /// <summary>
+        /// The current group reward received by the agent.
+        /// </summary>
+        public float groupReward;
+
         /// <summary>
         /// Whether the agent is done or not.
         /// </summary>
@@ -50,6 +55,11 @@ internal struct AgentInfo
         /// </summary>
         public int episodeId;
 
+        /// <summary>
+        /// MultiAgentGroup identifier.
+        /// </summary>
+        public int groupId;
+
         public void ClearActions()
         {
             storedActions.Clear();
@@ -243,6 +253,9 @@ internal struct AgentParameters
         /// Additionally, the magnitude of the reward should not exceed 1.0
         float m_Reward;
 
+        /// Represents the group reward the agent accumulated during the current step.
+        float m_GroupReward;
+
         /// Keeps track of the cumulative reward in this episode.
         float m_CumulativeReward;
 
@@ -317,6 +330,13 @@ internal struct AgentParameters
         /// </summary>
         float[] m_LegacyHeuristicCache;
 
+        /// Currect MultiAgentGroup ID. Default to 0 (meaning no group)
+        int m_GroupId;
+
+        /// Delegate for the agent to unregister itself from the MultiAgentGroup without cyclic reference
+        /// between agent and the group
+        internal event Action<Agent> OnAgentDisabled;
+
         /// <summary>
         /// Called when the attached [GameObject] becomes enabled and active.
         /// [GameObject]: https://docs.unity3d.com/Manual/GameObjects.html
@@ -448,6 +468,8 @@ public void LazyInitialize()
                 new int[m_ActuatorManager.NumDiscreteActions]
             );
 
+            m_Info.groupId = m_GroupId;
+
             // The first time the Academy resets, all Agents in the scene will be
             // forced to reset through the <see cref="AgentForceReset"/> event.
             // To avoid the Agent resetting twice, the Agents will not begin their
@@ -516,6 +538,7 @@ protected virtual void OnDisable()
                 NotifyAgentDone(DoneReason.Disabled);
             }
             m_Brain?.Dispose();
+            OnAgentDisabled?.Invoke(this);
             m_Initialized = false;
         }
 
@@ -528,8 +551,10 @@ void NotifyAgentDone(DoneReason doneReason)
             }
             m_Info.episodeId = m_EpisodeId;
             m_Info.reward = m_Reward;
+            m_Info.groupReward = m_GroupReward;
             m_Info.done = true;
             m_Info.maxStepReached = doneReason == DoneReason.MaxStepReached;
+            m_Info.groupId = m_GroupId;
             if (collectObservationsSensor != null)
             {
                 // Make sure the latest observations are being passed to training.
@@ -559,6 +584,7 @@ void NotifyAgentDone(DoneReason doneReason)
             }
 
             m_Reward = 0f;
+            m_GroupReward = 0f;
             m_CumulativeReward = 0f;
             m_RequestAction = false;
             m_RequestDecision = false;
@@ -698,6 +724,22 @@ public void AddReward(float increment)
             m_CumulativeReward += increment;
         }
 
+        internal void SetGroupReward(float reward)
+        {
+#if DEBUG
+            Utilities.DebugCheckNanAndInfinity(reward, nameof(reward), nameof(SetGroupReward));
+#endif
+            m_GroupReward = reward;
+        }
+
+        internal void AddGroupReward(float increment)
+        {
+#if DEBUG
+            Utilities.DebugCheckNanAndInfinity(increment, nameof(increment), nameof(AddGroupReward));
+#endif
+            m_GroupReward += increment;
+        }
+
         /// <summary>
         /// Retrieves the episode reward for the Agent.
         /// </summary>
@@ -1054,9 +1096,11 @@ void SendInfoToBrain()
 
             m_Info.discreteActionMasks = m_ActuatorManager.DiscreteActionMask?.GetMask();
             m_Info.reward = m_Reward;
+            m_Info.groupReward = m_GroupReward;
             m_Info.done = false;
             m_Info.maxStepReached = false;
             m_Info.episodeId = m_EpisodeId;
+            m_Info.groupId = m_GroupId;
 
             using (TimerStack.Instance.Scoped("RequestDecision"))
             {
@@ -1323,6 +1367,7 @@ void SendInfo()
             {
                 SendInfoToBrain();
                 m_Reward = 0f;
+                m_GroupReward = 0f;
                 m_RequestDecision = false;
             }
         }
@@ -1358,5 +1403,25 @@ void DecideAction()
             m_Info.CopyActions(actions);
             m_ActuatorManager.UpdateActions(actions);
         }
+
+        internal void SetMultiAgentGroup(IMultiAgentGroup multiAgentGroup)
+        {
+            if (multiAgentGroup == null)
+            {
+                m_GroupId = 0;
+            }
+            else
+            {
+                var newGroupId = multiAgentGroup.GetId();
+                if (m_GroupId == 0 || m_GroupId == newGroupId)
+                {
+                    m_GroupId = newGroupId;
+                }
+                else
+                {
+                    throw new UnityAgentsException("Agent is already registered with a group. Unregister it first.");
+                }
+            }
+        }
     }
 }

com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs

@@ -58,9 +58,11 @@ public static AgentInfoProto ToAgentInfoProto(this AgentInfo ai)
             var agentInfoProto = new AgentInfoProto
             {
                 Reward = ai.reward,
+                GroupReward = ai.groupReward,
                 MaxStepReached = ai.maxStepReached,
                 Done = ai.done,
                 Id = ai.episodeId,
+                GroupId = ai.groupId,
             };
 
             if (ai.discreteActionMasks != null)

com.unity.ml-agents/Runtime/Grpc/CommunicatorObjects/AgentInfo.cs

@@ -26,17 +26,18 @@ static AgentInfoReflection() {
           string.Concat(
             "CjNtbGFnZW50c19lbnZzL2NvbW11bmljYXRvcl9vYmplY3RzL2FnZW50X2lu",
             "Zm8ucHJvdG8SFGNvbW11bmljYXRvcl9vYmplY3RzGjRtbGFnZW50c19lbnZz",
-            "L2NvbW11bmljYXRvcl9vYmplY3RzL29ic2VydmF0aW9uLnByb3RvItEBCg5B",
+            "L2NvbW11bmljYXRvcl9vYmplY3RzL29ic2VydmF0aW9uLnByb3RvIvkBCg5B",
             "Z2VudEluZm9Qcm90bxIOCgZyZXdhcmQYByABKAISDAoEZG9uZRgIIAEoCBIY",
             "ChBtYXhfc3RlcF9yZWFjaGVkGAkgASgIEgoKAmlkGAogASgFEhMKC2FjdGlv",
             "bl9tYXNrGAsgAygIEjwKDG9ic2VydmF0aW9ucxgNIAMoCzImLmNvbW11bmlj",
-            "YXRvcl9vYmplY3RzLk9ic2VydmF0aW9uUHJvdG9KBAgBEAJKBAgCEANKBAgD",
-            "EARKBAgEEAVKBAgFEAZKBAgGEAdKBAgMEA1CJaoCIlVuaXR5Lk1MQWdlbnRz",
-            "LkNvbW11bmljYXRvck9iamVjdHNiBnByb3RvMw=="));
+            "YXRvcl9vYmplY3RzLk9ic2VydmF0aW9uUHJvdG8SEAoIZ3JvdXBfaWQYDiAB",
+            "KAUSFAoMZ3JvdXBfcmV3YXJkGA8gASgCSgQIARACSgQIAhADSgQIAxAESgQI",
+            "BBAFSgQIBRAGSgQIBhAHSgQIDBANQiWqAiJVbml0eS5NTEFnZW50cy5Db21t",
+            "dW5pY2F0b3JPYmplY3RzYgZwcm90bzM="));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { global::Unity.MLAgents.CommunicatorObjects.ObservationReflection.Descriptor, },
           new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {
-            new pbr::GeneratedClrTypeInfo(typeof(global::Unity.MLAgents.CommunicatorObjects.AgentInfoProto), global::Unity.MLAgents.CommunicatorObjects.AgentInfoProto.Parser, new[]{ "Reward", "Done", "MaxStepReached", "Id", "ActionMask", "Observations" }, null, null, null)
+            new pbr::GeneratedClrTypeInfo(typeof(global::Unity.MLAgents.CommunicatorObjects.AgentInfoProto), global::Unity.MLAgents.CommunicatorObjects.AgentInfoProto.Parser, new[]{ "Reward", "Done", "MaxStepReached", "Id", "ActionMask", "Observations", "GroupId", "GroupReward" }, null, null, null)
           }));
     }
     #endregion
@@ -74,6 +75,8 @@ public AgentInfoProto(AgentInfoProto other) : this() {
       id_ = other.id_;
       actionMask_ = other.actionMask_.Clone();
       observations_ = other.observations_.Clone();
+      groupId_ = other.groupId_;
+      groupReward_ = other.groupReward_;
       _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
     }
 
@@ -146,6 +149,28 @@ public int Id {
       get { return observations_; }
     }
 
+    /// <summary>Field number for the "group_id" field.</summary>
+    public const int GroupIdFieldNumber = 14;
+    private int groupId_;
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public int GroupId {
+      get { return groupId_; }
+      set {
+        groupId_ = value;
+      }
+    }
+
+    /// <summary>Field number for the "group_reward" field.</summary>
+    public const int GroupRewardFieldNumber = 15;
+    private float groupReward_;
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public float GroupReward {
+      get { return groupReward_; }
+      set {
+        groupReward_ = value;
+      }
+    }
+
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
     public override bool Equals(object other) {
       return Equals(other as AgentInfoProto);
@@ -165,6 +190,8 @@ public bool Equals(AgentInfoProto other) {
       if (Id != other.Id) return false;
       if(!actionMask_.Equals(other.actionMask_)) return false;
       if(!observations_.Equals(other.observations_)) return false;
+      if (GroupId != other.GroupId) return false;
+      if (!pbc::ProtobufEqualityComparers.BitwiseSingleEqualityComparer.Equals(GroupReward, other.GroupReward)) return false;
       return Equals(_unknownFields, other._unknownFields);
     }
 
@@ -177,6 +204,8 @@ public override int GetHashCode() {
       if (Id != 0) hash ^= Id.GetHashCode();
       hash ^= actionMask_.GetHashCode();
       hash ^= observations_.GetHashCode();
+      if (GroupId != 0) hash ^= GroupId.GetHashCode();
+      if (GroupReward != 0F) hash ^= pbc::ProtobufEqualityComparers.BitwiseSingleEqualityComparer.GetHashCode(GroupReward);
       if (_unknownFields != null) {
         hash ^= _unknownFields.GetHashCode();
       }
@@ -208,6 +237,14 @@ public void WriteTo(pb::CodedOutputStream output) {
       }
       actionMask_.WriteTo(output, _repeated_actionMask_codec);
       observations_.WriteTo(output, _repeated_observations_codec);
+      if (GroupId != 0) {
+        output.WriteRawTag(112);
+        output.WriteInt32(GroupId);
+      }
+      if (GroupReward != 0F) {
+        output.WriteRawTag(125);
+        output.WriteFloat(GroupReward);
+      }
       if (_unknownFields != null) {
         _unknownFields.WriteTo(output);
       }
@@ -230,6 +267,12 @@ public int CalculateSize() {
       }
       size += actionMask_.CalculateSize(_repeated_actionMask_codec);
       size += observations_.CalculateSize(_repeated_observations_codec);
+      if (GroupId != 0) {
+        size += 1 + pb::CodedOutputStream.ComputeInt32Size(GroupId);
+      }
+      if (GroupReward != 0F) {
+        size += 1 + 4;
+      }
       if (_unknownFields != null) {
         size += _unknownFields.CalculateSize();
       }
@@ -255,6 +298,12 @@ public void MergeFrom(AgentInfoProto other) {
       }
       actionMask_.Add(other.actionMask_);
       observations_.Add(other.observations_);
+      if (other.GroupId != 0) {
+        GroupId = other.GroupId;
+      }
+      if (other.GroupReward != 0F) {
+        GroupReward = other.GroupReward;
+      }
       _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
     }
 
@@ -291,6 +340,14 @@ public void MergeFrom(pb::CodedInputStream input) {
             observations_.AddEntriesFrom(input, _repeated_observations_codec);
             break;
           }
+          case 112: {
+            GroupId = input.ReadInt32();
+            break;
+          }
+          case 125: {
+            GroupReward = input.ReadFloat();
+            break;
+          }
         }
       }
     }

com.unity.ml-agents/Runtime/IMultiAgentGroup.cs

@@ -0,0 +1,26 @@
+namespace Unity.MLAgents
+{
+    /// <summary>
+    /// MultiAgentGroup interface for grouping agents to support multi-agent training.
+    /// </summary>
+    public interface IMultiAgentGroup
+    {
+        /// <summary>
+        /// Get the ID of MultiAgentGroup.
+        /// </summary>
+        /// <returns>
+        /// MultiAgentGroup ID.
+        /// </returns>
+        int GetId();
+
+        /// <summary>
+        /// Register agent to the MultiAgentGroup.
+        /// </summary>
+        void RegisterAgent(Agent agent);
+
+        /// <summary>
+        /// Unregister agent from the MultiAgentGroup.
+        /// </summary>
+        void UnregisterAgent(Agent agent);
+    }
+}

com.unity.ml-agents/Runtime/IMultiAgentGroup.cs.meta

@@ -0,0 +1,13 @@
+using System.Threading;
+
+namespace Unity.MLAgents
+{
+    internal static class MultiAgentGroupIdCounter
+    {
+        static int s_Counter;
+        public static int GetGroupId()
+        {
+            return Interlocked.Increment(ref s_Counter); ;
+        }
+    }
+}