CSHARP-5493: Fix stackoverflow in EnumRepresentationConvention (#1616)

Author: papafe

Diff for: src/MongoDB.Bson/Serialization/Conventions/EnumRepresentationConvention.cs

@@ -14,6 +14,7 @@
 */
 
 using System;
+using System.Collections;
 
 namespace MongoDB.Bson.Serialization.Conventions
 {
@@ -67,6 +68,13 @@ public EnumRepresentationConvention(BsonType representation, bool topLevelOnly)
         /// <param name="memberMap">The member map.</param>
         public void Apply(BsonMemberMap memberMap)
         {
+            var memberType = memberMap.MemberType;
+
+            if (!(memberType.IsEnum || memberType.IsNullableEnum() || memberType.IsArray || typeof(IEnumerable).IsAssignableFrom(memberType)))
+            {
+                return;
+            }
+
             var serializer = memberMap.GetSerializer();
             var reconfiguredSerializer = _topLevelOnly && !serializer.ValueType.IsNullableEnum() ?
                 Reconfigure(serializer) :

Diff for: src/MongoDB.Bson/Serialization/Conventions/ObjectSerializerAllowedTypesConvention.cs

@@ -14,6 +14,7 @@
  */
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using System.Reflection;
@@ -152,6 +153,14 @@ public bool AllowDefaultFrameworkTypes
         /// <param name="memberMap">The member map.</param>
         public void Apply(BsonMemberMap memberMap)
         {
+            var memberType = memberMap.MemberType;
+
+            if (memberType != typeof(object) && Nullable.GetUnderlyingType(memberType) == null && !memberType.IsArray &&
+                !typeof(IEnumerable).IsAssignableFrom(memberType))
+            {
+                return;
+            }
+
             var serializer = memberMap.GetSerializer();
 
             var reconfiguredSerializer = Reconfigure(serializer);

Diff for: tests/MongoDB.Bson.Tests/Serialization/Conventions/EnumRepresentationConventionTests.cs

@@ -41,6 +41,7 @@ public class C
             public int I { get; set; }
             public int NI { get; set; }
             public int[] ArrayInt { get; set; }
+            public C RecursiveProp { get; set; }
         }
 
         [Theory]
@@ -204,6 +205,17 @@ public void Apply_should_do_nothing_when_member_is_not_an_enum_collection()
             memberMap.GetSerializer().Should().BeSameAs(serializer);
         }
 
+        [Fact]
+        public void Convention_should_work_with_recursive_type()
+        {
+            var pack = new ConventionPack { new EnumRepresentationConvention(BsonType.String) };
+            ConventionRegistry.Register("enumRecursive", pack, t => t == typeof(C));
+
+            _ = new BsonClassMap<C>(cm => cm.AutoMap()).Freeze();
+
+            ConventionRegistry.Remove("enumRecursive");
+        }
+
         [Theory]
         [InlineData((BsonType)0)]
         [InlineData(BsonType.Int32)]

Diff for: tests/MongoDB.Bson.Tests/Serialization/Conventions/ObjectSerializerAllowedTypesConventionTests.cs

@@ -31,6 +31,7 @@ private class TestClass
             public object ObjectProp { get; set; }
             public object[] ArrayOfObjectProp { get; set; }
             public object[][] ArrayOfArrayOfObjectProp { get; set; }
+            public TestClass RecursiveProp { get; set; }
         }
 
         [Fact]
@@ -344,6 +345,17 @@ public void Convention_should_be_applied_during_automapping()
             ConventionRegistry.Remove(conventionName);
         }
 
+        [Fact]
+        public void Convention_should_work_with_recursive_type()
+        {
+            var pack = new ConventionPack { new ObjectSerializerAllowedTypesConvention() };
+            ConventionRegistry.Register("objectRecursive", pack, t => t == typeof(TestClass));
+
+            _ = new BsonClassMap<TestClass>(cm => cm.AutoMap()).Freeze();
+
+            ConventionRegistry.Remove("enumRecursive");
+        }
+
         // private methods
         private static BsonMemberMap CreateMemberMap<TMember>(Expression<Func<TestClass, TMember>> member)
         {