Fix DeepEquals_DeepJsonDocument stack overflow that manifests in xunit.console and netfx. (#105309)

* Fix DeepEquals_DeepJsonDocument stack overflow that manifests in xunit.console and netfx.

* Guard DeepEquals calls with EnsureSufficientExecutionStack().

src/libraries/System.Text.Json/src/Resources/Strings.resx

@@ -246,6 +246,9 @@
   <data name="JsonElementHasWrongType" xml:space="preserve">
     <value>The requested operation requires an element of type '{0}', but the target element has type '{1}'.</value>
   </data>
+  <data name="JsonElementDeepEqualsInsufficientExecutionStack" xml:space="preserve">
+    <value>Insufficient stack to continue executing 'JsonElement.DeepEquals'. This can happen either because the 'JsonElement' values are too deep or 'DeepEquals' is being called too deep in the stack.</value>
+  </data>
   <data name="JsonNumberExponentTooLarge" xml:space="preserve">
     <value>The exponent value in the specified JSON number is too large.</value>
   </data>

src/libraries/System.Text.Json/src/System.Text.Json.csproj

@@ -161,6 +161,7 @@ The System.Text.Json library is built-in as part of the shared framework in .NET
     <Compile Include="System\Text\Json\Serialization\Metadata\JsonTypeInfoResolverWithAddedModifiers.cs" />
     <Compile Include="System\Text\Json\Serialization\Metadata\PropertyRefCacheBuilder.cs" />
     <Compile Include="System\Text\Json\Serialization\PolymorphicSerializationState.cs" />
+    <Compile Include="System\Text\Json\StackHelper.cs" />
     <Compile Include="System\Text\Json\ValueQueue.cs" />
     <Compile Include="System\Text\Json\Writer\Utf8JsonWriterCache.cs" />
     <Compile Include="System\Text\Json\Serialization\ReferenceEqualsWrapper.cs" />

src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs

@@ -1262,6 +1262,11 @@ namespace System.Text.Json
         /// </remarks>
         public static bool DeepEquals(JsonElement element1, JsonElement element2)
         {
+            if (!StackHelper.TryEnsureSufficientExecutionStack())
+            {
+                ThrowHelper.ThrowInsufficientExecutionStackException_JsonElementDeepEqualsInsufficientExecutionStack();
+            }
+
             element1.CheckValidInstance();
             element2.CheckValidInstance();
 

src/libraries/System.Text.Json/src/System/Text/Json/StackHelper.cs

@@ -0,0 +1,29 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.CompilerServices;
+
+namespace System.Text.Json
+{
+    /// <summary>Provides tools for avoiding stack overflows.</summary>
+    internal static class StackHelper
+    {
+        /// <summary>Tries to ensure there is sufficient stack to execute the average .NET function.</summary>
+        public static bool TryEnsureSufficientExecutionStack()
+        {
+#if NET
+            return RuntimeHelpers.TryEnsureSufficientExecutionStack();
+#else
+            try
+            {
+                RuntimeHelpers.EnsureSufficientExecutionStack();
+                return true;
+            }
+            catch
+            {
+                return false;
+            }
+#endif
+        }
+    }
+}

src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs

@@ -719,6 +719,12 @@ namespace System.Text.Json
         {
             throw new ObjectDisposedException(nameof(JsonDocument));
         }
+
+        [DoesNotReturn]
+        public static void ThrowInsufficientExecutionStackException_JsonElementDeepEqualsInsufficientExecutionStack()
+        {
+            throw new InsufficientExecutionStackException(SR.JsonElementDeepEqualsInsufficientExecutionStack);
+        }
     }
 
     internal enum ExceptionResource

src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonElementTests.cs

@@ -221,11 +221,26 @@ namespace System.Text.Json.Serialization.Tests
         [Theory]
         [InlineData(10)]
         [InlineData(100)]
-        [InlineData(1000)]
+        [InlineData(500)]
         public static void DeepEquals_DeepJsonDocument(int depth)
+        {
+            using JsonDocument jDoc = CreateDeepJsonDocument(depth);
+            JsonElement element = jDoc.RootElement;
+            Assert.True(JsonElement.DeepEquals(element, element));
+        }
+
+        [Fact]
+        public static void DeepEquals_TooDeepJsonDocument_ThrowsInsufficientExecutionStackException()
+        {
+            using JsonDocument jDoc = CreateDeepJsonDocument(10_000);
+            JsonElement element = jDoc.RootElement;
+            Assert.Throws<InsufficientExecutionStackException>(() => JsonElement.DeepEquals(element, element));
+        }
+
+        private static JsonDocument CreateDeepJsonDocument(int depth)
         {
             ArrayBufferWriter<byte> bufferWriter = new();
-            using Utf8JsonWriter writer = new(bufferWriter);
+            using Utf8JsonWriter writer = new(bufferWriter, new() { MaxDepth = depth + 1 });
 
             for (int i = 0; i < depth; i++)
             {
@@ -257,10 +272,7 @@ namespace System.Text.Json.Serialization.Tests
             writer.Flush();
 
             JsonDocumentOptions options = new JsonDocumentOptions { MaxDepth = depth };
-            using JsonDocument jDoc = JsonDocument.Parse(bufferWriter.WrittenSpan.ToArray(), options);
-            JsonElement element = jDoc.RootElement;
-
-            Assert.True(JsonElement.DeepEquals(element, element));
+            return JsonDocument.Parse(bufferWriter.WrittenSpan.ToArray(), options);
         }
     }
 }