Add new Pipeline for Running Libs Tests with TestReadyToRun (#91229)

2025-06-11 02:13:38 +09:00 · 2023-12-14 11:08:46 -08:00 · 2023-12-14 11:08:46 -08:00 · 07b6e338bb
commit 07b6e338bb
parent 40c550a378
13 changed files with 199 additions and 6 deletions
--- a/eng/pipelines/common/platform-matrix.yml
+++ b/eng/pipelines/common/platform-matrix.yml
@ -25,6 +25,7 @@ parameters:
 jobs:

 # Linux arm
+
 - ${{ if or(containsValue(parameters.platforms, 'linux_arm'), in(parameters.platformGroup, 'all', 'gcstress')) }}:
  - template: xplat-setup.yml
    parameters:
@ -45,6 +46,7 @@ jobs:
        ${{ insert }}: ${{ parameters.jobParameters }}

 # Linux armv6
+
 - ${{ if containsValue(parameters.platforms, 'linux_armv6') }}:
  - template: xplat-setup.yml
    parameters:
--- a/eng/pipelines/coreclr/crossgen2.yml
+++ b/eng/pipelines/coreclr/crossgen2.yml
@ -60,6 +60,29 @@ extends:
            displayNameArgs: R2R_CG2
            liveLibrariesBuildConfig: Release

+      - template: /eng/pipelines/common/platform-matrix.yml
+        parameters:
+          jobTemplate: /eng/pipelines/common/global-build-job.yml
+          helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
+          buildConfig: Release
+          platforms:
+            - linux_x64
+            - osx_x64
+            - windows_x64
+          jobParameters:
+            testGroup: innerloop
+            buildArgs: -s clr+libs+libs.tests
+                       -c $(_BuildConfig)
+                       /p:TestReadyToRun=true
+                       /p:ArchiveTests=true
+            nameSuffix: TestReadyToRun_Libraries
+            timeoutInMinutes: 360
+            postBuildSteps:
+              - template: /eng/pipelines/libraries/helix.yml
+                parameters:
+                  creator: dotnet-bot
+                  testRunNamePrefixSuffix: TestReadyToRun_$(_BuildConfig)
+
      # Run pri0 tests with hot/cold splitting enabled (only supported on x64 at the moment)
      # TODO: test on arm64 once supported
      - template: /eng/pipelines/common/platform-matrix.yml
--- a/eng/testing/tests.singlefile.targets
+++ b/eng/testing/tests.singlefile.targets
@ -70,6 +70,10 @@
    <DefineConstants>$(DefineConstants);SINGLE_FILE_TEST_RUNNER</DefineConstants>
  </PropertyGroup>

+  <PropertyGroup Condition="'$(TestReadyToRun)' == 'true'">
+    <DefineConstants>$(DefineConstants);TEST_READY_TO_RUN_COMPILED</DefineConstants>
+  </PropertyGroup>
+
  <Import Project="$(CoreCLRBuildIntegrationDir)Microsoft.DotNet.ILCompiler.SingleEntry.targets" Condition="'$(TestNativeAot)' == 'true'" />

  <ItemGroup Condition="'$(TestNativeAot)' == 'true'">
@ -106,6 +110,114 @@
    </ItemGroup>
  </Target>

+  <!--
+      For TestReadyToRun, we need the whole framework to be R2R-compiled besides
+      the actual test assembly. However, this is a very lengthy process and it's
+      unnecessary in this case because we already have an R2R-compiled framework.
+      So, we have to tell the build that we already have these binaries so that it
+      doesn't build them again for each test.
+  -->
+  <Target Name="ExcludeExistingR2RBinaries"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          BeforeTargets="_PrepareForReadyToRunCompilation">
+    <PropertyGroup>
+      <ArtifactsNetCoreAppBundlePath>$(ArtifactsObjDir)Microsoft.NETCore.App.Bundle/</ArtifactsNetCoreAppBundlePath>
+      <ArtifactsNetCoreAppBundlePath>$(ArtifactsNetCoreAppBundlePath)$(Configuration)/$(NetCoreAppCurrent)/$(OutputRID)/output/</ArtifactsNetCoreAppBundlePath>
+      <ArtifactsNetCoreAppBundlePath>$(ArtifactsNetCoreAppBundlePath)shared/$(MicrosoftNetCoreAppFrameworkName)/$(PackageVersion)/</ArtifactsNetCoreAppBundlePath>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <_BundleAssembliesToCopy Include="$(ArtifactsNetCoreAppBundlePath)*.dll" />
+      <ResolvedFileToPublish Remove="@(_BundleAssembliesToCopy)" MatchOnMetadata="Filename" />
+    </ItemGroup>
+  </Target>
+
+  <!--
+      For TestReadyToRun, each crossgen'd assembly needs to reference the whole
+      framework. For this, it looks at the contents of the same list that contains
+      all the assemblies we're going to R2R-compile. However, since we removed those
+      belonging to the framework we have ready to use in the previous target, then
+      the references list generated in _PrepareForReadyToRunCompilation is incomplete.
+      So, we add those missing assemblies only to the references list in this target.
+  -->
+  <Target Name="AddExistingR2RBinariesReferencesForCrossgen2"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          AfterTargets="_PrepareForReadyToRunCompilation">
+    <ItemGroup>
+      <_ReadyToRunAssembliesToReference Include="@(_BundleAssembliesToCopy)" />
+    </ItemGroup>
+  </Target>
+
+  <!--
+      For TestReadyToRun, debugging binaries bloat the test sizes way too much and
+      makes the Helix machines run out of disk. Since we don't need them for the
+      TestReadyToRun test runs, we remove them from the list that is later on copied
+      to the final location.
+  -->
+  <Target Name="RemoveDbgBinsFromTestR2ROutput"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          BeforeTargets="_CopyFilesMarkedCopyLocal">
+    <ItemGroup>
+      <ReferenceCopyLocalPaths
+          Remove="@(ReferenceCopyLocalPaths->WithMetadataValue('Extension', '.dbg'))" />
+    </ItemGroup>
+  </Target>
+
+  <!--
+      Very similarly to the previous target, we need to get rid of the debugging
+      binaries from the publishing directory as well.
+  -->
+  <Target Name="RemoveDbgBinsFromTestR2RPublish"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          BeforeTargets="_CopyResolvedFilesToPublishPreserveNewest">
+    <ItemGroup>
+      <_ResolvedFileToPublishPreserveNewest
+          Remove="@(_ResolvedFileToPublishPreserveNewest->WithMetadataValue('Extension', '.dbg'))" />
+    </ItemGroup>
+  </Target>
+
+  <!--
+      As explained in Target 'ExcludeExistingR2RBinaries' up above, for TestReadyToRun
+      we need the fully R2R-compiled framework, but we already have it elsewhere. So,
+      once the test's specific stuff is constructed, we copy the R2R-compiled framework
+      to the test's self-contained directory so the test can use it when called.
+  -->
+  <Target Name="CopyExistingR2RBinaries"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          AfterTargets="_CopyResolvedFilesToPublishAlways">
+
+    <Copy SourceFiles="@(_BundleAssembliesToCopy)"
+          DestinationFolder="$(PublishDir)"
+          OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
+          Retries="$(CopyRetryCount)"
+          RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
+          UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
+          UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)" />
+
+  </Target>
+
+  <!--
+      There are a few tests that need a 'live-ref-pack', which is missing from the
+      publish directory in TestReadyToRun builds. This target copies it there.
+  -->
+  <Target Name="CopyLiveRefPackIfPresent"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          AfterTargets="CopyExistingR2RBinaries">
+
+    <ItemGroup>
+      <OutDirLiveRefPackFiles Include="$(OutDir)live-ref-pack/*" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(OutDirLiveRefPackFiles)"
+          DestinationFolder="$(PublishDir)live-ref-pack"
+          OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
+          Retries="$(CopyRetryCount)"
+          RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
+          UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
+          UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)" />
+
+  </Target>
+
  <Target Name="__UpdateExcludedAssembliesFromSingleFile"
          Inputs="ExcludeFromSingleFile"
          Outputs="ResolvedFileToPublish"
--- a/src/installer/pkg/sfx/bundle/Microsoft.NETCore.App.Bundle.bundleproj
+++ b/src/installer/pkg/sfx/bundle/Microsoft.NETCore.App.Bundle.bundleproj
@ -25,8 +25,8 @@

  <ItemGroup>
    <BundleComponentReference Include="../Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj" />
-    <BundleComponentReference Include="../installers/dotnet-hostfxr.proj" />
-    <BundleComponentReference Include="../installers/dotnet-host.proj" />
+    <BundleComponentReference Include="../installers/dotnet-hostfxr.proj" Condition="'$(TestReadyToRun)' != 'true'" />
+    <BundleComponentReference Include="../installers/dotnet-host.proj" Condition="'$(TestReadyToRun)' != 'true'" />
  </ItemGroup>

  <Target Name="PublishToDisk">
--- a/src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs
+++ b/src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs
@ -30,6 +30,17 @@ public class SingleFileTestRunner : XunitTestFramework
        // The current RemoteExecutor implementation is not compatible with the SingleFileTestRunner.
        Environment.SetEnvironmentVariable("DOTNET_REMOTEEXECUTOR_SUPPORTED", "0");

+        // To detect ReadyToRun testing mode, we set a constant in
+        // eng/testing/tests.singlefile.targets, which we use in the following
+        // preprocessor directive. In the case that it is defined, we set an
+        // environment variable that we consume later to implement
+        // PlatformDetection.IsReadyToRunCompiled. This last value is used for the
+        // [ActiveIssue] annotations designed to exclude tests from running.
+
+#if TEST_READY_TO_RUN_COMPILED
+        Environment.SetEnvironmentVariable("TEST_READY_TO_RUN_MODE" ,"1");
+#endif
+
        var diagnosticSink = new ConsoleDiagnosticMessageSink();
        var testsFinished = new TaskCompletionSource();
        var testSink = new TestMessageSink();
--- a/src/libraries/Common/tests/TestUtilities/System/PlatformDetection.cs
+++ b/src/libraries/Common/tests/TestUtilities/System/PlatformDetection.cs
@ -211,6 +211,8 @@ namespace System
        public static bool HasHostExecutable => HasAssemblyFiles; // single-file don't have a host
        public static bool IsSingleFile => !HasAssemblyFiles;

+        public static bool IsReadyToRunCompiled => Environment.GetEnvironmentVariable("TEST_READY_TO_RUN_MODE") == "1";
+
        private static volatile Tuple<bool> s_lazyNonZeroLowerBoundArraySupported;
        public static bool IsNonZeroLowerBoundArraySupported
        {
--- a/src/libraries/Directory.Build.props
+++ b/src/libraries/Directory.Build.props
@ -85,6 +85,10 @@
    <EnableCoverageSupport Condition="'$(ContinuousIntegrationBuild)' != 'true'">true</EnableCoverageSupport>
  </PropertyGroup>

+  <PropertyGroup Condition="'$(TestReadyToRun)' == 'true'">
+    <UseLocalAppHostPack>true</UseLocalAppHostPack>
+  </PropertyGroup>
+
  <!-- To enable the interpreter for mono desktop, we need to pass an env switch -->
  <PropertyGroup>
    <MonoEnvOptions Condition="'$(MonoEnvOptions)' == '' and '$(TargetsMobile)' != 'true' and '$(MonoForceInterpreter)' == 'true'">--interpreter</MonoEnvOptions>
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/EnumTests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/EnumTests.cs
@ -1599,6 +1599,7 @@ namespace System.Tests
        private class ClassWithEnumConstraint<T> where T : Enum { }

        [Fact]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/94189", typeof(PlatformDetection), nameof(PlatformDetection.IsReadyToRunCompiled))]
        public void EnumConstraint_ThrowsArgumentException()
        {
            Type genericArgumentWithEnumConstraint = typeof(ClassWithEnumConstraint<>).GetGenericArguments()[0];
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Runtime/JitInfoTests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Runtime/JitInfoTests.cs
@ -100,6 +100,7 @@ namespace System.Runtime.Tests

        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsReflectionEmitSupported))]
        [SkipOnMono("Mono does not track thread specific JIT information")]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/94189", typeof(PlatformDetection), nameof(PlatformDetection.IsReadyToRunCompiled))]
        public void JitInfoCurrentThreadIsPopulated()
        {
            TimeSpan t1_beforeCompilationTime = TimeSpan.Zero;
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Type/TypeTests.Get.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Type/TypeTests.Get.cs
@ -55,6 +55,7 @@ namespace System.Tests

        [Theory]
        [MemberData(nameof(GetInterface_TestData))]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/94189", typeof(PlatformDetection), nameof(PlatformDetection.IsReadyToRunCompiled))]
        public void GetInterface_Invoke_ReturnsExpected(Type type, string name, bool ignoreCase, Type expected)
        {
            if (!ignoreCase)
--- a/src/libraries/pretest.proj
+++ b/src/libraries/pretest.proj
@ -26,7 +26,14 @@
    <ProjectReference Include="$(MonoProjectRoot)wasm\symbolicator\WasmSymbolicator.csproj" Condition="'$(TargetOS)' == 'browser'" />

    <!-- needed to test workloads for wasm -->
-    <ProjectReference Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\Microsoft.NETCore.App.Runtime.sfxproj" Pack="true" Condition="'$(TargetOS)' == 'browser' or '$(TargetOS)' == 'wasi'" />
+    <ProjectReference Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\Microsoft.NETCore.App.Runtime.sfxproj"
+                      Pack="true"
+                      Condition="'$(TargetOS)' == 'browser' or '$(TargetOS)' == 'wasi'" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(TestReadyToRun)' == 'true'">
+    <ProjectReference Include="$(InstallerProjectRoot)pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj" />
+    <ProjectReference Include="$(InstallerProjectRoot)pkg/sfx/bundle/Microsoft.NETCore.App.Bundle.bundleproj" />
  </ItemGroup>

  <Import Project="$(RepositoryEngineeringDir)testing\wasm-provisioning.targets"
--- a/src/libraries/tests.proj
+++ b/src/libraries/tests.proj
@ -611,6 +611,31 @@
    -->
  </ItemGroup>

+  <!--
+      There is a decent number of hidden tests that fail with TestReadyToRun, and
+      it's proven to be a neverending task to flush all of them at once. So, we will
+      start by only enabling the tests we have confirmed work correctly, and we will
+      gradually enable the rest in subsequent PR's.
+
+      Tracking Issue for this work item: https://github.com/dotnet/runtime/issues/95928
+  -->
+  <ItemGroup Condition="'$(TestReadyToRun)' == 'true'">
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)Microsoft.Extensions.DependencyInjection/tests/DI.External.Tests/Microsoft.Extensions.DependencyInjection.ExternalContainers.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.ComponentModel.Composition/tests/System.ComponentModel.Composition.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Composition/tests/System.Composition.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Console/tests/System.Console.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Runtime/tests/System.Dynamic.Runtime.Tests/System.Dynamic.Runtime.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.FileSystem.Watcher/tests/System.IO.FileSystem.Watcher.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Runtime/tests/System.Reflection.Tests/System.Reflection.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Reflection.TypeExtensions/tests/System.Reflection.TypeExtensions.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Runtime/tests/System.Runtime.Extensions.Tests/System.Runtime.Extensions.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Runtime.Loader/tests/DefaultContext/System.Runtime.Loader.DefaultContext.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Runtime.Loader/tests/System.Runtime.Loader.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Text.RegularExpressions/tests/FunctionalTests/System.Text.RegularExpressions.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Threading.Thread/tests/System.Threading.Thread.Tests.csproj" />
+  </ItemGroup>
+
  <ItemGroup>
    <ProjectReference Condition="'$(RunSmokeTestsOnly)' == 'true'"
                      Include="@(SmokeTestProject)" />
@ -621,7 +646,11 @@
                      Include="@(GrpcTestProject)" />
  </ItemGroup>

-  <ItemGroup Condition="'$(RunSmokeTestsOnly)' != 'true' and '$(RunGrpcTestsOnly)' != 'true' and '$(RunHighAOTResourceRequiringTestsOnly)' != 'true' and '$(RunLimitedSetOfTests)' != 'true'">
+  <ItemGroup Condition="'$(RunSmokeTestsOnly)' != 'true'
+                        and '$(RunGrpcTestsOnly)' != 'true'
+                        and '$(RunHighAOTResourceRequiringTestsOnly)' != 'true'
+                        and '$(RunLimitedSetOfTests)' != 'true'">
+
    <ProjectReference Include="$(MSBuildThisFileDirectory)*\tests\**\*.Tests.csproj"
                      Exclude="@(ProjectExclusions)"
                      Condition="'$(TestAssemblies)' == 'true'"
--- a/src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/Microsoft.NET.CrossGen.props
+++ b/src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/Microsoft.NET.CrossGen.props
@ -1,6 +1,6 @@
 <!--
 ***********************************************************************************************
-Microsoft.NET.CrossGen.targets
+Microsoft.NET.CrossGen.props

 WARNING:  DO NOT MODIFY this file unless you are knowledgeable about MSBuild and have
          created a backup copy.  Incorrect changes to this file will make it
@ -17,4 +17,4 @@ Copyright (c) .NET Foundation. All rights reserved.
  <UsingTask TaskName="PrepareForReadyToRunCompilation" AssemblyFile="$(MSBuildThisFileDirectory)Crossgen2Tasks.dll" />
  <UsingTask TaskName="ResolveReadyToRunCompilers" AssemblyFile="$(MSBuildThisFileDirectory)Crossgen2Tasks.dll" />
  <UsingTask TaskName="RunReadyToRunCompiler" AssemblyFile="$(MSBuildThisFileDirectory)Crossgen2Tasks.dll" />
-</Project>
+</Project>