From d8eecb714fcc29556b0bd573d540ad49b4a8bcce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?=
 <MichalStrehovsky@users.noreply.github.com>
Date: Mon, 22 Apr 2024 18:44:17 +0900
Subject: [PATCH] Run trimming tests as AOT tests (#101229)

Not everything is passing, so I baselined this. Some we'll probably exclude permanently, others are more concerning and we need to determine if it's test issues or product issues.
---
 Directory.Build.props                         | 10 ++++------
 .../coreclr/nativeaot-post-build-steps.yml    |  8 --------
 .../execute-trimming-tests-steps.yml          |  6 ++++++
 eng/pipelines/runtime-linker-tests.yml        |  1 +
 .../linker/SupportFiles/Directory.Build.props |  1 +
 eng/testing/linker/trimmingTests.props        |  4 +++-
 eng/testing/linker/trimmingTests.targets      |  5 ++++-
 src/libraries/Directory.Build.props           |  2 +-
 src/libraries/Directory.Build.targets         |  2 +-
 ...stics.DiagnosticSource.NativeAotTests.proj | 10 ----------
 .../DiagnosticSourceEventSourceTests.cs       |  0
 .../System.Diagnostics.DiagnosticSource.proj  |  2 ++
 src/libraries/tests.proj                      | 19 +++++++++++++------
 13 files changed, 36 insertions(+), 34 deletions(-)
 delete mode 100644 src/libraries/System.Diagnostics.DiagnosticSource/tests/NativeAotTests/System.Diagnostics.DiagnosticSource.NativeAotTests.proj
 rename src/libraries/System.Diagnostics.DiagnosticSource/tests/{NativeAotTests => TrimmingTests}/DiagnosticSourceEventSourceTests.cs (100%)

diff --git a/Directory.Build.props b/Directory.Build.props
index b1ac2559f9e..ef0e8c1b44d 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -397,12 +397,10 @@
   <PropertyGroup Condition="$(MSBuildProjectFullPath.Contains('$([System.IO.Path]::DirectorySeparatorChar)tests$([System.IO.Path]::DirectorySeparatorChar)'))">
     <IsTestProject Condition="$(MSBuildProjectName.EndsWith('.UnitTests')) or $(MSBuildProjectName.EndsWith('.Tests'))">true</IsTestProject>
     <IsTrimmingTestProject Condition="$(MSBuildProjectName.EndsWith('.TrimmingTests'))">true</IsTrimmingTestProject>
-    <IsNativeAotTestProject Condition="$(MSBuildProjectName.EndsWith('.NativeAotTests'))">true</IsNativeAotTestProject>
-    <IsPublishedAppTestProject Condition="'$(IsTrimmingTestProject)' == 'true' or '$(IsNativeAotTestProject)' == 'true'">true</IsPublishedAppTestProject>
-    <IsTestSupportProject Condition="'$(IsTestProject)' != 'true' and '$(IsPublishedAppTestProject)' != 'true'">true</IsTestSupportProject>
+    <IsTestSupportProject Condition="'$(IsTestProject)' != 'true' and '$(IsTrimmingTestProject)' != 'true'">true</IsTestSupportProject>
 
     <!-- Treat test assemblies as non-shipping (do not publish or sign them). -->
-    <IsShipping Condition="'$(IsTestProject)' == 'true' or '$(IsTestSupportProject)' == 'true' or '$(IsPublishedAppTestProject)' == 'true'">false</IsShipping>
+    <IsShipping Condition="'$(IsTestProject)' == 'true' or '$(IsTestSupportProject)' == 'true' or '$(IsTrimmingTestProject)' == 'true'">false</IsShipping>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -413,7 +411,7 @@
                                 '$(IsReferenceAssemblyProject)' != 'true' and
                                 '$(IsGeneratorProject)' != 'true' and
                                 '$(IsTestProject)' != 'true' and
-                                '$(IsPublishedAppTestProject)' != 'true' and
+                                '$(IsTrimmingTestProject)' != 'true' and
                                 '$(IsTestSupportProject)' != 'true' and
                                 '$(UsingMicrosoftDotNetSharedFrameworkSdk)' != 'true' and
                                 '$(MSBuildProjectExtension)' != '.pkgproj' and
@@ -464,7 +462,7 @@
   </PropertyGroup>
 
   <!-- Warnings that should be disabled in our test projects. -->
-  <PropertyGroup Condition="'$(IsTestProject)' == 'true' or '$(IsTestSupportProject)' == 'true' or '$(IsPublishedAppTestProject)' == 'true'">
+  <PropertyGroup Condition="'$(IsTestProject)' == 'true' or '$(IsTestSupportProject)' == 'true' or '$(IsTrimmingTestProject)' == 'true'">
     <!-- we need to re-enable BinaryFormatter within test projects since some tests exercise these code paths to ensure compat -->
     <EnableUnsafeBinaryFormatterSerialization>true</EnableUnsafeBinaryFormatterSerialization>
     <!-- don't warn on usage of BinaryFormatter or legacy serialization infrastructure from test projects -->
diff --git a/eng/pipelines/coreclr/nativeaot-post-build-steps.yml b/eng/pipelines/coreclr/nativeaot-post-build-steps.yml
index 94761028f48..bc29a657c45 100644
--- a/eng/pipelines/coreclr/nativeaot-post-build-steps.yml
+++ b/eng/pipelines/coreclr/nativeaot-post-build-steps.yml
@@ -21,11 +21,3 @@ steps:
       nativeAotTest: true
       helixQueues: ${{ parameters.helixQueues }}
       liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
-
-  # Can't run arm/arm64 tests on x64 build machines
-  - ${{ if and(ne(parameters.archType, 'arm'), ne(parameters.archType, 'arm64')) }}:
-
-    # Publishing tooling doesn't support different configs between runtime and libs, so only run tests in Release config
-    - ${{ if eq(parameters.buildConfig, 'release') }}:
-      - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci -arch ${{ parameters.archType }} $(_osParameter) -s libs.tests -c $(_BuildConfig) $(crossArg) $(_nativeSanitizersArg) /p:TestAssemblies=false /p:RunNativeAotTestApps=true $(_officialBuildParameter) /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfigUpper)/NativeAotTests.binlog ${{ parameters.extraTestArgs }}
-        displayName: Run NativeAot Library Tests
diff --git a/eng/pipelines/libraries/execute-trimming-tests-steps.yml b/eng/pipelines/libraries/execute-trimming-tests-steps.yml
index 567abab0bb9..13778ee0ab6 100644
--- a/eng/pipelines/libraries/execute-trimming-tests-steps.yml
+++ b/eng/pipelines/libraries/execute-trimming-tests-steps.yml
@@ -1,8 +1,14 @@
 parameters:
   archType: ''
   extraTestArgs: ''
+  runAotTests: true
 
 steps:
   # Execute tests
   - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci -arch ${{ parameters.archType }} $(_osParameter) -s libs.tests -c $(_BuildConfig) $(crossArg) /p:TestAssemblies=false /p:TestTrimming=true $(_officialBuildParameter) /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfigUpper)/TrimmingTests.binlog ${{ parameters.extraTestArgs }}
     displayName: Run Trimming Tests
+
+  # Execute AOT test app tests
+  - ${{ if eq(parameters.runAotTests, true) }}:
+    - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci -arch ${{ parameters.archType }} $(_osParameter) -s libs.tests -c $(_BuildConfig) $(crossArg) /p:TestAssemblies=false /p:RunNativeAotTestApps=true $(_officialBuildParameter) /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfigUpper)/NativeAotTestAppTests.binlog ${{ parameters.extraTestArgs }}
+      displayName: Run Native AOT Test App Tests
diff --git a/eng/pipelines/runtime-linker-tests.yml b/eng/pipelines/runtime-linker-tests.yml
index aec5e1057ac..2b001b769f5 100644
--- a/eng/pipelines/runtime-linker-tests.yml
+++ b/eng/pipelines/runtime-linker-tests.yml
@@ -135,3 +135,4 @@ extends:
               - template: /eng/pipelines/libraries/execute-trimming-tests-steps.yml
                 parameters:
                     extraTestArgs: '/p:WasmBuildNative=false'
+                    runAotTests: false
diff --git a/eng/testing/linker/SupportFiles/Directory.Build.props b/eng/testing/linker/SupportFiles/Directory.Build.props
index 5a54c83e569..4e33801ab12 100644
--- a/eng/testing/linker/SupportFiles/Directory.Build.props
+++ b/eng/testing/linker/SupportFiles/Directory.Build.props
@@ -11,6 +11,7 @@
     <!-- Suppress analyzer and trimming warnings as these are tests -->
     <EnableTrimAnalyzer>false</EnableTrimAnalyzer>
     <SuppressTrimAnalysisWarnings>true</SuppressTrimAnalysisWarnings>
+    <SuppressAotAnalysisWarnings>true</SuppressAotAnalysisWarnings>
     <NoWarn>$(NoWarn);IL2121</NoWarn>
   </PropertyGroup>
 </Project>
diff --git a/eng/testing/linker/trimmingTests.props b/eng/testing/linker/trimmingTests.props
index b917cd5fe38..b822294a93d 100644
--- a/eng/testing/linker/trimmingTests.props
+++ b/eng/testing/linker/trimmingTests.props
@@ -1,6 +1,8 @@
 <Project>
   <PropertyGroup>
-    <TrimmingTestDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'trimmingTests'))</TrimmingTestDir>
+    <TrimmingTestDir Condition="'$(TestTrimming)' == 'true'">$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'trimmingTests'))</TrimmingTestDir>
+    <TrimmingTestDir Condition="'$(RunNativeAotTestApps)' == 'true'">$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'aotTests'))</TrimmingTestDir>
+
     <TrimmingTestProjectsDir>$([MSBuild]::NormalizeDirectory('$(TrimmingTestDir)', 'projects'))</TrimmingTestProjectsDir>
     <ProjectTemplate>$(MSBuildThisFileDirectory)project.csproj.template</ProjectTemplate>
     <UseLocalAppHostPack>true</UseLocalAppHostPack>
diff --git a/eng/testing/linker/trimmingTests.targets b/eng/testing/linker/trimmingTests.targets
index b29bc1dc75c..9ff325e8297 100644
--- a/eng/testing/linker/trimmingTests.targets
+++ b/eng/testing/linker/trimmingTests.targets
@@ -81,6 +81,9 @@
       <_additionalPropertiesString>@(_propertiesAsItems->'&lt;%(Identity)&gt;%(Value)&lt;/%(Identity)&gt;', '%0a    ')</_additionalPropertiesString>
     </PropertyGroup>
 
+    <!-- RunNativeAotTestApps trumps TestTrimming, same as PublishAot trumps PublishTrimmed, but it's very likely not what the user intended. -->
+    <Error Condition="'$(RunNativeAotTestApps)' == 'true' and '$(TestTrimming)' == 'true'" Text="Not possible to test both IL trimming and AOT at the same time" />
+
     <MakeDir Directories="$(_projectDir)" />
     <WriteLinesToFile File="$(_projectFile)"
                       Lines="$([System.IO.File]::ReadAllText('$(ProjectTemplate)')
@@ -88,7 +91,7 @@
                                                  .Replace('{NetCoreAppMaximumVersion}', '$(NetCoreAppMaximumVersion)')
                                                  .Replace('{UseMonoRuntime}','$(UseMonoRuntime)')
                                                  .Replace('{RuntimeIdentifier}','%(TestConsoleApps.TestRuntimeIdentifier)')
-                                                 .Replace('{PublishAot}','$(IsNativeAotTestProject)')
+                                                 .Replace('{PublishAot}','$(RunNativeAotTestApps)')
                                                  .Replace('{ExtraTrimmerArgs}', '%(TestConsoleApps.ExtraTrimmerArgs)')
                                                  .Replace('{AdditionalProperties}', '$(_additionalPropertiesString)')
                                                  .Replace('{ToolsILLinkDir}', '$(ToolsILLinkDir)')
diff --git a/src/libraries/Directory.Build.props b/src/libraries/Directory.Build.props
index c8d1737bea1..7ccc19377a2 100644
--- a/src/libraries/Directory.Build.props
+++ b/src/libraries/Directory.Build.props
@@ -25,7 +25,7 @@
   </ItemGroup>
 
   <Import Project="NetCoreAppLibrary.props" />
-  <Import Project="$(RepositoryEngineeringDir)testing\linker\trimmingTests.props" Condition="'$(IsPublishedAppTestProject)' == 'true'" />
+  <Import Project="$(RepositoryEngineeringDir)testing\linker\trimmingTests.props" Condition="'$(IsTrimmingTestProject)' == 'true'" />
 
   <PropertyGroup>
     <!-- Default any assembly not specifying a key to use the Open Key -->
diff --git a/src/libraries/Directory.Build.targets b/src/libraries/Directory.Build.targets
index b219e7cf41a..e3d599b2d82 100644
--- a/src/libraries/Directory.Build.targets
+++ b/src/libraries/Directory.Build.targets
@@ -131,7 +131,7 @@
   <Import Project="$(RepositoryEngineeringDir)codeOptimization.targets" />
   <Import Project="$(RepositoryEngineeringDir)references.targets" />
   <Import Project="$(RepositoryEngineeringDir)testing\tests.targets" Condition="'$(EnableTestSupport)' == 'true'" />
-  <Import Project="$(RepositoryEngineeringDir)testing\linker\trimmingTests.targets" Condition="'$(IsPublishedAppTestProject)' == 'true'" />
+  <Import Project="$(RepositoryEngineeringDir)testing\linker\trimmingTests.targets" Condition="'$(IsTrimmingTestProject)' == 'true'" />
   <Import Project="$(RepositoryEngineeringDir)testing\runtimeConfiguration.targets" />
   <Import Project="$(RepositoryEngineeringDir)testing\runsettings.targets" Condition="'$(EnableRunSettingsSupport)' == 'true'" />
   <Import Project="$(RepositoryEngineeringDir)testing\coverage.targets" Condition="'$(EnableRunSettingsSupport)' == 'true' or '$(EnableCoverageSupport)' == 'true'" />
diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/tests/NativeAotTests/System.Diagnostics.DiagnosticSource.NativeAotTests.proj b/src/libraries/System.Diagnostics.DiagnosticSource/tests/NativeAotTests/System.Diagnostics.DiagnosticSource.NativeAotTests.proj
deleted file mode 100644
index 8001203352b..00000000000
--- a/src/libraries/System.Diagnostics.DiagnosticSource/tests/NativeAotTests/System.Diagnostics.DiagnosticSource.NativeAotTests.proj
+++ /dev/null
@@ -1,10 +0,0 @@
-<Project DefaultTargets="Build">
-  <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.props))" />
-
-  <ItemGroup>
-    <TestConsoleAppSourceFiles Include="DiagnosticSourceEventSourceTests.cs"
-                               EnabledProperties="EventSourceSupport" />
-  </ItemGroup>
-
-  <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.targets))" />
-</Project>
diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/tests/NativeAotTests/DiagnosticSourceEventSourceTests.cs b/src/libraries/System.Diagnostics.DiagnosticSource/tests/TrimmingTests/DiagnosticSourceEventSourceTests.cs
similarity index 100%
rename from src/libraries/System.Diagnostics.DiagnosticSource/tests/NativeAotTests/DiagnosticSourceEventSourceTests.cs
rename to src/libraries/System.Diagnostics.DiagnosticSource/tests/TrimmingTests/DiagnosticSourceEventSourceTests.cs
diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/tests/TrimmingTests/System.Diagnostics.DiagnosticSource.proj b/src/libraries/System.Diagnostics.DiagnosticSource/tests/TrimmingTests/System.Diagnostics.DiagnosticSource.proj
index ca9483e224f..919c0b7614b 100644
--- a/src/libraries/System.Diagnostics.DiagnosticSource/tests/TrimmingTests/System.Diagnostics.DiagnosticSource.proj
+++ b/src/libraries/System.Diagnostics.DiagnosticSource/tests/TrimmingTests/System.Diagnostics.DiagnosticSource.proj
@@ -2,6 +2,8 @@
   <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.props))" />
 
   <ItemGroup>
+    <TestConsoleAppSourceFiles Include="DiagnosticSourceEventSourceTests.cs"
+                               EnabledProperties="EventSourceSupport" />
     <TestConsoleAppSourceFiles Include="WritePreservesAnonymousProperties.cs" />
   </ItemGroup>
 
diff --git a/src/libraries/tests.proj b/src/libraries/tests.proj
index 659fda6a8d7..f89150c3e25 100644
--- a/src/libraries/tests.proj
+++ b/src/libraries/tests.proj
@@ -646,16 +646,23 @@
                       BuildInParallel="$(BuildTestInParallel)" />
     <ProjectReference Include="$(MSBuildThisFileDirectory)testPackages\testPackages.proj"
                       Condition="'$(TestPackages)' == 'true'" />
+
+    <!-- We need to go over these disablements: https://github.com/dotnet/runtime/issues/101228 -->
+    <ProjectExclusions Condition="'$(RunNativeAotTestApps)' == 'true'" Include="$(MSBuildThisFileDirectory)\System.Diagnostics.Tracing\tests\TrimmingTests\System.Diagnostics.Tracing.TrimmingTests.proj" />
+    <ProjectExclusions Condition="'$(RunNativeAotTestApps)' == 'true'" Include="$(MSBuildThisFileDirectory)\System.Linq.Queryable\tests\TrimmingTests\System.Linq.Queryable.TrimmingTests.proj" />
+    <ProjectExclusions Condition="'$(RunNativeAotTestApps)' == 'true'" Include="$(MSBuildThisFileDirectory)\System.Private.Xml.Linq\tests\TrimmingTests\System.Xml.Linq.TrimmingTests.proj" />
+    <ProjectExclusions Condition="'$(RunNativeAotTestApps)' == 'true'" Include="$(MSBuildThisFileDirectory)\System.Net.Http\tests\TrimmingTests\System.Net.Http.TrimmingTests.proj" />
+    <ProjectExclusions Condition="'$(RunNativeAotTestApps)' == 'true'" Include="$(MSBuildThisFileDirectory)\System.Reflection.DispatchProxy\tests\TrimmingTests\System.Reflection.DispatchProxy.TrimmingTests.proj" />
+    <ProjectExclusions Condition="'$(RunNativeAotTestApps)' == 'true'" Include="$(MSBuildThisFileDirectory)\System.Linq.Expressions\tests\TrimmingTests\System.Linq.Expressions.TrimmingTests.proj" />
+    <ProjectExclusions Condition="'$(RunNativeAotTestApps)' == 'true'" Include="$(MSBuildThisFileDirectory)\System.Private.Xml\tests\TrimmingTests\System.Private.Xml.TrimmingTests.proj" />
+    <ProjectExclusions Condition="'$(RunNativeAotTestApps)' == 'true'" Include="$(MSBuildThisFileDirectory)\System.Runtime\tests\System.Runtime.Tests\TrimmingTests\System.Runtime.TrimmingTests.proj" />
+    <ProjectExclusions Condition="'$(RunNativeAotTestApps)' == 'true'" Include="$(MSBuildThisFileDirectory)\System.Text.Json\tests\System.Text.Json.Tests\TrimmingTests\System.Text.Json.TrimmingTests.proj" />
+
     <TrimmingTestProjects Include="$(MSBuildThisFileDirectory)*\tests\**\*.TrimmingTests.proj"
                       Exclude="@(ProjectExclusions)"
-                      Condition="'$(TestTrimming)' == 'true'"
+                      Condition="'$(TestTrimming)' == 'true' or '$(RunNativeAotTestApps)' == 'true'"
                       AdditionalProperties="%(AdditionalProperties);SkipTrimmingProjectsRestore=true" />
     <ProjectReference Include="@(TrimmingTestProjects)" />
-    <NativeAotTestAppProjects Include="$(MSBuildThisFileDirectory)*\tests\**\*.NativeAotTests.proj"
-                              Exclude="@(ProjectExclusions)"
-                              Condition="'$(RunNativeAotTestApps)' == 'true'"
-                              AdditionalProperties="%(AdditionalProperties);SkipTrimmingProjectsRestore=true" />
-    <ProjectReference Include="@(NativeAotTestAppProjects)" />
 
     <!-- wasm.build.tests are run on _WasmBuildTests job on CI, and with library tests locally. -->
     <ProjectReference Include="$(WasmProjectRoot)Wasm.Build.Tests\*.Tests.csproj"