Add support to produce WebAssembly runtime assets (dotnet/corefx#42128)

* Add support to produce WebAssembly runtime assets

* Add support to build in OSX and address PR feedback

* Change AND -> and in Directory.Build.props change region

* PR Feedback: run build in rolling CI builds as well

* Add TODO comment for WebAssembly OSGroup

* Fix indentation.

* Fix wrong #include as apis defined in that include are used outside of ETHTOOL_H #if-def


Commit migrated from 1790133578

eng/configurations/osgroups.props

@@ -35,8 +35,10 @@
       <PackageTargetRuntime>netbsd</PackageTargetRuntime>
     </OSGroups>
     <OSGroups Include="WebAssembly">
-      <Imports>Unix</Imports>
+      <!-- TODO: we need to change this to import Unix instead whenever
-      <TargetsUnix>true</TargetsUnix>
+      we want to start using managed implementation for WebAssembly -->
+      <Imports>Linux</Imports>
+      <TargetsLinux>true</TargetsLinux>
       <TargetsWebAssembly>true</TargetsWebAssembly>
     </OSGroups>
     <OSGroups Include="AnyOS" />

eng/pipelines/libraries/.azure-ci.yml

@@ -55,6 +55,9 @@ resources:
   - container: ubuntu_1604_arm_cross_container
     image: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548
 
+  - container: ubuntu_1604
+    image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-bd0fe7c-20190923200211
+
 variables:
   - ${{ if and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}:
     - group: DotNet-Blob-Feed

eng/pipelines/libraries/linux.yml

@@ -134,35 +134,57 @@ stages:
             - linuxArm64Queues: \(Ubuntu.1604.Arm64.Open\)Ubuntu.1604.Arm64.Docker.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-helix-arm64v8-a45aeeb-20190620160300
 
       # Legs without helix testing
-      # Only run this leg in PRs.
+      # Don't run these legs in outerloop builds
-      - ${{ if and(eq(parameters.fullMatrix, 'false'), and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all'))) }}:
+      - ${{ if and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all')) }}:
         - job: LinuxNoTest
           displayName: Build
           strategy:
             matrix:
-              arm_Debug:
+              wasm_Release:
-                _BuildConfig: Debug
+                _BuildConfig: Release
-                _architecture: arm
+                _architecture: wasm
                 _framework: netcoreapp
-                _buildExtraArguments: /p:RuntimeOS=ubuntu.16.04 -warnAsError false
+                _dockerContainer: ubuntu_1604
-                _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm '
+                _emsdkPath: $(Build.BinariesDirectory)/emsdk
-                _dockerContainer: ubuntu_1604_arm_cross_container
+                _buildScriptPrefix: 'EMSDK_PATH=$(_emsdkPath) '
+                _buildExtraArguments: -os WebAssembly -warnAsError false
+                _installEmscripten: true
 
-              musl_arm64_Debug:
+              # Only run these legs in PRs.
-                _BuildConfig: Debug
+              ${{ if eq(parameters.fullMatrix, 'false') }}:
-                _architecture: arm64
+                arm_Debug:
-                _framework: netcoreapp
+                  _BuildConfig: Debug
-                _dockerContainer: alpine_37_arm64_container
+                  _architecture: arm
-                _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 '
+                  _framework: netcoreapp
-                _buildExtraArguments: -warnAsError false /p:BuildNativeCompiler=--clang5.0 /p:RuntimeOS=linux-musl
+                  _buildExtraArguments: /p:RuntimeOS=ubuntu.16.04 -warnAsError false
+                  _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm '
+                  _dockerContainer: ubuntu_1604_arm_cross_container
 
-              arm64_Debug:
+                musl_arm64_Debug:
-                _BuildConfig: Debug
+                  _BuildConfig: Debug
-                _architecture: arm64
+                  _architecture: arm64
-                _framework: netcoreapp
+                  _framework: netcoreapp
-                _dockerContainer: ubuntu_1604_arm64_cross_container
+                  _dockerContainer: alpine_37_arm64_container
-                _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 '
+                  _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 '
-                _buildExtraArguments: --warnAsError false
+                  _buildExtraArguments: -warnAsError false /p:BuildNativeCompiler=--clang5.0 /p:RuntimeOS=linux-musl
+
+                arm64_Debug:
+                  _BuildConfig: Debug
+                  _architecture: arm64
+                  _framework: netcoreapp
+                  _dockerContainer: ubuntu_1604_arm64_cross_container
+                  _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 '
+                  _buildExtraArguments: --warnAsError false
+
+          preBuildSteps:
+            - script: |
+                EMSCRIPTEN_VERSION=1.38.47
+                git clone https://github.com/emscripten-core/emsdk.git $(_emsdkPath)
+                cd $(_emsdkPath)
+                ./emsdk install ${EMSCRIPTEN_VERSION}-upstream
+                ./emsdk activate --embedded ${EMSCRIPTEN_VERSION}-upstream
+              displayName: Install Emscripten
+              condition: and(succeeded(), eq(variables['_installEmscripten'], 'true'))
 
           pool:
             name: Hosted Ubuntu 1604

eng/restore/runtime/runtime.depproj

@@ -1,6 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <RuntimeIdentifier>$(PackageRID)</RuntimeIdentifier>
+    <!-- We're using ToolRuntimeRID as a placeholder for the real corelib/runtime components until we have an actual set of runtime bits to consume for webassembly. -->
+    <RuntimeIdentifier Condition="'$(RuntimeOS)' == 'webassembly'">$(ToolRuntimeRID)</RuntimeIdentifier>
     <NoWarn>$(NoWarn);NU1603;NU1605</NoWarn>
     <SwapNativeForIL Condition="'$(SwapNativeForIL)' == '' AND ('$(ConfigurationGroup)' == 'Debug' OR '$(Coverage)' == 'true')">true</SwapNativeForIL>
   </PropertyGroup>

src/libraries/Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj

@@ -3,7 +3,7 @@
     <DefineConstants>$(DefineConstants);XMLSERIALIZERGENERATORTESTS</DefineConstants>
     <Configurations>netcoreapp-Debug;netcoreapp-Release</Configurations>
     <CoverageSupported>false</CoverageSupported>
-    <SkipTestsOnPlatform Condition="'$(ArchGroup)' == 'arm' OR '$(ArchGroup)' == 'arm64' OR '$(ArchGroup)' == 'armel'">true</SkipTestsOnPlatform>
+    <SkipTestsOnPlatform Condition="'$(ArchGroup)' == 'arm' or '$(ArchGroup)' == 'arm64' or '$(ArchGroup)' == 'armel' or '$(ArchGroup)' == 'wasm'">true</SkipTestsOnPlatform>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Reuse the same runtimeconfig used by MSBuild. -->

src/libraries/Native/Unix/CMakeLists.txt

@@ -18,7 +18,6 @@ add_compile_options(-Wno-empty-translation-unit)
 add_compile_options(-Wno-cast-align)
 add_compile_options(-Wno-typedef-redefinition)
 add_compile_options(-Wno-c11-extensions)
-add_compile_options(-fPIC)
 add_compile_options(-I${CMAKE_CURRENT_SOURCE_DIR}/Common)
 add_compile_options(-I${CMAKE_CURRENT_BINARY_DIR}/Common)
 add_compile_options(-g)
@@ -29,9 +28,19 @@ endif()
 add_compile_options(-Werror)
 
 if(CMAKE_SYSTEM_NAME STREQUAL Emscripten)
+    # Build a static library so no -fPIC
     set(CLR_CMAKE_PLATFORM_WASM 1)
     add_definitions(-D_WASM_)
+    # The emscripten build has additional warnings so -Werror breaks
+    add_compile_options(-Wno-unused-parameter)
+    add_compile_options(-Wno-unused-function)
+    add_compile_options(-Wno-alloca)
+    add_compile_options(-Wno-implicit-int-float-conversion)
+else()
+  add_compile_options(-fPIC)
+  set(GEN_SHARED_LIB 1)
 endif(CMAKE_SYSTEM_NAME STREQUAL Emscripten)
+
 if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64)
     add_definitions(-DBIT64=1)
     add_definitions(-D_AMD64_)
@@ -95,7 +104,8 @@ else ()
     message(FATAL_ERROR "Unknown build type. Set CMAKE_BUILD_TYPE to DEBUG or RELEASE.")
 endif ()
 
-if (APPLE)
+if(CMAKE_SYSTEM_NAME STREQUAL Emscripten)
+elseif (APPLE)
     add_definitions(-D__APPLE_USE_RFC_3542)
 
    # We cannot enable "stack-protector-strong" on OS X due to a bug in clang compiler (current version 7.0.2)

src/libraries/Native/Unix/Common/pal_config.h.in

@@ -67,6 +67,7 @@
 #cmakedefine01 HAVE_SYS_SYSCTL_H
 #cmakedefine01 HAVE_NET_IFMEDIA_H
 #cmakedefine01 HAVE_LINUX_RTNETLINK_H
+#cmakedefine01 HAVE_LINUX_CAN_H
 #cmakedefine01 HAVE_GETDOMAINNAME_SIZET
 #cmakedefine01 HAVE_INOTIFY
 #cmakedefine01 HAVE_CLOCK_MONOTONIC

src/libraries/Native/Unix/System.Native/CMakeLists.txt

@@ -31,12 +31,25 @@ if (CMAKE_SYSTEM_NAME STREQUAL Linux)
     endif ()
 endif ()
 
-add_library(System.Native
+if (GEN_SHARED_LIB)
-    SHARED
+    add_library(System.Native
-    ${NATIVE_SOURCES}
+        SHARED
-    ${VERSION_FILE_PATH}
+        ${NATIVE_SOURCES}
-)
+        ${VERSION_FILE_PATH}
+    )
+    if (CMAKE_SYSTEM_NAME STREQUAL Linux AND NOT CLR_CMAKE_PLATFORM_ANDROID)
+        target_link_libraries(System.Native rt)
+    endif ()
 
+    if (CMAKE_SYSTEM_NAME STREQUAL FreeBSD)
+        target_link_libraries(System.Native pthread)
+    if (HAVE_INOTIFY)
+        find_library(INOTIFY_LIBRARY inotify HINTS /usr/local/lib)
+        target_link_libraries(System.Native ${INOTIFY_LIBRARY})
+    endif ()
+    endif ()
+    install_library_and_symbols (System.Native)
+endif ()
 
 add_library(System.Native-Static
     STATIC
@@ -48,17 +61,4 @@ add_library(System.Native-Static
 set_target_properties(System.Native-Static PROPERTIES PREFIX "")
 set_target_properties(System.Native-Static PROPERTIES OUTPUT_NAME System.Native CLEAN_DIRECT_OUTPUT 1)
 
-if (CMAKE_SYSTEM_NAME STREQUAL Linux AND NOT CLR_CMAKE_PLATFORM_ANDROID)
-    target_link_libraries(System.Native rt)
-endif ()
-
-if (CMAKE_SYSTEM_NAME STREQUAL FreeBSD)
-    target_link_libraries(System.Native pthread)
-if (HAVE_INOTIFY)
-    find_library(INOTIFY_LIBRARY inotify HINTS /usr/local/lib)
-    target_link_libraries(System.Native ${INOTIFY_LIBRARY})
-endif()
-endif ()
-
-install_library_and_symbols (System.Native)
 install (TARGETS System.Native-Static DESTINATION .)

src/libraries/Native/Unix/System.Native/pal_interfaceaddresses.c

@@ -22,12 +22,12 @@
 #endif
 #if HAVE_ETHTOOL_H
 #include <linux/ethtool.h>
-#include <sys/ioctl.h>
 #include <linux/sockios.h>
 #include <arpa/inet.h>
 #endif
 
 #if defined(AF_PACKET)
+#include <sys/ioctl.h>
 #if defined(_WASM_)
 #include <netpacket/packet.h>
 #else // _WASM_

src/libraries/Native/Unix/System.Native/pal_networking.c

@@ -55,7 +55,7 @@
 #include <ifaddrs.h>
 #endif
 #endif
-#ifdef AF_CAN
+#if HAVE_LINUX_CAN_H
 #include <linux/can.h>
 #endif
 #if HAVE_KQUEUE
@@ -2001,7 +2001,7 @@ static bool TryConvertProtocolTypePalToPlatform(int32_t palAddressFamily, int32_
             *platformProtocolType = palProtocolType;
             return true;
 #endif
-#ifdef AF_CAN
+#if HAVE_LINUX_CAN_H
         case AddressFamily_AF_CAN:
             switch (palProtocolType)
             {
@@ -2492,8 +2492,7 @@ static int32_t WaitForSocketEventsInner(int32_t port, SocketEvent* buffer, int32
 }
 
 #else
-#warning epoll/kqueue not detected; building with stub socket events support
+static const size_t SocketEventBufferElementSize = 0;
-static const size_t SocketEventBufferElementSize = sizeof(struct pollfd);
 
 static SocketEvents GetSocketEvents(int16_t filter, uint16_t flags)
 {

src/libraries/Native/Unix/System.Native/pal_process.c

@@ -154,7 +154,7 @@ static int compare_groups(const void * a, const void * b)
 
 static int SetGroups(uint32_t* userGroups, int32_t userGroupsLength, uint32_t* processGroups)
 {
-#ifdef __linux__
+#if defined(__linux__) || defined(_WASM_)
     size_t platformGroupsLength = Int32ToSizeT(userGroupsLength);
 #else // BSD
     int platformGroupsLength = userGroupsLength;

src/libraries/Native/Unix/configure.cmake

@@ -655,6 +655,10 @@ check_include_files(
     linux/rtnetlink.h
     HAVE_LINUX_RTNETLINK_H)
 
+check_include_files(
+    linux/can.h
+    HAVE_LINUX_CAN_H)
+
 check_symbol_exists(
     getpeereid
     unistd.h

src/libraries/Native/Unix/gen-buildsys-clang.sh

@@ -44,6 +44,7 @@ else
   buildtype="$5"
 fi
 
+cmake_cmd=cmake
 cmake_extra_defines="-DCMAKE_BUILD_TYPE=$buildtype"
 if [[ -n "$CROSSCOMPILE" ]]; then
     if ! [[ -n "$ROOTFS_DIR" ]]; then
@@ -60,6 +61,13 @@ fi
 if [ "$build_arch" == "armel" ]; then
     cmake_extra_defines="$cmake_extra_defines -DARM_SOFTFP=1"
 fi
+if [ "$build_arch" == "wasm" ]; then
+   if [ "$EMSCRIPTEN_ROOT" == "" ]; then
+       EMSCRIPTEN_ROOT="$EMSDK_PATH/upstream/emscripten"
+   fi
+   cmake_cmd="emcmake cmake"
+   cmake_extra_defines="$cmake_extra_defines -DCMAKE_TOOLCHAIN_FILE=$EMSCRIPTEN_ROOT/cmake/Modules/Platform/Emscripten.cmake -DEMSCRIPTEN_GENERATE_BITCODE_STATIC_LIBRARIES=1"
+fi
 
 __UnprocessedCMakeArgs=""
 if [ -z "$6" ]; then
@@ -68,6 +76,7 @@ else
     __UnprocessedCMakeArgs="$6"
 fi
 
-cmake $cmake_extra_defines \
+echo "Invoking \"$cmake_cmd $cmake_extra_defines $__UnprocessedCMakeArgs $1\""
+$cmake_cmd $cmake_extra_defines \
     $__UnprocessedCMakeArgs \
     $1

src/libraries/Native/Windows/gen-buildsys-win.bat

@@ -36,6 +36,14 @@ popd
 :DoGen
 
 if "%3" == "wasm" (
+    if "%EMSDK_PATH%" == "" (
+       echo "Error: Should set EMSDK_PATH environment variable pointing to emsdk root."
+       exit /B 1
+    )
+
+    if "%EMSCRIPTEN_ROOT%" == "" (
+      set EMSCRIPTEN_ROOT="%EMSDK_PATH/upstream/emscripten%"
+    )
     emcmake cmake "-DEMSCRIPTEN_GENERATE_BITCODE_STATIC_LIBRARIES=1" "-DCMAKE_TOOLCHAIN_FILE=%EMSCRIPTEN%/cmake/Modules/Platform/Emscripten.cmake" "-DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE%" -G "NMake Makefiles" %__sourceDir%
 ) else (
     "%CMakePath%" %__SDKVersion% "-DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE%" "-DCMAKE_INSTALL_PREFIX=%__CMakeBinDir%" -G "Visual Studio %__VSString%" -B. -H%1 %__ExtraCmakeParams%

src/libraries/Native/build-native.cmd

@@ -160,10 +160,6 @@ IF ERRORLEVEL 1 (
     goto :Failure
 )
 
-:: Copy results to native_aot since packaging expects a copy there too
-mkdir "%__artifactsDir%\bin\native\%__outConfig%-aot"
-copy "%__artifactsDir%\bin\native\%__outConfig%\*" "%__artifactsDir%\bin\native\%__outConfig%-aot"
-
 exit /B 0
 
 :Failure

src/libraries/Native/build-native.proj

@@ -20,7 +20,7 @@
         then we should consider calling Environment.ProcessorCount
       -->
       <_ProcessorCountArg> --numproc $(MSBuildNodeCount)</_ProcessorCountArg>
-      <_StripSymbolsArg Condition="'$(BuildNativeStripSymbols)' == 'true'"> stripsymbols</_StripSymbolsArg>
+      <_StripSymbolsArg Condition="'$(BuildNativeStripSymbols)' == 'true' and '$(OSGroup)' != 'WebAssembly'"> stripsymbols</_StripSymbolsArg>
       <_PortableBuildArg Condition="'$(PortableBuild)' == 'true'"> -portable</_PortableBuildArg>
 
       <!--

src/libraries/Native/build-native.sh

@@ -137,6 +137,14 @@ build_native()
     echo "Commencing build of corefx native components for $__BuildOS.$__BuildArch.$__BuildType"
     cd "$__IntermediatesDir"
 
+    if [ "$__BuildArch" == "wasm" ]; then
+        if [ "$EMSDK_PATH" == "" ]; then
+            echo "Error: Should set EMSDK_PATH environment variable pointing to emsdk root."
+            exit 1
+        fi
+        source $EMSDK_PATH/emsdk_env.sh
+    fi
+
     # Regenerate the CMake solution
     if [ "$__GccBuild" = 0 ]; then
         echo "Invoking \"$__nativeroot/gen-buildsys-clang.sh\" \"$__nativeroot\" \"$__ClangMajorVersion\" \"$__ClangMinorVersion\" \"$__BuildArch\" \"$__CMakeArgs\" \"$__CMakeExtraArgs\""
@@ -260,6 +268,9 @@ while :; do
         arm64|-arm64)
             __BuildArch=arm64
             ;;
+        wasm|-wasm)
+            __BuildArch=wasm
+            ;;
         debug|-debug)
             __BuildType=Debug
             ;;
@@ -388,24 +399,26 @@ while :; do
     shift
 done
 
-__CMakeExtraArgs="$__CMakeExtraArgs -DFEATURE_DISTRO_AGNOSTIC_SSL=$__PortableBuild"
-__CMakeExtraArgs="$__CMakeExtraArgs -DCMAKE_STATIC_LIB_LINK=$__StaticLibLink"
-
 # Set cross build
-case $CPUName in
+if [ $__BuildArch != wasm ]; then
-    i686)
+    __CMakeExtraArgs="$__CMakeExtraArgs -DFEATURE_DISTRO_AGNOSTIC_SSL=$__PortableBuild"
-        if [ $__BuildArch != x86 ]; then
+    __CMakeExtraArgs="$__CMakeExtraArgs -DCMAKE_STATIC_LIB_LINK=$__StaticLibLink"
-            __CrossBuild=1
+
-            echo "Set CrossBuild for $__BuildArch build"
+    case $CPUName in
-        fi
+        i686)
-        ;;
+            if [ $__BuildArch != x86 ]; then
-    x86_64)
+                __CrossBuild=1
-        if [ $__BuildArch != x64 ]; then
+                echo "Set CrossBuild for $__BuildArch build"
-            __CrossBuild=1
+            fi
-            echo "Set CrossBuild for $__BuildArch build"
+            ;;
-        fi
+        x86_64)
-        ;;
+            if [ $__BuildArch != x64 ]; then
-esac
+                __CrossBuild=1
+                echo "Set CrossBuild for $__BuildArch build"
+            fi
+            ;;
+    esac
+fi
 
 # Set the default clang version if not already set
 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then

src/libraries/Native/native-binplace.proj

@@ -19,6 +19,7 @@
       <BinPlaceItem Include="$(NativeBinDir)/*.pdb" />
       <BinPlaceItem Include="$(NativeBinDir)/*.lib" />
       <BinPlaceItem Include="$(NativeBinDir)/*.a" />
+      <BinPlaceItem Include="$(NativeBinDir)/*.bc" />
       <BinPlaceItem Include="$(NativeBinDir)/*.so" />
       <BinPlaceItem Include="$(NativeBinDir)/*.dbg" />
       <BinPlaceItem Include="$(NativeBinDir)/*.dylib" />

src/libraries/System.Data.Odbc/src/System.Data.Odbc.csproj

@@ -121,7 +121,7 @@
       <Link>Common\Interop\Interop.Odbc.cs</Link>
     </Compile>
   </ItemGroup>
-  <ItemGroup Condition="'$(TargetsLinux)' == 'true' OR '$(TargetsFreeBSD)' == 'true' OR '$(TargetsWebAssembly)'=='true'">
+  <ItemGroup Condition="'$(TargetsLinux)' == 'true' OR '$(TargetsFreeBSD)' == 'true'">
     <Compile Include="$(CommonPath)\Interop\Linux\Interop.Libraries.cs">
       <Link>Common\Interop\Linux\Interop.Libraries.cs</Link>
     </Compile>