Add support to produce WebAssembly runtime assets (dotnet/corefx#42128)

* Add support to produce WebAssembly runtime assets

* Add support to build in OSX and address PR feedback

* Change AND -> and in Directory.Build.props change region

* PR Feedback: run build in rolling CI builds as well

* Add TODO comment for WebAssembly OSGroup

* Fix indentation.

* Fix wrong #include as apis defined in that include are used outside of ETHTOOL_H #if-def


Commit migrated from 1790133578

eng/configurations/osgroups.props

@@ -35,8 +35,10 @@
       <PackageTargetRuntime>netbsd</PackageTargetRuntime>
     </OSGroups>
     <OSGroups Include="WebAssembly">
-      <Imports>Unix</Imports>
-      <TargetsUnix>true</TargetsUnix>
+      <!-- TODO: we need to change this to import Unix instead whenever
+      we want to start using managed implementation for WebAssembly -->
+      <Imports>Linux</Imports>
+      <TargetsLinux>true</TargetsLinux>
       <TargetsWebAssembly>true</TargetsWebAssembly>
     </OSGroups>
     <OSGroups Include="AnyOS" />

eng/pipelines/libraries/.azure-ci.yml

@@ -55,6 +55,9 @@ resources:
   - container: ubuntu_1604_arm_cross_container
     image: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548
 
+  - container: ubuntu_1604
+    image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-bd0fe7c-20190923200211
+
 variables:
   - ${{ if and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}:
     - group: DotNet-Blob-Feed

eng/pipelines/libraries/linux.yml

@@ -134,12 +134,24 @@ stages:
             - linuxArm64Queues: \(Ubuntu.1604.Arm64.Open\)Ubuntu.1604.Arm64.Docker.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-helix-arm64v8-a45aeeb-20190620160300
 
       # Legs without helix testing
-      # Only run this leg in PRs.
-      - ${{ if and(eq(parameters.fullMatrix, 'false'), and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all'))) }}:
+      # Don't run these legs in outerloop builds
+      - ${{ if and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all')) }}:
         - job: LinuxNoTest
           displayName: Build
           strategy:
             matrix:
+              wasm_Release:
+                _BuildConfig: Release
+                _architecture: wasm
+                _framework: netcoreapp
+                _dockerContainer: ubuntu_1604
+                _emsdkPath: $(Build.BinariesDirectory)/emsdk
+                _buildScriptPrefix: 'EMSDK_PATH=$(_emsdkPath) '
+                _buildExtraArguments: -os WebAssembly -warnAsError false
+                _installEmscripten: true
+
+              # Only run these legs in PRs.
+              ${{ if eq(parameters.fullMatrix, 'false') }}:
                 arm_Debug:
                   _BuildConfig: Debug
                   _architecture: arm
@@ -164,6 +176,16 @@ stages:
                   _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 '
                   _buildExtraArguments: --warnAsError false
 
+          preBuildSteps:
+            - script: |
+                EMSCRIPTEN_VERSION=1.38.47
+                git clone https://github.com/emscripten-core/emsdk.git $(_emsdkPath)
+                cd $(_emsdkPath)
+                ./emsdk install ${EMSCRIPTEN_VERSION}-upstream
+                ./emsdk activate --embedded ${EMSCRIPTEN_VERSION}-upstream
+              displayName: Install Emscripten
+              condition: and(succeeded(), eq(variables['_installEmscripten'], 'true'))
+
           pool:
             name: Hosted Ubuntu 1604
 

eng/restore/runtime/runtime.depproj

@@ -1,6 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <RuntimeIdentifier>$(PackageRID)</RuntimeIdentifier>
+    <!-- We're using ToolRuntimeRID as a placeholder for the real corelib/runtime components until we have an actual set of runtime bits to consume for webassembly. -->
+    <RuntimeIdentifier Condition="'$(RuntimeOS)' == 'webassembly'">$(ToolRuntimeRID)</RuntimeIdentifier>
     <NoWarn>$(NoWarn);NU1603;NU1605</NoWarn>
     <SwapNativeForIL Condition="'$(SwapNativeForIL)' == '' AND ('$(ConfigurationGroup)' == 'Debug' OR '$(Coverage)' == 'true')">true</SwapNativeForIL>
   </PropertyGroup>

src/libraries/Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj

@@ -3,7 +3,7 @@
     <DefineConstants>$(DefineConstants);XMLSERIALIZERGENERATORTESTS</DefineConstants>
     <Configurations>netcoreapp-Debug;netcoreapp-Release</Configurations>
     <CoverageSupported>false</CoverageSupported>
-    <SkipTestsOnPlatform Condition="'$(ArchGroup)' == 'arm' OR '$(ArchGroup)' == 'arm64' OR '$(ArchGroup)' == 'armel'">true</SkipTestsOnPlatform>
+    <SkipTestsOnPlatform Condition="'$(ArchGroup)' == 'arm' or '$(ArchGroup)' == 'arm64' or '$(ArchGroup)' == 'armel' or '$(ArchGroup)' == 'wasm'">true</SkipTestsOnPlatform>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Reuse the same runtimeconfig used by MSBuild. -->

src/libraries/Native/Unix/CMakeLists.txt

@@ -18,7 +18,6 @@ add_compile_options(-Wno-empty-translation-unit)
 add_compile_options(-Wno-cast-align)
 add_compile_options(-Wno-typedef-redefinition)
 add_compile_options(-Wno-c11-extensions)
-add_compile_options(-fPIC)
 add_compile_options(-I${CMAKE_CURRENT_SOURCE_DIR}/Common)
 add_compile_options(-I${CMAKE_CURRENT_BINARY_DIR}/Common)
 add_compile_options(-g)
@@ -29,9 +28,19 @@ endif()
 add_compile_options(-Werror)
 
 if(CMAKE_SYSTEM_NAME STREQUAL Emscripten)
+    # Build a static library so no -fPIC
     set(CLR_CMAKE_PLATFORM_WASM 1)
     add_definitions(-D_WASM_)
+    # The emscripten build has additional warnings so -Werror breaks
+    add_compile_options(-Wno-unused-parameter)
+    add_compile_options(-Wno-unused-function)
+    add_compile_options(-Wno-alloca)
+    add_compile_options(-Wno-implicit-int-float-conversion)
+else()
+  add_compile_options(-fPIC)
+  set(GEN_SHARED_LIB 1)
 endif(CMAKE_SYSTEM_NAME STREQUAL Emscripten)
+
 if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64)
     add_definitions(-DBIT64=1)
     add_definitions(-D_AMD64_)
@@ -95,7 +104,8 @@ else ()
     message(FATAL_ERROR "Unknown build type. Set CMAKE_BUILD_TYPE to DEBUG or RELEASE.")
 endif ()
 
-if (APPLE)
+if(CMAKE_SYSTEM_NAME STREQUAL Emscripten)
+elseif (APPLE)
     add_definitions(-D__APPLE_USE_RFC_3542)
 
    # We cannot enable "stack-protector-strong" on OS X due to a bug in clang compiler (current version 7.0.2)

src/libraries/Native/Unix/Common/pal_config.h.in

@@ -67,6 +67,7 @@
 #cmakedefine01 HAVE_SYS_SYSCTL_H
 #cmakedefine01 HAVE_NET_IFMEDIA_H
 #cmakedefine01 HAVE_LINUX_RTNETLINK_H
+#cmakedefine01 HAVE_LINUX_CAN_H
 #cmakedefine01 HAVE_GETDOMAINNAME_SIZET
 #cmakedefine01 HAVE_INOTIFY
 #cmakedefine01 HAVE_CLOCK_MONOTONIC

src/libraries/Native/Unix/System.Native/CMakeLists.txt

@@ -31,23 +31,12 @@ if (CMAKE_SYSTEM_NAME STREQUAL Linux)
     endif ()
 endif ()
 
+if (GEN_SHARED_LIB)
     add_library(System.Native
         SHARED
         ${NATIVE_SOURCES}
         ${VERSION_FILE_PATH}
     )
-
-
-add_library(System.Native-Static
-    STATIC
-    ${NATIVE_SOURCES}
-    ${VERSION_FILE_PATH}
-)
-
-# Disable the "lib" prefix and override default name
-set_target_properties(System.Native-Static PROPERTIES PREFIX "")
-set_target_properties(System.Native-Static PROPERTIES OUTPUT_NAME System.Native CLEAN_DIRECT_OUTPUT 1)
-
     if (CMAKE_SYSTEM_NAME STREQUAL Linux AND NOT CLR_CMAKE_PLATFORM_ANDROID)
         target_link_libraries(System.Native rt)
     endif ()
@@ -59,6 +48,17 @@ if (HAVE_INOTIFY)
         target_link_libraries(System.Native ${INOTIFY_LIBRARY})
     endif ()
     endif ()
-
     install_library_and_symbols (System.Native)
+endif ()
+
+add_library(System.Native-Static
+    STATIC
+    ${NATIVE_SOURCES}
+    ${VERSION_FILE_PATH}
+)
+
+# Disable the "lib" prefix and override default name
+set_target_properties(System.Native-Static PROPERTIES PREFIX "")
+set_target_properties(System.Native-Static PROPERTIES OUTPUT_NAME System.Native CLEAN_DIRECT_OUTPUT 1)
+
 install (TARGETS System.Native-Static DESTINATION .)

src/libraries/Native/Unix/System.Native/pal_interfaceaddresses.c

@@ -22,12 +22,12 @@
 #endif
 #if HAVE_ETHTOOL_H
 #include <linux/ethtool.h>
-#include <sys/ioctl.h>
 #include <linux/sockios.h>
 #include <arpa/inet.h>
 #endif
 
 #if defined(AF_PACKET)
+#include <sys/ioctl.h>
 #if defined(_WASM_)
 #include <netpacket/packet.h>
 #else // _WASM_

src/libraries/Native/Unix/System.Native/pal_networking.c

@@ -55,7 +55,7 @@
 #include <ifaddrs.h>
 #endif
 #endif
-#ifdef AF_CAN
+#if HAVE_LINUX_CAN_H
 #include <linux/can.h>
 #endif
 #if HAVE_KQUEUE
@@ -2001,7 +2001,7 @@ static bool TryConvertProtocolTypePalToPlatform(int32_t palAddressFamily, int32_
             *platformProtocolType = palProtocolType;
             return true;
 #endif
-#ifdef AF_CAN
+#if HAVE_LINUX_CAN_H
         case AddressFamily_AF_CAN:
             switch (palProtocolType)
             {
@@ -2492,8 +2492,7 @@ static int32_t WaitForSocketEventsInner(int32_t port, SocketEvent* buffer, int32
 }
 
 #else
-#warning epoll/kqueue not detected; building with stub socket events support
-static const size_t SocketEventBufferElementSize = sizeof(struct pollfd);
+static const size_t SocketEventBufferElementSize = 0;
 
 static SocketEvents GetSocketEvents(int16_t filter, uint16_t flags)
 {

src/libraries/Native/Unix/System.Native/pal_process.c

@@ -154,7 +154,7 @@ static int compare_groups(const void * a, const void * b)
 
 static int SetGroups(uint32_t* userGroups, int32_t userGroupsLength, uint32_t* processGroups)
 {
-#ifdef __linux__
+#if defined(__linux__) || defined(_WASM_)
     size_t platformGroupsLength = Int32ToSizeT(userGroupsLength);
 #else // BSD
     int platformGroupsLength = userGroupsLength;

src/libraries/Native/Unix/configure.cmake

@@ -655,6 +655,10 @@ check_include_files(
     linux/rtnetlink.h
     HAVE_LINUX_RTNETLINK_H)
 
+check_include_files(
+    linux/can.h
+    HAVE_LINUX_CAN_H)
+
 check_symbol_exists(
     getpeereid
     unistd.h

src/libraries/Native/Unix/gen-buildsys-clang.sh

@@ -44,6 +44,7 @@ else
   buildtype="$5"
 fi
 
+cmake_cmd=cmake
 cmake_extra_defines="-DCMAKE_BUILD_TYPE=$buildtype"
 if [[ -n "$CROSSCOMPILE" ]]; then
     if ! [[ -n "$ROOTFS_DIR" ]]; then
@@ -60,6 +61,13 @@ fi
 if [ "$build_arch" == "armel" ]; then
     cmake_extra_defines="$cmake_extra_defines -DARM_SOFTFP=1"
 fi
+if [ "$build_arch" == "wasm" ]; then
+   if [ "$EMSCRIPTEN_ROOT" == "" ]; then
+       EMSCRIPTEN_ROOT="$EMSDK_PATH/upstream/emscripten"
+   fi
+   cmake_cmd="emcmake cmake"
+   cmake_extra_defines="$cmake_extra_defines -DCMAKE_TOOLCHAIN_FILE=$EMSCRIPTEN_ROOT/cmake/Modules/Platform/Emscripten.cmake -DEMSCRIPTEN_GENERATE_BITCODE_STATIC_LIBRARIES=1"
+fi
 
 __UnprocessedCMakeArgs=""
 if [ -z "$6" ]; then
@@ -68,6 +76,7 @@ else
     __UnprocessedCMakeArgs="$6"
 fi
 
-cmake $cmake_extra_defines \
+echo "Invoking \"$cmake_cmd $cmake_extra_defines $__UnprocessedCMakeArgs $1\""
+$cmake_cmd $cmake_extra_defines \
     $__UnprocessedCMakeArgs \
     $1

src/libraries/Native/Windows/gen-buildsys-win.bat

@@ -36,6 +36,14 @@ popd
 :DoGen
 
 if "%3" == "wasm" (
+    if "%EMSDK_PATH%" == "" (
+       echo "Error: Should set EMSDK_PATH environment variable pointing to emsdk root."
+       exit /B 1
+    )
+
+    if "%EMSCRIPTEN_ROOT%" == "" (
+      set EMSCRIPTEN_ROOT="%EMSDK_PATH/upstream/emscripten%"
+    )
     emcmake cmake "-DEMSCRIPTEN_GENERATE_BITCODE_STATIC_LIBRARIES=1" "-DCMAKE_TOOLCHAIN_FILE=%EMSCRIPTEN%/cmake/Modules/Platform/Emscripten.cmake" "-DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE%" -G "NMake Makefiles" %__sourceDir%
 ) else (
     "%CMakePath%" %__SDKVersion% "-DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE%" "-DCMAKE_INSTALL_PREFIX=%__CMakeBinDir%" -G "Visual Studio %__VSString%" -B. -H%1 %__ExtraCmakeParams%

src/libraries/Native/build-native.cmd

@@ -160,10 +160,6 @@ IF ERRORLEVEL 1 (
     goto :Failure
 )
 
-:: Copy results to native_aot since packaging expects a copy there too
-mkdir "%__artifactsDir%\bin\native\%__outConfig%-aot"
-copy "%__artifactsDir%\bin\native\%__outConfig%\*" "%__artifactsDir%\bin\native\%__outConfig%-aot"
-
 exit /B 0
 
 :Failure

src/libraries/Native/build-native.proj

@@ -20,7 +20,7 @@
         then we should consider calling Environment.ProcessorCount
       -->
       <_ProcessorCountArg> --numproc $(MSBuildNodeCount)</_ProcessorCountArg>
-      <_StripSymbolsArg Condition="'$(BuildNativeStripSymbols)' == 'true'"> stripsymbols</_StripSymbolsArg>
+      <_StripSymbolsArg Condition="'$(BuildNativeStripSymbols)' == 'true' and '$(OSGroup)' != 'WebAssembly'"> stripsymbols</_StripSymbolsArg>
       <_PortableBuildArg Condition="'$(PortableBuild)' == 'true'"> -portable</_PortableBuildArg>
 
       <!--

src/libraries/Native/build-native.sh

@@ -137,6 +137,14 @@ build_native()
     echo "Commencing build of corefx native components for $__BuildOS.$__BuildArch.$__BuildType"
     cd "$__IntermediatesDir"
 
+    if [ "$__BuildArch" == "wasm" ]; then
+        if [ "$EMSDK_PATH" == "" ]; then
+            echo "Error: Should set EMSDK_PATH environment variable pointing to emsdk root."
+            exit 1
+        fi
+        source $EMSDK_PATH/emsdk_env.sh
+    fi
+
     # Regenerate the CMake solution
     if [ "$__GccBuild" = 0 ]; then
         echo "Invoking \"$__nativeroot/gen-buildsys-clang.sh\" \"$__nativeroot\" \"$__ClangMajorVersion\" \"$__ClangMinorVersion\" \"$__BuildArch\" \"$__CMakeArgs\" \"$__CMakeExtraArgs\""
@@ -260,6 +268,9 @@ while :; do
         arm64|-arm64)
             __BuildArch=arm64
             ;;
+        wasm|-wasm)
+            __BuildArch=wasm
+            ;;
         debug|-debug)
             __BuildType=Debug
             ;;
@@ -388,10 +399,11 @@ while :; do
     shift
 done
 
+# Set cross build
+if [ $__BuildArch != wasm ]; then
     __CMakeExtraArgs="$__CMakeExtraArgs -DFEATURE_DISTRO_AGNOSTIC_SSL=$__PortableBuild"
     __CMakeExtraArgs="$__CMakeExtraArgs -DCMAKE_STATIC_LIB_LINK=$__StaticLibLink"
 
-# Set cross build
     case $CPUName in
         i686)
             if [ $__BuildArch != x86 ]; then
@@ -406,6 +418,7 @@ case $CPUName in
             fi
             ;;
     esac
+fi
 
 # Set the default clang version if not already set
 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then

src/libraries/Native/native-binplace.proj

@@ -19,6 +19,7 @@
       <BinPlaceItem Include="$(NativeBinDir)/*.pdb" />
       <BinPlaceItem Include="$(NativeBinDir)/*.lib" />
       <BinPlaceItem Include="$(NativeBinDir)/*.a" />
+      <BinPlaceItem Include="$(NativeBinDir)/*.bc" />
       <BinPlaceItem Include="$(NativeBinDir)/*.so" />
       <BinPlaceItem Include="$(NativeBinDir)/*.dbg" />
       <BinPlaceItem Include="$(NativeBinDir)/*.dylib" />

src/libraries/System.Data.Odbc/src/System.Data.Odbc.csproj

@@ -121,7 +121,7 @@
       <Link>Common\Interop\Interop.Odbc.cs</Link>
     </Compile>
   </ItemGroup>
-  <ItemGroup Condition="'$(TargetsLinux)' == 'true' OR '$(TargetsFreeBSD)' == 'true' OR '$(TargetsWebAssembly)'=='true'">
+  <ItemGroup Condition="'$(TargetsLinux)' == 'true' OR '$(TargetsFreeBSD)' == 'true'">
     <Compile Include="$(CommonPath)\Interop\Linux\Interop.Libraries.cs">
       <Link>Common\Interop\Linux\Interop.Libraries.cs</Link>
     </Compile>