Add statically linked libnuma support for AOT (#80671)

* Add statically linked libnuma support for AOT

* Fix windows build

* Update src/coreclr/nativeaot/docs/compiling.md

* Align formatting in doc

* Address CR feedback

* Add ability to skip static PIE

* Cleanup pal.cpp

* Highlight LGPL with a note in doc

Co-authored-by: Jan Kotas <jkotas@microsoft.com>

src/coreclr/gc/unix/CMakeLists.txt

@@ -5,6 +5,7 @@ include(configure.cmake)
 
 set(GC_PAL_SOURCES
     gcenv.unix.cpp
+    numasupport.dynamic.cpp
     events.cpp
     cgroup.cpp)
 

src/coreclr/gc/unix/gcenv.unix.cpp

@@ -20,6 +20,7 @@
 #include "gcenv.os.h"
 #include "gcenv.unix.inl"
 #include "volatile.h"
+#include "numasupport.h"
 
 #if HAVE_SWAPCTL
 #include <sys/swap.h>
@@ -135,36 +136,6 @@ typedef cpuset_t cpu_set_t;
 #endif
 #endif // __APPLE__
 
-#if HAVE_NUMA_H
-
-#include <numa.h>
-#include <numaif.h>
-#include <dlfcn.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-// List of all functions from the numa library that are used
-#define FOR_ALL_NUMA_FUNCTIONS \
-    PER_FUNCTION_BLOCK(mbind) \
-    PER_FUNCTION_BLOCK(numa_available) \
-    PER_FUNCTION_BLOCK(numa_max_node) \
-    PER_FUNCTION_BLOCK(numa_node_of_cpu)
-
-// Declare pointers to all the used numa functions
-#define PER_FUNCTION_BLOCK(fn) extern decltype(fn)* fn##_ptr;
-FOR_ALL_NUMA_FUNCTIONS
-#undef PER_FUNCTION_BLOCK
-
-// Redefine all calls to numa functions as calls through pointers that are set
-// to the functions of libnuma in the initialization.
-#define mbind(...) mbind_ptr(__VA_ARGS__)
-#define numa_available() numa_available_ptr()
-#define numa_max_node() numa_max_node_ptr()
-#define numa_node_of_cpu(...) numa_node_of_cpu_ptr(__VA_ARGS__)
-
-#endif // HAVE_NUMA_H
-
 #if defined(HOST_ARM) || defined(HOST_ARM64) || defined(HOST_LOONGARCH64)
 #define SYSCONF_GET_NUMPROCS _SC_NPROCESSORS_CONF
 #else
@@ -233,120 +204,8 @@ uint32_t g_pageSizeUnixInl = 0;
 
 AffinitySet g_processAffinitySet;
 
-// The highest NUMA node available
+extern "C" int g_highestNumaNode;
-int g_highestNumaNode = 0;
+extern "C" bool g_numaAvailable;
-// Is numa available
-bool g_numaAvailable = false;
-
-void* g_numaHandle = nullptr;
-
-#if HAVE_NUMA_H
-#define PER_FUNCTION_BLOCK(fn) decltype(fn)* fn##_ptr;
-FOR_ALL_NUMA_FUNCTIONS
-#undef PER_FUNCTION_BLOCK
-
-#if defined(__linux__)
-static bool ShouldOpenLibNuma()
-{
-    // This is a simple heuristic to determine if libnuma.so should be opened.  There's
-    // no point in linking and resolving everything in this library if we're running on
-    // a system that's not NUMA-capable.
-    int fd = open("/sys/devices/system/node/possible", O_RDONLY | O_CLOEXEC);
-
-    if (fd == -1)
-    {
-        // sysfs might not be mounted, not available, or the interface might have
-        // changed.  Return `true' here so NUMASupportInitialize() can try initializing
-        // NUMA support with libnuma.
-        return true;
-    }
-
-    while (true)
-    {
-        char buffer[32];
-        ssize_t bytesRead = read(fd, buffer, 32);
-
-        if (bytesRead == -1 && errno == EINTR)
-        {
-            continue;
-        }
-
-        close(fd);
-
-        // If an unknown error happened (bytesRead < 0), or the file was empty
-        // (bytesRead = 0), let libnuma handle this.  Otherwise, if there's just
-        // one NUMA node, don't bother linking in libnuma.
-        return (bytesRead <= 0) ? true : strncmp(buffer, "0\n", bytesRead) != 0;
-    }
-}
-#else
-static bool ShouldOpenLibNuma()
-{
-    return true;
-}
-#endif // __linux__
-
-#endif // HAVE_NUMA_H
-
-// Initialize data structures for getting and setting thread affinities to processors and
-// querying NUMA related processor information.
-// On systems with no NUMA support, it behaves as if there was a single NUMA node with
-// a single group of processors.
-void NUMASupportInitialize()
-{
-#if HAVE_NUMA_H
-    if (!ShouldOpenLibNuma())
-    {
-        g_numaAvailable = false;
-        g_highestNumaNode = 0;
-        return;
-    }
-
-    g_numaHandle = dlopen("libnuma.so.1", RTLD_LAZY);
-    if (g_numaHandle == 0)
-    {
-        g_numaHandle = dlopen("libnuma.so.1.0.0", RTLD_LAZY);
-        if (g_numaHandle == 0)
-        {
-            g_numaHandle = dlopen("libnuma.so", RTLD_LAZY);
-        }
-    }
-    if (g_numaHandle != 0)
-    {
-#define PER_FUNCTION_BLOCK(fn) \
-    fn##_ptr = (decltype(fn)*)dlsym(g_numaHandle, #fn); \
-    if (fn##_ptr == NULL) { fprintf(stderr, "Cannot get symbol " #fn " from libnuma\n"); abort(); }
-FOR_ALL_NUMA_FUNCTIONS
-#undef PER_FUNCTION_BLOCK
-
-        if (numa_available() == -1)
-        {
-            dlclose(g_numaHandle);
-        }
-        else
-        {
-            g_numaAvailable = true;
-            g_highestNumaNode = numa_max_node();
-        }
-    }
-#endif // HAVE_NUMA_H
-    if (!g_numaAvailable)
-    {
-        // No NUMA
-        g_highestNumaNode = 0;
-    }
-}
-
-// Cleanup of the NUMA support data structures
-void NUMASupportCleanup()
-{
-#if HAVE_NUMA_H
-    if (g_numaAvailable)
-    {
-        dlclose(g_numaHandle);
-    }
-#endif // HAVE_NUMA_H
-}
 
 // Initialize the interface implementation
 // Return:

src/coreclr/gc/unix/numasupport.dynamic.cpp

@@ -0,0 +1,124 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#include "numasupport.h"
+
+// The highest NUMA node available
+int g_highestNumaNode = 0;
+// Is numa available
+bool g_numaAvailable = false;
+
+#if HAVE_NUMA_H
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <dlfcn.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#define PER_FUNCTION_BLOCK(fn) decltype(fn)* fn##_ptr;
+FOR_ALL_NUMA_FUNCTIONS
+#undef PER_FUNCTION_BLOCK
+
+void* g_numaHandle = nullptr;
+
+static bool ShouldOpenLibNuma()
+{
+#ifdef TARGET_LINUX
+    // This is a simple heuristic to determine if libnuma.so should be opened.  There's
+    // no point in linking and resolving everything in this library if we're running on
+    // a system that's not NUMA-capable.
+    int fd = open("/sys/devices/system/node/possible", O_RDONLY | O_CLOEXEC);
+
+    if (fd == -1)
+    {
+        // sysfs might not be mounted, not available, or the interface might have
+        // changed.  Return `true' here so NUMASupportInitialize() can try initializing
+        // NUMA support with libnuma.
+        return true;
+    }
+
+    while (true)
+    {
+        char buffer[32];
+        ssize_t bytesRead = read(fd, buffer, 32);
+
+        if (bytesRead == -1 && errno == EINTR)
+        {
+            continue;
+        }
+
+        close(fd);
+
+        // If an unknown error happened (bytesRead < 0), or the file was empty
+        // (bytesRead = 0), let libnuma handle this.  Otherwise, if there's just
+        // one NUMA node, don't bother linking in libnuma.
+        return (bytesRead <= 0) ? true : strncmp(buffer, "0\n", bytesRead) != 0;
+    }
+#else
+    return true;
+#endif // TARGET_LINUX
+}
+
+#endif // HAVE_NUMA_H
+
+// Initialize data structures for getting and setting thread affinities to processors and
+// querying NUMA related processor information.
+// On systems with no NUMA support, it behaves as if there was a single NUMA node with
+// a single group of processors.
+void NUMASupportInitialize()
+{
+#if HAVE_NUMA_H
+    if (!ShouldOpenLibNuma())
+    {
+        g_numaAvailable = false;
+        g_highestNumaNode = 0;
+        return;
+    }
+
+    g_numaHandle = dlopen("libnuma.so.1", RTLD_LAZY);
+    if (g_numaHandle == 0)
+    {
+        g_numaHandle = dlopen("libnuma.so.1.0.0", RTLD_LAZY);
+        if (g_numaHandle == 0)
+        {
+            g_numaHandle = dlopen("libnuma.so", RTLD_LAZY);
+        }
+    }
+    if (g_numaHandle != 0)
+    {
+#define PER_FUNCTION_BLOCK(fn) \
+    fn##_ptr = (decltype(fn)*)dlsym(g_numaHandle, #fn); \
+    if (fn##_ptr == NULL) { fprintf(stderr, "Cannot get symbol " #fn " from libnuma\n"); abort(); }
+FOR_ALL_NUMA_FUNCTIONS
+#undef PER_FUNCTION_BLOCK
+
+        if (numa_available() == -1)
+        {
+            dlclose(g_numaHandle);
+        }
+        else
+        {
+            g_numaAvailable = true;
+            g_highestNumaNode = numa_max_node();
+        }
+    }
+#endif // HAVE_NUMA_H
+    if (!g_numaAvailable)
+    {
+        // No NUMA
+        g_highestNumaNode = 0;
+    }
+}
+
+// Cleanup of the NUMA support data structures
+void NUMASupportCleanup()
+{
+#if HAVE_NUMA_H
+    if (g_numaAvailable)
+    {
+        dlclose(g_numaHandle);
+    }
+#endif // HAVE_NUMA_H
+}

src/coreclr/gc/unix/numasupport.h

@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#ifndef __NUMASUPPORT_H__
+#define __NUMASUPPORT_H__
+
+#include "config.gc.h"
+
+#if HAVE_NUMA_H
+
+#include <numa.h>
+#include <numaif.h>
+
+#endif // HAVE_NUMA_H
+
+void NUMASupportInitialize();
+void NUMASupportCleanup();
+
+#if HAVE_NUMA_H
+
+// List of all functions from the numa library that are used
+#define FOR_ALL_NUMA_FUNCTIONS \
+    PER_FUNCTION_BLOCK(mbind) \
+    PER_FUNCTION_BLOCK(numa_available) \
+    PER_FUNCTION_BLOCK(numa_max_node) \
+    PER_FUNCTION_BLOCK(numa_node_of_cpu)
+
+// Declare pointers to all the used numa functions
+#define PER_FUNCTION_BLOCK(fn) extern decltype(fn)* fn##_ptr;
+FOR_ALL_NUMA_FUNCTIONS
+#undef PER_FUNCTION_BLOCK
+
+// Redefine all calls to numa functions as calls through pointers that are set
+// to the functions of libnuma in the initialization.
+#define mbind(...) mbind_ptr(__VA_ARGS__)
+#define numa_available() numa_available_ptr()
+#define numa_max_node() numa_max_node_ptr()
+#define numa_node_of_cpu(...) numa_node_of_cpu_ptr(__VA_ARGS__)
+
+#endif // HAVE_NUMA_H
+
+#endif // __NUMASUPPORT_H__

src/coreclr/gc/unix/numasupport.static.cpp

@@ -0,0 +1,32 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#include "numasupport.h"
+
+#if HAVE_NUMA_H
+#define PER_FUNCTION_BLOCK(fn) decltype(fn)* fn##_ptr = fn;
+FOR_ALL_NUMA_FUNCTIONS
+#undef PER_FUNCTION_BLOCK
+
+#endif // HAVE_NUMA_H
+
+// The highest NUMA node available
+int g_highestNumaNode = 0;
+// Is numa available
+bool g_numaAvailable = false;
+
+void NUMASupportInitialize()
+{
+#if HAVE_NUMA_H
+    if (numa_available() != -1)
+    {
+        g_numaAvailable = true;
+        g_highestNumaNode = numa_max_node();
+    }
+#endif // HAVE_NUMA_H
+}
+
+void NUMASupportCleanup()
+{
+    // nop
+}

src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets

@@ -50,6 +50,8 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeLibrary Condition="'$(NativeLib)' != ''" Include="$(IlcSdkPath)libbootstrapperdll.a" />
       <NativeLibrary Include="$(IlcSdkPath)$(FullRuntimeName).a" />
       <NativeLibrary Condition="'$(LinkStandardCPlusPlusLibrary)' != 'true' and '$(StaticICULinking)' != 'true'" Include="$(IlcSdkPath)libstdc++compat.a" />
+      <NativeLibrary Condition="'$(StaticNumaLinking)' == 'true'" Include="$(IlcSdkPath)libnumasupportstatic.a" />
+      <NativeLibrary Condition="'$(StaticNumaLinking)' != 'true'" Include="$(IlcSdkPath)libnumasupportdynamic.a" />
     </ItemGroup>
 
     <ItemGroup>
@@ -69,6 +71,12 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeLibrary Include="@(NetCoreAppNativeLibrary->'%(EscapedPath)')" />
     </ItemGroup>
 
+    <ItemGroup Condition="'$(StaticNumaLinking)' == 'true' and '$(NativeLib)' != 'Static'">
+      <StaticNumaLibs Include="-Wl,-Bstatic" Condition="'$(StaticExecutable)' != 'true'" />
+      <StaticNumaLibs Include="-lnuma" />
+      <StaticNumaLibs Include="-Wl,-Bdynamic" Condition="'$(StaticExecutable)' != 'true'" />
+    </ItemGroup>
+
     <ItemGroup Condition="'$(StaticICULinking)' == 'true' and '$(NativeLib)' != 'Static'">
       <NativeLibrary Include="$(IntermediateOutputPath)/libs/System.Globalization.Native/build/libSystem.Globalization.Native.a"/>
       <DirectPInvoke Include="libSystem.Globalization.Native" />
@@ -121,12 +129,14 @@ The .NET Foundation licenses this file to you under the MIT license.
       <LinkerArg Include="-lrt" Condition="'$(TargetOS)' != 'osx'" />
       <LinkerArg Include="-licucore" Condition="'$(TargetOS)' == 'osx'" />
       <LinkerArg Include="-L/usr/lib/swift" Condition="'$(TargetOS)' == 'osx'" />
+      <LinkerArg Include="@(StaticNumaLibs)" Condition="'$(StaticNumaLinking)' == 'true'" />
       <LinkerArg Include="@(StaticICULibs)" Condition="'$(StaticICULinking)' == 'true'" />
       <LinkerArg Include="@(StaticSslLibs)" Condition="'$(StaticOpenSslLinking)' == 'true'" />
       <LinkerArg Include="-dynamiclib" Condition="'$(TargetOS)' == 'osx' and '$(NativeLib)' == 'Shared'" />
       <LinkerArg Include="-shared" Condition="'$(TargetOS)' != 'osx' and '$(NativeLib)' == 'Shared'" />
       <!-- binskim warning BA3001 PIE disabled on executable -->
-      <LinkerArg Include="-pie" Condition="'$(TargetOS)' != 'osx' and '$(NativeLib)' == '' and '$(PositionIndependentExecutable)' != 'false'" />
+      <LinkerArg Include="-pie" Condition="'$(TargetOS)' != 'osx' and '$(NativeLib)' == '' and '$(StaticExecutable)' != 'true' and '$(PositionIndependentExecutable)' != 'false'" />
+      <LinkerArg Include="-static-pie" Condition="'$(StaticExecutable)' == 'true' and '$(PositionIndependentExecutable)' != 'false'" />
       <!-- binskim warning BA3010 The GNU_RELRO segment is missing -->
       <LinkerArg Include="-Wl,-z,relro" Condition="'$(TargetOS)' != 'osx'" />
       <!-- binskim warning BA3011 The BIND_NOW flag is missing -->

src/coreclr/nativeaot/Runtime/CMakeLists.txt

@@ -263,3 +263,11 @@ if(NOT CLR_CMAKE_TARGET_ARCH_WASM)
 else()
   add_subdirectory(Portable)
 endif()
+
+if (CLR_CMAKE_TARGET_UNIX)
+  add_library(numasupportdynamic STATIC ${GC_DIR}/unix/numasupport.dynamic.cpp)
+  install_static_library(numasupportdynamic aotsdk nativeaot)
+
+  add_library(numasupportstatic STATIC ${GC_DIR}/unix/numasupport.static.cpp)
+  install_static_library(numasupportstatic aotsdk nativeaot)
+endif(CLR_CMAKE_TARGET_UNIX)

src/coreclr/nativeaot/docs/compiling.md

@@ -84,15 +84,17 @@ You can use this feature by adding the `StaticICULinking` property to your proje
 
 This feature is only supported on Linux. This feature is not supported when crosscompiling.
 
+License (Unicode): https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE
+
 ### Prerequisites
 
-Ubuntu (20.04+)
+Ubuntu
-```
+```sh
-sudo apt-get install libicu-dev cmake
+apt install libicu-dev cmake
 ```
 
 Alpine
-```
+```sh
 apk add cmake icu-static icu-dev
 ```
 
@@ -111,9 +113,43 @@ You can use this feature by adding the `StaticOpenSslLinking` property to your p
 
 This feature is only supported on Linux. This feature is not supported when crosscompiling.
 
+License for OpenSSL v3+ (Apache v2.0): https://github.com/openssl/openssl/blob/master/LICENSE.txt
+License for OpenSSL releases prior to v3 (dual OpenSSL and SSLeay license): https://www.openssl.org/source/license-openssl-ssleay.txt
+
 ### Prerequisites
 
-Alpine
+Ubuntu
+```sh
+apt install libssl-dev cmake
 ```
+
+Alpine
+```sh
 apk add cmake openssl-dev openssl-libs-static
 ```
+
+## Using statically linked NUMA
+This feature can statically link NUMA library (libnuma.a) into your applications at build time.
+NativeAOT binaries built with this feature can run even when NUMA libraries are not installed.
+
+You can use this feature by adding the `StaticNumaLinking` property to your project file as follows:
+
+```xml
+<PropertyGroup>
+  <StaticNumaLinking>true</StaticNumaLinking>
+</PropertyGroup>
+```
+
+License (LGPL v2.1): https://github.com/numactl/numactl/blob/master/LICENSE.LGPL2.1. Note that this license imposes specific requirements on distribution of statically linked binaries.
+
+### Prerequisites
+
+Ubuntu
+```sh
+apt install libnuma-dev
+```
+
+Alpine
+```sh
+apk add numactl-dev
+```

src/coreclr/pal/src/include/pal/numa.h

@@ -1,38 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-
-/*++
-
-
-
-Module Name:
-
-    include/pal/numa.h
-
-Abstract:
-
-    Header file for the NUMA functions.
-
-
-
---*/
-
-#ifndef _PAL_NUMA_H_
-#define _PAL_NUMA_H_
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif // __cplusplus
-
-BOOL
-NUMASupportInitialize();
-
-VOID
-NUMASupportCleanup();
-
-#ifdef __cplusplus
-}
-#endif // __cplusplus
-
-#endif /* _PAL_CRITSECT_H_ */

src/coreclr/pal/src/init/pal.cpp

@@ -36,7 +36,6 @@ SET_DEFAULT_DEBUG_CHANNEL(PAL); // some headers have code with asserts, so do th
 #include "pal/utils.h"
 #include "pal/debug.h"
 #include "pal/init.h"
-#include "pal/numa.h"
 #include "pal/stackstring.hpp"
 #include "pal/cgroup.h"
 #include <minipal/getexepath.h>
@@ -692,7 +691,6 @@ Initialize(
     }
     goto done;
 
-    NUMASupportCleanup();
     /* No cleanup required for CRTInitStdStreams */
 CLEANUP15:
     FILECleanupStdHandles();