1
0
Fork 0
mirror of https://github.com/VSadov/Satori.git synced 2025-06-09 09:34:49 +09:00

Simplify numasupport (#84207)

* Simplify numasupport

* short-circuit

* Cleanup from QUIC readme

* Address CR feedback: early bail for < 2 NUMA nodes

* Fix node numbering, which is 0-based
This commit is contained in:
Adeel Mujahid 2023-04-07 02:23:56 +03:00 committed by GitHub
parent aca37006c4
commit de5a04b2d7
Signed by: github
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 101 additions and 252 deletions

View file

@ -23,7 +23,6 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
libicu-dev \
liblttng-ust-dev \
libssl-dev \
libnuma-dev \
libkrb5-dev \
zlib1g-dev \
ninja-build

View file

@ -23,7 +23,6 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
libicu-dev \
liblttng-ust-dev \
libssl-dev \
libnuma-dev \
libkrb5-dev \
zlib1g-dev \
ninja-build

View file

@ -37,14 +37,13 @@ Install the following packages for the toolchain:
* liblttng-ust-dev
* libssl-dev
* libkrb5-dev
* libnuma-dev (optional, enables numa support)
* zlib1g-dev
* ninja-build (optional, enables building native code with ninja instead of make)
```bash
sudo apt install -y cmake llvm lld clang build-essential \
python-is-python3 curl git lldb libicu-dev liblttng-ust-dev \
libssl-dev libnuma-dev libkrb5-dev zlib1g-dev ninja-build
libssl-dev libkrb5-dev zlib1g-dev ninja-build
```
You now have all the required components.

View file

@ -28,7 +28,7 @@ case "$os" in
apt update
apt install -y build-essential gettext locales cmake llvm clang lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \
libssl-dev libkrb5-dev libnuma-dev zlib1g-dev
libssl-dev libkrb5-dev zlib1g-dev
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
;;

View file

@ -5,7 +5,7 @@ include(configure.cmake)
set(GC_PAL_SOURCES
gcenv.unix.cpp
numasupport.dynamic.cpp
numasupport.cpp
events.cpp
cgroup.cpp)

View file

@ -11,7 +11,6 @@
#cmakedefine01 HAVE_VM_FLAGS_SUPERPAGE_SIZE_ANY
#cmakedefine01 HAVE_MAP_HUGETLB
#cmakedefine01 HAVE_SCHED_GETCPU
#cmakedefine01 HAVE_NUMA_H
#cmakedefine01 HAVE_VM_ALLOCATE
#cmakedefine01 HAVE_SWAPCTL
#cmakedefine01 HAVE_SYSCTLBYNAME

View file

@ -10,7 +10,6 @@ include(CheckLibraryExists)
check_include_files(sys/time.h HAVE_SYS_TIME_H)
check_include_files(sys/mman.h HAVE_SYS_MMAN_H)
check_include_files(numa.h HAVE_NUMA_H)
check_include_files(pthread_np.h HAVE_PTHREAD_NP_H)
check_function_exists(vm_allocate HAVE_VM_ALLOCATE)

View file

@ -318,7 +318,6 @@ void GCToOSInterface::Shutdown()
munmap(g_helperPage, OS_PAGE_SIZE);
CleanupCGroup();
NUMASupportCleanup();
}
// Get numeric id of the current thread if possible on the
@ -615,7 +614,7 @@ bool GCToOSInterface::VirtualCommit(void* address, size_t size, uint16_t node)
}
#endif
#if HAVE_NUMA_H
#ifdef TARGET_LINUX
if (success && g_numaAvailable && (node != NUMA_NODE_UNDEFINED))
{
if ((int)node <= g_highestNumaNode)
@ -628,12 +627,12 @@ bool GCToOSInterface::VirtualCommit(void* address, size_t size, uint16_t node)
int index = node / sizeof(unsigned long);
nodeMask[index] = ((unsigned long)1) << (node & (sizeof(unsigned long) - 1));
int st = mbind(address, size, MPOL_PREFERRED, nodeMask, usedNodeMaskBits, 0);
int st = BindMemoryPolicy(address, size, nodeMask, usedNodeMaskBits);
assert(st == 0);
// If the mbind fails, we still return the allocated memory since the node is just a hint
}
}
#endif // HAVE_NUMA_H
#endif // TARGET_LINUX
return success;
}
@ -1430,14 +1429,14 @@ bool GCToOSInterface::GetProcessorForHeap(uint16_t heap_number, uint16_t* proc_n
if (availableProcNumber == heap_number)
{
*proc_no = procNumber;
#if HAVE_NUMA_H
#ifdef TARGET_LINUX
if (GCToOSInterface::CanEnableGCNumaAware())
{
int result = numa_node_of_cpu(procNumber);
int result = GetNumaNodeNumByCpu(procNumber);
*node_no = (result >= 0) ? (uint16_t)result : NUMA_NODE_UNDEFINED;
}
else
#endif // HAVE_NUMA_H
#endif // TARGET_LINUX
{
*node_no = NUMA_NODE_UNDEFINED;
}

View file

@ -0,0 +1,86 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#include "numasupport.h"
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <stdio.h>
#include <dirent.h>
#include <string.h>
#include <sys/syscall.h>
#include <minipal/utils.h>
// The highest NUMA node available
int g_highestNumaNode = 0;
// Is numa available
bool g_numaAvailable = false;
#ifdef TARGET_LINUX
static int GetNodeNum(const char* path, bool firstOnly)
{
DIR *dir;
struct dirent *entry;
int result = -1;
dir = opendir(path);
if (dir)
{
while ((entry = readdir(dir)) != NULL)
{
if (strncmp(entry->d_name, "node", STRING_LENGTH("node")))
continue;
int nodeNum = strtoul(entry->d_name + STRING_LENGTH("node"), NULL, 0);
if (result < nodeNum)
result = nodeNum;
if (firstOnly)
break;
}
closedir(dir);
}
return result;
}
#endif
void NUMASupportInitialize()
{
#ifdef TARGET_LINUX
if (syscall(__NR_get_mempolicy, NULL, NULL, 0, 0, 0) < 0 && errno == ENOSYS)
return;
int highestNumaNode = GetNodeNum("/sys/devices/system/node", false);
// we only use this implementation when there are two or more NUMA nodes available
if (highestNumaNode < 1)
return;
g_numaAvailable = true;
g_highestNumaNode = highestNumaNode;
#endif
}
int GetNumaNodeNumByCpu(int cpu)
{
#ifdef TARGET_LINUX
char path[64];
if (snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%d", cpu) < 0)
return -1;
return GetNodeNum(path, true);
#else
return -1;
#endif
}
long BindMemoryPolicy(void* start, unsigned long len, const unsigned long* nodemask, unsigned long maxnode)
{
#ifdef TARGET_LINUX
return syscall(__NR_mbind, (long)start, len, 1, (long)nodemask, maxnode, 0);
#else
return -1;
#endif
}

View file

@ -1,124 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#include "numasupport.h"
// The highest NUMA node available
int g_highestNumaNode = 0;
// Is numa available
bool g_numaAvailable = false;
#if HAVE_NUMA_H
#include <unistd.h>
#include <errno.h>
#include <stdio.h>
#include <dlfcn.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#define PER_FUNCTION_BLOCK(fn) decltype(fn)* fn##_ptr;
FOR_ALL_NUMA_FUNCTIONS
#undef PER_FUNCTION_BLOCK
void* g_numaHandle = nullptr;
static bool ShouldOpenLibNuma()
{
#ifdef TARGET_LINUX
// This is a simple heuristic to determine if libnuma.so should be opened. There's
// no point in linking and resolving everything in this library if we're running on
// a system that's not NUMA-capable.
int fd = open("/sys/devices/system/node/possible", O_RDONLY | O_CLOEXEC);
if (fd == -1)
{
// sysfs might not be mounted, not available, or the interface might have
// changed. Return `true' here so NUMASupportInitialize() can try initializing
// NUMA support with libnuma.
return true;
}
while (true)
{
char buffer[32];
ssize_t bytesRead = read(fd, buffer, 32);
if (bytesRead == -1 && errno == EINTR)
{
continue;
}
close(fd);
// If an unknown error happened (bytesRead < 0), or the file was empty
// (bytesRead = 0), let libnuma handle this. Otherwise, if there's just
// one NUMA node, don't bother linking in libnuma.
return (bytesRead <= 0) ? true : strncmp(buffer, "0\n", bytesRead) != 0;
}
#else
return true;
#endif // TARGET_LINUX
}
#endif // HAVE_NUMA_H
// Initialize data structures for getting and setting thread affinities to processors and
// querying NUMA related processor information.
// On systems with no NUMA support, it behaves as if there was a single NUMA node with
// a single group of processors.
void NUMASupportInitialize()
{
#if HAVE_NUMA_H
if (!ShouldOpenLibNuma())
{
g_numaAvailable = false;
g_highestNumaNode = 0;
return;
}
g_numaHandle = dlopen("libnuma.so.1", RTLD_LAZY);
if (g_numaHandle == 0)
{
g_numaHandle = dlopen("libnuma.so.1.0.0", RTLD_LAZY);
if (g_numaHandle == 0)
{
g_numaHandle = dlopen("libnuma.so", RTLD_LAZY);
}
}
if (g_numaHandle != 0)
{
#define PER_FUNCTION_BLOCK(fn) \
fn##_ptr = (decltype(fn)*)dlsym(g_numaHandle, #fn); \
if (fn##_ptr == NULL) { fprintf(stderr, "Cannot get symbol " #fn " from libnuma\n"); abort(); }
FOR_ALL_NUMA_FUNCTIONS
#undef PER_FUNCTION_BLOCK
if (numa_available() == -1)
{
dlclose(g_numaHandle);
}
else
{
g_numaAvailable = true;
g_highestNumaNode = numa_max_node();
}
}
#endif // HAVE_NUMA_H
if (!g_numaAvailable)
{
// No NUMA
g_highestNumaNode = 0;
}
}
// Cleanup of the NUMA support data structures
void NUMASupportCleanup()
{
#if HAVE_NUMA_H
if (g_numaAvailable)
{
dlclose(g_numaHandle);
}
#endif // HAVE_NUMA_H
}

View file

@ -4,39 +4,8 @@
#ifndef __NUMASUPPORT_H__
#define __NUMASUPPORT_H__
#include "config.gc.h"
#if HAVE_NUMA_H
#include <numa.h>
#include <numaif.h>
#endif // HAVE_NUMA_H
void NUMASupportInitialize();
void NUMASupportCleanup();
#if HAVE_NUMA_H
// List of all functions from the numa library that are used
#define FOR_ALL_NUMA_FUNCTIONS \
PER_FUNCTION_BLOCK(mbind) \
PER_FUNCTION_BLOCK(numa_available) \
PER_FUNCTION_BLOCK(numa_max_node) \
PER_FUNCTION_BLOCK(numa_node_of_cpu)
// Declare pointers to all the used numa functions
#define PER_FUNCTION_BLOCK(fn) extern decltype(fn)* fn##_ptr;
FOR_ALL_NUMA_FUNCTIONS
#undef PER_FUNCTION_BLOCK
// Redefine all calls to numa functions as calls through pointers that are set
// to the functions of libnuma in the initialization.
#define mbind(...) mbind_ptr(__VA_ARGS__)
#define numa_available() numa_available_ptr()
#define numa_max_node() numa_max_node_ptr()
#define numa_node_of_cpu(...) numa_node_of_cpu_ptr(__VA_ARGS__)
#endif // HAVE_NUMA_H
int GetNumaNodeNumByCpu(int cpu);
long BindMemoryPolicy(void* start, unsigned long len, const unsigned long* nodemask, unsigned long maxnode);
#endif // __NUMASUPPORT_H__

View file

@ -1,32 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#include "numasupport.h"
#if HAVE_NUMA_H
#define PER_FUNCTION_BLOCK(fn) decltype(fn)* fn##_ptr = fn;
FOR_ALL_NUMA_FUNCTIONS
#undef PER_FUNCTION_BLOCK
#endif // HAVE_NUMA_H
// The highest NUMA node available
int g_highestNumaNode = 0;
// Is numa available
bool g_numaAvailable = false;
void NUMASupportInitialize()
{
#if HAVE_NUMA_H
if (numa_available() != -1)
{
g_numaAvailable = true;
g_highestNumaNode = numa_max_node();
}
#endif // HAVE_NUMA_H
}
void NUMASupportCleanup()
{
// nop
}

View file

@ -58,8 +58,6 @@ The .NET Foundation licenses this file to you under the MIT license.
<NativeLibrary Include="$(IlcSdkPath)$(FullRuntimeName).a" />
<NativeLibrary Include="$(IlcSdkPath)$(EventPipeName)$(LibFileExt)" />
<NativeLibrary Condition="'$(LinkStandardCPlusPlusLibrary)' != 'true' and '$(StaticICULinking)' != 'true'" Include="$(IlcSdkPath)libstdc++compat.a" />
<NativeLibrary Condition="'$(StaticNumaLinking)' == 'true'" Include="$(IlcSdkPath)libnumasupportstatic.a" />
<NativeLibrary Condition="'$(StaticNumaLinking)' != 'true'" Include="$(IlcSdkPath)libnumasupportdynamic.a" />
</ItemGroup>
<ItemGroup>
@ -81,12 +79,6 @@ The .NET Foundation licenses this file to you under the MIT license.
<NativeLibrary Include="@(NetCoreAppNativeLibrary->'%(EscapedPath)')" />
</ItemGroup>
<ItemGroup Condition="'$(StaticNumaLinking)' == 'true' and '$(NativeLib)' != 'Static'">
<StaticNumaLibs Include="-Wl,-Bstatic" Condition="'$(StaticExecutable)' != 'true'" />
<StaticNumaLibs Include="-lnuma" />
<StaticNumaLibs Include="-Wl,-Bdynamic" Condition="'$(StaticExecutable)' != 'true'" />
</ItemGroup>
<ItemGroup Condition="'$(StaticICULinking)' == 'true' and '$(NativeLib)' != 'Static'">
<NativeLibrary Include="$(IntermediateOutputPath)libs/System.Globalization.Native/build/libSystem.Globalization.Native.a" />
<DirectPInvoke Include="libSystem.Globalization.Native" />
@ -134,7 +126,6 @@ The .NET Foundation licenses this file to you under the MIT license.
<LinkerArg Include="-lrt" Condition="'$(_IsApplePlatform)' != 'true'" />
<LinkerArg Include="-licucore" Condition="'$(_IsApplePlatform)' == 'true'" />
<LinkerArg Include="-L/usr/lib/swift" Condition="'$(_IsApplePlatform)' == 'true'" />
<LinkerArg Include="@(StaticNumaLibs)" Condition="'$(StaticNumaLinking)' == 'true'" />
<LinkerArg Include="@(StaticICULibs)" Condition="'$(StaticICULinking)' == 'true'" />
<LinkerArg Include="@(StaticSslLibs)" Condition="'$(StaticOpenSslLinking)' == 'true'" />
<LinkerArg Include="-lm" />

View file

@ -148,6 +148,7 @@ else()
list(APPEND COMMON_RUNTIME_SOURCES
unix/PalRedhawkUnix.cpp
${GC_DIR}/unix/gcenv.unix.cpp
${GC_DIR}/unix/numasupport.cpp
${GC_DIR}/unix/events.cpp
${GC_DIR}/unix/cgroup.cpp
)
@ -276,11 +277,3 @@ endif()
if(FEATURE_PERFTRACING)
add_subdirectory(eventpipe)
endif()
if (CLR_CMAKE_TARGET_UNIX)
add_library(numasupportdynamic STATIC ${GC_DIR}/unix/numasupport.dynamic.cpp)
install_static_library(numasupportdynamic aotsdk nativeaot)
add_library(numasupportstatic STATIC ${GC_DIR}/unix/numasupport.static.cpp)
install_static_library(numasupportstatic aotsdk nativeaot)
endif(CLR_CMAKE_TARGET_UNIX)

View file

@ -127,29 +127,3 @@ Alpine
```sh
apk add cmake openssl-dev openssl-libs-static
```
## Using statically linked NUMA
This feature can statically link NUMA library (libnuma.a) into your applications at build time.
NativeAOT binaries built with this feature can run even when NUMA libraries are not installed.
You can use this feature by adding the `StaticNumaLinking` property to your project file as follows:
```xml
<PropertyGroup>
<StaticNumaLinking>true</StaticNumaLinking>
</PropertyGroup>
```
License (LGPL v2.1): https://github.com/numactl/numactl/blob/master/LICENSE.LGPL2.1. Note that this license imposes specific requirements on distribution of statically linked binaries.
### Prerequisites
Ubuntu
```sh
apt install libnuma-dev
```
Alpine
```sh
apk add numactl-dev
```

View file

@ -19,7 +19,6 @@
#cmakedefine01 HAVE_RUNETYPE_H
#cmakedefine01 HAVE_GNU_LIBNAMES_H
#cmakedefine01 HAVE_PRCTL_H
#cmakedefine01 HAVE_NUMA_H
#cmakedefine01 HAVE_PTHREAD_NP_H
#cmakedefine01 HAVE_AUXV_HWCAP_H
#cmakedefine01 HAVE_SYS_PTRACE_H

View file

@ -46,7 +46,6 @@ check_include_files(lwp.h HAVE_LWP_H)
check_include_files(runetype.h HAVE_RUNETYPE_H)
check_include_files(semaphore.h HAVE_SEMAPHORE_H)
check_include_files(sys/prctl.h HAVE_PRCTL_H)
check_include_files(numa.h HAVE_NUMA_H)
check_include_files("sys/auxv.h;asm/hwcap.h" HAVE_AUXV_HWCAP_H)
check_include_files("sys/ptrace.h" HAVE_SYS_PTRACE_H)
check_symbol_exists(getauxval sys/auxv.h HAVE_GETAUXVAL)

View file

@ -5,7 +5,7 @@ FROM $SDK_BASE_IMAGE
WORKDIR /msquic
RUN apt-get update -y && \
apt-get upgrade -y && \
apt-get install -y cmake clang ruby-dev gem lttng-tools libssl-dev libnuma-dev && \
apt-get install -y cmake clang ruby-dev gem lttng-tools libssl-dev && \
gem install fpm
RUN git clone --recursive https://github.com/dotnet/msquic
RUN cd msquic/src/msquic && \

View file

@ -92,7 +92,7 @@ To consume the current main branch of msquic, we pull code from [dotnet/msquic](
WORKDIR /msquic
RUN apt-get update -y && \
apt-get upgrade -y && \
apt-get install -y cmake clang ruby-dev gem lttng-tools libssl-dev libnuma-dev && \
apt-get install -y cmake clang ruby-dev gem lttng-tools libssl-dev && \
gem install fpm
RUN git clone --recursive https://github.com/dotnet/msquic
RUN cd msquic/src/msquic && \
@ -120,4 +120,4 @@ Officially released `msquic.dll` is published to NuGet.org, see [Microsoft.Nativ
To consume MsQuic from the current main branch, we use [dotnet/msquic](https://github.com/dotnet/msquic) repository which will build and publish `msquic.dll` to the transport feed, e.g. [dotnet8-transport](https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet8-transport). And from there, it'll get flown into this repository via [Darc subscription](https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md). See https://github.com/dotnet/runtime/blob/bd540938a4830ee91dec5ee2d39545b2f69a19d5/eng/Version.Details.xml#L7-L10 and maestro-bot PR: https://github.com/dotnet/runtime/pull/71900.
System.Net.Quic [project file](https://github.com/dotnet/runtime/blob/0304f1f5157a8280fa093bdfc7cfb8d9f62e016f/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj) allows switching between those two options with [`UseQuicTransportPackage` property](https://github.com/dotnet/runtime/blob/0304f1f5157a8280fa093bdfc7cfb8d9f62e016f/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj#L15).
System.Net.Quic [project file](https://github.com/dotnet/runtime/blob/0304f1f5157a8280fa093bdfc7cfb8d9f62e016f/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj) allows switching between those two options with [`UseQuicTransportPackage` property](https://github.com/dotnet/runtime/blob/0304f1f5157a8280fa093bdfc7cfb8d9f62e016f/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj#L15).