From b0c010860006ecc6a53bcb2d32ea7f2b83a95064 Mon Sep 17 00:00:00 2001 From: Alex Williams-Ferreira Date: Thu, 9 Jul 2026 10:55:16 -0700 Subject: [PATCH] Fix intermittent Graph500 hang on ARM64 by forcing a stable libfabric provider Graph500 runs as a single MPI rank and needs no network transport, yet on systems where MPICH selects the OFI (libfabric) 'sockets' provider (e.g. Ubuntu 22.04 / libfabric 1.11) that provider intermittently deadlocks in MPI_Ibarrier progress -- most often on ARM64 -- leaving the benchmark process spinning indefinitely with no output. Because the executor waits on the process with no per-process timeout, Virtual Client stalls until the outer step timeout, so the QoS PERF-GRAPH500 arm64 goal never completes. Root cause was confirmed on an ARM64 VM: a gdb backtrace of the hung process showed run_sssp -> aml_barrier -> MPI_Test spinning inside libfabric's sockets provider (epoll_pwait) while a single-rank MPI_Ibarrier never completes. Force the maintained 'tcp' provider (FI_PROVIDER=tcp) via the process beforeExecution hook when launching the workload binary. This bypasses the buggy sockets provider without changing results and applies to both linux-x64 and linux-arm64. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- VERSION | 2 +- .../Graph500/Graph500ExecutorTests.cs | 19 ++++++++++++ .../Graph500/Graph500Executor.cs | 29 ++++++++++++++++++- 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index d93b12b9c9..38ec40252d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.16 \ No newline at end of file +3.3.17 \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Graph500/Graph500ExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Graph500/Graph500ExecutorTests.cs index 090c29e98f..966b241a81 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Graph500/Graph500ExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Graph500/Graph500ExecutorTests.cs @@ -97,6 +97,25 @@ await executor.ExecuteAsync(EventContext.None, CancellationToken.None) } } + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task Graph500ExecutorForcesAStableFabricProviderOnTheWorkloadProcess(PlatformID platform, Architecture architecture) + { + this.SetupTest(platform, architecture); + mockFixture.Parameters["Scale"] = Scale; + mockFixture.Parameters["EdgeFactor"] = EdgeFactor; + using (TestGraph500Executor executor = new TestGraph500Executor(this.mockFixture)) + { + await executor.ExecuteAsync(EventContext.None, CancellationToken.None) + .ConfigureAwait(false); + + // The single-rank benchmark hangs intermittently (most often on ARM64) when MPICH selects the + // libfabric 'sockets' provider, so the executor forces the maintained 'tcp' provider. + Assert.AreEqual("tcp", this.mockFixture.Process.EnvironmentVariables["FI_PROVIDER"]); + } + } + [Test] public async Task Graph500ExecutorLogsTheExpectedWorkloadMetrics() { diff --git a/src/VirtualClient/VirtualClient.Actions/Graph500/Graph500Executor.cs b/src/VirtualClient/VirtualClient.Actions/Graph500/Graph500Executor.cs index 3a97d2133e..05ab368c4e 100644 --- a/src/VirtualClient/VirtualClient.Actions/Graph500/Graph500Executor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Graph500/Graph500Executor.cs @@ -24,6 +24,19 @@ namespace VirtualClient.Actions [SupportedPlatforms("linux-arm64,linux-x64")] public class Graph500Executor : VirtualClientComponent { + /// + /// The libfabric provider selection environment variable honored by MPICH's OFI netmod. + /// + private const string FabricProviderVariable = "FI_PROVIDER"; + + /// + /// The libfabric provider forced for the Graph500 process. The default OFI 'sockets' provider + /// (e.g. on Ubuntu 22.04's libfabric 1.11) intermittently deadlocks in MPI_Ibarrier progress, + /// most often on ARM64, hanging the single-rank benchmark indefinitely. The maintained 'tcp' + /// provider does not exhibit the hang. + /// + private const string FabricProvider = "tcp"; + private IFileSystem fileSystem; private ISystemManagement systemManagement; private IPackageManager packageManager; @@ -108,7 +121,21 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel { await this.ExecuteCommandAsync("make", this.CompilerFlags, this.PackageDirectory, cancellationToken); - using (IProcessProxy process = await this.ExecuteCommandAsync(this.ExecutableFilePath, this.Scale + " " + this.EdgeFactor, this.PackageDirectory, telemetryContext, cancellationToken)) + using (IProcessProxy process = await this.ExecuteCommandAsync( + this.ExecutableFilePath, + this.Scale + " " + this.EdgeFactor, + this.PackageDirectory, + telemetryContext, + cancellationToken, + beforeExecution: workloadProcess => + { + // Graph500 runs as a single MPI rank and requires no network transport. On systems where + // MPICH selects the OFI (libfabric) 'sockets' provider (e.g. Ubuntu 22.04 / libfabric 1.11), + // that provider intermittently deadlocks in MPI_Ibarrier progress -- most often on ARM64 -- + // leaving the benchmark spinning indefinitely with no output. Forcing the maintained 'tcp' + // provider avoids the hang without affecting results. + workloadProcess.EnvironmentVariables[Graph500Executor.FabricProviderVariable] = Graph500Executor.FabricProvider; + })) { if (!cancellationToken.IsCancellationRequested) {