diff --git a/VERSION b/VERSION index d93b12b9c9..38ec40252d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.16 \ No newline at end of file +3.3.17 \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Graph500/Graph500ExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Graph500/Graph500ExecutorTests.cs index 090c29e98f..966b241a81 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Graph500/Graph500ExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Graph500/Graph500ExecutorTests.cs @@ -97,6 +97,25 @@ await executor.ExecuteAsync(EventContext.None, CancellationToken.None) } } + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task Graph500ExecutorForcesAStableFabricProviderOnTheWorkloadProcess(PlatformID platform, Architecture architecture) + { + this.SetupTest(platform, architecture); + mockFixture.Parameters["Scale"] = Scale; + mockFixture.Parameters["EdgeFactor"] = EdgeFactor; + using (TestGraph500Executor executor = new TestGraph500Executor(this.mockFixture)) + { + await executor.ExecuteAsync(EventContext.None, CancellationToken.None) + .ConfigureAwait(false); + + // The single-rank benchmark hangs intermittently (most often on ARM64) when MPICH selects the + // libfabric 'sockets' provider, so the executor forces the maintained 'tcp' provider. + Assert.AreEqual("tcp", this.mockFixture.Process.EnvironmentVariables["FI_PROVIDER"]); + } + } + [Test] public async Task Graph500ExecutorLogsTheExpectedWorkloadMetrics() { diff --git a/src/VirtualClient/VirtualClient.Actions/Graph500/Graph500Executor.cs b/src/VirtualClient/VirtualClient.Actions/Graph500/Graph500Executor.cs index 3a97d2133e..05ab368c4e 100644 --- a/src/VirtualClient/VirtualClient.Actions/Graph500/Graph500Executor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Graph500/Graph500Executor.cs @@ -24,6 +24,19 @@ namespace VirtualClient.Actions [SupportedPlatforms("linux-arm64,linux-x64")] public class Graph500Executor : VirtualClientComponent { + /// + /// The libfabric provider selection environment variable honored by MPICH's OFI netmod. + /// + private const string FabricProviderVariable = "FI_PROVIDER"; + + /// + /// The libfabric provider forced for the Graph500 process. The default OFI 'sockets' provider + /// (e.g. on Ubuntu 22.04's libfabric 1.11) intermittently deadlocks in MPI_Ibarrier progress, + /// most often on ARM64, hanging the single-rank benchmark indefinitely. The maintained 'tcp' + /// provider does not exhibit the hang. + /// + private const string FabricProvider = "tcp"; + private IFileSystem fileSystem; private ISystemManagement systemManagement; private IPackageManager packageManager; @@ -108,7 +121,21 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel { await this.ExecuteCommandAsync("make", this.CompilerFlags, this.PackageDirectory, cancellationToken); - using (IProcessProxy process = await this.ExecuteCommandAsync(this.ExecutableFilePath, this.Scale + " " + this.EdgeFactor, this.PackageDirectory, telemetryContext, cancellationToken)) + using (IProcessProxy process = await this.ExecuteCommandAsync( + this.ExecutableFilePath, + this.Scale + " " + this.EdgeFactor, + this.PackageDirectory, + telemetryContext, + cancellationToken, + beforeExecution: workloadProcess => + { + // Graph500 runs as a single MPI rank and requires no network transport. On systems where + // MPICH selects the OFI (libfabric) 'sockets' provider (e.g. Ubuntu 22.04 / libfabric 1.11), + // that provider intermittently deadlocks in MPI_Ibarrier progress -- most often on ARM64 -- + // leaving the benchmark spinning indefinitely with no output. Forcing the maintained 'tcp' + // provider avoids the hang without affecting results. + workloadProcess.EnvironmentVariables[Graph500Executor.FabricProviderVariable] = Graph500Executor.FabricProvider; + })) { if (!cancellationToken.IsCancellationRequested) {