Fix intermittent Graph500 hang on ARM64 by forcing a stable libfabric provider

VERSION

@@ -1 +1 @@
-3.3.16
+3.3.17

src/VirtualClient/VirtualClient.Actions.UnitTests/Graph500/Graph500ExecutorTests.cs

@@ -97,6 +97,25 @@ await executor.ExecuteAsync(EventContext.None, CancellationToken.None)
             }
         }
 
+        [Test]
+        [TestCase(PlatformID.Unix, Architecture.X64)]
+        [TestCase(PlatformID.Unix, Architecture.Arm64)]
+        public async Task Graph500ExecutorForcesAStableFabricProviderOnTheWorkloadProcess(PlatformID platform, Architecture architecture)
+        {
+            this.SetupTest(platform, architecture);
+            mockFixture.Parameters["Scale"] = Scale;
+            mockFixture.Parameters["EdgeFactor"] = EdgeFactor;
+            using (TestGraph500Executor executor = new TestGraph500Executor(this.mockFixture))
+            {
+                await executor.ExecuteAsync(EventContext.None, CancellationToken.None)
+                    .ConfigureAwait(false);
+
+                // The single-rank benchmark hangs intermittently (most often on ARM64) when MPICH selects the
+                // libfabric 'sockets' provider, so the executor forces the maintained 'tcp' provider.
+                Assert.AreEqual("tcp", this.mockFixture.Process.EnvironmentVariables["FI_PROVIDER"]);
+            }
+        }
+
         [Test]
         public async Task Graph500ExecutorLogsTheExpectedWorkloadMetrics()
         {

src/VirtualClient/VirtualClient.Actions/Graph500/Graph500Executor.cs

@@ -24,6 +24,19 @@ namespace VirtualClient.Actions
     [SupportedPlatforms("linux-arm64,linux-x64")]
     public class Graph500Executor : VirtualClientComponent
     {
+        /// <summary>
+        /// The libfabric provider selection environment variable honored by MPICH's OFI netmod.
+        /// </summary>
+        private const string FabricProviderVariable = "FI_PROVIDER";
+
+        /// <summary>
+        /// The libfabric provider forced for the Graph500 process. The default OFI 'sockets' provider
+        /// (e.g. on Ubuntu 22.04's libfabric 1.11) intermittently deadlocks in MPI_Ibarrier progress,
+        /// most often on ARM64, hanging the single-rank benchmark indefinitely. The maintained 'tcp'
+        /// provider does not exhibit the hang.
+        /// </summary>
+        private const string FabricProvider = "tcp";
+
         private IFileSystem fileSystem;
         private ISystemManagement systemManagement;
         private IPackageManager packageManager;
@@ -108,7 +121,21 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel
             {
                 await this.ExecuteCommandAsync("make", this.CompilerFlags, this.PackageDirectory, cancellationToken);
 
-                using (IProcessProxy process = await this.ExecuteCommandAsync(this.ExecutableFilePath, this.Scale + " " + this.EdgeFactor, this.PackageDirectory, telemetryContext, cancellationToken))
+                using (IProcessProxy process = await this.ExecuteCommandAsync(
+                    this.ExecutableFilePath,
+                    this.Scale + " " + this.EdgeFactor,
+                    this.PackageDirectory,
+                    telemetryContext,
+                    cancellationToken,
+                    beforeExecution: workloadProcess =>
+                    {
+                        // Graph500 runs as a single MPI rank and requires no network transport. On systems where
+                        // MPICH selects the OFI (libfabric) 'sockets' provider (e.g. Ubuntu 22.04 / libfabric 1.11),
+                        // that provider intermittently deadlocks in MPI_Ibarrier progress -- most often on ARM64 --
+                        // leaving the benchmark spinning indefinitely with no output. Forcing the maintained 'tcp'
+                        // provider avoids the hang without affecting results.
+                        workloadProcess.EnvironmentVariables[Graph500Executor.FabricProviderVariable] = Graph500Executor.FabricProvider;
+                    }))
                 {
                     if (!cancellationToken.IsCancellationRequested)
                     {