diff --git a/VERSION b/VERSION index fbb5333616..a9c846e62f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.14 \ No newline at end of file +3.3.15 diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/ScriptExecutor/ScriptExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/ScriptExecutor/ScriptExecutorTests.cs index 9a64117514..3b722bcac5 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/ScriptExecutor/ScriptExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/ScriptExecutor/ScriptExecutorTests.cs @@ -11,9 +11,11 @@ namespace VirtualClient.Actions using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Moq; using NUnit.Framework; + using VirtualClient.Common.Contracts; using VirtualClient.Common.Telemetry; using VirtualClient.Contracts; @@ -380,6 +382,147 @@ public void ScriptExecutorMovesTheLogFilesToCorrectDirectory_Unix(PlatformID pla } } + [Test] + [TestCase(PlatformID.Win32NT)] + [TestCase(PlatformID.Unix)] + public void ScriptExecutorPreservesTheLogDirectoryStructureOnDiskWhenRequested(PlatformID platform) + { + this.SetupTest(platform); + this.mockFixture.Parameters[nameof(ScriptExecutor.LogPaths)] = "*.log"; + this.mockFixture.Parameters[nameof(ScriptExecutor.PreserveDirectories)] = true; + + // A log file that exists within a sub-directory of the script/executable directory. + string nestedLogFile = "subfolder/file.log"; + bool logDirectoryStructurePreserved = false; + + // The relative path (from the script directory) that should be retained under the central logs directory. + this.mockFixture.FileSystem.Setup(fe => fe.Path.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns(nestedLogFile); + + // Only the file-name matching path (search pattern '*.log') should return the nested log file. + this.mockFixture.FileSystem.Setup(fe => fe.Directory.GetFiles(It.IsAny(), "*.log", SearchOption.AllDirectories)) + .Returns(new[] { nestedLogFile }); + + using (TestScriptExecutor executor = new TestScriptExecutor(this.mockFixture)) + { + this.mockFixture.File.Setup(fe => fe.Move(It.IsAny(), It.IsAny(), true)) + .Callback((sourcePath, destinitionPath, overwrite) => + { + if (sourcePath.EndsWith("file.log")) + { + logDirectoryStructurePreserved = destinitionPath.Contains("subfolder"); + } + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => this.mockFixture.Process; + + Assert.DoesNotThrowAsync(() => executor.ExecuteAsync(CancellationToken.None)); + Assert.IsTrue(logDirectoryStructurePreserved); + } + } + + [Test] + [TestCase(PlatformID.Win32NT)] + [TestCase(PlatformID.Unix)] + public void ScriptExecutorFlattensTheLogDirectoryStructureOnDiskByDefault(PlatformID platform) + { + this.SetupTest(platform); + this.mockFixture.Parameters[nameof(ScriptExecutor.LogPaths)] = "*.log"; + + // A log file that exists within a sub-directory of the script/executable directory. + string nestedLogFile = "subfolder/file.log"; + bool logDirectoryStructurePreserved = true; + + this.mockFixture.FileSystem.Setup(fe => fe.Path.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns(nestedLogFile); + + // Only the file-name matching path (search pattern '*.log') should return the nested log file. + this.mockFixture.FileSystem.Setup(fe => fe.Directory.GetFiles(It.IsAny(), "*.log", SearchOption.AllDirectories)) + .Returns(new[] { nestedLogFile }); + + using (TestScriptExecutor executor = new TestScriptExecutor(this.mockFixture)) + { + this.mockFixture.File.Setup(fe => fe.Move(It.IsAny(), It.IsAny(), true)) + .Callback((sourcePath, destinitionPath, overwrite) => + { + if (sourcePath.EndsWith("file.log")) + { + logDirectoryStructurePreserved = destinitionPath.Contains("subfolder"); + } + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => this.mockFixture.Process; + + Assert.DoesNotThrowAsync(() => executor.ExecuteAsync(CancellationToken.None)); + Assert.IsFalse(logDirectoryStructurePreserved); + } + } + + [Test] + [TestCase(PlatformID.Win32NT)] + [TestCase(PlatformID.Unix)] + public void ScriptExecutorPreservesTheDirectoryStructureInTheUploadPathWhenRequested(PlatformID platform) + { + this.SetupTest(platform); + this.mockFixture.Parameters[nameof(ScriptExecutor.PreserveDirectories)] = true; + + // A content store must exist for file uploads to be requested. + this.mockFixture.Dependencies.AddSingleton>(new List { this.mockFixture.ContentBlobManager.Object }); + + // A log file that was moved into a subfolder within the central logs directory. + string logsRootDirectory = this.mockFixture.Combine(this.mockFixture.PlatformSpecifics.LogsDirectory, "generictool"); + string movedLogFile = this.mockFixture.Combine(logsRootDirectory, "subfolder", "file.log"); + FileUploadDescriptor capturedDescriptor = null; + + // The upload request writes the descriptor as JSON. Capture it to inspect the blob path. + this.mockFixture.File.Setup(fe => fe.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, contents, token) => + { + capturedDescriptor = contents.FromJson(); + }) + .Returns(Task.CompletedTask); + + using (TestScriptExecutor executor = new TestScriptExecutor(this.mockFixture)) + { + Assert.DoesNotThrowAsync(() => executor.RequestLogUploadsAsync(new[] { movedLogFile }, logsRootDirectory)); + Assert.IsNotNull(capturedDescriptor); + Assert.IsTrue( + capturedDescriptor.BlobPath.Replace('\\', '/').Contains("/subfolder/"), + $"Expected the upload blob path to preserve the 'subfolder' directory but was '{capturedDescriptor.BlobPath}'."); + } + } + + [Test] + [TestCase(PlatformID.Win32NT)] + [TestCase(PlatformID.Unix)] + public void ScriptExecutorFlattensTheDirectoryStructureInTheUploadPathByDefault(PlatformID platform) + { + this.SetupTest(platform); + + // A content store must exist for file uploads to be requested. + this.mockFixture.Dependencies.AddSingleton>(new List { this.mockFixture.ContentBlobManager.Object }); + + string logsRootDirectory = this.mockFixture.Combine(this.mockFixture.PlatformSpecifics.LogsDirectory, "generictool"); + string movedLogFile = this.mockFixture.Combine(logsRootDirectory, "subfolder", "file.log"); + FileUploadDescriptor capturedDescriptor = null; + + this.mockFixture.File.Setup(fe => fe.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, contents, token) => + { + capturedDescriptor = contents.FromJson(); + }) + .Returns(Task.CompletedTask); + + using (TestScriptExecutor executor = new TestScriptExecutor(this.mockFixture)) + { + Assert.DoesNotThrowAsync(() => executor.RequestLogUploadsAsync(new[] { movedLogFile }, logsRootDirectory)); + Assert.IsNotNull(capturedDescriptor); + Assert.IsFalse( + capturedDescriptor.BlobPath.Replace('\\', '/').Contains("/subfolder/"), + $"Expected the upload blob path to be flattened but was '{capturedDescriptor.BlobPath}'."); + } + } + [Test] [TestCase(PlatformID.Win32NT)] [TestCase(PlatformID.Unix)] @@ -459,6 +602,11 @@ public TestScriptExecutor(MockFixture fixture) { return base.ExecuteAsync(telemetryContext, cancellationToken); } + + public new Task RequestLogUploadsAsync(IEnumerable logPaths, string logsRootDirectory = null) + { + return base.RequestLogUploadsAsync(logPaths, logsRootDirectory); + } } } } diff --git a/src/VirtualClient/VirtualClient.Actions/ScriptExecutor/ScriptExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ScriptExecutor/ScriptExecutor.cs index 734615bf95..1f467a5757 100644 --- a/src/VirtualClient/VirtualClient.Actions/ScriptExecutor/ScriptExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/ScriptExecutor/ScriptExecutor.cs @@ -111,6 +111,25 @@ public bool RunElevated } } + /// + /// True to preserve the directory structure of the script-generated log files (relative to the + /// script directory) both on the file system and within the content/blob store virtual paths + /// when uploaded. False to flatten all matched log files into the central logs directory root. + /// Default = false. + /// + public bool PreserveDirectories + { + get + { + return this.Parameters.GetValue(nameof(this.PreserveDirectories), false); + } + + set + { + this.Parameters[nameof(this.PreserveDirectories)] = value; + } + } + /// /// The full path to the script executable. /// @@ -313,15 +332,20 @@ protected async Task CaptureLogsAsync(CancellationToken cancellationToken) foreach (string logFilePath in this.fileSystem.Directory.GetFiles(fullLogPath, "*", SearchOption.AllDirectories)) { var logs = await this.MoveLogsAsync(logFilePath, destinationLogsDir, cancellationToken, sourceRootDirectory: fullLogPath); - await this.RequestLogUploadsAsync(logs); + await this.RequestLogUploadsAsync(logs, destinationLogsDir); } } // Check for Matching FileNames foreach (string logFilePath in this.fileSystem.Directory.GetFiles(this.ExecutableDirectory, logPath, SearchOption.AllDirectories)) { - var logs = await this.MoveLogsAsync(logFilePath, destinationLogsDir, cancellationToken); - await this.RequestLogUploadsAsync(logs); + var logs = await this.MoveLogsAsync( + logFilePath, + destinationLogsDir, + cancellationToken, + sourceRootDirectory: this.PreserveDirectories ? this.ExecutableDirectory : null); + + await this.RequestLogUploadsAsync(logs, destinationLogsDir); } } } @@ -330,19 +354,37 @@ protected async Task CaptureLogsAsync(CancellationToken cancellationToken) if (this.fileSystem.File.Exists(this.MetricsFilePath)) { var logs = await this.MoveLogsAsync(this.MetricsFilePath, destinationLogsDir, cancellationToken); - await this.RequestLogUploadsAsync(logs); + await this.RequestLogUploadsAsync(logs, destinationLogsDir); } } /// /// Requests a file upload for each of the log file paths provided. /// - protected async Task RequestLogUploadsAsync(IEnumerable logPaths) + /// The set of (already moved) log file paths to upload. + /// + /// The central logs directory the files were moved into. When is true, the + /// relative subdirectory of each file (under this root) is preserved in the content/blob store virtual path. + /// + protected async Task RequestLogUploadsAsync(IEnumerable logPaths, string logsRootDirectory = null) { if (logPaths?.Any() == true && this.TryGetContentStoreManager(out IBlobManager blobManager)) { foreach (string logPath in logPaths) { + string subPath = null; + + if (this.PreserveDirectories && !string.IsNullOrWhiteSpace(logsRootDirectory)) + { + // Preserve the relative subdirectory (under the central logs directory) in the + // blob/content store virtual path so the uploaded logs retain the directory structure. + string relativeSubdirectory = this.fileSystem.GetRelativeSubdirectory(logsRootDirectory, logPath); + if (!string.IsNullOrWhiteSpace(relativeSubdirectory)) + { + subPath = relativeSubdirectory; + } + } + FileUploadDescriptor descriptor = this.CreateFileUploadDescriptor( new FileContext( this.fileSystem.FileInfo.New(logPath), @@ -353,7 +395,8 @@ protected async Task RequestLogUploadsAsync(IEnumerable logPaths) this.ToolName, this.Scenario, null, - this.Roles?.FirstOrDefault())); + this.Roles?.FirstOrDefault()), + subPath: subPath); await this.RequestFileUploadAsync(descriptor); } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/EXAMPLE-EXECUTE-SCRIPT.json b/src/VirtualClient/VirtualClient.Main/profiles/EXAMPLE-EXECUTE-SCRIPT.json index 90cbea20fc..f0c0e3d8a1 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/EXAMPLE-EXECUTE-SCRIPT.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/EXAMPLE-EXECUTE-SCRIPT.json @@ -13,7 +13,8 @@ "PackageName": "exampleWorkload", "WorkloadPackage": "exampleworkload.1.1.0.zip", "FailFast": false, - "RunElevated": false + "RunElevated": false, + "PreserveDirectories": false }, "Actions": [ { @@ -27,6 +28,7 @@ "PackageName": "$.Parameters.PackageName", "FailFast": "$.Parameters.FailFast", "RunElevated": "$.Parameters.RunElevated", + "PreserveDirectories": "$.Parameters.PreserveDirectories", "Tags": "Test,VC,Script" } }