From 77295ac81a55ebb07b4f33a97adbc1379422c803 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Thu, 22 Jan 2026 11:46:17 -0600 Subject: [PATCH 1/5] Init Commit --- .../NCPS/NCPS_Example_Results_Client.txt | 49 +++ .../NCPS/NCPS_Example_Results_Server.txt | 42 ++ .../Network/NCPS/NCPSClientExecutorTests.cs | 187 +++++++++ .../Network/NCPS/NCPSMetricsParserTests.cs | 181 +++++++++ .../Network/NCPS/NCPSServerExecutorTests.cs | 153 ++++++++ .../NCPS/NCPSClientExecutor.cs | 70 ++++ .../NetworkingWorkload/NCPS/NCPSExecutor.cs | 364 ++++++++++++++++++ .../NCPS/NCPSMetricsParser.cs | 338 ++++++++++++++++ .../NCPS/NCPSServerExecutor.cs | 63 +++ .../NetworkingWorkloadExecutor.cs | 16 + .../NetworkingWorkloadState.cs | 7 +- .../NetworkConfigurationSetup.cs | 8 +- 12 files changed, 1475 insertions(+), 3 deletions(-) create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Client.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Server.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSClientExecutorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSServerExecutorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSClientExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSMetricsParser.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSServerExecutor.cs diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Client.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Client.txt new file mode 100644 index 0000000000..808f0c16ff --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Client.txt @@ -0,0 +1,49 @@ + T(sec) N Pend Failed IOFail Conn/s Close/s RXkbyte/s TXkbyte/s RT/i c0/i c0rtt/i cR/i cRrtt/i + 1.001 0 1600 0 0 24152.0 22552.0 24151.0 24153.0 0 24152 10543 0 0 + 2.001 0 1600 0 0 32069.0 32069.0 32069.0 32069.0 0 32069 9567 0 0 + 3.002 10 1590 0 0 30695.7 30685.7 30695.7 30695.7 0 30695 8778 0 0 + 4.002 4 1596 0 0 29827.0 29833.0 29827.0 29828.0 0 29827 8967 0 0 + 5.003 6 1594 0 0 28873.6 28873.6 28873.6 28873.6 0 28873 10034 0 0 + 6.003 8 1592 0 0 27841.0 27839.0 27841.0 27841.0 0 27841 9456 0 0 + 7.003 7 1593 0 0 26934.0 26935.0 26934.0 26934.0 0 26934 10234 0 0 + 8.003 0 1600 0 0 26123.0 26130.0 26123.0 26123.0 0 26123 11234 0 0 + 9.003 5 1595 0 0 25389.0 25384.0 25389.0 25389.0 0 25389 11875 0 0 + 10.004 0 1600 0 0 24720.9 24725.9 24720.9 24720.9 0 24720 13132 0 0 + +=== CMDLINE: ./ncps -c 10.1.0.5 -r 16 -bp 9800 -np 16 -N 1600 -P 1600 -D 0 -M 1 -i 1 -wt 30 -t 330 + +=== VERSION 1.0 + +###RXGBPS 0.65 +###TXGBPS 0.65 + +=== Time (ms) to Nth connection establishment for first 1250000 connections: +=== N T(ms) CPS +=== 100000 3925 25477 +=== 200000 7901 25316 +=== 300000 11932 25142 +=== 400000 15987 25021 +=== 500000 20062 24922 +=== 600000 24161 24828 +=== 700000 28262 24775 +=== 800000 32401 24690 +=== 900000 36541 24631 +=== 1000000 40707 24566 +=== 1100000 44883 24509 +=== 1200000 49081 24448 +=== 1250000 51188 24416 + +###ENDCPS 24416 + +###CPS,100000:3925,200000:7901,300000:11932,400000:15987,500000:20062,600000:24161,700000:28262,800000:32401,900000:36541,1000000:40707,1100000:44883,1200000:49081,1250000:51188 + +=== SYN RTT (us) stats for first 1250000 connections: +=== P25 Median Mean P75 P90 P95 P99 P99.9 P99.99 +=== 3424 6221 18737 10893 17628 24607 54222 2027000 4057000 + +###SYNRTT,25:3424,Median:6221,Mean:18737,75:10893,90:17628,95:24607,99:54222,99.9:2027000,99.99:4057000 + +=== Percentage of connections with retransmits in the first 1250000 connections: 3.2825% +=== Average retransmit count per connection (excluding 0-retransmit cases): 1.1436 + +###REXMIT,rtconnpercentage:3.2825,rtperconn:1.1436 diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Server.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Server.txt new file mode 100644 index 0000000000..891c88c77f --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Server.txt @@ -0,0 +1,42 @@ + T(sec) N Pend Failed IOFail Conn/s Close/s RXkbyte/s TXkbyte/s RT/i c0/i c0rtt/i cR/i cRrtt/i + 1.002 7 1593 0 0 69065.9 69058.9 69062.9 69065.9 0 69204 12888 0 0 + 2.002 14 1586 0 0 69987.0 69980.0 69978.0 69984.0 606 69381 12962 606 161224 + 3.002 0 1600 0 0 67533.0 67547.0 67545.0 67536.0 618 66915 11499 618 434771 + 4.002 8 1592 0 0 66157.0 66149.0 66149.0 66157.0 925 65332 12544 825 407273 + 5.002 12 1588 0 0 64957.0 64953.0 64953.0 64956.0 711 64301 13859 656 524382 + 6.003 23 1577 0 0 63437.6 63426.6 63430.6 63431.6 669 62916 14764 585 537452 + 7.003 3 1597 0 0 54750.0 54770.0 54766.0 54757.0 727 54029 9112 721 419237 + 8.003 6 1594 0 0 56852.0 56849.0 56849.0 56852.0 857 56003 10083 849 426095 + 9.003 2 1598 0 0 55323.0 55327.0 55328.0 55322.0 1143 54181 9149 1142 261028 + 10.003 0 1600 0 0 56011.0 56013.0 56012.0 56012.0 1197 54848 9935 1163 237388 + +=== CMDLINE: ./ncps -c 10.1.0.5 -wt 5 -t 15 + +=== VERSION 1.0 + +###RXGBPS 0.45 +###TXGBPS 0.45 + +=== Time (ms) to Nth connection establishment for first 556947 connections: +=== N T(ms) CPS +=== 100000 1678 59594 +=== 200000 3467 57686 +=== 300000 5248 57164 +=== 400000 7093 56393 +=== 500000 8919 56060 +=== 556947 10002 55683 + +###ENDCPS 55683 + +###CPS,100000:1678,200000:3467,300000:5248,400000:7093,500000:8919,556947:10002 + +=== SYN RTT (us) stats for first 556947 connections: +=== P25 Median Mean P75 P90 P95 P99 P99.9 P99.99 +=== 2342 4441 15996 13249 24443 38198 79657 1038000 1057000 + +###SYNRTT,25:2342,Median:4441,Mean:15996,75:13249,90:24443,95:38198,99:79657,99.9:1038000,99.99:1057000 + +=== Percentage of connections with retransmits in the first 556947 connections: 1.7278% +=== Average retransmit count per connection (excluding 0-retransmit cases): 1.0138 + +###REXMIT,rtconnpercentage:1.7278,rtperconn:1.0138 diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSClientExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSClientExecutorTests.cs new file mode 100644 index 0000000000..efa0566202 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSClientExecutorTests.cs @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Actions.NetworkPerformance; + using VirtualClient.Contracts; + using Polly; + using System.Net.Http; + using System.Net; + using System.IO; + using System.Reflection; + using VirtualClient.Common.Telemetry; + using VirtualClient.Common.Contracts; + + [TestFixture] + [Category("Unit")] + public class NCPSClientExecutorTests + { + private MockFixture mockFixture; + private DependencyPath mockPath; + private NetworkingWorkloadState networkingWorkloadState; + + [SetUp] + public void SetupTest() + { + this.mockFixture = new MockFixture(); + this.mockPath = new DependencyPath("NetworkingWorkload", this.mockFixture.PlatformSpecifics.GetPackagePath("networkingworkload")); + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(this.mockPath); + + this.mockFixture.Directory.Setup(d => d.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.Parameters["PackageName"] = "Networking"; + this.mockFixture.Parameters["ThreadCount"] = "16"; + this.mockFixture.Parameters["TestDuration"] = "00:05:00"; + this.mockFixture.Parameters["WarmupTime"] = "00:00:30"; + this.mockFixture.Parameters["Delaytime"] = "00:00:00"; + this.mockFixture.Parameters["ConfidenceLevel"] = "99"; + + string currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + string resultsPath = Path.Combine(currentDirectory, "Examples", "NCPS", "NCPS_Example_Results_Server.txt"); + string results = File.ReadAllText(resultsPath); + + this.mockFixture.Process.StandardOutput.Append(results); + + this.mockFixture.FileSystem.Setup(rt => rt.File.ReadAllTextAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(results); + + this.SetupNetworkingWorkloadState(); + } + + [Test] + public void NCPSClientExecutorThrowsOnUnsupportedOS() + { + this.mockFixture.SystemManagement.SetupGet(sm => sm.Platform).Returns(PlatformID.Other); + TestNCPSClientExecutor component = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.ThrowsAsync(() => component.ExecuteAsync(CancellationToken.None)); + } + + [Test] + public async Task NCPSClientExecutorExecutesAsExpected() + { + NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor networkingWorkloadExecutor = new NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + await networkingWorkloadExecutor.OnInitialize.Invoke(EventContext.None, CancellationToken.None); + + int processExecuted = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (file, arguments, workingDirectory) => + { + processExecuted++; + this.networkingWorkloadState.ToolState = NetworkingWorkloadToolState.Stopped; + var expectedStateItem = new Item(nameof(NetworkingWorkloadState), this.networkingWorkloadState); + + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, expectedStateItem)); + + return this.mockFixture.Process; + }; + + TestNCPSClientExecutor component = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + await component.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(1, processExecuted); + } + + [Test] + public async Task NCPSClientExecutorExecutesAsExpectedWithIntegerTimeParameters() + { + this.mockFixture.Parameters["TestDuration"] = 120; + this.mockFixture.Parameters["WarmupTime"] = 30; + this.mockFixture.Parameters["Delaytime"] = 15; + + NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor networkingWorkloadExecutor = new NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + await networkingWorkloadExecutor.OnInitialize.Invoke(EventContext.None, CancellationToken.None); + + int processExecuted = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (file, arguments, workingDirectory) => + { + processExecuted++; + this.networkingWorkloadState.ToolState = NetworkingWorkloadToolState.Stopped; + var expectedStateItem = new Item(nameof(NetworkingWorkloadState), this.networkingWorkloadState); + + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, expectedStateItem)); + + return this.mockFixture.Process; + }; + + TestNCPSClientExecutor component = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.AreEqual(TimeSpan.FromSeconds(120), component.TestDuration); + Assert.AreEqual(TimeSpan.FromSeconds(30), component.WarmupTime); + Assert.AreEqual(TimeSpan.FromSeconds(15), component.DelayTime); + + await component.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(1, processExecuted); + } + + [Test] + public void NCPSClientExecutorSupportsIntegerAndTimeSpanTimeFormats() + { + this.mockFixture.Parameters["TestDuration"] = 300; + this.mockFixture.Parameters["WarmupTime"] = 60; + this.mockFixture.Parameters["Delaytime"] = 30; + + TestNCPSClientExecutor executor = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.AreEqual(TimeSpan.FromSeconds(300), executor.TestDuration); + Assert.AreEqual(TimeSpan.FromSeconds(60), executor.WarmupTime); + Assert.AreEqual(TimeSpan.FromSeconds(30), executor.DelayTime); + + this.mockFixture.Parameters["TestDuration"] = "00:05:00"; + this.mockFixture.Parameters["WarmupTime"] = "00:01:00"; + this.mockFixture.Parameters["Delaytime"] = "00:00:30"; + + executor = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.AreEqual(TimeSpan.FromMinutes(5), executor.TestDuration); + Assert.AreEqual(TimeSpan.FromMinutes(1), executor.WarmupTime); + Assert.AreEqual(TimeSpan.FromSeconds(30), executor.DelayTime); + + this.mockFixture.Parameters["TestDuration"] = 180; + executor = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + TimeSpan integerBasedDuration = executor.TestDuration; + + this.mockFixture.Parameters["TestDuration"] = "00:03:00"; + executor = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + TimeSpan timespanBasedDuration = executor.TestDuration; + + Assert.AreEqual(integerBasedDuration, timespanBasedDuration); + } + + private void SetupNetworkingWorkloadState() + { + this.networkingWorkloadState = new NetworkingWorkloadState(); + this.networkingWorkloadState.Scenario = "AnyScenario"; + this.networkingWorkloadState.Tool = NetworkingWorkloadTool.NCPS; + this.networkingWorkloadState.ToolState = NetworkingWorkloadToolState.Running; + this.networkingWorkloadState.Protocol = "TCP"; + this.networkingWorkloadState.TestMode = "MockTestMode"; + + var expectedStateItem = new Item(nameof(NetworkingWorkloadState), this.networkingWorkloadState); + + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, expectedStateItem)); + } + + private class TestNCPSClientExecutor : NCPSClientExecutor + { + public TestNCPSClientExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs new file mode 100644 index 0000000000..86b5de8598 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs @@ -0,0 +1,181 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions.NetworkPerformance +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using NUnit.Framework; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class NCPSMetricsParserTests + { + private MockFixture mockFixture; + + [SetUp] + public void SetupDefaults() + { + this.mockFixture = new MockFixture(); + } + + [Test] + public void NcpsParserParsesExpectedMetricsFromValidServerSideResults() + { + this.mockFixture.Setup(PlatformID.Win32NT); + string results = NCPSMetricsParserTests.GetFileContents("NCPS_Example_Results_Server.txt"); + + NCPSMetricsParser parser = new NCPSMetricsParser(results, 90, 5); + IList metrics = parser.Parse(); + + Assert.IsNotEmpty(metrics); + Assert.IsTrue(metrics.Count == 30); // 28 + 2 throughput metrics + + // Throughput metrics (new in NCPS) + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RxGbps" && m.Value == 0.45)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "TxGbps" && m.Value == 0.45)); + + // CPS metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Cps" && m.Value == 55683)); + + // SYN RTT metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMean" && m.Value == 15996)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMedian" && m.Value == 4441)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP25" && m.Value == 2342)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP75" && m.Value == 13249)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP90" && m.Value == 24443)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP95" && m.Value == 38198)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99" && m.Value == 79657)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99_9" && m.Value == 1038000)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99_99" && m.Value == 1057000)); + + // Retransmit metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitConnPercentage" && m.Value == 1.7278)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitPerConn" && m.Value == 1.0138)); + + // Statistical metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_Min")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_Max")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_Med")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_Avg")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_P25")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_P50")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_P75")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_P90")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_P99")); + } + + [Test] + public void NcpsParserParsesExpectedMetricsFromValidClientSideResults() + { + this.mockFixture.Setup(PlatformID.Win32NT); + string results = NCPSMetricsParserTests.GetFileContents("NCPS_Example_Results_Client.txt"); + + NCPSMetricsParser parser = new NCPSMetricsParser(results, 90, 30); + IList metrics = parser.Parse(); + + Assert.IsNotEmpty(metrics); + // Client results have no periodic data after warmup (all 10 rows are within warmup period of 30s) + // So we only get: 2 throughput + 1 CPS + 9 SYN RTT + 2 Retransmit = 14 metrics + Assert.IsTrue(metrics.Count == 14, $"Expected 14 metrics but got {metrics.Count}"); + + // Throughput metrics (new in NCPS) + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RxGbps" && m.Value == 0.65)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "TxGbps" && m.Value == 0.65)); + + // CPS metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Cps" && m.Value == 24416)); + + // SYN RTT metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMean" && m.Value == 18737)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMedian" && m.Value == 6221)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP25" && m.Value == 3424)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP75" && m.Value == 10893)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP90" && m.Value == 17628)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP95" && m.Value == 24607)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99" && m.Value == 54222)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99_9" && m.Value == 2027000)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99_99" && m.Value == 4057000)); + + // Retransmit metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitConnPercentage" && m.Value == 3.2825)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitPerConn" && m.Value == 1.1436)); + + // No statistical metrics because all periodic data falls within warmup period + Assert.IsNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_Min")); + } + + [Test] + public void NcpsParserHandlesMissingThroughputMetrics() + { + this.mockFixture.Setup(PlatformID.Win32NT); + string results = @" +###ENDCPS 12345 + +###CPS,100000:1000 + +###SYNRTT,25:100,Median:200,Mean:150,75:300,90:400,95:500,99:600,99.9:700,99.99:800 + +###REXMIT,rtconnpercentage:1.5,rtperconn:1.2 +"; + + NCPSMetricsParser parser = new NCPSMetricsParser(results, 90, 5); + IList metrics = parser.Parse(); + + Assert.IsNotEmpty(metrics); + + // Should have CPS, RTT, and Retransmit metrics, but no throughput metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Cps")); + Assert.IsNull(metrics.FirstOrDefault(m => m.Name == "RxGbps")); + Assert.IsNull(metrics.FirstOrDefault(m => m.Name == "TxGbps")); + } + + [Test] + public void NcpsParserThrowsOnInvalidResults() + { + this.mockFixture.Setup(PlatformID.Win32NT); + string results = "Invalid results content"; + + NCPSMetricsParser parser = new NCPSMetricsParser(results, 90, 5); + + // The parser should throw WorkloadResultsException when parsing invalid content + Assert.Throws(() => parser.Parse()); + } + + [Test] + public void NcpsParserHandlesDifferentConfidenceLevels() + { + this.mockFixture.Setup(PlatformID.Win32NT); + string results = NCPSMetricsParserTests.GetFileContents("NCPS_Example_Results_Server.txt"); + + // Test with 95% confidence level + NCPSMetricsParser parser95 = new NCPSMetricsParser(results, 95, 5); + IList metrics95 = parser95.Parse(); + + // Test with 99% confidence level + NCPSMetricsParser parser99 = new NCPSMetricsParser(results, 99, 5); + IList metrics99 = parser99.Parse(); + + // Both should parse successfully with different confidence intervals + Assert.IsNotEmpty(metrics95); + Assert.IsNotEmpty(metrics99); + + // Confidence intervals should be different + var lowerCI95 = metrics95.FirstOrDefault(m => m.Name == "ConnectsPerSec_LowerCI"); + var lowerCI99 = metrics99.FirstOrDefault(m => m.Name == "ConnectsPerSec_LowerCI"); + + Assert.IsNotNull(lowerCI95); + Assert.IsNotNull(lowerCI99); + Assert.AreNotEqual(lowerCI95.Value, lowerCI99.Value); + } + + private static string GetFileContents(string fileName) + { + string outputPath = Path.Combine(MockFixture.TestAssemblyDirectory, "Examples", "NCPS", fileName); + return File.ReadAllText(outputPath); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSServerExecutorTests.cs new file mode 100644 index 0000000000..c4b915e6a6 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSServerExecutorTests.cs @@ -0,0 +1,153 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using Polly; + using System; + using System.Collections.Generic; + using System.IO; + using System.Net; + using System.Net.Http; + using System.Reflection; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Actions.NetworkPerformance; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class NCPSServerExecutorTests + { + private static readonly string ExamplesDirectory = MockFixture.GetDirectory(typeof(NTttcpExecutorTests2), "Examples", "NCPS"); + + private MockFixture mockFixture; + private DependencyPath mockPackage; + private NetworkingWorkloadState workloadState; + + public void SetupApiCalls() + { + this.workloadState = new NetworkingWorkloadState(); + this.workloadState.Scenario = "AnyScenario"; + this.workloadState.Tool = NetworkingWorkloadTool.NCPS; + this.workloadState.ToolState = NetworkingWorkloadToolState.Running; + this.workloadState.Protocol = "TCP"; + this.workloadState.TestMode = "MockTestMode"; + + var expectedStateItem = new Item(nameof(NetworkingWorkloadState), this.workloadState); + + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, expectedStateItem)); + } + + [SetUp] + public void SetupTest() + { + this.mockFixture = new MockFixture(); + this.mockPackage = new DependencyPath("ncps", this.mockFixture.PlatformSpecifics.GetPackagePath("ncps")); + this.mockFixture.SetupPackage(this.mockPackage); + + this.mockFixture.Directory.Setup(d => d.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.Parameters["PackageName"] = "ncps"; + this.mockFixture.Parameters["ThreadCount"] = "16"; + this.mockFixture.Parameters["TestDuration"] = "00:05:00"; + this.mockFixture.Parameters["WarmupTime"] = "00:00:30"; + this.mockFixture.Parameters["Delaytime"] = "00:00:00"; + this.mockFixture.Parameters["ConfidenceLevel"] = "99"; + + string exampleResults = File.ReadAllText(this.mockFixture.Combine(NCPSServerExecutorTests.ExamplesDirectory, "NCPS_Example_Results_Server.txt")); + + this.mockFixture.Process.StandardOutput.Append(exampleResults); + + this.mockFixture.FileSystem.Setup(rt => rt.File.ReadAllTextAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(exampleResults); + + this.SetupApiCalls(); + } + + [Test] + public void NCPSServerExecutorThrowsOnUnsupportedOS() + { + this.mockFixture.SystemManagement.SetupGet(sm => sm.Platform).Returns(PlatformID.Other); + TestNCPSServerExecutor component = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.ThrowsAsync(() => component.ExecuteAsync(CancellationToken.None)); + } + + [Test] + [Platform(Exclude = "Unix,Linux,MacOsX")] + public async Task NCPSServerExecutorExecutesAsExpected() + { + NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor networkingWorkloadExecutor = new NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + await networkingWorkloadExecutor.OnInitialize.Invoke(EventContext.None, CancellationToken.None); + string agentId = $"{Environment.MachineName}-Server"; + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(agentId); + + int processExecuted = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (file, arguments, workingDirectory) => + { + processExecuted++; + + return this.mockFixture.Process; + }; + + TestNCPSServerExecutor component = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + await component.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(1, processExecuted); + } + + [Test] + public void NCPSServerExecutorUsesExpectedDefaultParameters() + { + TestNCPSServerExecutor executor = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.AreEqual(16, executor.ThreadCount); + Assert.AreEqual(9800, executor.Port); + Assert.AreEqual(16, executor.PortCount); + Assert.AreEqual("1", executor.DataTransferMode); + } + + [Test] + public void NCPSServerExecutorSupportsCustomDataTransferModes() + { + // Test continuous send mode + this.mockFixture.Parameters["DataTransferMode"] = "s"; + TestNCPSServerExecutor executor = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + Assert.AreEqual("s", executor.DataTransferMode); + + // Test continuous receive mode + this.mockFixture.Parameters["DataTransferMode"] = "r"; + executor = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + Assert.AreEqual("r", executor.DataTransferMode); + + // Test ping-pong mode + this.mockFixture.Parameters["DataTransferMode"] = "p"; + executor = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + Assert.AreEqual("p", executor.DataTransferMode); + } + + private class TestNCPSServerExecutor : NCPSServerExecutor + { + public TestNCPSServerExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + + protected override bool IsProcessRunning(string processName) + { + return true; + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSClientExecutor.cs new file mode 100644 index 0000000000..b6d0a71910 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSClientExecutor.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions.NetworkPerformance +{ + using System; + using System.Collections.Generic; + using System.Linq; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient.Contracts; + + /// + /// NCPS(New Connections Per Second) Tool Client Executor. + /// + public class NCPSClientExecutor : NCPSExecutor + { + /// + /// Initializes a new instance of the class. + /// + /// Component to copy. + public NCPSClientExecutor(VirtualClientComponent component) + : base(component) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public NCPSClientExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + + /// + /// Returns the NCPS client-side command line arguments. + /// + protected override string GetCommandLineArguments() + { + string serverIPAddress = this.GetLayoutClientInstances(ClientRole.Server).First().IPAddress; + + // Build the command line based on NCPS recipe + // ncps -c -wt 30 -t 330 + string command = $"-c {serverIPAddress} " + + $"-r {this.ThreadCount} " + + $"-bp {this.Port} " + + $"-np {this.PortCount} " + + $"-N {this.TotalConnectionsToOpen} " + + $"-P {this.MaxPendingRequests} " + + $"-D {this.ConnectionDuration} " + + $"-M {this.DataTransferMode} " + + $"-i {this.DisplayInterval} " + + $"-wt {this.WarmupTime.TotalSeconds} " + + $"-t {this.TestDuration.TotalSeconds} "; + + if (this.DelayTime != TimeSpan.Zero) + { + command += $"-ds {this.DelayTime.TotalSeconds} "; + } + + if (!string.IsNullOrWhiteSpace(this.AdditionalParams)) + { + command += this.AdditionalParams; + } + + return command.Trim(); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSExecutor.cs new file mode 100644 index 0000000000..84eb72c942 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSExecutor.cs @@ -0,0 +1,364 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions.NetworkPerformance +{ + using System; + using System.Collections.Generic; + using System.IO.Abstractions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + using VirtualClient.Contracts.Metadata; + + /// + /// NCPS(New Connections Per Second) Tool Executor. + /// + public class NCPSExecutor : NetworkingWorkloadToolExecutor + { + private IFileSystem fileSystem; + + /// + /// Initializes a new instance of the class. + /// + /// Component to copy. + public NCPSExecutor(VirtualClientComponent component) + : base(component) + { + this.ProcessStartRetryPolicy = Policy.Handle(exc => exc.Message.Contains("sockwiz")).Or() + .WaitAndRetryAsync(5, retries => TimeSpan.FromSeconds(retries * 3)); + } + + /// + /// Initializes a new instance of the class. + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public NCPSExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.fileSystem = dependencies.GetService(); + this.ProcessStartRetryPolicy = Policy.Handle(exc => exc.Message.Contains("sockwiz")).Or() + .WaitAndRetryAsync(5, retries => TimeSpan.FromSeconds(retries * 3)); + } + + /// + /// The thread count for the workload. + /// + public int ThreadCount + { + get + { + return this.Parameters.GetValue(nameof(this.ThreadCount), 16); + } + } + + /// + /// The total number of connections to keep open (client-side only). + /// + public int TotalConnectionsToOpen + { + get + { + return this.Parameters.GetValue(nameof(this.TotalConnectionsToOpen), this.ThreadCount * 100); + } + } + + /// + /// The max number of pending connect requests at any given time (client-side only). + /// + public int MaxPendingRequests + { + get + { + return this.Parameters.GetValue(nameof(this.MaxPendingRequests), this.TotalConnectionsToOpen); + } + } + + /// + /// The duration (in milliseconds) for each connection. + /// + public int ConnectionDuration + { + get + { + return this.Parameters.GetValue(nameof(this.ConnectionDuration), 0); + } + } + + /// + /// The data transfer mode for each connection. + /// 0: no send/receive, 1: one send/receive, p: ping/pong (continuous send/receive) + /// s: continuous send, r: continuous receive + /// + public string DataTransferMode + { + get + { + return this.Parameters.GetValue(nameof(this.DataTransferMode), "1"); + } + } + + /// + /// The interval (in seconds) to display NCPS stats. + /// + public int DisplayInterval + { + get + { + return this.Parameters.GetValue(nameof(NCPSExecutor.DisplayInterval), 1); + } + } + + /// + /// The starting port for the range of ports that will be used for client/server + /// network connections. + /// + public int Port + { + get + { + return this.Parameters.GetValue(nameof(this.Port), 9800); + } + } + + /// + /// Number of TCP ports to listen on or connect to. + /// + public int PortCount + { + get + { + return this.Parameters.GetValue(nameof(this.PortCount), this.ThreadCount); + } + } + + /// + /// Parameter defines the duration for running the NCPS workload. + /// Note: NCPS -t parameter includes warmup time specified by -wt parameter. + /// + public TimeSpan TestDuration + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.TestDuration), TimeSpan.FromSeconds(60)); + } + } + + /// + /// Gets test warmup time values. + /// + public TimeSpan WarmupTime + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.WarmupTime), TimeSpan.FromSeconds(5)); + } + } + + /// + /// Gets test delay time values. + /// + public TimeSpan DelayTime + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.DelayTime), TimeSpan.Zero); + } + } + + /// + /// Gets the confidence level used for calculating the confidence intervals. + /// + public double ConfidenceLevel + { + get + { + return this.Parameters.GetValue(nameof(this.ConfidenceLevel), 99); + } + } + + /// + /// Gets additional optional parameters. + /// + public string AdditionalParams + { + get + { + return this.Parameters.GetValue(nameof(this.AdditionalParams), string.Empty); + } + } + + /// + /// The retry policy to apply to the startup of the NCPS workload to handle + /// transient issues. + /// + protected IAsyncPolicy ProcessStartRetryPolicy { get; set; } + + /// + /// Initializes the environment and dependencies for running the tool. + /// + protected override Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(this.Scenario)) + { + throw new WorkloadException( + $"Scenario parameter missing. The profile supplied is missing the required '{nameof(this.Scenario)}' parameter " + + $"for one or more of the '{nameof(NCPSExecutor)}' steps.", + ErrorReason.InvalidProfileDefinition); + } + + DependencyPath workloadPackage = this.GetDependencyPath(this.PackageName, cancellationToken); + + this.IsInClientRole = this.IsInRole(ClientRole.Client); + this.IsInServerRole = !this.IsInClientRole; + this.Role = this.IsInClientRole ? ClientRole.Client : ClientRole.Server; + + // e.g. + // NCPS_T16 Client, NCPS_T16 Server + this.Name = $"{this.Scenario} {this.Role}"; + this.ProcessName = "ncps"; + this.Tool = NetworkingWorkloadTool.NCPS; + + if (this.Platform == PlatformID.Win32NT) + { + this.ExecutablePath = this.Combine(workloadPackage.Path, "ncps.exe"); + } + else if (this.Platform == PlatformID.Unix) + { + this.ExecutablePath = this.Combine(workloadPackage.Path, "ncps"); + } + else + { + throw new NotSupportedException($"{this.Platform} is not supported"); + } + + // Validating NCPS parameters + if (this.TestDuration <= TimeSpan.Zero) + { + throw new WorkloadException("Test duration cannot be equal or less than zero for NCPS workload", ErrorReason.InstructionsNotValid); + } + else if (this.WarmupTime >= this.TestDuration) + { + throw new WorkloadException("WarmUp time must be less than test duration for NCPS workload", ErrorReason.InstructionsNotValid); + } + else if (this.DelayTime >= this.TestDuration) + { + throw new WorkloadException("Delay time must be less than test duration for NCPS workload", ErrorReason.InstructionsNotValid); + } + else if ((this.DelayTime + this.WarmupTime) >= this.TestDuration) + { + throw new WorkloadException("Sum of delay time and WarmUp time must be less than test duration for NCPS workload", ErrorReason.InstructionsNotValid); + } + + return this.SystemManagement.MakeFileExecutableAsync(this.ExecutablePath, this.Platform, cancellationToken); + } + + /// + /// Returns the NCPS command line arguments. + /// + protected override string GetCommandLineArguments() + { + return null; + } + + /// + protected override Task ExecuteWorkloadAsync(string commandArguments, EventContext telemetryContext, CancellationToken cancellationToken, TimeSpan? timeout = null) + { + IProcessProxy process = null; + + EventContext relatedContext = telemetryContext.Clone() + .AddContext("command", this.ExecutablePath) + .AddContext("commandArguments", commandArguments); + + return this.Logger.LogMessageAsync($"{this.TypeName}.ExecuteWorkload", relatedContext, async () => + { + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await this.ProcessStartRetryPolicy.ExecuteAsync(async () => + { + using (process = this.SystemManagement.ProcessManager.CreateProcess(this.ExecutablePath, commandArguments)) + { + try + { + this.CleanupTasks.Add(() => process.SafeKill(this.Logger)); + await process.StartAndWaitAsync(cancellationToken, timeout, withExitConfirmation: true); + await this.LogProcessDetailsAsync(process, relatedContext, "NCPS"); + process.ThrowIfWorkloadFailed(); + + this.CaptureMetrics( + process.StandardOutput.ToString(), + process.FullCommand(), + process.StartTime, + process.ExitTime, + relatedContext); + } + catch (OperationCanceledException) + { + // Expected when the client signals a cancellation. + } + catch (TimeoutException exc) + { + // We give this a best effort but do not want it to prevent the next workload + // from executing. + this.Logger.LogMessage($"{this.GetType().Name}.WorkloadTimeout", LogLevel.Warning, relatedContext.AddError(exc)); + process.SafeKill(this.Logger); + + throw new WorkloadException($"NCPS workload did not exit within the timeout period defined (timeout={timeout}).", exc, ErrorReason.WorkloadFailed); + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.GetType().Name}.WorkloadStartupError", LogLevel.Warning, relatedContext.AddError(exc)); + throw new WorkloadException($"NCPS workload failed to start successfully", exc, ErrorReason.WorkloadFailed); + } + } + }); + } + + return process; + }); + } + + /// + /// Logs the workload metrics to the telemetry. + /// + protected override void CaptureMetrics(string results, string commandArguments, DateTime startTime, DateTime endTime, EventContext telemetryContext) + { + if (!string.IsNullOrWhiteSpace(results)) + { + this.MetadataContract.AddForScenario( + this.Tool.ToString(), + commandArguments, + toolVersion: null); + + this.MetadataContract.Apply(telemetryContext); + + MetricsParser parser = new NCPSMetricsParser(results, this.ConfidenceLevel, this.WarmupTime.TotalSeconds); + IList metrics = parser.Parse(); + + this.Logger.LogMetrics( + this.Tool.ToString(), + this.Name, + startTime, + endTime, + metrics, + string.Empty, + commandArguments, + this.Tags, + telemetryContext); + } + else + { + throw new WorkloadException( + $"Workload results missing. The NCPS workload did not produce valid results.", + ErrorReason.WorkloadResultsNotFound); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSMetricsParser.cs new file mode 100644 index 0000000000..80ea8f8c75 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSMetricsParser.cs @@ -0,0 +1,338 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions.NetworkPerformance +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Text.RegularExpressions; + using MathNet.Numerics.Distributions; + using MathNet.Numerics.Statistics; + using VirtualClient.Contracts; + + /// + /// NCPS toolset results parser. + /// + public class NCPSMetricsParser : MetricsParser + { + private static readonly Regex CpsExpression = new Regex(@"^###ENDCPS\s(?\d+)", RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex SynRttExpression = new Regex(@"^###SYNRTT,(?.*)", RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex RetransmitsExpression = new Regex(@"^###REXMIT,(?.*)", RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex RxGbpsExpression = new Regex(@"^###RXGBPS\s(?[\d.]+)", RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex TxGbpsExpression = new Regex(@"^###TXGBPS\s(?[\d.]+)", RegexOptions.Compiled | RegexOptions.Multiline); + + private double confidenceLevel; + private double warmupTimeInSeconds; + + /// + /// Initializes a new instance of the class. + /// + public NCPSMetricsParser(string results, double confidenceLevel, double warmupTimeSeconds) + : base(results) + { + this.confidenceLevel = confidenceLevel; + this.warmupTimeInSeconds = warmupTimeSeconds; + } + + /// + /// Parses the NCPS results and returns a list of metrics from them. + /// + public override IList Parse() + { + try + { + IList metrics = new List(); + + // Parse RXGBPS and TXGBPS metrics + MatchCollection matches = NCPSMetricsParser.RxGbpsExpression.Matches(this.RawText); + NCPSMetricsParser.AddThroughputMetrics(metrics, matches, "RxGbps", "Receive throughput in Gbps"); + + matches = NCPSMetricsParser.TxGbpsExpression.Matches(this.RawText); + NCPSMetricsParser.AddThroughputMetrics(metrics, matches, "TxGbps", "Transmit throughput in Gbps"); + + // Parse CPS metrics + matches = NCPSMetricsParser.CpsExpression.Matches(this.RawText); + NCPSMetricsParser.AddConnectionPerSecondMetrics(metrics, matches); + + // Parse SYN RTT metrics + matches = NCPSMetricsParser.SynRttExpression.Matches(this.RawText); + NCPSMetricsParser.AddSynRttMetrics(metrics, matches); + + // Parse retransmit metrics + matches = NCPSMetricsParser.RetransmitsExpression.Matches(this.RawText); + NCPSMetricsParser.AddRetransmitMetrics(metrics, matches); + + // Parse periodic output for statistical metrics + KeyValuePair, List> connectionsPerSec = NCPSMetricsParser.GetTimestampsAndConnectionsPerSec(this.RawText, this.warmupTimeInSeconds); + + if (connectionsPerSec.Value.Count > 0) + { + NCPSMetricsParser.AddStatisticalMetrics(metrics, connectionsPerSec.Key, connectionsPerSec.Value, this.confidenceLevel); + } + + return metrics; + } + catch (Exception exc) + { + throw new WorkloadResultsException( + $"Results parsing operation failed. The NCPS parser failed to parse the results of the NCPS workload.", + exc, + ErrorReason.WorkloadResultsParsingFailed); + } + } + + private static void AddThroughputMetrics(IList metrics, MatchCollection matches, string metricName, string description) + { + if (matches?.Any() == true) + { + if (double.TryParse(matches[0].Groups[1].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out double throughput)) + { + metrics.Add(new Metric(metricName, throughput, "Gbps", MetricRelativity.HigherIsBetter, description: description)); + } + } + } + + private static void AddStatisticalMetrics(IList metrics, List timestamps, List connectsPerSec, double confidenceLevel) + { + Normal normal = new Normal(); + double theta = ((confidenceLevel / 100.0) + 1.0) / 2; + double mean = connectsPerSec.Mean(); + double sd = connectsPerSec.StandardDeviation(); + double inverserCDF = normal.InverseCumulativeDistribution(theta); + double sem = sd / (double)Math.Sqrt(connectsPerSec.Count); + double t = inverserCDF * sem; + double lowerCI = mean - t; + double upperCI = mean + t; + + metrics.Add(new Metric("ConnectsPerSec_Min", connectsPerSec.Min())); + metrics.Add(new Metric("ConnectsPerSec_Max", connectsPerSec.Max())); + metrics.Add(new Metric("ConnectsPerSec_Med", connectsPerSec.Median())); + metrics.Add(new Metric("ConnectsPerSec_Avg", connectsPerSec.Average(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric("ConnectsPerSec_P25", connectsPerSec.Percentile(25))); + metrics.Add(new Metric("ConnectsPerSec_P50", connectsPerSec.Percentile(50))); + metrics.Add(new Metric("ConnectsPerSec_P75", connectsPerSec.Percentile(75))); + metrics.Add(new Metric("ConnectsPerSec_P90", connectsPerSec.Percentile(90))); + metrics.Add(new Metric("ConnectsPerSec_P99", connectsPerSec.Percentile(99))); + metrics.Add(new Metric("ConnectsPerSec_P99_9", Statistics.QuantileCustom(connectsPerSec, 1d - 0.001d, QuantileDefinition.R3))); + metrics.Add(new Metric("ConnectsPerSec_P99_99", Statistics.QuantileCustom(connectsPerSec, 1d - 0.0001d, QuantileDefinition.R3))); + metrics.Add(new Metric("ConnectsPerSec_P99_999", Statistics.QuantileCustom(connectsPerSec, 1d - 0.00001d, QuantileDefinition.R3))); + double median = Statistics.Median(connectsPerSec); + double[] absoluteDeviations = connectsPerSec.Select(x => Math.Abs(x - median)).ToArray(); + metrics.Add(new Metric("ConnectsPerSec_Mad", Statistics.Median(absoluteDeviations), MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_StandardErrorMean", sem, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_LowerCI", lowerCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_UpperCI", upperCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + } + + /// + /// Parses the NCPS results and returns a list of pairs of {Timestamp, Conn/s} for workloads duration (excluding warmupTime). + /// + private static KeyValuePair, List> GetTimestampsAndConnectionsPerSec(string content, double warmupTime) + { + bool appendResult = true; + int valueIndex = 0; + + // The timestamps and connectionsPerSecond co-relate on the index. + List timestamps = new List(); + List connectionsPerSec = new List(); + StringReader strReader = new StringReader(content); + + try + { + double excludeCount = warmupTime; + + while (true) + { + string line = strReader.ReadLine(); + + if (string.IsNullOrEmpty(line)) + { + break; + } + else if (appendResult) + { + string[] stringList = line.Split(" ", StringSplitOptions.RemoveEmptyEntries); + + if (stringList.Contains("Conn/s")) + { + valueIndex = Array.IndexOf(stringList, "Conn/s"); + continue; + } + else if (stringList.Length == 0) + { + appendResult = false; + } + else + { + string formattedTimestamp = string.Format("{0:0.##}", stringList[0]); + string formattedConnectRate = string.Format("{0:0.#}", stringList[valueIndex]); + double timestamp = Convert.ToDouble(formattedTimestamp); + double connectRate = Convert.ToDouble(formattedConnectRate); + + if (excludeCount < Math.Floor(timestamp)) + { + timestamps.Add(Convert.ToDouble(formattedTimestamp)); + connectionsPerSec.Add(Convert.ToDouble(formattedConnectRate)); + } + } + } + } + } + catch (ArgumentNullException exc) + { + throw new WorkloadResultsException( + "Results parsing operation failed. The NCPS parser failed to parse the results of the NCPS workload. Error getting values of Conn/s", + exc, + ErrorReason.WorkloadResultsNotFound); + } + + return new KeyValuePair, List>(timestamps, connectionsPerSec); + } + + private static void AddConnectionPerSecondMetrics(IList metrics, MatchCollection matches) + { + if (matches?.Any() == true) + { + if (double.TryParse(matches[0].Groups[1].Value, out double cps)) + { + metrics.Add(new Metric("Cps", cps, MetricRelativity.HigherIsBetter, description: "Connections per second")); + } + } + } + + private static void AddRetransmitMetrics(IList metrics, MatchCollection matches) + { + if (matches?.Any() == true) + { + Match match = matches.First(); + string[] tokens = match.Groups[1].Value.Split(','); + + foreach (string token in tokens) + { + string[] tokenPair = token.Split(':'); + if (tokenPair?.Any() == true && tokenPair.Length == 2) + { + string metricName = tokenPair[0].Trim(); + string metricValue = tokenPair[1].Trim(); + + switch (metricName.ToLowerInvariant()) + { + case "rtconnpercentage": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double median)) + { + metrics.Add(new Metric("RexmitConnPercentage", median, MetricRelativity.LowerIsBetter, description: "Connection retransmits percentage.")); + } + + break; + + case "rtperconn": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double mean)) + { + metrics.Add(new Metric("RexmitPerConn", mean, MetricRelativity.LowerIsBetter, description: "Retransmits per connection.")); + } + + break; + } + } + } + } + } + + private static void AddSynRttMetrics(IList metrics, MatchCollection matches) + { + if (matches?.Any() == true) + { + Match match = matches.First(); + string[] tokens = match.Groups[1].Value.Split(','); + + foreach (string token in tokens) + { + string[] tokenPair = token.Split(':'); + if (tokenPair?.Any() == true && tokenPair.Length == 2) + { + string metricName = tokenPair[0].Trim(); + string metricValue = tokenPair[1].Trim(); + + switch (metricName.ToLowerInvariant()) + { + case "median": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double median)) + { + metrics.Add(new Metric("SynRttMedian", median, MetricRelativity.LowerIsBetter, description: "Synchronous (SYN) median round-trip time.")); + } + + break; + + case "mean": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double mean)) + { + metrics.Add(new Metric("SynRttMean", mean, MetricRelativity.LowerIsBetter, description: "Synchronous (SYN) mean round-trip time.")); + } + + break; + + case "25": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p25)) + { + metrics.Add(new Metric("SynRttP25", p25, MetricRelativity.LowerIsBetter, description: "P25 Synchronous (SYN) round-trip time.")); + } + + break; + + case "75": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p75)) + { + metrics.Add(new Metric("SynRttP75", p75, MetricRelativity.LowerIsBetter, description: "P75 Synchronous (SYN) round-trip time.")); + } + + break; + + case "90": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p90)) + { + metrics.Add(new Metric("SynRttP90", p90, MetricRelativity.LowerIsBetter, description: "P90 Synchronous (SYN) round-trip time.")); + } + + break; + + case "95": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p95)) + { + metrics.Add(new Metric("SynRttP95", p95, MetricRelativity.LowerIsBetter, description: "P95 Synchronous (SYN) round-trip time.")); + } + + break; + + case "99": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p99)) + { + metrics.Add(new Metric("SynRttP99", p99, MetricRelativity.LowerIsBetter, description: "P99 Synchronous (SYN) round-trip time.")); + } + + break; + + case "99.9": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p99_9)) + { + metrics.Add(new Metric("SynRttP99_9", p99_9, MetricRelativity.LowerIsBetter, description: "P99.9 Synchronous (SYN) round-trip time.")); + } + + break; + + case "99.99": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p99_99)) + { + metrics.Add(new Metric("SynRttP99_99", p99_99, MetricRelativity.LowerIsBetter, description: "P99.99 Synchronous (SYN) round-trip time.")); + } + + break; + } + } + } + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSServerExecutor.cs new file mode 100644 index 0000000000..460e7409f4 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSServerExecutor.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions.NetworkPerformance +{ + using System; + using System.Collections.Generic; + using System.Linq; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient.Contracts; + + /// + /// NCPS(New Connections Per Second) Tool Server Executor + /// + public class NCPSServerExecutor : NCPSExecutor + { + /// + /// Initializes a new instance of the class. + /// + /// Component to copy. + public NCPSServerExecutor(VirtualClientComponent component) + : base(component) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public NCPSServerExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + + /// + /// Returns the NCPS server-side command line arguments. + /// + protected override string GetCommandLineArguments() + { + // Build the command line based on NCPS recipe + // ncps -s + string command = $"-s " + + $"-r {this.ThreadCount} " + + $"-bp {this.Port} " + + $"-np {this.PortCount} "; + + if (!string.IsNullOrWhiteSpace(this.DataTransferMode) && + (this.DataTransferMode.Equals("s", StringComparison.OrdinalIgnoreCase) || + this.DataTransferMode.Equals("r", StringComparison.OrdinalIgnoreCase))) + { + command += $"-M {this.DataTransferMode} "; + } + + if (!string.IsNullOrWhiteSpace(this.AdditionalParams)) + { + command += this.AdditionalParams; + } + + return command.Trim(); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadExecutor.cs index 6e6ce8183d..e2ad4d3495 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadExecutor.cs @@ -611,6 +611,9 @@ protected override bool IsSupported() case "cps": isSupported = this.Platform == PlatformID.Win32NT || this.Platform == PlatformID.Unix; break; + case "ncps": + isSupported = this.Platform == PlatformID.Win32NT || this.Platform == PlatformID.Unix; + break; case "latte": isSupported = this.Platform == PlatformID.Win32NT; @@ -668,6 +671,10 @@ protected virtual NetworkingWorkloadToolExecutor CreateWorkloadExecutor(Networki action = new CPSClientExecutor(this); break; + case NetworkingWorkloadTool.NCPS: + action = new NCPSClientExecutor(this); + break; + case NetworkingWorkloadTool.NTttcp: action = new NTttcpClientExecutor(this); break; @@ -692,6 +699,10 @@ protected virtual NetworkingWorkloadToolExecutor CreateWorkloadExecutor(Networki action = new CPSServerExecutor(this); break; + case NetworkingWorkloadTool.NCPS: + action = new NCPSServerExecutor(this); + break; + case NetworkingWorkloadTool.NTttcp: action = new NTttcpServerExecutor(this); break; @@ -869,6 +880,11 @@ private async Task ExecuteClientAsync(EventContext telemetryContext, Cancellatio await this.ExecuteClientToolAsync(NetworkingWorkloadTool.CPS, telemetryContext, cancellationToken) .ConfigureAwait(false); } + else if (string.Equals(this.ToolName, NetworkingWorkloadTool.NCPS.ToString(), ignoreCase)) + { + await this.ExecuteClientToolAsync(NetworkingWorkloadTool.NCPS, telemetryContext, cancellationToken) + .ConfigureAwait(false); + } else if (string.Equals(this.ToolName, NetworkingWorkloadTool.Latte.ToString(), ignoreCase)) { if (this.Platform != PlatformID.Win32NT) diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadState.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadState.cs index f3bb6e4f7d..955835d76f 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadState.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadState.cs @@ -42,7 +42,12 @@ public enum NetworkingWorkloadTool /// /// Socket Performance Tool. /// - SockPerf + SockPerf, + + /// + /// NCPS Tool. + /// + NCPS } /// diff --git a/src/VirtualClient/VirtualClient.Dependencies/NetworkConfigurationSetup.cs b/src/VirtualClient/VirtualClient.Dependencies/NetworkConfigurationSetup.cs index 9a77b8340a..7b5a696cb2 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/NetworkConfigurationSetup.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/NetworkConfigurationSetup.cs @@ -292,6 +292,9 @@ private async Task ApplyRcLocalFileCommandsAsync(CancellationToken cancellationT rcLocalContents.Add($"sysctl -w fs.file-max={NetworkConfigurationSetup.NoFileLimit}"); rcLocalContents.Add("sysctl -w net.ipv4.tcp_tw_reuse=1 # TIME_WAIT work-around"); rcLocalContents.Add("sysctl -w net.ipv4.ip_local_port_range=\"10000 60000\" # ephemeral ports increased"); + rcLocalContents.Add("sysctl -w net.ipv4.tcp_syncookies=0 # disable SYN cookies for CPS/NCPS"); + rcLocalContents.Add("sysctl -w net.ipv4.tcp_max_syn_backlog=2048 # increase SYN backlog for CPS/NCPS"); + rcLocalContents.Add("sysctl -w net.ipv4.conf.all.rp_filter=0 # disable reverse path filtering for CPS/NCPS"); if (this.EnableBusyPoll) { @@ -301,8 +304,9 @@ private async Task ApplyRcLocalFileCommandsAsync(CancellationToken cancellationT if (this.DisableFirewall) { - rcLocalContents.Add("iptables -I OUTPUT -j NOTRACK # disable connection tracking"); - rcLocalContents.Add("iptables -I PREROUTING -j NOTRACK # disable connection tracking"); + rcLocalContents.Add("iptables -t raw -I OUTPUT -j NOTRACK # disable connection tracking"); + rcLocalContents.Add("iptables -t raw -I PREROUTING -j NOTRACK # disable connection tracking"); + rcLocalContents.Add("sysctl -w net.netfilter.nf_conntrack_max=0 # disable connection tracking (NCPS)"); rcLocalContents.Add("iptables -P INPUT ACCEPT # accept all inbound traffic"); rcLocalContents.Add("iptables -P OUTPUT ACCEPT # accept all outbound traffic"); rcLocalContents.Add("iptables -P FORWARD ACCEPT # accept all forward traffic"); From ae77c11519f0e11f935599bcf7f3c1adb5133b36 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 23 Jan 2026 11:33:56 -0600 Subject: [PATCH 2/5] Commit Changes --- .../NCPS/NCPS_Example_Results_Server.txt | 10 +-- .../Network/NCPS/NCPSMetricsParserTests.cs | 14 ++-- .../NCPS/NCPSServerExecutor.cs | 5 +- .../profiles/PERF-NETWORK.json | 73 ++++++++++++------- .../network-suite/network-suite-profiles.md | 5 +- .../workloads/network-suite/network-suite.md | 8 +- 6 files changed, 71 insertions(+), 44 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Server.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Server.txt index 891c88c77f..dd1a9b104c 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Server.txt +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Server.txt @@ -10,12 +10,12 @@ 9.003 2 1598 0 0 55323.0 55327.0 55328.0 55322.0 1143 54181 9149 1142 261028 10.003 0 1600 0 0 56011.0 56013.0 56012.0 56012.0 1197 54848 9935 1163 237388 -=== CMDLINE: ./ncps -c 10.1.0.5 -wt 5 -t 15 - -=== VERSION 1.0 +=== CMDLINE: ./ncps -s -r 16 -bp 9800 -np 16 -t 300 + +=== VERSION 1.4 -###RXGBPS 0.45 -###TXGBPS 0.45 +###RXGBPS 0.24 +###TXGBPS 0.24 === Time (ms) to Nth connection establishment for first 556947 connections: === N T(ms) CPS diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs index 86b5de8598..8402dff6fb 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs @@ -35,8 +35,8 @@ public void NcpsParserParsesExpectedMetricsFromValidServerSideResults() Assert.IsTrue(metrics.Count == 30); // 28 + 2 throughput metrics // Throughput metrics (new in NCPS) - Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RxGbps" && m.Value == 0.45)); - Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "TxGbps" && m.Value == 0.45)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RxGbps" && m.Value == 0.24)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "TxGbps" && m.Value == 0.24)); // CPS metrics Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Cps" && m.Value == 55683)); @@ -113,14 +113,14 @@ public void NcpsParserHandlesMissingThroughputMetrics() { this.mockFixture.Setup(PlatformID.Win32NT); string results = @" -###ENDCPS 12345 + ###ENDCPS 12345 -###CPS,100000:1000 + ###CPS,100000:1000 -###SYNRTT,25:100,Median:200,Mean:150,75:300,90:400,95:500,99:600,99.9:700,99.99:800 + ###SYNRTT,25:100,Median:200,Mean:150,75:300,90:400,95:500,99:600,99.9:700,99.99:800 -###REXMIT,rtconnpercentage:1.5,rtperconn:1.2 -"; + ###REXMIT,rtconnpercentage:1.5,rtperconn:1.2 + "; NCPSMetricsParser parser = new NCPSMetricsParser(results, 90, 5); IList metrics = parser.Parse(); diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSServerExecutor.cs index 460e7409f4..71e8698a9a 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSServerExecutor.cs @@ -39,11 +39,12 @@ public NCPSServerExecutor(IServiceCollection dependencies, IDictionary Date: Fri, 23 Jan 2026 13:14:16 -0600 Subject: [PATCH 3/5] Update Docs --- .../workloads/network-suite/network-suite.md | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/website/docs/workloads/network-suite/network-suite.md b/website/docs/workloads/network-suite/network-suite.md index 89a5747970..a53b2b9598 100644 --- a/website/docs/workloads/network-suite/network-suite.md +++ b/website/docs/workloads/network-suite/network-suite.md @@ -27,6 +27,7 @@ The following performance analysis scenarios are covered as part of the network * NCPS * Enhanced network connection establishment performance with detailed metrics on both Unix/Linux and Windows systems. + * Connection throughput with RX/TX bandwidth measurements in Gbps. * NTttcp * Network throughput and bandwidth with TCP communications on both Unix/Linux and Windows systems. @@ -98,6 +99,69 @@ For the Latte and SockPerf workloads, the client measurements are the only ones | CPS Server | SynRttP99_9 | 74810.0 | 367477.0 | 86169.82155477032 | | | CPS Server | SynRttP99_99 | 91533.0 | 627165.0 | 114658.16077738516 | | +| Scenario | Metric Name | Example Value (min) | Example Value (max) | Example Value (avg) | Unit | +|----------|-------------|---------------------|---------------------|---------------------|------| +| NCPS Client | RxGbps | 0.5 | 0.75 | 0.65 | Gbps | +| NCPS Client | TxGbps | 0.5 | 0.75 | 0.65 | Gbps | +| NCPS Client | Cps | 15000.0 | 32000.0 | 24416.0 | connections/sec | +| NCPS Client | ConnectsPerSec_Min | 24152.0 | 32069.0 | 25389.0 | connections/sec | +| NCPS Client | ConnectsPerSec_Max | 24152.0 | 32069.0 | 32069.0 | connections/sec | +| NCPS Client | ConnectsPerSec_Med | 24152.0 | 32069.0 | 27500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_Avg | 24152.0 | 32069.0 | 27840.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P25 | 24152.0 | 32069.0 | 26500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P50 | 24152.0 | 32069.0 | 27500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P75 | 24152.0 | 32069.0 | 28500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P90 | 24152.0 | 32069.0 | 29500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P99 | 24152.0 | 32069.0 | 31000.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P99_9 | 24152.0 | 32069.0 | 31500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P99_99 | 24152.0 | 32069.0 | 31800.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P99_999 | 24152.0 | 32069.0 | 31900.0 | connections/sec | +| NCPS Client | ConnectsPerSec_Mad | 500.0 | 1500.0 | 1000.0 | connections/sec | +| NCPS Client | ConnectsPerSec_StandardErrorMean | 100.0 | 300.0 | 200.0 | connections/sec | +| NCPS Client | ConnectsPerSec_LowerCI | 24000.0 | 31500.0 | 27500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_UpperCI | 25000.0 | 32500.0 | 28200.0 | connections/sec | +| NCPS Client | RexmitConnPercentage | 0.0 | 5.0 | 3.2825 | % | +| NCPS Client | RexmitPerConn | 0.0 | 2.0 | 1.1436 | retransmits/connection | +| NCPS Client | SynRttMean | 5000.0 | 25000.0 | 18737.0 | microseconds | +| NCPS Client | SynRttMedian | 2000.0 | 10000.0 | 6221.0 | microseconds | +| NCPS Client | SynRttP25 | 1000.0 | 5000.0 | 3424.0 | microseconds | +| NCPS Client | SynRttP75 | 5000.0 | 15000.0 | 10893.0 | microseconds | +| NCPS Client | SynRttP90 | 10000.0 | 25000.0 | 17628.0 | microseconds | +| NCPS Client | SynRttP95 | 15000.0 | 35000.0 | 24607.0 | microseconds | +| NCPS Client | SynRttP99 | 30000.0 | 75000.0 | 54222.0 | microseconds | +| NCPS Client | SynRttP99_9 | 1000000.0 | 3000000.0 | 2027000.0 | microseconds | +| NCPS Client | SynRttP99_99 | 2000000.0 | 5000000.0 | 4057000.0 | microseconds | +| NCPS Server | RxGbps | 0.5 | 0.75 | 0.65 | Gbps | +| NCPS Server | TxGbps | 0.5 | 0.75 | 0.65 | Gbps | +| NCPS Server | Cps | 15000.0 | 32000.0 | 24416.0 | connections/sec | +| NCPS Server | ConnectsPerSec_Min | 24152.0 | 32069.0 | 25389.0 | connections/sec | +| NCPS Server | ConnectsPerSec_Max | 24152.0 | 32069.0 | 32069.0 | connections/sec | +| NCPS Server | ConnectsPerSec_Med | 24152.0 | 32069.0 | 27500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_Avg | 24152.0 | 32069.0 | 27840.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P25 | 24152.0 | 32069.0 | 26500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P50 | 24152.0 | 32069.0 | 27500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P75 | 24152.0 | 32069.0 | 28500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P90 | 24152.0 | 32069.0 | 29500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P99 | 24152.0 | 32069.0 | 31000.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P99_9 | 24152.0 | 32069.0 | 31500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P99_99 | 24152.0 | 32069.0 | 31800.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P99_999 | 24152.0 | 32069.0 | 31900.0 | connections/sec | +| NCPS Server | ConnectsPerSec_Mad | 500.0 | 1500.0 | 1000.0 | connections/sec | +| NCPS Server | ConnectsPerSec_StandardErrorMean | 100.0 | 300.0 | 200.0 | connections/sec | +| NCPS Server | ConnectsPerSec_LowerCI | 24000.0 | 31500.0 | 27500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_UpperCI | 25000.0 | 32500.0 | 28200.0 | connections/sec | +| NCPS Server | RexmitConnPercentage | 0.0 | 3.0 | 1.5 | % | +| NCPS Server | RexmitPerConn | 0.0 | 1.5 | 1.0 | retransmits/connection | +| NCPS Server | SynRttMean | 3000.0 | 20000.0 | 12000.0 | microseconds | +| NCPS Server | SynRttMedian | 1500.0 | 8000.0 | 4500.0 | microseconds | +| NCPS Server | SynRttP25 | 800.0 | 4000.0 | 2200.0 | microseconds | +| NCPS Server | SynRttP75 | 4000.0 | 12000.0 | 8000.0 | microseconds | +| NCPS Server | SynRttP90 | 8000.0 | 20000.0 | 14000.0 | microseconds | +| NCPS Server | SynRttP95 | 12000.0 | 28000.0 | 20000.0 | microseconds | +| NCPS Server | SynRttP99 | 25000.0 | 60000.0 | 45000.0 | microseconds | +| NCPS Server | SynRttP99_9 | 800000.0 | 2500000.0 | 1500000.0 | microseconds | +| NCPS Server | SynRttP99_99 | 1500000.0 | 4000000.0 | 3000000.0 | microseconds | + | Scenario | Metric Name | Example Value (min) | Example Value (max) | Example Value (avg) | Unit | |-----------|-------------|---------------------|---------------------|---------------------|------| | Latte Client | Latency-Average | 0.0 | 502.28 | 248.65472527472529 | microseconds | From 62e4cfc4fae49d9aa639c0fd69b85e502693b91b Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 23 Jan 2026 13:14:52 -0600 Subject: [PATCH 4/5] Update pkg ref --- src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json index e2827b9ecf..35ab5311b2 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json @@ -341,7 +341,7 @@ "Parameters": { "Scenario": "InstallNetworkToolsetPackage", "BlobContainer": "packages", - "BlobName": "networking.3.1.0.zip", + "BlobName": "networking.3.1.1.zip", "PackageName": "networking", "Extract": true } From 7811aba5378ba2e9b08af9bb18cca03ea6cfa219 Mon Sep 17 00:00:00 2001 From: Rakeshwar Reddy Kambaiahgari Date: Fri, 23 Jan 2026 13:28:55 -0600 Subject: [PATCH 5/5] Update Tests --- .../Network/NCPS/NCPSMetricsParserTests.cs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs index 8402dff6fb..858b2d5171 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs @@ -113,24 +113,33 @@ public void NcpsParserHandlesMissingThroughputMetrics() { this.mockFixture.Setup(PlatformID.Win32NT); string results = @" - ###ENDCPS 12345 +###ENDCPS 12345 - ###CPS,100000:1000 +###SYNRTT,25:100,Median:200,Mean:150,75:300,90:400,95:500,99:600,99.9:700,99.99:800 - ###SYNRTT,25:100,Median:200,Mean:150,75:300,90:400,95:500,99:600,99.9:700,99.99:800 - - ###REXMIT,rtconnpercentage:1.5,rtperconn:1.2 - "; +###REXMIT,rtconnpercentage:1.5,rtperconn:1.2 +"; NCPSMetricsParser parser = new NCPSMetricsParser(results, 90, 5); IList metrics = parser.Parse(); Assert.IsNotEmpty(metrics); + // Should have CPS, RTT (9 metrics), and Retransmit (2 metrics) = 12 total metrics + Assert.AreEqual(12, metrics.Count); + // Should have CPS, RTT, and Retransmit metrics, but no throughput metrics Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Cps")); Assert.IsNull(metrics.FirstOrDefault(m => m.Name == "RxGbps")); Assert.IsNull(metrics.FirstOrDefault(m => m.Name == "TxGbps")); + + // Verify specific RTT metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMean")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMedian")); + + // Verify retransmit metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitConnPercentage")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitPerConn")); } [Test]