diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Client.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Client.txt new file mode 100644 index 0000000000..808f0c16ff --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Client.txt @@ -0,0 +1,49 @@ + T(sec) N Pend Failed IOFail Conn/s Close/s RXkbyte/s TXkbyte/s RT/i c0/i c0rtt/i cR/i cRrtt/i + 1.001 0 1600 0 0 24152.0 22552.0 24151.0 24153.0 0 24152 10543 0 0 + 2.001 0 1600 0 0 32069.0 32069.0 32069.0 32069.0 0 32069 9567 0 0 + 3.002 10 1590 0 0 30695.7 30685.7 30695.7 30695.7 0 30695 8778 0 0 + 4.002 4 1596 0 0 29827.0 29833.0 29827.0 29828.0 0 29827 8967 0 0 + 5.003 6 1594 0 0 28873.6 28873.6 28873.6 28873.6 0 28873 10034 0 0 + 6.003 8 1592 0 0 27841.0 27839.0 27841.0 27841.0 0 27841 9456 0 0 + 7.003 7 1593 0 0 26934.0 26935.0 26934.0 26934.0 0 26934 10234 0 0 + 8.003 0 1600 0 0 26123.0 26130.0 26123.0 26123.0 0 26123 11234 0 0 + 9.003 5 1595 0 0 25389.0 25384.0 25389.0 25389.0 0 25389 11875 0 0 + 10.004 0 1600 0 0 24720.9 24725.9 24720.9 24720.9 0 24720 13132 0 0 + +=== CMDLINE: ./ncps -c 10.1.0.5 -r 16 -bp 9800 -np 16 -N 1600 -P 1600 -D 0 -M 1 -i 1 -wt 30 -t 330 + +=== VERSION 1.0 + +###RXGBPS 0.65 +###TXGBPS 0.65 + +=== Time (ms) to Nth connection establishment for first 1250000 connections: +=== N T(ms) CPS +=== 100000 3925 25477 +=== 200000 7901 25316 +=== 300000 11932 25142 +=== 400000 15987 25021 +=== 500000 20062 24922 +=== 600000 24161 24828 +=== 700000 28262 24775 +=== 800000 32401 24690 +=== 900000 36541 24631 +=== 1000000 40707 24566 +=== 1100000 44883 24509 +=== 1200000 49081 24448 +=== 1250000 51188 24416 + +###ENDCPS 24416 + +###CPS,100000:3925,200000:7901,300000:11932,400000:15987,500000:20062,600000:24161,700000:28262,800000:32401,900000:36541,1000000:40707,1100000:44883,1200000:49081,1250000:51188 + +=== SYN RTT (us) stats for first 1250000 connections: +=== P25 Median Mean P75 P90 P95 P99 P99.9 P99.99 +=== 3424 6221 18737 10893 17628 24607 54222 2027000 4057000 + +###SYNRTT,25:3424,Median:6221,Mean:18737,75:10893,90:17628,95:24607,99:54222,99.9:2027000,99.99:4057000 + +=== Percentage of connections with retransmits in the first 1250000 connections: 3.2825% +=== Average retransmit count per connection (excluding 0-retransmit cases): 1.1436 + +###REXMIT,rtconnpercentage:3.2825,rtperconn:1.1436 diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Server.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Server.txt new file mode 100644 index 0000000000..dd1a9b104c --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/NCPS/NCPS_Example_Results_Server.txt @@ -0,0 +1,42 @@ + T(sec) N Pend Failed IOFail Conn/s Close/s RXkbyte/s TXkbyte/s RT/i c0/i c0rtt/i cR/i cRrtt/i + 1.002 7 1593 0 0 69065.9 69058.9 69062.9 69065.9 0 69204 12888 0 0 + 2.002 14 1586 0 0 69987.0 69980.0 69978.0 69984.0 606 69381 12962 606 161224 + 3.002 0 1600 0 0 67533.0 67547.0 67545.0 67536.0 618 66915 11499 618 434771 + 4.002 8 1592 0 0 66157.0 66149.0 66149.0 66157.0 925 65332 12544 825 407273 + 5.002 12 1588 0 0 64957.0 64953.0 64953.0 64956.0 711 64301 13859 656 524382 + 6.003 23 1577 0 0 63437.6 63426.6 63430.6 63431.6 669 62916 14764 585 537452 + 7.003 3 1597 0 0 54750.0 54770.0 54766.0 54757.0 727 54029 9112 721 419237 + 8.003 6 1594 0 0 56852.0 56849.0 56849.0 56852.0 857 56003 10083 849 426095 + 9.003 2 1598 0 0 55323.0 55327.0 55328.0 55322.0 1143 54181 9149 1142 261028 + 10.003 0 1600 0 0 56011.0 56013.0 56012.0 56012.0 1197 54848 9935 1163 237388 + +=== CMDLINE: ./ncps -s -r 16 -bp 9800 -np 16 -t 300 + +=== VERSION 1.4 + +###RXGBPS 0.24 +###TXGBPS 0.24 + +=== Time (ms) to Nth connection establishment for first 556947 connections: +=== N T(ms) CPS +=== 100000 1678 59594 +=== 200000 3467 57686 +=== 300000 5248 57164 +=== 400000 7093 56393 +=== 500000 8919 56060 +=== 556947 10002 55683 + +###ENDCPS 55683 + +###CPS,100000:1678,200000:3467,300000:5248,400000:7093,500000:8919,556947:10002 + +=== SYN RTT (us) stats for first 556947 connections: +=== P25 Median Mean P75 P90 P95 P99 P99.9 P99.99 +=== 2342 4441 15996 13249 24443 38198 79657 1038000 1057000 + +###SYNRTT,25:2342,Median:4441,Mean:15996,75:13249,90:24443,95:38198,99:79657,99.9:1038000,99.99:1057000 + +=== Percentage of connections with retransmits in the first 556947 connections: 1.7278% +=== Average retransmit count per connection (excluding 0-retransmit cases): 1.0138 + +###REXMIT,rtconnpercentage:1.7278,rtperconn:1.0138 diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSClientExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSClientExecutorTests.cs new file mode 100644 index 0000000000..efa0566202 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSClientExecutorTests.cs @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Actions.NetworkPerformance; + using VirtualClient.Contracts; + using Polly; + using System.Net.Http; + using System.Net; + using System.IO; + using System.Reflection; + using VirtualClient.Common.Telemetry; + using VirtualClient.Common.Contracts; + + [TestFixture] + [Category("Unit")] + public class NCPSClientExecutorTests + { + private MockFixture mockFixture; + private DependencyPath mockPath; + private NetworkingWorkloadState networkingWorkloadState; + + [SetUp] + public void SetupTest() + { + this.mockFixture = new MockFixture(); + this.mockPath = new DependencyPath("NetworkingWorkload", this.mockFixture.PlatformSpecifics.GetPackagePath("networkingworkload")); + this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(this.mockPath); + + this.mockFixture.Directory.Setup(d => d.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.Parameters["PackageName"] = "Networking"; + this.mockFixture.Parameters["ThreadCount"] = "16"; + this.mockFixture.Parameters["TestDuration"] = "00:05:00"; + this.mockFixture.Parameters["WarmupTime"] = "00:00:30"; + this.mockFixture.Parameters["Delaytime"] = "00:00:00"; + this.mockFixture.Parameters["ConfidenceLevel"] = "99"; + + string currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + string resultsPath = Path.Combine(currentDirectory, "Examples", "NCPS", "NCPS_Example_Results_Server.txt"); + string results = File.ReadAllText(resultsPath); + + this.mockFixture.Process.StandardOutput.Append(results); + + this.mockFixture.FileSystem.Setup(rt => rt.File.ReadAllTextAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(results); + + this.SetupNetworkingWorkloadState(); + } + + [Test] + public void NCPSClientExecutorThrowsOnUnsupportedOS() + { + this.mockFixture.SystemManagement.SetupGet(sm => sm.Platform).Returns(PlatformID.Other); + TestNCPSClientExecutor component = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.ThrowsAsync(() => component.ExecuteAsync(CancellationToken.None)); + } + + [Test] + public async Task NCPSClientExecutorExecutesAsExpected() + { + NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor networkingWorkloadExecutor = new NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + await networkingWorkloadExecutor.OnInitialize.Invoke(EventContext.None, CancellationToken.None); + + int processExecuted = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (file, arguments, workingDirectory) => + { + processExecuted++; + this.networkingWorkloadState.ToolState = NetworkingWorkloadToolState.Stopped; + var expectedStateItem = new Item(nameof(NetworkingWorkloadState), this.networkingWorkloadState); + + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, expectedStateItem)); + + return this.mockFixture.Process; + }; + + TestNCPSClientExecutor component = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + await component.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(1, processExecuted); + } + + [Test] + public async Task NCPSClientExecutorExecutesAsExpectedWithIntegerTimeParameters() + { + this.mockFixture.Parameters["TestDuration"] = 120; + this.mockFixture.Parameters["WarmupTime"] = 30; + this.mockFixture.Parameters["Delaytime"] = 15; + + NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor networkingWorkloadExecutor = new NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + await networkingWorkloadExecutor.OnInitialize.Invoke(EventContext.None, CancellationToken.None); + + int processExecuted = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (file, arguments, workingDirectory) => + { + processExecuted++; + this.networkingWorkloadState.ToolState = NetworkingWorkloadToolState.Stopped; + var expectedStateItem = new Item(nameof(NetworkingWorkloadState), this.networkingWorkloadState); + + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, expectedStateItem)); + + return this.mockFixture.Process; + }; + + TestNCPSClientExecutor component = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.AreEqual(TimeSpan.FromSeconds(120), component.TestDuration); + Assert.AreEqual(TimeSpan.FromSeconds(30), component.WarmupTime); + Assert.AreEqual(TimeSpan.FromSeconds(15), component.DelayTime); + + await component.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(1, processExecuted); + } + + [Test] + public void NCPSClientExecutorSupportsIntegerAndTimeSpanTimeFormats() + { + this.mockFixture.Parameters["TestDuration"] = 300; + this.mockFixture.Parameters["WarmupTime"] = 60; + this.mockFixture.Parameters["Delaytime"] = 30; + + TestNCPSClientExecutor executor = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.AreEqual(TimeSpan.FromSeconds(300), executor.TestDuration); + Assert.AreEqual(TimeSpan.FromSeconds(60), executor.WarmupTime); + Assert.AreEqual(TimeSpan.FromSeconds(30), executor.DelayTime); + + this.mockFixture.Parameters["TestDuration"] = "00:05:00"; + this.mockFixture.Parameters["WarmupTime"] = "00:01:00"; + this.mockFixture.Parameters["Delaytime"] = "00:00:30"; + + executor = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.AreEqual(TimeSpan.FromMinutes(5), executor.TestDuration); + Assert.AreEqual(TimeSpan.FromMinutes(1), executor.WarmupTime); + Assert.AreEqual(TimeSpan.FromSeconds(30), executor.DelayTime); + + this.mockFixture.Parameters["TestDuration"] = 180; + executor = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + TimeSpan integerBasedDuration = executor.TestDuration; + + this.mockFixture.Parameters["TestDuration"] = "00:03:00"; + executor = new TestNCPSClientExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + TimeSpan timespanBasedDuration = executor.TestDuration; + + Assert.AreEqual(integerBasedDuration, timespanBasedDuration); + } + + private void SetupNetworkingWorkloadState() + { + this.networkingWorkloadState = new NetworkingWorkloadState(); + this.networkingWorkloadState.Scenario = "AnyScenario"; + this.networkingWorkloadState.Tool = NetworkingWorkloadTool.NCPS; + this.networkingWorkloadState.ToolState = NetworkingWorkloadToolState.Running; + this.networkingWorkloadState.Protocol = "TCP"; + this.networkingWorkloadState.TestMode = "MockTestMode"; + + var expectedStateItem = new Item(nameof(NetworkingWorkloadState), this.networkingWorkloadState); + + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, expectedStateItem)); + } + + private class TestNCPSClientExecutor : NCPSClientExecutor + { + public TestNCPSClientExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs new file mode 100644 index 0000000000..858b2d5171 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSMetricsParserTests.cs @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions.NetworkPerformance +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using NUnit.Framework; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class NCPSMetricsParserTests + { + private MockFixture mockFixture; + + [SetUp] + public void SetupDefaults() + { + this.mockFixture = new MockFixture(); + } + + [Test] + public void NcpsParserParsesExpectedMetricsFromValidServerSideResults() + { + this.mockFixture.Setup(PlatformID.Win32NT); + string results = NCPSMetricsParserTests.GetFileContents("NCPS_Example_Results_Server.txt"); + + NCPSMetricsParser parser = new NCPSMetricsParser(results, 90, 5); + IList metrics = parser.Parse(); + + Assert.IsNotEmpty(metrics); + Assert.IsTrue(metrics.Count == 30); // 28 + 2 throughput metrics + + // Throughput metrics (new in NCPS) + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RxGbps" && m.Value == 0.24)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "TxGbps" && m.Value == 0.24)); + + // CPS metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Cps" && m.Value == 55683)); + + // SYN RTT metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMean" && m.Value == 15996)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMedian" && m.Value == 4441)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP25" && m.Value == 2342)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP75" && m.Value == 13249)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP90" && m.Value == 24443)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP95" && m.Value == 38198)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99" && m.Value == 79657)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99_9" && m.Value == 1038000)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99_99" && m.Value == 1057000)); + + // Retransmit metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitConnPercentage" && m.Value == 1.7278)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitPerConn" && m.Value == 1.0138)); + + // Statistical metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_Min")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_Max")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_Med")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_Avg")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_P25")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_P50")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_P75")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_P90")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_P99")); + } + + [Test] + public void NcpsParserParsesExpectedMetricsFromValidClientSideResults() + { + this.mockFixture.Setup(PlatformID.Win32NT); + string results = NCPSMetricsParserTests.GetFileContents("NCPS_Example_Results_Client.txt"); + + NCPSMetricsParser parser = new NCPSMetricsParser(results, 90, 30); + IList metrics = parser.Parse(); + + Assert.IsNotEmpty(metrics); + // Client results have no periodic data after warmup (all 10 rows are within warmup period of 30s) + // So we only get: 2 throughput + 1 CPS + 9 SYN RTT + 2 Retransmit = 14 metrics + Assert.IsTrue(metrics.Count == 14, $"Expected 14 metrics but got {metrics.Count}"); + + // Throughput metrics (new in NCPS) + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RxGbps" && m.Value == 0.65)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "TxGbps" && m.Value == 0.65)); + + // CPS metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Cps" && m.Value == 24416)); + + // SYN RTT metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMean" && m.Value == 18737)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMedian" && m.Value == 6221)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP25" && m.Value == 3424)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP75" && m.Value == 10893)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP90" && m.Value == 17628)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP95" && m.Value == 24607)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99" && m.Value == 54222)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99_9" && m.Value == 2027000)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttP99_99" && m.Value == 4057000)); + + // Retransmit metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitConnPercentage" && m.Value == 3.2825)); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitPerConn" && m.Value == 1.1436)); + + // No statistical metrics because all periodic data falls within warmup period + Assert.IsNull(metrics.FirstOrDefault(m => m.Name == "ConnectsPerSec_Min")); + } + + [Test] + public void NcpsParserHandlesMissingThroughputMetrics() + { + this.mockFixture.Setup(PlatformID.Win32NT); + string results = @" +###ENDCPS 12345 + +###SYNRTT,25:100,Median:200,Mean:150,75:300,90:400,95:500,99:600,99.9:700,99.99:800 + +###REXMIT,rtconnpercentage:1.5,rtperconn:1.2 +"; + + NCPSMetricsParser parser = new NCPSMetricsParser(results, 90, 5); + IList metrics = parser.Parse(); + + Assert.IsNotEmpty(metrics); + + // Should have CPS, RTT (9 metrics), and Retransmit (2 metrics) = 12 total metrics + Assert.AreEqual(12, metrics.Count); + + // Should have CPS, RTT, and Retransmit metrics, but no throughput metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "Cps")); + Assert.IsNull(metrics.FirstOrDefault(m => m.Name == "RxGbps")); + Assert.IsNull(metrics.FirstOrDefault(m => m.Name == "TxGbps")); + + // Verify specific RTT metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMean")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "SynRttMedian")); + + // Verify retransmit metrics + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitConnPercentage")); + Assert.IsNotNull(metrics.FirstOrDefault(m => m.Name == "RexmitPerConn")); + } + + [Test] + public void NcpsParserThrowsOnInvalidResults() + { + this.mockFixture.Setup(PlatformID.Win32NT); + string results = "Invalid results content"; + + NCPSMetricsParser parser = new NCPSMetricsParser(results, 90, 5); + + // The parser should throw WorkloadResultsException when parsing invalid content + Assert.Throws(() => parser.Parse()); + } + + [Test] + public void NcpsParserHandlesDifferentConfidenceLevels() + { + this.mockFixture.Setup(PlatformID.Win32NT); + string results = NCPSMetricsParserTests.GetFileContents("NCPS_Example_Results_Server.txt"); + + // Test with 95% confidence level + NCPSMetricsParser parser95 = new NCPSMetricsParser(results, 95, 5); + IList metrics95 = parser95.Parse(); + + // Test with 99% confidence level + NCPSMetricsParser parser99 = new NCPSMetricsParser(results, 99, 5); + IList metrics99 = parser99.Parse(); + + // Both should parse successfully with different confidence intervals + Assert.IsNotEmpty(metrics95); + Assert.IsNotEmpty(metrics99); + + // Confidence intervals should be different + var lowerCI95 = metrics95.FirstOrDefault(m => m.Name == "ConnectsPerSec_LowerCI"); + var lowerCI99 = metrics99.FirstOrDefault(m => m.Name == "ConnectsPerSec_LowerCI"); + + Assert.IsNotNull(lowerCI95); + Assert.IsNotNull(lowerCI99); + Assert.AreNotEqual(lowerCI95.Value, lowerCI99.Value); + } + + private static string GetFileContents(string fileName) + { + string outputPath = Path.Combine(MockFixture.TestAssemblyDirectory, "Examples", "NCPS", fileName); + return File.ReadAllText(outputPath); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSServerExecutorTests.cs new file mode 100644 index 0000000000..c4b915e6a6 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Network/NCPS/NCPSServerExecutorTests.cs @@ -0,0 +1,153 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using Polly; + using System; + using System.Collections.Generic; + using System.IO; + using System.Net; + using System.Net.Http; + using System.Reflection; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Actions.NetworkPerformance; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class NCPSServerExecutorTests + { + private static readonly string ExamplesDirectory = MockFixture.GetDirectory(typeof(NTttcpExecutorTests2), "Examples", "NCPS"); + + private MockFixture mockFixture; + private DependencyPath mockPackage; + private NetworkingWorkloadState workloadState; + + public void SetupApiCalls() + { + this.workloadState = new NetworkingWorkloadState(); + this.workloadState.Scenario = "AnyScenario"; + this.workloadState.Tool = NetworkingWorkloadTool.NCPS; + this.workloadState.ToolState = NetworkingWorkloadToolState.Running; + this.workloadState.Protocol = "TCP"; + this.workloadState.TestMode = "MockTestMode"; + + var expectedStateItem = new Item(nameof(NetworkingWorkloadState), this.workloadState); + + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, expectedStateItem)); + } + + [SetUp] + public void SetupTest() + { + this.mockFixture = new MockFixture(); + this.mockPackage = new DependencyPath("ncps", this.mockFixture.PlatformSpecifics.GetPackagePath("ncps")); + this.mockFixture.SetupPackage(this.mockPackage); + + this.mockFixture.Directory.Setup(d => d.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.Parameters["PackageName"] = "ncps"; + this.mockFixture.Parameters["ThreadCount"] = "16"; + this.mockFixture.Parameters["TestDuration"] = "00:05:00"; + this.mockFixture.Parameters["WarmupTime"] = "00:00:30"; + this.mockFixture.Parameters["Delaytime"] = "00:00:00"; + this.mockFixture.Parameters["ConfidenceLevel"] = "99"; + + string exampleResults = File.ReadAllText(this.mockFixture.Combine(NCPSServerExecutorTests.ExamplesDirectory, "NCPS_Example_Results_Server.txt")); + + this.mockFixture.Process.StandardOutput.Append(exampleResults); + + this.mockFixture.FileSystem.Setup(rt => rt.File.ReadAllTextAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(exampleResults); + + this.SetupApiCalls(); + } + + [Test] + public void NCPSServerExecutorThrowsOnUnsupportedOS() + { + this.mockFixture.SystemManagement.SetupGet(sm => sm.Platform).Returns(PlatformID.Other); + TestNCPSServerExecutor component = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.ThrowsAsync(() => component.ExecuteAsync(CancellationToken.None)); + } + + [Test] + [Platform(Exclude = "Unix,Linux,MacOsX")] + public async Task NCPSServerExecutorExecutesAsExpected() + { + NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor networkingWorkloadExecutor = new NetworkingWorkloadExecutorTests.TestNetworkingWorkloadExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + await networkingWorkloadExecutor.OnInitialize.Invoke(EventContext.None, CancellationToken.None); + string agentId = $"{Environment.MachineName}-Server"; + this.mockFixture.SystemManagement.SetupGet(obj => obj.AgentId).Returns(agentId); + + int processExecuted = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (file, arguments, workingDirectory) => + { + processExecuted++; + + return this.mockFixture.Process; + }; + + TestNCPSServerExecutor component = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + await component.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + Assert.AreEqual(1, processExecuted); + } + + [Test] + public void NCPSServerExecutorUsesExpectedDefaultParameters() + { + TestNCPSServerExecutor executor = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + + Assert.AreEqual(16, executor.ThreadCount); + Assert.AreEqual(9800, executor.Port); + Assert.AreEqual(16, executor.PortCount); + Assert.AreEqual("1", executor.DataTransferMode); + } + + [Test] + public void NCPSServerExecutorSupportsCustomDataTransferModes() + { + // Test continuous send mode + this.mockFixture.Parameters["DataTransferMode"] = "s"; + TestNCPSServerExecutor executor = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + Assert.AreEqual("s", executor.DataTransferMode); + + // Test continuous receive mode + this.mockFixture.Parameters["DataTransferMode"] = "r"; + executor = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + Assert.AreEqual("r", executor.DataTransferMode); + + // Test ping-pong mode + this.mockFixture.Parameters["DataTransferMode"] = "p"; + executor = new TestNCPSServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters); + Assert.AreEqual("p", executor.DataTransferMode); + } + + private class TestNCPSServerExecutor : NCPSServerExecutor + { + public TestNCPSServerExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + + protected override bool IsProcessRunning(string processName) + { + return true; + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSClientExecutor.cs new file mode 100644 index 0000000000..b6d0a71910 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSClientExecutor.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions.NetworkPerformance +{ + using System; + using System.Collections.Generic; + using System.Linq; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient.Contracts; + + /// + /// NCPS(New Connections Per Second) Tool Client Executor. + /// + public class NCPSClientExecutor : NCPSExecutor + { + /// + /// Initializes a new instance of the class. + /// + /// Component to copy. + public NCPSClientExecutor(VirtualClientComponent component) + : base(component) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public NCPSClientExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + + /// + /// Returns the NCPS client-side command line arguments. + /// + protected override string GetCommandLineArguments() + { + string serverIPAddress = this.GetLayoutClientInstances(ClientRole.Server).First().IPAddress; + + // Build the command line based on NCPS recipe + // ncps -c -wt 30 -t 330 + string command = $"-c {serverIPAddress} " + + $"-r {this.ThreadCount} " + + $"-bp {this.Port} " + + $"-np {this.PortCount} " + + $"-N {this.TotalConnectionsToOpen} " + + $"-P {this.MaxPendingRequests} " + + $"-D {this.ConnectionDuration} " + + $"-M {this.DataTransferMode} " + + $"-i {this.DisplayInterval} " + + $"-wt {this.WarmupTime.TotalSeconds} " + + $"-t {this.TestDuration.TotalSeconds} "; + + if (this.DelayTime != TimeSpan.Zero) + { + command += $"-ds {this.DelayTime.TotalSeconds} "; + } + + if (!string.IsNullOrWhiteSpace(this.AdditionalParams)) + { + command += this.AdditionalParams; + } + + return command.Trim(); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSExecutor.cs new file mode 100644 index 0000000000..84eb72c942 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSExecutor.cs @@ -0,0 +1,364 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions.NetworkPerformance +{ + using System; + using System.Collections.Generic; + using System.IO.Abstractions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + using VirtualClient.Contracts.Metadata; + + /// + /// NCPS(New Connections Per Second) Tool Executor. + /// + public class NCPSExecutor : NetworkingWorkloadToolExecutor + { + private IFileSystem fileSystem; + + /// + /// Initializes a new instance of the class. + /// + /// Component to copy. + public NCPSExecutor(VirtualClientComponent component) + : base(component) + { + this.ProcessStartRetryPolicy = Policy.Handle(exc => exc.Message.Contains("sockwiz")).Or() + .WaitAndRetryAsync(5, retries => TimeSpan.FromSeconds(retries * 3)); + } + + /// + /// Initializes a new instance of the class. + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public NCPSExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.fileSystem = dependencies.GetService(); + this.ProcessStartRetryPolicy = Policy.Handle(exc => exc.Message.Contains("sockwiz")).Or() + .WaitAndRetryAsync(5, retries => TimeSpan.FromSeconds(retries * 3)); + } + + /// + /// The thread count for the workload. + /// + public int ThreadCount + { + get + { + return this.Parameters.GetValue(nameof(this.ThreadCount), 16); + } + } + + /// + /// The total number of connections to keep open (client-side only). + /// + public int TotalConnectionsToOpen + { + get + { + return this.Parameters.GetValue(nameof(this.TotalConnectionsToOpen), this.ThreadCount * 100); + } + } + + /// + /// The max number of pending connect requests at any given time (client-side only). + /// + public int MaxPendingRequests + { + get + { + return this.Parameters.GetValue(nameof(this.MaxPendingRequests), this.TotalConnectionsToOpen); + } + } + + /// + /// The duration (in milliseconds) for each connection. + /// + public int ConnectionDuration + { + get + { + return this.Parameters.GetValue(nameof(this.ConnectionDuration), 0); + } + } + + /// + /// The data transfer mode for each connection. + /// 0: no send/receive, 1: one send/receive, p: ping/pong (continuous send/receive) + /// s: continuous send, r: continuous receive + /// + public string DataTransferMode + { + get + { + return this.Parameters.GetValue(nameof(this.DataTransferMode), "1"); + } + } + + /// + /// The interval (in seconds) to display NCPS stats. + /// + public int DisplayInterval + { + get + { + return this.Parameters.GetValue(nameof(NCPSExecutor.DisplayInterval), 1); + } + } + + /// + /// The starting port for the range of ports that will be used for client/server + /// network connections. + /// + public int Port + { + get + { + return this.Parameters.GetValue(nameof(this.Port), 9800); + } + } + + /// + /// Number of TCP ports to listen on or connect to. + /// + public int PortCount + { + get + { + return this.Parameters.GetValue(nameof(this.PortCount), this.ThreadCount); + } + } + + /// + /// Parameter defines the duration for running the NCPS workload. + /// Note: NCPS -t parameter includes warmup time specified by -wt parameter. + /// + public TimeSpan TestDuration + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.TestDuration), TimeSpan.FromSeconds(60)); + } + } + + /// + /// Gets test warmup time values. + /// + public TimeSpan WarmupTime + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.WarmupTime), TimeSpan.FromSeconds(5)); + } + } + + /// + /// Gets test delay time values. + /// + public TimeSpan DelayTime + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.DelayTime), TimeSpan.Zero); + } + } + + /// + /// Gets the confidence level used for calculating the confidence intervals. + /// + public double ConfidenceLevel + { + get + { + return this.Parameters.GetValue(nameof(this.ConfidenceLevel), 99); + } + } + + /// + /// Gets additional optional parameters. + /// + public string AdditionalParams + { + get + { + return this.Parameters.GetValue(nameof(this.AdditionalParams), string.Empty); + } + } + + /// + /// The retry policy to apply to the startup of the NCPS workload to handle + /// transient issues. + /// + protected IAsyncPolicy ProcessStartRetryPolicy { get; set; } + + /// + /// Initializes the environment and dependencies for running the tool. + /// + protected override Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(this.Scenario)) + { + throw new WorkloadException( + $"Scenario parameter missing. The profile supplied is missing the required '{nameof(this.Scenario)}' parameter " + + $"for one or more of the '{nameof(NCPSExecutor)}' steps.", + ErrorReason.InvalidProfileDefinition); + } + + DependencyPath workloadPackage = this.GetDependencyPath(this.PackageName, cancellationToken); + + this.IsInClientRole = this.IsInRole(ClientRole.Client); + this.IsInServerRole = !this.IsInClientRole; + this.Role = this.IsInClientRole ? ClientRole.Client : ClientRole.Server; + + // e.g. + // NCPS_T16 Client, NCPS_T16 Server + this.Name = $"{this.Scenario} {this.Role}"; + this.ProcessName = "ncps"; + this.Tool = NetworkingWorkloadTool.NCPS; + + if (this.Platform == PlatformID.Win32NT) + { + this.ExecutablePath = this.Combine(workloadPackage.Path, "ncps.exe"); + } + else if (this.Platform == PlatformID.Unix) + { + this.ExecutablePath = this.Combine(workloadPackage.Path, "ncps"); + } + else + { + throw new NotSupportedException($"{this.Platform} is not supported"); + } + + // Validating NCPS parameters + if (this.TestDuration <= TimeSpan.Zero) + { + throw new WorkloadException("Test duration cannot be equal or less than zero for NCPS workload", ErrorReason.InstructionsNotValid); + } + else if (this.WarmupTime >= this.TestDuration) + { + throw new WorkloadException("WarmUp time must be less than test duration for NCPS workload", ErrorReason.InstructionsNotValid); + } + else if (this.DelayTime >= this.TestDuration) + { + throw new WorkloadException("Delay time must be less than test duration for NCPS workload", ErrorReason.InstructionsNotValid); + } + else if ((this.DelayTime + this.WarmupTime) >= this.TestDuration) + { + throw new WorkloadException("Sum of delay time and WarmUp time must be less than test duration for NCPS workload", ErrorReason.InstructionsNotValid); + } + + return this.SystemManagement.MakeFileExecutableAsync(this.ExecutablePath, this.Platform, cancellationToken); + } + + /// + /// Returns the NCPS command line arguments. + /// + protected override string GetCommandLineArguments() + { + return null; + } + + /// + protected override Task ExecuteWorkloadAsync(string commandArguments, EventContext telemetryContext, CancellationToken cancellationToken, TimeSpan? timeout = null) + { + IProcessProxy process = null; + + EventContext relatedContext = telemetryContext.Clone() + .AddContext("command", this.ExecutablePath) + .AddContext("commandArguments", commandArguments); + + return this.Logger.LogMessageAsync($"{this.TypeName}.ExecuteWorkload", relatedContext, async () => + { + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await this.ProcessStartRetryPolicy.ExecuteAsync(async () => + { + using (process = this.SystemManagement.ProcessManager.CreateProcess(this.ExecutablePath, commandArguments)) + { + try + { + this.CleanupTasks.Add(() => process.SafeKill(this.Logger)); + await process.StartAndWaitAsync(cancellationToken, timeout, withExitConfirmation: true); + await this.LogProcessDetailsAsync(process, relatedContext, "NCPS"); + process.ThrowIfWorkloadFailed(); + + this.CaptureMetrics( + process.StandardOutput.ToString(), + process.FullCommand(), + process.StartTime, + process.ExitTime, + relatedContext); + } + catch (OperationCanceledException) + { + // Expected when the client signals a cancellation. + } + catch (TimeoutException exc) + { + // We give this a best effort but do not want it to prevent the next workload + // from executing. + this.Logger.LogMessage($"{this.GetType().Name}.WorkloadTimeout", LogLevel.Warning, relatedContext.AddError(exc)); + process.SafeKill(this.Logger); + + throw new WorkloadException($"NCPS workload did not exit within the timeout period defined (timeout={timeout}).", exc, ErrorReason.WorkloadFailed); + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.GetType().Name}.WorkloadStartupError", LogLevel.Warning, relatedContext.AddError(exc)); + throw new WorkloadException($"NCPS workload failed to start successfully", exc, ErrorReason.WorkloadFailed); + } + } + }); + } + + return process; + }); + } + + /// + /// Logs the workload metrics to the telemetry. + /// + protected override void CaptureMetrics(string results, string commandArguments, DateTime startTime, DateTime endTime, EventContext telemetryContext) + { + if (!string.IsNullOrWhiteSpace(results)) + { + this.MetadataContract.AddForScenario( + this.Tool.ToString(), + commandArguments, + toolVersion: null); + + this.MetadataContract.Apply(telemetryContext); + + MetricsParser parser = new NCPSMetricsParser(results, this.ConfidenceLevel, this.WarmupTime.TotalSeconds); + IList metrics = parser.Parse(); + + this.Logger.LogMetrics( + this.Tool.ToString(), + this.Name, + startTime, + endTime, + metrics, + string.Empty, + commandArguments, + this.Tags, + telemetryContext); + } + else + { + throw new WorkloadException( + $"Workload results missing. The NCPS workload did not produce valid results.", + ErrorReason.WorkloadResultsNotFound); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSMetricsParser.cs new file mode 100644 index 0000000000..80ea8f8c75 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSMetricsParser.cs @@ -0,0 +1,338 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions.NetworkPerformance +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Text.RegularExpressions; + using MathNet.Numerics.Distributions; + using MathNet.Numerics.Statistics; + using VirtualClient.Contracts; + + /// + /// NCPS toolset results parser. + /// + public class NCPSMetricsParser : MetricsParser + { + private static readonly Regex CpsExpression = new Regex(@"^###ENDCPS\s(?\d+)", RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex SynRttExpression = new Regex(@"^###SYNRTT,(?.*)", RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex RetransmitsExpression = new Regex(@"^###REXMIT,(?.*)", RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex RxGbpsExpression = new Regex(@"^###RXGBPS\s(?[\d.]+)", RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex TxGbpsExpression = new Regex(@"^###TXGBPS\s(?[\d.]+)", RegexOptions.Compiled | RegexOptions.Multiline); + + private double confidenceLevel; + private double warmupTimeInSeconds; + + /// + /// Initializes a new instance of the class. + /// + public NCPSMetricsParser(string results, double confidenceLevel, double warmupTimeSeconds) + : base(results) + { + this.confidenceLevel = confidenceLevel; + this.warmupTimeInSeconds = warmupTimeSeconds; + } + + /// + /// Parses the NCPS results and returns a list of metrics from them. + /// + public override IList Parse() + { + try + { + IList metrics = new List(); + + // Parse RXGBPS and TXGBPS metrics + MatchCollection matches = NCPSMetricsParser.RxGbpsExpression.Matches(this.RawText); + NCPSMetricsParser.AddThroughputMetrics(metrics, matches, "RxGbps", "Receive throughput in Gbps"); + + matches = NCPSMetricsParser.TxGbpsExpression.Matches(this.RawText); + NCPSMetricsParser.AddThroughputMetrics(metrics, matches, "TxGbps", "Transmit throughput in Gbps"); + + // Parse CPS metrics + matches = NCPSMetricsParser.CpsExpression.Matches(this.RawText); + NCPSMetricsParser.AddConnectionPerSecondMetrics(metrics, matches); + + // Parse SYN RTT metrics + matches = NCPSMetricsParser.SynRttExpression.Matches(this.RawText); + NCPSMetricsParser.AddSynRttMetrics(metrics, matches); + + // Parse retransmit metrics + matches = NCPSMetricsParser.RetransmitsExpression.Matches(this.RawText); + NCPSMetricsParser.AddRetransmitMetrics(metrics, matches); + + // Parse periodic output for statistical metrics + KeyValuePair, List> connectionsPerSec = NCPSMetricsParser.GetTimestampsAndConnectionsPerSec(this.RawText, this.warmupTimeInSeconds); + + if (connectionsPerSec.Value.Count > 0) + { + NCPSMetricsParser.AddStatisticalMetrics(metrics, connectionsPerSec.Key, connectionsPerSec.Value, this.confidenceLevel); + } + + return metrics; + } + catch (Exception exc) + { + throw new WorkloadResultsException( + $"Results parsing operation failed. The NCPS parser failed to parse the results of the NCPS workload.", + exc, + ErrorReason.WorkloadResultsParsingFailed); + } + } + + private static void AddThroughputMetrics(IList metrics, MatchCollection matches, string metricName, string description) + { + if (matches?.Any() == true) + { + if (double.TryParse(matches[0].Groups[1].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out double throughput)) + { + metrics.Add(new Metric(metricName, throughput, "Gbps", MetricRelativity.HigherIsBetter, description: description)); + } + } + } + + private static void AddStatisticalMetrics(IList metrics, List timestamps, List connectsPerSec, double confidenceLevel) + { + Normal normal = new Normal(); + double theta = ((confidenceLevel / 100.0) + 1.0) / 2; + double mean = connectsPerSec.Mean(); + double sd = connectsPerSec.StandardDeviation(); + double inverserCDF = normal.InverseCumulativeDistribution(theta); + double sem = sd / (double)Math.Sqrt(connectsPerSec.Count); + double t = inverserCDF * sem; + double lowerCI = mean - t; + double upperCI = mean + t; + + metrics.Add(new Metric("ConnectsPerSec_Min", connectsPerSec.Min())); + metrics.Add(new Metric("ConnectsPerSec_Max", connectsPerSec.Max())); + metrics.Add(new Metric("ConnectsPerSec_Med", connectsPerSec.Median())); + metrics.Add(new Metric("ConnectsPerSec_Avg", connectsPerSec.Average(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric("ConnectsPerSec_P25", connectsPerSec.Percentile(25))); + metrics.Add(new Metric("ConnectsPerSec_P50", connectsPerSec.Percentile(50))); + metrics.Add(new Metric("ConnectsPerSec_P75", connectsPerSec.Percentile(75))); + metrics.Add(new Metric("ConnectsPerSec_P90", connectsPerSec.Percentile(90))); + metrics.Add(new Metric("ConnectsPerSec_P99", connectsPerSec.Percentile(99))); + metrics.Add(new Metric("ConnectsPerSec_P99_9", Statistics.QuantileCustom(connectsPerSec, 1d - 0.001d, QuantileDefinition.R3))); + metrics.Add(new Metric("ConnectsPerSec_P99_99", Statistics.QuantileCustom(connectsPerSec, 1d - 0.0001d, QuantileDefinition.R3))); + metrics.Add(new Metric("ConnectsPerSec_P99_999", Statistics.QuantileCustom(connectsPerSec, 1d - 0.00001d, QuantileDefinition.R3))); + double median = Statistics.Median(connectsPerSec); + double[] absoluteDeviations = connectsPerSec.Select(x => Math.Abs(x - median)).ToArray(); + metrics.Add(new Metric("ConnectsPerSec_Mad", Statistics.Median(absoluteDeviations), MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_StandardErrorMean", sem, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_LowerCI", lowerCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_UpperCI", upperCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + } + + /// + /// Parses the NCPS results and returns a list of pairs of {Timestamp, Conn/s} for workloads duration (excluding warmupTime). + /// + private static KeyValuePair, List> GetTimestampsAndConnectionsPerSec(string content, double warmupTime) + { + bool appendResult = true; + int valueIndex = 0; + + // The timestamps and connectionsPerSecond co-relate on the index. + List timestamps = new List(); + List connectionsPerSec = new List(); + StringReader strReader = new StringReader(content); + + try + { + double excludeCount = warmupTime; + + while (true) + { + string line = strReader.ReadLine(); + + if (string.IsNullOrEmpty(line)) + { + break; + } + else if (appendResult) + { + string[] stringList = line.Split(" ", StringSplitOptions.RemoveEmptyEntries); + + if (stringList.Contains("Conn/s")) + { + valueIndex = Array.IndexOf(stringList, "Conn/s"); + continue; + } + else if (stringList.Length == 0) + { + appendResult = false; + } + else + { + string formattedTimestamp = string.Format("{0:0.##}", stringList[0]); + string formattedConnectRate = string.Format("{0:0.#}", stringList[valueIndex]); + double timestamp = Convert.ToDouble(formattedTimestamp); + double connectRate = Convert.ToDouble(formattedConnectRate); + + if (excludeCount < Math.Floor(timestamp)) + { + timestamps.Add(Convert.ToDouble(formattedTimestamp)); + connectionsPerSec.Add(Convert.ToDouble(formattedConnectRate)); + } + } + } + } + } + catch (ArgumentNullException exc) + { + throw new WorkloadResultsException( + "Results parsing operation failed. The NCPS parser failed to parse the results of the NCPS workload. Error getting values of Conn/s", + exc, + ErrorReason.WorkloadResultsNotFound); + } + + return new KeyValuePair, List>(timestamps, connectionsPerSec); + } + + private static void AddConnectionPerSecondMetrics(IList metrics, MatchCollection matches) + { + if (matches?.Any() == true) + { + if (double.TryParse(matches[0].Groups[1].Value, out double cps)) + { + metrics.Add(new Metric("Cps", cps, MetricRelativity.HigherIsBetter, description: "Connections per second")); + } + } + } + + private static void AddRetransmitMetrics(IList metrics, MatchCollection matches) + { + if (matches?.Any() == true) + { + Match match = matches.First(); + string[] tokens = match.Groups[1].Value.Split(','); + + foreach (string token in tokens) + { + string[] tokenPair = token.Split(':'); + if (tokenPair?.Any() == true && tokenPair.Length == 2) + { + string metricName = tokenPair[0].Trim(); + string metricValue = tokenPair[1].Trim(); + + switch (metricName.ToLowerInvariant()) + { + case "rtconnpercentage": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double median)) + { + metrics.Add(new Metric("RexmitConnPercentage", median, MetricRelativity.LowerIsBetter, description: "Connection retransmits percentage.")); + } + + break; + + case "rtperconn": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double mean)) + { + metrics.Add(new Metric("RexmitPerConn", mean, MetricRelativity.LowerIsBetter, description: "Retransmits per connection.")); + } + + break; + } + } + } + } + } + + private static void AddSynRttMetrics(IList metrics, MatchCollection matches) + { + if (matches?.Any() == true) + { + Match match = matches.First(); + string[] tokens = match.Groups[1].Value.Split(','); + + foreach (string token in tokens) + { + string[] tokenPair = token.Split(':'); + if (tokenPair?.Any() == true && tokenPair.Length == 2) + { + string metricName = tokenPair[0].Trim(); + string metricValue = tokenPair[1].Trim(); + + switch (metricName.ToLowerInvariant()) + { + case "median": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double median)) + { + metrics.Add(new Metric("SynRttMedian", median, MetricRelativity.LowerIsBetter, description: "Synchronous (SYN) median round-trip time.")); + } + + break; + + case "mean": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double mean)) + { + metrics.Add(new Metric("SynRttMean", mean, MetricRelativity.LowerIsBetter, description: "Synchronous (SYN) mean round-trip time.")); + } + + break; + + case "25": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p25)) + { + metrics.Add(new Metric("SynRttP25", p25, MetricRelativity.LowerIsBetter, description: "P25 Synchronous (SYN) round-trip time.")); + } + + break; + + case "75": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p75)) + { + metrics.Add(new Metric("SynRttP75", p75, MetricRelativity.LowerIsBetter, description: "P75 Synchronous (SYN) round-trip time.")); + } + + break; + + case "90": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p90)) + { + metrics.Add(new Metric("SynRttP90", p90, MetricRelativity.LowerIsBetter, description: "P90 Synchronous (SYN) round-trip time.")); + } + + break; + + case "95": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p95)) + { + metrics.Add(new Metric("SynRttP95", p95, MetricRelativity.LowerIsBetter, description: "P95 Synchronous (SYN) round-trip time.")); + } + + break; + + case "99": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p99)) + { + metrics.Add(new Metric("SynRttP99", p99, MetricRelativity.LowerIsBetter, description: "P99 Synchronous (SYN) round-trip time.")); + } + + break; + + case "99.9": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p99_9)) + { + metrics.Add(new Metric("SynRttP99_9", p99_9, MetricRelativity.LowerIsBetter, description: "P99.9 Synchronous (SYN) round-trip time.")); + } + + break; + + case "99.99": + if (double.TryParse(metricValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double p99_99)) + { + metrics.Add(new Metric("SynRttP99_99", p99_99, MetricRelativity.LowerIsBetter, description: "P99.99 Synchronous (SYN) round-trip time.")); + } + + break; + } + } + } + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSServerExecutor.cs new file mode 100644 index 0000000000..71e8698a9a --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NCPS/NCPSServerExecutor.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions.NetworkPerformance +{ + using System; + using System.Collections.Generic; + using System.Linq; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient.Contracts; + + /// + /// NCPS(New Connections Per Second) Tool Server Executor + /// + public class NCPSServerExecutor : NCPSExecutor + { + /// + /// Initializes a new instance of the class. + /// + /// Component to copy. + public NCPSServerExecutor(VirtualClientComponent component) + : base(component) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public NCPSServerExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + + /// + /// Returns the NCPS server-side command line arguments. + /// + protected override string GetCommandLineArguments() + { + // Build the command line based on NCPS recipe + string command = $"-s " + + $"-r {this.ThreadCount} " + + $"-bp {this.Port} " + + $"-np {this.PortCount} " + + $"-i {this.DisplayInterval} -wt {this.WarmupTime.TotalSeconds} -t {this.TestDuration.TotalSeconds} " + + $"{((this.DelayTime != TimeSpan.Zero) ? $"-ds {this.DelayTime.TotalSeconds}" : string.Empty)}"; + + if (!string.IsNullOrWhiteSpace(this.DataTransferMode) && + (this.DataTransferMode.Equals("s", StringComparison.OrdinalIgnoreCase) || + this.DataTransferMode.Equals("r", StringComparison.OrdinalIgnoreCase))) + { + command += $"-M {this.DataTransferMode} "; + } + + if (!string.IsNullOrWhiteSpace(this.AdditionalParams)) + { + command += this.AdditionalParams; + } + + return command.Trim(); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadExecutor.cs index 6e6ce8183d..e2ad4d3495 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadExecutor.cs @@ -611,6 +611,9 @@ protected override bool IsSupported() case "cps": isSupported = this.Platform == PlatformID.Win32NT || this.Platform == PlatformID.Unix; break; + case "ncps": + isSupported = this.Platform == PlatformID.Win32NT || this.Platform == PlatformID.Unix; + break; case "latte": isSupported = this.Platform == PlatformID.Win32NT; @@ -668,6 +671,10 @@ protected virtual NetworkingWorkloadToolExecutor CreateWorkloadExecutor(Networki action = new CPSClientExecutor(this); break; + case NetworkingWorkloadTool.NCPS: + action = new NCPSClientExecutor(this); + break; + case NetworkingWorkloadTool.NTttcp: action = new NTttcpClientExecutor(this); break; @@ -692,6 +699,10 @@ protected virtual NetworkingWorkloadToolExecutor CreateWorkloadExecutor(Networki action = new CPSServerExecutor(this); break; + case NetworkingWorkloadTool.NCPS: + action = new NCPSServerExecutor(this); + break; + case NetworkingWorkloadTool.NTttcp: action = new NTttcpServerExecutor(this); break; @@ -869,6 +880,11 @@ private async Task ExecuteClientAsync(EventContext telemetryContext, Cancellatio await this.ExecuteClientToolAsync(NetworkingWorkloadTool.CPS, telemetryContext, cancellationToken) .ConfigureAwait(false); } + else if (string.Equals(this.ToolName, NetworkingWorkloadTool.NCPS.ToString(), ignoreCase)) + { + await this.ExecuteClientToolAsync(NetworkingWorkloadTool.NCPS, telemetryContext, cancellationToken) + .ConfigureAwait(false); + } else if (string.Equals(this.ToolName, NetworkingWorkloadTool.Latte.ToString(), ignoreCase)) { if (this.Platform != PlatformID.Win32NT) diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadState.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadState.cs index f3bb6e4f7d..955835d76f 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadState.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NetworkingWorkloadState.cs @@ -42,7 +42,12 @@ public enum NetworkingWorkloadTool /// /// Socket Performance Tool. /// - SockPerf + SockPerf, + + /// + /// NCPS Tool. + /// + NCPS } /// diff --git a/src/VirtualClient/VirtualClient.Dependencies/NetworkConfigurationSetup.cs b/src/VirtualClient/VirtualClient.Dependencies/NetworkConfigurationSetup.cs index 9a77b8340a..7b5a696cb2 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/NetworkConfigurationSetup.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/NetworkConfigurationSetup.cs @@ -292,6 +292,9 @@ private async Task ApplyRcLocalFileCommandsAsync(CancellationToken cancellationT rcLocalContents.Add($"sysctl -w fs.file-max={NetworkConfigurationSetup.NoFileLimit}"); rcLocalContents.Add("sysctl -w net.ipv4.tcp_tw_reuse=1 # TIME_WAIT work-around"); rcLocalContents.Add("sysctl -w net.ipv4.ip_local_port_range=\"10000 60000\" # ephemeral ports increased"); + rcLocalContents.Add("sysctl -w net.ipv4.tcp_syncookies=0 # disable SYN cookies for CPS/NCPS"); + rcLocalContents.Add("sysctl -w net.ipv4.tcp_max_syn_backlog=2048 # increase SYN backlog for CPS/NCPS"); + rcLocalContents.Add("sysctl -w net.ipv4.conf.all.rp_filter=0 # disable reverse path filtering for CPS/NCPS"); if (this.EnableBusyPoll) { @@ -301,8 +304,9 @@ private async Task ApplyRcLocalFileCommandsAsync(CancellationToken cancellationT if (this.DisableFirewall) { - rcLocalContents.Add("iptables -I OUTPUT -j NOTRACK # disable connection tracking"); - rcLocalContents.Add("iptables -I PREROUTING -j NOTRACK # disable connection tracking"); + rcLocalContents.Add("iptables -t raw -I OUTPUT -j NOTRACK # disable connection tracking"); + rcLocalContents.Add("iptables -t raw -I PREROUTING -j NOTRACK # disable connection tracking"); + rcLocalContents.Add("sysctl -w net.netfilter.nf_conntrack_max=0 # disable connection tracking (NCPS)"); rcLocalContents.Add("iptables -P INPUT ACCEPT # accept all inbound traffic"); rcLocalContents.Add("iptables -P OUTPUT ACCEPT # accept all outbound traffic"); rcLocalContents.Add("iptables -P FORWARD ACCEPT # accept all forward traffic"); diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json index a4b2e4004b..35ab5311b2 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json @@ -5,21 +5,23 @@ "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows,AwsLinux" }, - "Parameters": { - "ConfigureNetwork": true, - "EnableBusyPoll": true, - "DisableFirewall": true, - "CpsPort": 7201, - "CpsDuration": "00:05:00", - "LattePort": 6100, - "NTttcpPort": 5500, - "NTttcpDuration": "00:01:00", - "SockPerfPort": 8201, - "SockPerfDuration": "00:01:00", - "ProfilingEnabled": false, - "ProfilingMode": "None", - "TestDuration": "00:01:00" - }, + "Parameters": { + "ConfigureNetwork": true, + "EnableBusyPoll": true, + "DisableFirewall": true, + "CpsPort": 7201, + "CpsDuration": "00:05:00", + "NcpsPort": 9800, + "NcpsDuration": "00:05:00", + "LattePort": 6100, + "NTttcpPort": 5500, + "NTttcpDuration": "00:01:00", + "SockPerfPort": 8201, + "SockPerfDuration": "00:01:00", + "ProfilingEnabled": false, + "ProfilingMode": "None", + "TestDuration": "00:01:00" + }, "Actions": [ { "Type": "NetworkingWorkloadExecutor", @@ -235,19 +237,34 @@ } }, { - "Type": "NetworkingWorkloadExecutor", - "Parameters": { - "Scenario": "CPS_T16", - "ToolName": "CPS", - "PackageName": "Networking", - "Connections": 16, - "TestDuration": "$.Parameters.CpsDuration", - "WarmupTime": "00:01:00", - "Port": "$.Parameters.CpsPort", - "ProfilingScenario": "CPS_T16", - "ProfilingEnabled": "$.Parameters.ProfilingEnabled", - "ProfilingMode": "$.Parameters.ProfilingMode" - } + "Type": "NetworkingWorkloadExecutor", + "Parameters": { + "Scenario": "CPS_T16", + "ToolName": "CPS", + "PackageName": "Networking", + "Connections": 16, + "TestDuration": "$.Parameters.CpsDuration", + "WarmupTime": "00:01:00", + "Port": "$.Parameters.CpsPort", + "ProfilingScenario": "CPS_T16", + "ProfilingEnabled": "$.Parameters.ProfilingEnabled", + "ProfilingMode": "$.Parameters.ProfilingMode" + } + }, + { + "Type": "NetworkingWorkloadExecutor", + "Parameters": { + "Scenario": "NCPS_T16", + "ToolName": "NCPS", + "PackageName": "Networking", + "Connections": 16, + "TestDuration": "$.Parameters.NcpsDuration", + "WarmupTime": "00:01:00", + "Port": "$.Parameters.NcpsPort", + "ProfilingScenario": "NCPS_T16", + "ProfilingEnabled": "$.Parameters.ProfilingEnabled", + "ProfilingMode": "$.Parameters.ProfilingMode" + } }, { "Type": "NetworkingWorkloadExecutor", @@ -324,7 +341,7 @@ "Parameters": { "Scenario": "InstallNetworkToolsetPackage", "BlobContainer": "packages", - "BlobName": "networking.3.1.0.zip", + "BlobName": "networking.3.1.1.zip", "PackageName": "networking", "Extract": true } diff --git a/website/docs/workloads/network-suite/network-suite-profiles.md b/website/docs/workloads/network-suite/network-suite-profiles.md index c24ca2d719..3d2f3ef344 100644 --- a/website/docs/workloads/network-suite/network-suite-profiles.md +++ b/website/docs/workloads/network-suite/network-suite-profiles.md @@ -1,5 +1,5 @@ # Network Workload Suite Profiles -The following profiles run customer-representative or benchmarking scenarios using the suite of network workloads (CPS, NTttcp, Latte and SockPerf). +The following profiles run customer-representative or benchmarking scenarios using the suite of network workloads (CPS, NCPS, NTttcp, Latte and SockPerf). * [Workload Details](./network-suite.md) * [Client/Server Workloads](../../guides/0020-client-server.md) @@ -100,6 +100,7 @@ sysctl -w net.core.busy_read=50 The following scenarios are covered by this workload profile. * CPS. Connection establishment speed and reliability, 16 concurrent connections. + * NCPS. Connection establishment speed and reliability, 16 concurrent connections. * NTttcp. Network communications throughput and bandwidth with the following scenarios. * TCP protocol, 4K network buffer, 1 thread * TCP protocol, 64K network buffer, 1 thread @@ -132,6 +133,8 @@ sysctl -w net.core.busy_read=50 | DisableFirewall | Optional. True to disable the firewall on Linux systems. False to make no changes. This setting depends upon 'ConfigureNetwork' = true. | True | | CpsDuration | Optional. The amount of time (in seconds) to run the CPS workload. | 300 secs | | CpsPort | Optional. The starting port on which connections will be established between client and server when running the CPS workload. The CPS workload will use connections on additional ports starting with this port for each of the number of connections (e.g. 16) defined in the component/action. | 7201 | + | NcpsDuration | Optional. The amount of time (in seconds) to run the NCPS workload. | 300 secs | + | NcpsPort | Optional. The starting port on which connections will be established between client and server when running the NCPS workload. The NCPS workload will use connections on additional ports starting with this port for each of the number of connections (e.g. 16) defined in the component/action. | 9800 | | LattePort | Optional. The starting port on which connections will be established between client and server when running the Latte workload. The Latte workload will use connections on additional ports starting with this port. | 6100 | | NTttcpDuration | Optional. The amount of time (in seconds) to run the NTttcp workload. | 60 secs | | NTttcpPort | Optional. The starting port on which connections will be established between client and server when running the NTttcp workload. The NTttcp workload will use connections on additional ports starting with this port. | 5500 | diff --git a/website/docs/workloads/network-suite/network-suite.md b/website/docs/workloads/network-suite/network-suite.md index c9e6de4e72..a53b2b9598 100644 --- a/website/docs/workloads/network-suite/network-suite.md +++ b/website/docs/workloads/network-suite/network-suite.md @@ -1,5 +1,5 @@ # Network Suite -The Networking workload suite is a set of 4 workloads that are the recommended benchmarks for the Azure Networking team. The workloads are each designed to test network performance +The Networking workload suite is a set of 5 workloads that are the recommended benchmarks for the Azure Networking team. The workloads are each designed to test network performance and reliability. The workloads that are a part of the suite include: @@ -7,6 +7,9 @@ The workloads that are a part of the suite include: * **CPS** This workload that is used to measure network socket connection establishment efficiencies and reliability between a client and a server. +* **NCPS** + This workload (New Connections Per Second) is an enhanced version of CPS that provides more granular connection establishment metrics, throughput measurements, and advanced statistical analysis between a client and a server. + * **[Latte](https://github.com/microsoft/latte)** This workload that is used to measure network communications latencies between a client and a server. This workload runs on Windows systems only. @@ -22,6 +25,10 @@ The following performance analysis scenarios are covered as part of the network * CPS * Network connection establishment reliability and speed on both Unix/Linux and Windows systems. +* NCPS + * Enhanced network connection establishment performance with detailed metrics on both Unix/Linux and Windows systems. + * Connection throughput with RX/TX bandwidth measurements in Gbps. + * NTttcp * Network throughput and bandwidth with TCP communications on both Unix/Linux and Windows systems. * Network throughput and bandwidth with UDP communications on both Unix/Linux and Windows systems. @@ -92,6 +99,69 @@ For the Latte and SockPerf workloads, the client measurements are the only ones | CPS Server | SynRttP99_9 | 74810.0 | 367477.0 | 86169.82155477032 | | | CPS Server | SynRttP99_99 | 91533.0 | 627165.0 | 114658.16077738516 | | +| Scenario | Metric Name | Example Value (min) | Example Value (max) | Example Value (avg) | Unit | +|----------|-------------|---------------------|---------------------|---------------------|------| +| NCPS Client | RxGbps | 0.5 | 0.75 | 0.65 | Gbps | +| NCPS Client | TxGbps | 0.5 | 0.75 | 0.65 | Gbps | +| NCPS Client | Cps | 15000.0 | 32000.0 | 24416.0 | connections/sec | +| NCPS Client | ConnectsPerSec_Min | 24152.0 | 32069.0 | 25389.0 | connections/sec | +| NCPS Client | ConnectsPerSec_Max | 24152.0 | 32069.0 | 32069.0 | connections/sec | +| NCPS Client | ConnectsPerSec_Med | 24152.0 | 32069.0 | 27500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_Avg | 24152.0 | 32069.0 | 27840.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P25 | 24152.0 | 32069.0 | 26500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P50 | 24152.0 | 32069.0 | 27500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P75 | 24152.0 | 32069.0 | 28500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P90 | 24152.0 | 32069.0 | 29500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P99 | 24152.0 | 32069.0 | 31000.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P99_9 | 24152.0 | 32069.0 | 31500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P99_99 | 24152.0 | 32069.0 | 31800.0 | connections/sec | +| NCPS Client | ConnectsPerSec_P99_999 | 24152.0 | 32069.0 | 31900.0 | connections/sec | +| NCPS Client | ConnectsPerSec_Mad | 500.0 | 1500.0 | 1000.0 | connections/sec | +| NCPS Client | ConnectsPerSec_StandardErrorMean | 100.0 | 300.0 | 200.0 | connections/sec | +| NCPS Client | ConnectsPerSec_LowerCI | 24000.0 | 31500.0 | 27500.0 | connections/sec | +| NCPS Client | ConnectsPerSec_UpperCI | 25000.0 | 32500.0 | 28200.0 | connections/sec | +| NCPS Client | RexmitConnPercentage | 0.0 | 5.0 | 3.2825 | % | +| NCPS Client | RexmitPerConn | 0.0 | 2.0 | 1.1436 | retransmits/connection | +| NCPS Client | SynRttMean | 5000.0 | 25000.0 | 18737.0 | microseconds | +| NCPS Client | SynRttMedian | 2000.0 | 10000.0 | 6221.0 | microseconds | +| NCPS Client | SynRttP25 | 1000.0 | 5000.0 | 3424.0 | microseconds | +| NCPS Client | SynRttP75 | 5000.0 | 15000.0 | 10893.0 | microseconds | +| NCPS Client | SynRttP90 | 10000.0 | 25000.0 | 17628.0 | microseconds | +| NCPS Client | SynRttP95 | 15000.0 | 35000.0 | 24607.0 | microseconds | +| NCPS Client | SynRttP99 | 30000.0 | 75000.0 | 54222.0 | microseconds | +| NCPS Client | SynRttP99_9 | 1000000.0 | 3000000.0 | 2027000.0 | microseconds | +| NCPS Client | SynRttP99_99 | 2000000.0 | 5000000.0 | 4057000.0 | microseconds | +| NCPS Server | RxGbps | 0.5 | 0.75 | 0.65 | Gbps | +| NCPS Server | TxGbps | 0.5 | 0.75 | 0.65 | Gbps | +| NCPS Server | Cps | 15000.0 | 32000.0 | 24416.0 | connections/sec | +| NCPS Server | ConnectsPerSec_Min | 24152.0 | 32069.0 | 25389.0 | connections/sec | +| NCPS Server | ConnectsPerSec_Max | 24152.0 | 32069.0 | 32069.0 | connections/sec | +| NCPS Server | ConnectsPerSec_Med | 24152.0 | 32069.0 | 27500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_Avg | 24152.0 | 32069.0 | 27840.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P25 | 24152.0 | 32069.0 | 26500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P50 | 24152.0 | 32069.0 | 27500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P75 | 24152.0 | 32069.0 | 28500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P90 | 24152.0 | 32069.0 | 29500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P99 | 24152.0 | 32069.0 | 31000.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P99_9 | 24152.0 | 32069.0 | 31500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P99_99 | 24152.0 | 32069.0 | 31800.0 | connections/sec | +| NCPS Server | ConnectsPerSec_P99_999 | 24152.0 | 32069.0 | 31900.0 | connections/sec | +| NCPS Server | ConnectsPerSec_Mad | 500.0 | 1500.0 | 1000.0 | connections/sec | +| NCPS Server | ConnectsPerSec_StandardErrorMean | 100.0 | 300.0 | 200.0 | connections/sec | +| NCPS Server | ConnectsPerSec_LowerCI | 24000.0 | 31500.0 | 27500.0 | connections/sec | +| NCPS Server | ConnectsPerSec_UpperCI | 25000.0 | 32500.0 | 28200.0 | connections/sec | +| NCPS Server | RexmitConnPercentage | 0.0 | 3.0 | 1.5 | % | +| NCPS Server | RexmitPerConn | 0.0 | 1.5 | 1.0 | retransmits/connection | +| NCPS Server | SynRttMean | 3000.0 | 20000.0 | 12000.0 | microseconds | +| NCPS Server | SynRttMedian | 1500.0 | 8000.0 | 4500.0 | microseconds | +| NCPS Server | SynRttP25 | 800.0 | 4000.0 | 2200.0 | microseconds | +| NCPS Server | SynRttP75 | 4000.0 | 12000.0 | 8000.0 | microseconds | +| NCPS Server | SynRttP90 | 8000.0 | 20000.0 | 14000.0 | microseconds | +| NCPS Server | SynRttP95 | 12000.0 | 28000.0 | 20000.0 | microseconds | +| NCPS Server | SynRttP99 | 25000.0 | 60000.0 | 45000.0 | microseconds | +| NCPS Server | SynRttP99_9 | 800000.0 | 2500000.0 | 1500000.0 | microseconds | +| NCPS Server | SynRttP99_99 | 1500000.0 | 4000000.0 | 3000000.0 | microseconds | + | Scenario | Metric Name | Example Value (min) | Example Value (max) | Example Value (avg) | Unit | |-----------|-------------|---------------------|---------------------|---------------------|------| | Latte Client | Latency-Average | 0.0 | 502.28 | 248.65472527472529 | microseconds |