dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(deploy): hold the Applied ack until the address-space apply outcome is known (#601)

Browse Source 
DriverHostActor no longer acks Applied the moment the revision advances. It
asks OpcUaPublishActor for the outcome (RebuildAddressSpace.ReportOutcome) and
acks Applied+Degraded(reason) when the apply was degraded; a missing outcome
times out (60 s, inside the coordinator's 2 min deadline) to a degraded ack,
never a clean one. Re-dispatching a revision whose apply was degraded or is
still unconfirmed re-applies instead of taking the clean idempotent
short-circuit - that re-apply is also what repairs the degraded nodes.
Tests with a bare publish probe use a clean-outcome auto-pilot probe.

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
2026-10-08 17:29:45 -04:00
parent 521e1eb89b
commit 90d7f6dfca
13 changed files with 421 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs
+2 -2
View File
@@ -67,7 +67,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
var blob = EmptyArtifact();
var cached = new CachedDeploymentArtifact(
DeploymentId.NewId().ToString(), CachedRev.Value, blob, DateTimeOffset.UtcNow.AddHours(-1));
var publishProbe = CreateTestProbe();
var publishProbe = this.CreateCleanOutcomePublishProbe();
Sys.ActorOf(DriverHostActor.Props(
new ThrowingDbFactory(), TestNode, ackRouter: null,
@@ -92,7 +92,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
var cached = new CachedDeploymentArtifact(
DeploymentId.NewId().ToString(), CachedRev.Value,
System.Text.Encoding.Latin1.GetBytes(text), DateTimeOffset.UtcNow.AddHours(-1));
var publishProbe = CreateTestProbe();
var publishProbe = this.CreateCleanOutcomePublishProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
new ThrowingDbFactory(), TestNode, ackRouter: null,
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDbLessTests.cs
+4 -4
View File
@@ -112,7 +112,7 @@ public sealed class DriverHostActorDbLessTests : RuntimeActorTestBase
var cache = new RecordingArtifactCache();
var fetcher = new RecordingFetcher(ArtifactBytes());
var deploymentId = DeploymentId.NewId();
var publish = CreateTestProbe();
var publish = this.CreateCleanOutcomePublishProbe();
var coordinator = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
@@ -141,7 +141,7 @@ public sealed class DriverHostActorDbLessTests : RuntimeActorTestBase
[Fact]
public void DbLessNode_WithLocalDbStore_SpawnsScriptedAlarmHost()
{
var publish = CreateTestProbe();
var publish = this.CreateCleanOutcomePublishProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
dbFactory: null,
TestNode,
@@ -162,7 +162,7 @@ public sealed class DriverHostActorDbLessTests : RuntimeActorTestBase
[Fact]
public void DbLessNode_WithNoStoreAndNoDbFactory_SkipsScriptedAlarmHost()
{
var publish = CreateTestProbe();
var publish = this.CreateCleanOutcomePublishProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
dbFactory: null,
TestNode,
@@ -185,7 +185,7 @@ public sealed class DriverHostActorDbLessTests : RuntimeActorTestBase
[Fact]
public void DbBackedNode_WithNoStore_SkipsScriptedAlarmHost()
{
var publish = CreateTestProbe();
var publish = this.CreateCleanOutcomePublishProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
dbFactory: NewInMemoryDbFactory(),
TestNode,
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs
+203
View File
@@ -0,0 +1,203 @@
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Gitea #601 (archreview 01/S-9) — the deploy ack must be honest about the address-space apply. It used
/// to be sent the moment the revision advanced, before <c>OpcUaPublishActor</c> had even dequeued the
/// rebuild, so a degraded apply (swallowed sink faults; <c>_lastApplied</c> deliberately not advanced)
/// still acked a clean <see cref="ApplyAckOutcome.Applied"/> and the deployment sealed green fleet-wide.
/// Now the Applied ack waits for the publish actor's <see cref="OpcUaPublishActor.RebuildCompleted"/> and
/// carries its <see cref="ApplyAck.Degraded"/> verdict; a missing verdict times out to degraded, never clean.
/// </summary>
[Collection("opcua-publish-meters")]
public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
[Fact]
public void The_Applied_ack_waits_for_the_address_space_outcome_and_a_clean_one_acks_clean()
{
var (db, deploymentId) = SeedOneTagDeployment();
var coordinator = CreateTestProbe();
var publish = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
opcUaPublishActor: publish.Ref));
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
rebuild.ReportOutcome.ShouldBeTrue();
// The ack is HELD until the outcome lands — the pre-#601 code had already sent it by now.
coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, false, null));
var ack = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
ack.Degraded.ShouldBeFalse();
ack.DegradedReason.ShouldBeNull();
}
[Fact]
public void A_degraded_address_space_outcome_acks_Applied_but_degraded_with_the_reason()
{
var (db, deploymentId) = SeedOneTagDeployment();
var coordinator = CreateTestProbe();
var publish = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
opcUaPublishActor: publish.Ref));
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
publish.Reply(new OpcUaPublishActor.RebuildCompleted(
rebuild.Correlation, rebuild.DeploymentId, Degraded: true, DegradedReason: "failedNodes=2"));
var ack = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
ack.Degraded.ShouldBeTrue();
ack.DegradedReason.ShouldBe("failedNodes=2");
ack.DeploymentId.ShouldBe(deploymentId);
}
[Fact]
public void A_real_publish_actor_whose_sink_faults_produces_a_degraded_ack_end_to_end()
{
// No hand-written verdict: the real OpcUaPublishActor + AddressSpaceApplier over a sink that faults on
// every node. This is the production shape of the defect — a degraded apply that sealed green.
var (db, deploymentId) = SeedOneTagDeployment();
var coordinator = CreateTestProbe();
var sink = new FaultingSink();
var publish = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
sink: sink, applier: new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance)));
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
opcUaPublishActor: publish));
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
var ack = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
ack.Degraded.ShouldBeTrue();
ack.DegradedReason.ShouldNotBeNull();
ack.DegradedReason.ShouldContain("failedNodes=");
}
[Fact]
public void A_real_publish_actor_with_a_healthy_sink_produces_a_clean_ack_end_to_end()
{
// The control for the test above: same wiring, a sink that does not fault ⇒ clean.
var (db, deploymentId) = SeedOneTagDeployment();
var coordinator = CreateTestProbe();
var sink = new FaultingSink(fault: false);
var publish = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
sink: sink, applier: new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance)));
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
opcUaPublishActor: publish));
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
var ack = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
ack.Degraded.ShouldBeFalse();
}
[Fact]
public void No_outcome_within_the_timeout_acks_degraded_never_clean()
{
var (db, deploymentId) = SeedOneTagDeployment();
var coordinator = CreateTestProbe();
var publish = CreateTestProbe(); // never replies — a hung / lost apply
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
opcUaPublishActor: publish.Ref, applyOutcomeTimeout: TimeSpan.FromMilliseconds(300)));
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
var ack = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
ack.Degraded.ShouldBeTrue();
ack.DegradedReason.ShouldNotBeNull();
ack.DegradedReason.ShouldContain("did not report an outcome");
// A late outcome after the timeout must not produce a second ack.
var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, false, null));
coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
}
[Fact]
public void Redispatching_a_degraded_revision_reapplies_instead_of_acking_clean()
{
var (db, deploymentId) = SeedOneTagDeployment();
var coordinator = CreateTestProbe();
var publish = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
opcUaPublishActor: publish.Ref));
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
var first = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
publish.Reply(new OpcUaPublishActor.RebuildCompleted(first.Correlation, first.DeploymentId, true, "failedNodes=1"));
coordinator.ExpectMsg<ApplyAck>(PresenceBudget).Degraded.ShouldBeTrue();
// Same revision again: the old idempotent short-circuit would ack a CLEAN Applied right here. It must
// re-apply instead (a second rebuild request), and ack whatever THAT apply reports.
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
var second = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
publish.Reply(new OpcUaPublishActor.RebuildCompleted(second.Correlation, second.DeploymentId, false, null));
coordinator.ExpectMsg<ApplyAck>(PresenceBudget).Degraded.ShouldBeFalse();
// Now clean: a third dispatch of the same revision takes the short-circuit again (no rebuild).
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
var third = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
third.Degraded.ShouldBeFalse();
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
}
private (IDbContextFactory<OtOpcUaConfigDbContext> Db, DeploymentId Id) SeedOneTagDeployment()
{
var db = NewInMemoryDbFactory();
var id = V3ArtifactSeeder.SeedV3Deployment(db, RevA,
rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false) },
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
return (db, id);
}
/// <summary>A sink that throws from every node-creating call when <c>fault</c> is set.</summary>
private sealed class FaultingSink(bool fault = true) : IOpcUaAddressSpaceSink
{
private void MaybeFault()
{
if (fault) throw new InvalidOperationException("simulated sink fault");
}
public void WriteValue(string nodeId, object? value, uint statusCode, DateTime sourceTimestampUtc, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
public void WriteAlarmQuality(string alarmNodeId, uint statusCode, DateTime sourceTimestampUtc, AddressSpaceRealm realm) { }
public void WriteAlarmCondition(string alarmNodeId, AlarmConditionSnapshot state, DateTime occurredUtc, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
public void MaterialiseAlarmCondition(string alarmNodeId, string equipmentNodeId, string displayName, string alarmType, int severity, AddressSpaceRealm realm, bool isNative = false) => MaybeFault();
public void EnsureFolder(string folderNodeId, string? parentNodeId, string displayName, AddressSpaceRealm realm = AddressSpaceRealm.Uns) => MaybeFault();
public void EnsureVariable(string variableNodeId, string? parentFolderNodeId, string displayName, string dataType, bool writable, AddressSpaceRealm realm, string? historianTagname = null, bool isArray = false, uint? arrayLength = null, bool refuseArrayOnScalar = false) => MaybeFault();
public void RebuildAddressSpace() { }
public void RaiseNodesAddedModelChange(string affectedNodeId, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
public void WireAlarmNotifiers(string alarmNodeId, AddressSpaceRealm alarmRealm, IReadOnlyList<string> notifierFolderNodeIds, AddressSpaceRealm notifierFolderRealm) { }
public void AddReference(string sourceNodeId, AddressSpaceRealm sourceRealm, string targetNodeId, AddressSpaceRealm targetRealm, string referenceType = "Organizes") { }
}
}
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorFetchAndCacheBootstrapTests.cs
+1 -1
View File
@@ -42,7 +42,7 @@ public sealed class DriverHostActorFetchAndCacheBootstrapTests : RuntimeActorTes
var cached = new CachedDeploymentArtifact(
DeploymentId.NewId().ToString(), CachedRev.Value, blob, DateTimeOffset.UtcNow.AddHours(-1));
var cache = new StubArtifactCache(cached);
var publishProbe = CreateTestProbe();
var publishProbe = this.CreateCleanOutcomePublishProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
new ThrowingDbFactory(), TestNode, ackRouter: null,
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorFetchAndCacheUnreadableTests.cs
+3 -3
View File
@@ -42,7 +42,7 @@ public sealed class DriverHostActorFetchAndCacheUnreadableTests : RuntimeActorTe
[RevA.Value] = Artifact(),
[RevB.Value] = null,
});
var publishProbe = CreateTestProbe();
var publishProbe = this.CreateCleanOutcomePublishProbe();
var coordinator = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
@@ -72,7 +72,7 @@ public sealed class DriverHostActorFetchAndCacheUnreadableTests : RuntimeActorTe
// empty blob handed to it directly: ReconcileDriversFromBlob's #485 guard fails the apply.
var db = NewInMemoryDbFactory();
var fetcher = new MapFetcher(new Dictionary<string, byte[]?> { [RevA.Value] = Array.Empty<byte>() });
var publishProbe = CreateTestProbe();
var publishProbe = this.CreateCleanOutcomePublishProbe();
var coordinator = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
@@ -95,7 +95,7 @@ public sealed class DriverHostActorFetchAndCacheUnreadableTests : RuntimeActorTe
// A truncated/corrupt cached artifact surfaces as GetCurrentUnkeyedAsync == null (the cache's
// integrity check is deliberately indistinguishable from a miss). Boot must land
// Steady-no-revision and apply NOTHING — never a partial address space.
var publishProbe = CreateTestProbe();
var publishProbe = this.CreateCleanOutcomePublishProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
new ThrowingDbFactory(), TestNode, ackRouter: null,
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorHistoryWriterTests.cs
+1 -1
View File
@@ -53,7 +53,7 @@ public sealed class DriverHostActorHistoryWriterTests : RuntimeActorTestBase
// A publish actor MUST be wired for SpawnVirtualTagHost to spawn the real host (its ctor
// requires a non-null sink). No virtualTagHostOverride ⇒ the real VirtualTagHostActor is
// spawned with _historyWriter threaded in.
var publish = CreateTestProbe();
var publish = this.CreateCleanOutcomePublishProbe();
var writer = new CapturingHistoryWriter();
var actor = Sys.ActorOf(DriverHostActor.Props(
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmTests.cs
+1 -1
View File
@@ -441,7 +441,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId)
{
var coordinator = CreateTestProbe();
var publish = CreateTestProbe();
var publish = this.CreateCleanOutcomePublishProbe();
var mux = CreateTestProbe();
var vtHost = CreateTestProbe();
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorPrimaryGateTests.cs
+1 -1
View File
@@ -251,7 +251,7 @@ public sealed class DriverHostActorPrimaryGateTests : RuntimeActorTestBase
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId dep, int driverMemberCount)
{
var coordinator = CreateTestProbe();
var publish = CreateTestProbe();
var publish = this.CreateCleanOutcomePublishProbe();
var mux = CreateTestProbe();
var vtHost = CreateTestProbe();
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorTests.cs
+1 -1
View File
@@ -226,7 +226,7 @@ public sealed class DriverHostActorTests : RuntimeActorTestBase
seeded.Length.ShouldBeGreaterThan(0);
var coordinator = CreateTestProbe();
var publish = CreateTestProbe();
var publish = this.CreateCleanOutcomePublishProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
localRoles: new HashSet<string> { "driver" },
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/PublishProbes.cs
+34
View File
@@ -0,0 +1,34 @@
using Akka.Actor;
using Akka.TestKit;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
/// <summary>
/// Gitea #601 — since the deploy ack waits for <see cref="OpcUaPublishActor"/>'s
/// <see cref="OpcUaPublishActor.RebuildCompleted"/> reply, a bare <see cref="TestProbe"/> standing in for the
/// publish actor would hold every Applied ack until the outcome timeout (and then ack it degraded). Tests
/// that are not ABOUT the ack's honesty use this probe instead: it still records every message exactly
/// like a plain probe (the auto-pilot runs in addition to queueing), and answers each
/// <see cref="OpcUaPublishActor.RebuildAddressSpace"/> that asks for an outcome with a CLEAN one — the
/// reply the real publish actor gives a clean apply.
/// </summary>
public static class PublishProbes
{
/// <summary>Creates a publish-actor probe that reports a clean address-space outcome for every rebuild
/// carrying <see cref="OpcUaPublishActor.RebuildAddressSpace.ReportOutcome"/>.</summary>
/// <param name="kit">The calling TestKit.</param>
/// <returns>The probe.</returns>
public static TestProbe CreateCleanOutcomePublishProbe(this TestKitBase kit)
{
var probe = kit.CreateTestProbe();
probe.SetAutoPilot(new DelegateAutoPilot((sender, message) =>
{
if (message is OpcUaPublishActor.RebuildAddressSpace { ReportOutcome: true } rebuild)
sender.Tell(new OpcUaPublishActor.RebuildCompleted(
rebuild.Correlation, rebuild.DeploymentId, Degraded: false, DegradedReason: null));
return AutoPilot.KeepRunning;
}));
return probe;
}
}
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs
+1 -1
View File
@@ -151,7 +151,7 @@ public static class V3ArtifactSeeder
TimeSpan? reprovisionBackoff = null,
TimeSpan? commsRecoveryAuditDelay = null)
{
var publish = kit.CreateTestProbe();
var publish = kit.CreateCleanOutcomePublishProbe();
var (actor, mux, coordinator) =
Spawn(kit, db, localNode, publish.Ref, driverFactory, reprovisionBackoff, commsRecoveryAuditDelay);
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Telemetry/PublishSeamEmitsToHubTests.cs
+1 -1
View File
@@ -232,7 +232,7 @@ public sealed class PublishSeamEmitsToHubTests : RuntimeActorTestBase
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, ITelemetryLocalHub hub)
{
var coordinator = CreateTestProbe();
var publish = CreateTestProbe();
var publish = this.CreateCleanOutcomePublishProbe();
var mux = CreateTestProbe();
var vtHost = CreateTestProbe();