fix(runtime): lifecycle housekeeping — reprovision stamp, Stale drops, off-thread DB probe, tombstone sweep (A4)
- DriverHostActor.StopChild removes the driver's _lastReprovisionUtc stamp (and its deferred-reprovision timer): the map grew per removed id and a re-added driver inherited a backoff it never earned (02/L2). - Stale now registers explicit drops for AttributeValueBatchPublished, AttributeAlarmPublished, ArtifactForApply and ApplyWedged instead of letting them hit Unhandled (02/H5). - DbHealthProbeActor runs its SELECT on the thread pool and pipes the outcome back (single-flight), with a 2 s command timeout on relational providers, so a hung SQL Server no longer pins the actor thread and the cached GetStatus answer stays reachable (02/P7). - TelemetryLocalHub eviction tombstones are timestamped and swept past a 10 min TTL on the next Evict; a TimeProvider ctor is the test seam (02/L4). - Tests: ValuePublishes.Single helper replaces the per-value AttributeValuePublished in CalculationDependencyFlowTests (02/U4, test half; the production receive is owned by Lane E). - R05/8 audit: PollGroupEngine.Unsubscribe has no production caller; every driver already uses UnsubscribeAsync (no change). Refs #731 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
+6
-6
@@ -64,7 +64,7 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
{
|
||||
published.Enqueue(e);
|
||||
var quality = e.Snapshot.StatusCode == 0u ? OpcUaStatus.Good : OpcUaStatus.Bad;
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished(
|
||||
mux.Tell(ValuePublishes.Single(
|
||||
"calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
|
||||
};
|
||||
|
||||
@@ -75,7 +75,7 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
// compute dedups) until the computed value has flowed all the way through.
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished(
|
||||
mux.Tell(ValuePublishes.Single(
|
||||
"src-drv", "src/a", 21.0, OpcUaStatus.Good, Ts));
|
||||
|
||||
// Leg 1 — dep flow: src/a=21 → A computes 42 and publishes on calc/engine/out.
|
||||
@@ -118,14 +118,14 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
{
|
||||
published.Enqueue(e);
|
||||
var quality = e.Snapshot.StatusCode == 0u ? OpcUaStatus.Good : OpcUaStatus.Bad;
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
|
||||
mux.Tell(ValuePublishes.Single("calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
|
||||
};
|
||||
var adapter = Sys.ActorOf(DependencyConsumerMuxAdapter.Props(driver, mux), "dep-adapter2");
|
||||
|
||||
// Leg 1 — before redeploy: src/a=2 → keep=20, remove=7.
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/a", 2.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(ValuePublishes.Single("src-drv", "src/a", 2.0, OpcUaStatus.Good, Ts));
|
||||
published.ShouldContain(e => e.FullReference == "calc/keep" && Equals(e.Snapshot.Value, 20.0));
|
||||
published.ShouldContain(e => e.FullReference == "calc/remove" && Equals(e.Snapshot.Value, 7.0));
|
||||
}, PresenceBudget, TimeSpan.FromMilliseconds(100));
|
||||
@@ -150,8 +150,8 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
// new result; the REMOVED tag never publishes a value derived from the post-redeploy input.
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/b", 50.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/a", 3.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(ValuePublishes.Single("src-drv", "src/b", 50.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(ValuePublishes.Single("src-drv", "src/a", 3.0, OpcUaStatus.Good, Ts));
|
||||
|
||||
// (a) new dep B flows through the RE-REGISTERED interest → new tag computes: 50 + 100 = 150.
|
||||
published.ShouldContain(e => e.FullReference == "calc/add" && Equals(e.Snapshot.Value, 150.0));
|
||||
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 L2: <c>DriverHostActor._lastReprovisionUtc</c> was never pruned — a driver removed by a deploy
|
||||
/// after a supervision crash kept its stamp, so the map grew with every id ever reprovisioned-then-removed,
|
||||
/// and (the observable half) a driver removed and later re-added inherited the OLD stamp: its first crash
|
||||
/// after the re-add was deferred by the backoff as if it were crash-looping, although the new incarnation had
|
||||
/// never crashed. <c>StopChild</c> now drops the stamp.
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorReprovisionStampTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
|
||||
private static readonly RevisionHash RevC = RevisionHash.Parse(new string('c', 64));
|
||||
private static readonly NodeId TestNode = NodeId.Parse("node-1:4053");
|
||||
private const string DriverId = "DI-1";
|
||||
private const string SpeedPath = "Plant/Modbus/dev1/speed";
|
||||
|
||||
[Fact]
|
||||
public void A_driver_removed_and_re_added_does_not_inherit_the_old_incarnations_reprovision_stamp()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
// Far longer than the presence budget below: an inherited stamp would defer the re-added driver's
|
||||
// reprovision past it, which is the failure this test reports.
|
||||
var backoff = TimeSpan.FromSeconds(60);
|
||||
var factory = new RecordingSubscribableDriverFactory("Modbus");
|
||||
var withDriver = new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) };
|
||||
var none = Array.Empty<(string, string, string, string, string?)>();
|
||||
|
||||
var revA = V3ArtifactSeeder.SeedV3Deployment(db, RevA, withDriver, none, enabled: true);
|
||||
var (host, _, _, coordinator) = V3ArtifactSeeder.SpawnHostAndApplyWithCoordinator(
|
||||
this, db, TestNode, revA, RevA, factory, reprovisionBackoff: backoff);
|
||||
AwaitAssert(() => factory.Created.Count.ShouldBe(1), PresenceBudget);
|
||||
AwaitAssert(() => factory.Created[0].SubscribeCount.ShouldBe(1), PresenceBudget);
|
||||
|
||||
// Crash 1 on the first incarnation: reprovisioned immediately, which stamps the id.
|
||||
factory.Created[0].FireMalformedDataChange(SpeedPath);
|
||||
AwaitAssert(() => factory.Created[0].InitializeCount.ShouldBe(2), PresenceBudget);
|
||||
|
||||
// Deploy B removes the driver (StopChild), deploy C re-adds it: a brand-new driver object.
|
||||
var revB = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevB, Array.Empty<(string, string, string, string, string, bool)>(), none, enabled: true);
|
||||
V3ArtifactSeeder.Dispatch(host, coordinator, revB, RevB);
|
||||
var revC = V3ArtifactSeeder.SeedV3Deployment(db, RevC, withDriver, none, enabled: true);
|
||||
V3ArtifactSeeder.Dispatch(host, coordinator, revC, RevC);
|
||||
AwaitAssert(() => factory.Created.Count.ShouldBe(2), PresenceBudget);
|
||||
var reAdded = factory.Created[1];
|
||||
AwaitAssert(() => reAdded.SubscribeCount.ShouldBe(1), PresenceBudget);
|
||||
|
||||
// Its FIRST crash must reprovision immediately — it has no history of its own.
|
||||
reAdded.FireMalformedDataChange(SpeedPath);
|
||||
AwaitAssert(
|
||||
() => reAdded.InitializeCount.ShouldBe(
|
||||
2, "the re-added driver's first reprovision was deferred by the REMOVED incarnation's stamp"),
|
||||
duration: TimeSpan.FromSeconds(5));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Event;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 H5: the <c>Stale</c> behaviour registered no receive for a driver child's value/alarm bursts (nor
|
||||
/// for a late <c>ArtifactForApply</c> / <c>ApplyWedged</c> pipe-back), so any that landed there hit
|
||||
/// <c>Unhandled</c> — dead-letter noise and a dropped message with no log naming why. They are now explicit
|
||||
/// drops. The two private pipe-back records cannot be constructed from a test; the two public child
|
||||
/// messages are pinned here, and the private pair by the same registration pattern (compile-time).
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorStaleDropTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly NodeId TestNode = NodeId.Parse("stale-drop-test");
|
||||
|
||||
[Fact]
|
||||
public void Child_value_and_alarm_bursts_landing_in_Stale_are_dropped_not_unhandled()
|
||||
{
|
||||
var unhandled = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(unhandled.Ref, typeof(UnhandledMessage));
|
||||
|
||||
// ThrowingDbFactory ⇒ Bootstrap's catch ⇒ Become(Stale).
|
||||
var host = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, CreateTestProbe().Ref,
|
||||
localRoles: new HashSet<string> { "driver" }));
|
||||
// Barrier: Stale answers RouteNodeWrite synchronously, so a reply proves the host is in Stale.
|
||||
var asker = CreateTestProbe();
|
||||
host.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, AddressSpaceRealm.Uns), asker.Ref);
|
||||
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(PresenceBudget).Reason!.ShouldContain("stale");
|
||||
|
||||
// Positive control: an unknown message IS reported as unhandled by this host.
|
||||
host.Tell("not a message the host knows");
|
||||
unhandled.ExpectMsg<UnhandledMessage>(m => ReferenceEquals(m.Recipient, host) || m.Recipient.Equals(host), PresenceBudget);
|
||||
|
||||
host.Tell(new DriverInstanceActor.AttributeValueBatchPublished(
|
||||
"DI-1", new[] { new DriverInstanceActor.AttributeValueItem("Plant/x/y", 1.0, OpcUaStatus.Good, DateTime.UtcNow) }));
|
||||
host.Tell(new DriverInstanceActor.AttributeAlarmPublished(
|
||||
"DI-1", new AlarmEventArgs(new Handle(), "Plant/x/alarm", "Plant/x/alarm", "AlarmCondition", "test",
|
||||
AlarmSeverity.High, DateTime.UtcNow)));
|
||||
// Barrier again, then the absence window.
|
||||
host.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, AddressSpaceRealm.Uns), asker.Ref);
|
||||
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(PresenceBudget);
|
||||
|
||||
unhandled.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
private sealed class Handle : IAlarmSubscriptionHandle
|
||||
{
|
||||
public string DiagnosticId => "stale-test";
|
||||
}
|
||||
|
||||
private sealed class ThrowingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
public OtOpcUaConfigDbContext CreateDbContext() => throw new InvalidOperationException("db unreachable");
|
||||
}
|
||||
}
|
||||
@@ -201,7 +201,37 @@ public static class V3ArtifactSeeder
|
||||
return (actor, mux, coordinator);
|
||||
}
|
||||
|
||||
private static void Dispatch(IActorRef actor, TestProbe coordinator, DeploymentId deploymentId, RevisionHash rev)
|
||||
/// <summary>The same arrangement as <see cref="SpawnHostAndApply"/>, also returning the coordinator probe so a
|
||||
/// test can <see cref="Dispatch"/> FURTHER deployments to the same host (remove a driver, re-add it).</summary>
|
||||
/// <param name="kit">The calling TestKit, supplying the ActorSystem + probes.</param>
|
||||
/// <param name="db">The factory whose deployment was seeded by <see cref="SeedV3Deployment"/>.</param>
|
||||
/// <param name="localNode">The host's local node id.</param>
|
||||
/// <param name="deploymentId">The deployment to dispatch.</param>
|
||||
/// <param name="rev">The revision hash the deployment was sealed under.</param>
|
||||
/// <param name="driverFactory">Optional driver factory forwarded to <c>DriverHostActor.Props</c>.</param>
|
||||
/// <param name="reprovisionBackoff">Optional per-driver supervision-reprovision rate limit.</param>
|
||||
/// <returns>The host actor plus the publish, mux and coordinator probes wired into it.</returns>
|
||||
public static (IActorRef Actor, TestProbe Publish, TestProbe Mux, TestProbe Coordinator) SpawnHostAndApplyWithCoordinator(
|
||||
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db,
|
||||
NodeId localNode, DeploymentId deploymentId, RevisionHash rev,
|
||||
IDriverFactory? driverFactory = null,
|
||||
TimeSpan? reprovisionBackoff = null)
|
||||
{
|
||||
var publish = kit.CreateCleanOutcomePublishProbe();
|
||||
var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref, driverFactory, reprovisionBackoff);
|
||||
|
||||
Dispatch(actor, coordinator, deploymentId, rev);
|
||||
publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(RuntimeActorTestBase.PresenceBudget);
|
||||
|
||||
return (actor, publish, mux, coordinator);
|
||||
}
|
||||
|
||||
/// <summary>Dispatches <paramref name="deploymentId"/> to <paramref name="actor"/> and waits for its Applied ack.</summary>
|
||||
/// <param name="actor">The host.</param>
|
||||
/// <param name="coordinator">The coordinator probe the host acks to.</param>
|
||||
/// <param name="deploymentId">The deployment to dispatch.</param>
|
||||
/// <param name="rev">Its revision hash.</param>
|
||||
public static void Dispatch(IActorRef actor, TestProbe coordinator, DeploymentId deploymentId, RevisionHash rev)
|
||||
{
|
||||
actor.Tell(new DispatchDeployment(deploymentId, rev, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(RuntimeActorTestBase.PresenceBudget)
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Builds the ONE message shape a driver child sends for its values — a
|
||||
/// <see cref="DriverInstanceActor.AttributeValueBatchPublished"/> — for tests that push a single value into the
|
||||
/// dependency mux by hand. The per-value <see cref="DriverInstanceActor.AttributeValuePublished"/> has no
|
||||
/// production sender (02/U4); tests should not keep one alive either.
|
||||
/// </summary>
|
||||
public static class ValuePublishes
|
||||
{
|
||||
/// <summary>A one-item batch for <paramref name="fullReference"/>.</summary>
|
||||
/// <param name="driverInstanceId">The publishing driver's id.</param>
|
||||
/// <param name="fullReference">The value's reference (RawPath).</param>
|
||||
/// <param name="value">The value.</param>
|
||||
/// <param name="statusCode">The full OPC UA status code.</param>
|
||||
/// <param name="timestampUtc">The source timestamp.</param>
|
||||
/// <returns>The batch message.</returns>
|
||||
public static DriverInstanceActor.AttributeValueBatchPublished Single(
|
||||
string driverInstanceId, string fullReference, object? value, uint statusCode, DateTime timestampUtc) =>
|
||||
new(driverInstanceId, new[] { new DriverInstanceActor.AttributeValueItem(fullReference, value, statusCode, timestampUtc) });
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 P7: <see cref="DbHealthProbeActor"/> ran its SELECT synchronously on the actor thread, so a
|
||||
/// hanging SQL connect/command held the dispatcher thread for the implicit 15 s / 30 s SqlClient timeouts and
|
||||
/// queued every later tick (and every <c>GetStatus</c> Ask) behind it. The probe now runs off the mailbox
|
||||
/// and pipes its outcome back, so the cached answer stays available while a probe hangs.
|
||||
/// </summary>
|
||||
public sealed class DbHealthProbeOffThreadTests : RuntimeActorTestBase
|
||||
{
|
||||
[Fact]
|
||||
public async Task GetStatus_answers_from_cache_while_a_probe_is_hung()
|
||||
{
|
||||
var factory = new HangingDbFactory();
|
||||
try
|
||||
{
|
||||
var actor = Sys.ActorOf(DbHealthProbeActor.Props(factory));
|
||||
AwaitCondition(() => factory.Entered > 0, PresenceBudget);
|
||||
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(3));
|
||||
|
||||
status.Reachable.ShouldBeFalse();
|
||||
status.LastError.ShouldBe("not probed yet");
|
||||
factory.Entered.ShouldBe(1, "a second tick must not stack a second probe behind a hung one");
|
||||
}
|
||||
finally
|
||||
{
|
||||
factory.Release();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_probe_that_completes_updates_the_cached_status()
|
||||
{
|
||||
// Positive control for the off-thread path: the piped-back outcome lands in the cache.
|
||||
var db = NewInMemoryDbFactory();
|
||||
var actor = Sys.ActorOf(DbHealthProbeActor.Props(db));
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(3));
|
||||
status.Reachable.ShouldBeTrue();
|
||||
}, PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>Blocks every <c>CreateDbContext</c> until released — a connect that never answers.</summary>
|
||||
private sealed class HangingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
private readonly ManualResetEventSlim _release = new(false);
|
||||
private int _entered;
|
||||
|
||||
public int Entered => Volatile.Read(ref _entered);
|
||||
|
||||
public void Release() => _release.Set();
|
||||
|
||||
public OtOpcUaConfigDbContext CreateDbContext()
|
||||
{
|
||||
Interlocked.Increment(ref _entered);
|
||||
_release.Wait();
|
||||
throw new InvalidOperationException("released without a database");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,18 +15,22 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
|
||||
|
||||
public sealed class HealthProbeActorTests : RuntimeActorTestBase
|
||||
{
|
||||
/// <summary>Verifies that the DB health probe actor returns reachable status against an in-memory database.</summary>
|
||||
/// <summary>Verifies that the DB health probe actor returns reachable status against an in-memory database.
|
||||
/// The probe runs off the mailbox (02/P7), so the first Ask may still see "not probed yet" — poll.</summary>
|
||||
[Fact]
|
||||
public async Task DbHealthProbeActor_returns_reachable_against_in_memory_db()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var actor = Sys.ActorOf(DbHealthProbeActor.Props(db));
|
||||
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, PresenceBudget);
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, PresenceBudget);
|
||||
|
||||
status.Reachable.ShouldBeTrue();
|
||||
status.LastError.ShouldBeNull();
|
||||
status.Reachable.ShouldBeTrue();
|
||||
status.LastError.ShouldBeNull();
|
||||
}, PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the peer OPC UA probe actor reports Ok true against a live listener.</summary>
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Telemetry;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 L4: <see cref="TelemetryLocalHub.Evict"/>'s tombstone set only ever shrank on a re-spawn of the
|
||||
/// SAME id, so a driver removed for good kept its id forever. The tombstone is load-bearing (#609 — it holds
|
||||
/// back the removed child's late writes), so it cannot simply be cleared; it is now timestamped and swept
|
||||
/// once older than <see cref="TelemetryLocalHub.EvictionTombstoneTtl"/>, which outlives any in-flight
|
||||
/// capability call by a wide margin.
|
||||
/// </summary>
|
||||
public sealed class TelemetryLocalHubTombstoneSweepTests
|
||||
{
|
||||
private static DriverHealthChanged Health(string instanceId) =>
|
||||
new("cluster-a", instanceId, "Healthy", null, null, 0, DateTime.UtcNow);
|
||||
|
||||
private static List<TelemetryItem> Drain(ITelemetrySubscription sub)
|
||||
{
|
||||
var items = new List<TelemetryItem>();
|
||||
while (sub.Reader.TryRead(out var item)) items.Add(item);
|
||||
return items;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_tombstone_holds_late_writes_until_the_ttl_then_is_swept()
|
||||
{
|
||||
var clock = new ManualTime { Now = new DateTimeOffset(2026, 10, 22, 12, 0, 0, TimeSpan.Zero) };
|
||||
var hub = new TelemetryLocalHub(clock);
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
hub.Evict("drv-a");
|
||||
|
||||
// Inside the TTL: the late write is dropped (the #609 contract) — and another eviction's sweep does
|
||||
// not lift a tombstone that is still young.
|
||||
clock.Now += TelemetryLocalHub.EvictionTombstoneTtl / 2;
|
||||
hub.Evict("drv-b");
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
using (var sub = hub.Subscribe(16))
|
||||
Drain(sub).ShouldBeEmpty("a late write inside the TTL must still be held back");
|
||||
|
||||
// Past the TTL: the next eviction sweeps drv-a's tombstone and its writes land again.
|
||||
clock.Now += TelemetryLocalHub.EvictionTombstoneTtl;
|
||||
hub.Evict("drv-c");
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
using (var sub = hub.Subscribe(16))
|
||||
Drain(sub).OfType<TelemetryItem.Health>().Single().E.DriverInstanceId.ShouldBe("drv-a");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Readmit_lifts_a_tombstone_before_its_ttl()
|
||||
{
|
||||
var clock = new ManualTime { Now = DateTimeOffset.UtcNow };
|
||||
var hub = new TelemetryLocalHub(clock);
|
||||
hub.Evict("drv-a");
|
||||
hub.Readmit("drv-a");
|
||||
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
using var sub = hub.Subscribe(16);
|
||||
Drain(sub).Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
private sealed class ManualTime : TimeProvider
|
||||
{
|
||||
public DateTimeOffset Now { get; set; }
|
||||
public override DateTimeOffset GetUtcNow() => Now;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user