Joseph Doherty
299 lines
16 KiB
C#
299 lines
16 KiB
C#
using System.Diagnostics.Metrics;
|
|
using Akka.Actor;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
using static ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness.V3ArtifactSeeder;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// Archreview <b>03/S15</b> — the value-node half of the comms-loss fan-out. Until this, a driver
|
|
/// declared comms-lost annotated its Part 9 conditions and pushed a synthetic Bad into the dependency
|
|
/// mux, but its <b>raw and UNS value variables</b> — the product's external surface — got nothing: an
|
|
/// OPC UA client subscribed to a value node saw the last value at <b>Good</b> quality for the whole
|
|
/// outage, for every driver whose failure shape is "reader stops producing" rather than "per-tag Bad
|
|
/// snapshot".
|
|
///
|
|
/// <para><b>Status-only, value retained.</b> The write goes through
|
|
/// <see cref="OpcUaPublishActor.AttributeStatusBatchUpdate"/> so the node keeps its last known value
|
|
/// under <c>BadNoCommunication</c> (OPC UA Part 8's convention). The host holds no value cache — v3's
|
|
/// single-source design — so only the node itself can retain it.</para>
|
|
///
|
|
/// <para><b>Nothing is published on reconnect</b>, deliberately: publishing Good would assert a
|
|
/// quality this actor has not observed. Recovery is the driver's own re-publish (the poll engine's
|
|
/// <c>forceRaise</c> on a fresh subscription, or the push-model initial push). That contract is
|
|
/// end-to-end tested in <see cref="DriverHostCommsLossRecoveryTests"/>.</para>
|
|
/// </summary>
|
|
public sealed class DriverHostActorCommsLossValueNodesTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("driver-commsloss-values");
|
|
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('c', 64));
|
|
private static readonly DateTime TsA = new(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
|
|
|
/// <summary>Audit delay used by the T9 cases — long enough that a same-turn prune wins the race,
|
|
/// short enough to observe.</summary>
|
|
private static readonly TimeSpan AuditDelay = TimeSpan.FromMilliseconds(400);
|
|
|
|
// Distinct driver ids per metered test: the meter tag is what the recorder filters on, and a
|
|
// MeterListener sees every measurement in the process — including a parallel test's (#503).
|
|
private const string SilentDriver = "drv-audit-silent";
|
|
private const string RepublishingDriver = "drv-audit-republish";
|
|
private const string PartialDriver = "drv-audit-partial";
|
|
|
|
/// <summary>
|
|
/// A disconnect produces ONE status batch naming exactly the disconnected driver's raw nodes plus
|
|
/// every referencing UNS mirror, all <c>BadNoCommunication</c>. The second driver's ref is the
|
|
/// control: a fan-out that walked every value ref on the node instead of the per-driver index would
|
|
/// mark a perfectly healthy driver's nodes Bad, and no assertion about the first driver's nodes
|
|
/// would notice.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Disconnect_marks_the_drivers_raw_and_uns_value_nodes_BadNoCommunication()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedV3Deployment(db, RevA,
|
|
rawTags: new[]
|
|
{
|
|
(Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false),
|
|
(Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "temp", DataType: "Double", Writable: false),
|
|
(Driver: "drv-2", DriverName: "S7", Device: "dev2", Tag: "other", DataType: "Double", Writable: false),
|
|
},
|
|
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
|
|
|
|
var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA);
|
|
|
|
actor.Tell(new DriverInstanceActor.ConnectivityChanged("drv-1", Connected: false));
|
|
|
|
var batch = publish.ExpectMsg<OpcUaPublishActor.AttributeStatusBatchUpdate>(PresenceBudget);
|
|
batch.Items.Select(i => i.NodeId).ShouldBe(new[]
|
|
{
|
|
"Plant/Modbus/dev1/speed",
|
|
"filling/line1/station1/speed",
|
|
"Plant/Modbus/dev1/temp",
|
|
}, ignoreOrder: true);
|
|
|
|
batch.Items.ShouldAllBe(i => i.StatusCode == OpcUaStatus.BadNoCommunication);
|
|
batch.Items.Single(i => i.NodeId == "Plant/Modbus/dev1/speed").Realm.ShouldBe(AddressSpaceRealm.Raw);
|
|
batch.Items.Single(i => i.NodeId == "filling/line1/station1/speed").Realm.ShouldBe(AddressSpaceRealm.Uns);
|
|
|
|
// The control: the OTHER driver's node is untouched by drv-1's outage.
|
|
batch.Items.ShouldNotContain(i => i.NodeId == "Plant/S7/dev2/other");
|
|
|
|
// ONE batch, not one message per node.
|
|
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
|
}
|
|
|
|
/// <summary>
|
|
/// A reconnect publishes <b>nothing</b> to the value nodes. Writing Good here would claim a quality
|
|
/// the host has not observed — the device may be answering the transport and still returning
|
|
/// garbage — and it would race the driver's own first real value.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Reconnect_publishes_no_value_or_status_message()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedV3Deployment(db, RevA,
|
|
rawTags: new[]
|
|
{
|
|
(Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false),
|
|
},
|
|
unsRefs: Array.Empty<(string, string, string, string, string?)>());
|
|
|
|
var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA);
|
|
|
|
actor.Tell(new DriverInstanceActor.ConnectivityChanged("drv-1", Connected: false));
|
|
publish.ExpectMsg<OpcUaPublishActor.AttributeStatusBatchUpdate>(PresenceBudget);
|
|
|
|
actor.Tell(new DriverInstanceActor.ConnectivityChanged("drv-1", Connected: true));
|
|
|
|
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------
|
|
// T9 — the recovery audit (the dedup-trap tripwire)
|
|
// ---------------------------------------------------------------------------------
|
|
|
|
/// <summary>
|
|
/// The host publishes nothing Good on reconnect, so the only way a stuck-Bad node becomes visible
|
|
/// is this audit: refs still latched some time after the driver reconnected are logged and metered.
|
|
/// Here the driver re-publishes nothing at all — the pre-STAB-18 MQTT shape, and the exact failure
|
|
/// any future push driver without an initial push would produce.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Refs_still_latched_after_a_reconnect_are_logged_and_metered()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedV3Deployment(db, RevA,
|
|
rawTags: new[]
|
|
{
|
|
(Driver: SilentDriver, DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false),
|
|
(Driver: SilentDriver, DriverName: "Modbus", Device: "dev1", Tag: "temp", DataType: "Double", Writable: false),
|
|
},
|
|
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
|
|
|
|
var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, commsRecoveryAuditDelay: AuditDelay);
|
|
using var meter = new MeterRecorder("otopcua.driver.comms_latched_refs");
|
|
|
|
actor.Tell(new DriverInstanceActor.ConnectivityChanged(SilentDriver, Connected: false));
|
|
publish.ExpectMsg<OpcUaPublishActor.AttributeStatusBatchUpdate>(PresenceBudget);
|
|
|
|
EventFilter.Warning(contains: "still BadNoCommunication").ExpectOne(TimeSpan.FromSeconds(10), () =>
|
|
actor.Tell(new DriverInstanceActor.ConnectivityChanged(SilentDriver, Connected: true)));
|
|
|
|
// Two REFS, not three nodes: the latch is keyed by RawPath, and the UNS mirror shares its
|
|
// backing tag's ref — the operator is being told how many tags did not come back, not how many
|
|
// address-space nodes they project onto.
|
|
AwaitCondition(() => meter.WithTag("driver_instance", SilentDriver) > 0, TimeSpan.FromSeconds(5));
|
|
meter.SumWithTag("driver_instance", SilentDriver).ShouldBe(2);
|
|
}
|
|
|
|
/// <summary>
|
|
/// <b>The control.</b> A driver that DOES re-publish every latched ref on reconnect trips nothing —
|
|
/// without this, the test above would pass against an audit that fires unconditionally, which would
|
|
/// make the Warning noise rather than signal.
|
|
/// </summary>
|
|
[Fact]
|
|
public void A_driver_that_republishes_every_ref_trips_no_audit()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedV3Deployment(db, RevA,
|
|
rawTags: new[]
|
|
{
|
|
(Driver: RepublishingDriver, DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false),
|
|
(Driver: RepublishingDriver, DriverName: "Modbus", Device: "dev1", Tag: "temp", DataType: "Double", Writable: false),
|
|
},
|
|
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
|
|
|
|
var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, commsRecoveryAuditDelay: AuditDelay);
|
|
using var meter = new MeterRecorder("otopcua.driver.comms_latched_refs");
|
|
|
|
actor.Tell(new DriverInstanceActor.ConnectivityChanged(RepublishingDriver, Connected: false));
|
|
publish.ExpectMsg<OpcUaPublishActor.AttributeStatusBatchUpdate>(PresenceBudget);
|
|
|
|
EventFilter.Warning(contains: "still BadNoCommunication").Expect(0, TimeSpan.FromSeconds(5), () =>
|
|
{
|
|
actor.Tell(new DriverInstanceActor.ConnectivityChanged(RepublishingDriver, Connected: true));
|
|
|
|
// The driver's own re-publish — identical values, which is precisely the case a value-only
|
|
// dedup would swallow; here it is the driver that force-raised, so both refs are pruned.
|
|
actor.Tell(new DriverInstanceActor.AttributeValueBatchPublished(RepublishingDriver, new[]
|
|
{
|
|
new DriverInstanceActor.AttributeValueItem("Plant/Modbus/dev1/speed", 42.0, OpcUaStatus.Good, TsA),
|
|
new DriverInstanceActor.AttributeValueItem("Plant/Modbus/dev1/temp", 7.5, OpcUaStatus.Good, TsA),
|
|
}));
|
|
publish.ExpectMsg<OpcUaPublishActor.AttributeValueBatchUpdate>(PresenceBudget);
|
|
});
|
|
|
|
meter.WithTag("driver_instance", RepublishingDriver).ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A partial recovery is still reported, and reports only what is missing: one ref comes back, the
|
|
/// other does not, and the audit counts exactly one. An audit that reported the whole latch (or
|
|
/// nothing at all once anything arrived) would be useless on precisely the mixed case an operator
|
|
/// needs it for.
|
|
/// </summary>
|
|
[Fact]
|
|
public void A_partial_recovery_reports_only_the_refs_that_did_not_come_back()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedV3Deployment(db, RevA,
|
|
rawTags: new[]
|
|
{
|
|
(Driver: PartialDriver, DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false),
|
|
(Driver: PartialDriver, DriverName: "Modbus", Device: "dev1", Tag: "temp", DataType: "Double", Writable: false),
|
|
},
|
|
unsRefs: Array.Empty<(string, string, string, string, string?)>());
|
|
|
|
var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, commsRecoveryAuditDelay: AuditDelay);
|
|
using var meter = new MeterRecorder("otopcua.driver.comms_latched_refs");
|
|
|
|
actor.Tell(new DriverInstanceActor.ConnectivityChanged(PartialDriver, Connected: false));
|
|
publish.ExpectMsg<OpcUaPublishActor.AttributeStatusBatchUpdate>(PresenceBudget);
|
|
|
|
EventFilter.Warning(contains: "still BadNoCommunication").ExpectOne(TimeSpan.FromSeconds(10), () =>
|
|
{
|
|
actor.Tell(new DriverInstanceActor.ConnectivityChanged(PartialDriver, Connected: true));
|
|
actor.Tell(new DriverInstanceActor.AttributeValueBatchPublished(PartialDriver, new[]
|
|
{
|
|
new DriverInstanceActor.AttributeValueItem("Plant/Modbus/dev1/speed", 42.0, OpcUaStatus.Good, TsA),
|
|
}));
|
|
publish.ExpectMsg<OpcUaPublishActor.AttributeValueBatchUpdate>(PresenceBudget);
|
|
});
|
|
|
|
AwaitCondition(() => meter.WithTag("driver_instance", PartialDriver) > 0, TimeSpan.FromSeconds(5));
|
|
meter.SumWithTag("driver_instance", PartialDriver).ShouldBe(1, "only the ref that never came back is reported");
|
|
}
|
|
|
|
/// <summary>
|
|
/// A driver that owns no value refs produces no message at all — an empty batch would cost a hop
|
|
/// and a sink call for nothing on every flap of an alarm-only or freshly-added driver.
|
|
/// </summary>
|
|
[Fact]
|
|
public void A_driver_with_no_value_refs_publishes_nothing()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedV3Deployment(db, RevA,
|
|
rawTags: new[]
|
|
{
|
|
(Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false),
|
|
},
|
|
unsRefs: Array.Empty<(string, string, string, string, string?)>());
|
|
|
|
var (actor, publish, _) = SpawnHostAndApply(this, db, TestNode, deploymentId, RevA);
|
|
|
|
actor.Tell(new DriverInstanceActor.ConnectivityChanged("drv-unknown", Connected: false));
|
|
|
|
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Sums one counter's measurements, <b>filtered by tag</b>. A <see cref="MeterListener"/> observes
|
|
/// every measurement in the process, so filtering on instrument name alone would fold in a parallel
|
|
/// test's (#503) — each test here therefore uses its own driver id and asserts on that tag.
|
|
/// </summary>
|
|
private sealed class MeterRecorder : IDisposable
|
|
{
|
|
private readonly MeterListener _listener;
|
|
private readonly List<(long Value, KeyValuePair<string, object?>[] Tags)> _measurements = new();
|
|
private readonly object _gate = new();
|
|
|
|
public MeterRecorder(string instrumentName)
|
|
{
|
|
_listener = new MeterListener
|
|
{
|
|
InstrumentPublished = (instrument, listener) =>
|
|
{
|
|
if (instrument.Meter.Name == OtOpcUaTelemetry.MeterName && instrument.Name == instrumentName)
|
|
listener.EnableMeasurementEvents(instrument);
|
|
},
|
|
};
|
|
_listener.SetMeasurementEventCallback<long>((_, value, tags, _) =>
|
|
{
|
|
lock (_gate) _measurements.Add((value, tags.ToArray()));
|
|
});
|
|
_listener.Start();
|
|
}
|
|
|
|
public int WithTag(string key, string value)
|
|
{
|
|
lock (_gate) return _measurements.Count(m => m.Tags.Any(t => t.Key == key && Equals(t.Value, value)));
|
|
}
|
|
|
|
public long SumWithTag(string key, string value)
|
|
{
|
|
lock (_gate)
|
|
return _measurements.Where(m => m.Tags.Any(t => t.Key == key && Equals(t.Value, value)))
|
|
.Sum(m => m.Value);
|
|
}
|
|
|
|
public void Dispose() => _listener.Dispose();
|
|
}
|
|
}
|