Merge branch 'master' into worktree-agent-a720278a57ae41792
Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -71,7 +71,7 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
/// <summary>Hand-computed Part 13 DurationBad of Q's eight intervals, seconds.</summary>
|
||||
private static readonly double[] QDurationBad = [3, 0, 0, 0, 5, 0, 0, 3];
|
||||
|
||||
/// <summary>What the 1.5.378.182 baseline answers for Q's PercentGood (Uncertain counted as Good, #796).</summary>
|
||||
/// <summary>What the untouched 1.5.378.182 baseline answers for Q's PercentGood (Uncertain counted as Good, #796; live before #809).</summary>
|
||||
private static readonly double[] QBaselinePercentGood = [70, 100, 100, 100, 50, 100, 100, 70];
|
||||
|
||||
/// <summary>
|
||||
@@ -182,7 +182,8 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
var pGood = (await src.ReadProcessedAsync("Q", At(0), At(80), Ten, HistoryAggregateType.PercentGood, Ct)).Samples;
|
||||
ShouldBeClose(Values(good), Values(pGood).Select(p => p * Ten.TotalMilliseconds / 100).ToArray(), "DurationGood vs PercentGood");
|
||||
|
||||
// The baseline (the live AggregateFilter calculator, #809) answers Q's DurationGood with the Uncertain time added.
|
||||
// The untouched baseline (the live AggregateFilter calculator before #809) answers Q's DurationGood with the
|
||||
// Uncertain time added; the live calculator now answers the Part 13 values above (SdkAggregateParityTests).
|
||||
var baseline = SdkAggregateOracle.RunSdkAsIs(HistoryAggregateType.DurationGood, At(0), At(80), Ten,
|
||||
Q.Select(r => (r.T, r.V, r.Status)));
|
||||
ShouldBeClose(baseline.Select(r => Convert.ToDouble(r.Value)).ToArray(),
|
||||
@@ -236,15 +237,16 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
public async Task Processed_reads_over_partial_data_match_the_Part_13_oracle(double from, double to, double intervalS)
|
||||
{
|
||||
// QP's start-of-data and end-of-data intervals against the oracle, forward and reverse (#821). Left out, each a documented or pinned divergence:
|
||||
// TimeAverage2 / Total2 (the SDK's 1 ms end-of-data region), the min/max family (the SDK sets Partial on a Bad
|
||||
// result), and TimeAverage / Total (pinned by Time_weighted_values_over_partial_data_cover_only_the_data below).
|
||||
// TimeAverage2 / Total2 (the SDK's 1 ms end-of-data region) and the min/max family (the SDK sets Partial on a Bad
|
||||
// result). TimeAverage / Total are in since #821: the live calculator no longer integrates the span before QP's
|
||||
// Uncertain first sample (the hand-computed values are pinned below as well).
|
||||
await B.SeedAsync("QP", QP);
|
||||
await AssertMatchesOracle("QP", Rounded(QP), from, to, intervalS,
|
||||
[
|
||||
HistoryAggregateType.PercentGood, HistoryAggregateType.PercentBad, HistoryAggregateType.DurationGood,
|
||||
HistoryAggregateType.DurationBad, HistoryAggregateType.Count,
|
||||
HistoryAggregateType.Average, HistoryAggregateType.Start, HistoryAggregateType.End, HistoryAggregateType.Delta,
|
||||
HistoryAggregateType.Interpolative,
|
||||
HistoryAggregateType.Interpolative, HistoryAggregateType.TimeAverage, HistoryAggregateType.Total,
|
||||
]);
|
||||
}
|
||||
|
||||
@@ -254,9 +256,9 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
// Part 13 §5.4.3.3/§5.4.3.8 over QP's [0,20) (data from +5): the interpolated-bounding signal is defined from +5
|
||||
// only — 10 → 11 over [5,12) (area 73.5) and 11 → 11 + 8/13 over [12,20) (area 4·(22 + 8/13)) — so
|
||||
// Total = 73.5 + 88 + 32/13 and TimeAverage = Total ÷ 15 s; Uncertain (an Uncertain value used, not fully covered),
|
||||
// Partial (the data starts inside). Divergence found by #810: the 1.5.378.182 baseline, whose first sample is
|
||||
// Uncertain here, integrates the uncovered [0,5) as a ramp from 0 (Total +25) — so these are pinned by hand, not
|
||||
// against the oracle.
|
||||
// Partial (the data starts inside). Found by #810: the untouched 1.5.378.182 baseline, whose first sample is
|
||||
// Uncertain here, integrates the uncovered [0,5) as a ramp from 0 (Total +25); since #821 the live calculator (the
|
||||
// oracle) does not.
|
||||
await B.SeedAsync("QP", QP);
|
||||
var src = B.CreateDataSource();
|
||||
const double total = 73.5 + 88 + 32d / 13;
|
||||
@@ -267,8 +269,10 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
((double)avg.Value!).ShouldBe(total / 15, 1e-6);
|
||||
avg.StatusCode.ShouldBe(UncertainCalc | Partial);
|
||||
|
||||
var sdk = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(0)), B.Round(At(20)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(sdk.Value).ShouldBe(total + 25, 1e-6, "baseline divergence — if this moves, compare QP's TimeAverage/Total to the oracle");
|
||||
var live = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(0)), B.Round(At(20)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(live.Value).ShouldBe(total, 1e-6, "live AggregateFilter equals HistoryRead (#821)");
|
||||
var before = SdkAggregateOracle.RunSdkAsIs(HistoryAggregateType.Total, B.Round(At(0)), B.Round(At(20)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(before.Value).ShouldBe(total + 25, 1e-6, "the untouched baseline (live before #821)");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -276,8 +280,8 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
{
|
||||
// #821: the reverse-order variant of the late-start case above — (0,20] read 20 → 0. The same Part 13 integral and
|
||||
// average over the covered [5,20); Uncertain; NOT Partial (a reverse read keys start-of-data on the latest sample,
|
||||
// which is past the window — the SDK's rule, SdkAggregateParityTests); stamped with the later edge (#677). The
|
||||
// baseline's reverse answer carries the same +25 ramp divergence.
|
||||
// which is past the window — the SDK's rule, SdkAggregateParityTests); stamped with the later edge (#677). The live
|
||||
// calculator agrees since #821; the untouched baseline's reverse answer carries the same +25 ramp.
|
||||
await B.SeedAsync("QP", QP);
|
||||
var src = B.CreateDataSource();
|
||||
const double total = 73.5 + 88 + 32d / 13;
|
||||
@@ -289,8 +293,10 @@ public abstract class QualityMixConformance : ConformanceBase
|
||||
((double)avg.Value!).ShouldBe(total / 15, 1e-6);
|
||||
avg.StatusCode.ShouldBe(UncertainCalc);
|
||||
|
||||
var sdk = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(20)), B.Round(At(0)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(sdk.Value).ShouldBe(total + 25, 1e-6, "baseline divergence (reverse) — #821");
|
||||
var live = SdkAggregateOracle.Run(HistoryAggregateType.Total, B.Round(At(20)), B.Round(At(0)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(live.Value).ShouldBe(total, 1e-6, "live AggregateFilter equals HistoryRead (reverse) — #821");
|
||||
var before = SdkAggregateOracle.RunSdkAsIs(HistoryAggregateType.Total, B.Round(At(20)), B.Round(At(0)), Twenty, Rounded(QP)).Single();
|
||||
Convert.ToDouble(before.Value).ShouldBe(total + 25, 1e-6, "the untouched baseline (live before #821, reverse)");
|
||||
}
|
||||
|
||||
private async Task AssertMatchesOracle(string tag, (DateTime, HistorianValue, uint)[] rows, double from, double to,
|
||||
|
||||
@@ -2,7 +2,7 @@ using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates;
|
||||
using BaselineAggregateCalculator = ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates.AggregateCalculator;
|
||||
using BaselineStatusAggregateCalculator = ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates.StatusAggregateCalculator;
|
||||
using BaselineAggregators = ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates.Aggregators;
|
||||
using Shouldly;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
@@ -18,17 +18,25 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
|
||||
/// <remarks>
|
||||
/// <para>Shared (source-linked) by <c>Sql.Common.Tests</c> and both SQL backends' integration suites.</para>
|
||||
/// <para>
|
||||
/// <b>PercentGood (and, since #820, DurationGood) are the aggregates not taken as-is (#796).</b> The 1.5.378.182 baseline changed
|
||||
/// <b>PercentGood / DurationGood (#796, #820, #809).</b> The 1.5.378.182 baseline changed
|
||||
/// <c>StatusAggregateCalculator.ComputeDurationGoodBad</c> (1.5.378.106 and earlier counted Good regions only):
|
||||
/// with TreatUncertainAsBad = false it counts Uncertain regions as <b>Good</b> time (as Bad when true). Part 13
|
||||
/// keeps Uncertain out of both (§5.4.3.31 DurationGood counts Good regions; §5.4.3.33 PercentGood =
|
||||
/// DurationGood ÷ interval; Mantis 0006166, closed "no change required": an all-Uncertain interval is
|
||||
/// PercentGood 0 <i>and</i> PercentBad 0), and so does ours. <see cref="Run"/> therefore answers PercentGood
|
||||
/// (and DurationGood) through <see cref="Part13GoodCalculator"/>: the SAME baseline calculator, region machinery (simple
|
||||
/// bounds, end-of-data 1 ms region, status and Partial bits) and server-default configuration, counting only
|
||||
/// Good regions — exactly what the pre-.182 oracle computed. PercentBad / DurationBad still count Bad regions only
|
||||
/// (unchanged by .182) and are compared as-is. <see cref="RunSdkAsIs"/> runs the baseline untouched, for the test that pins
|
||||
/// the divergence.
|
||||
/// PercentGood 0 <i>and</i> PercentBad 0), and so does ours. Since #809 the LIVE calculators follow Part 13 too:
|
||||
/// <c>CompatibilityAggregateFactory</c> serves those two through the application's
|
||||
/// <c>GoodOnlyStatusAggregateCalculator</c> (the same baseline region machinery — simple bounds, end-of-data 1 ms
|
||||
/// region, status and Partial bits — counting only Good regions), so <see cref="Run"/> / <see cref="RunLive"/> are
|
||||
/// exactly what a live AggregateFilter computes, for every aggregate. PercentBad / DurationBad count Bad regions only
|
||||
/// (unchanged by .182). <see cref="RunSdkAsIs"/> runs the untouched baseline (what live served before #809 / #821),
|
||||
/// for the tests that pin those divergences.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>TimeAverage / Total with no start bound (#821).</b> When the data starts inside an interval with an
|
||||
/// <b>Uncertain</b> first value, the baseline integrates the uncovered span before it as a ramp from 0 (its region
|
||||
/// machinery overwrites that region's Bad status with Uncertain). Part 13 ("No Start Bound: Calculate Partial
|
||||
/// Interval") and ours integrate only the covered part, as the baseline itself does after a Good first value. Since
|
||||
/// #821 the live calculators follow Part 13 here too (<c>CoveredTimeAverageAggregateCalculator</c>).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Not <c>100 − PercentBad</c> under TreatUncertainAsBad = true: that flag also changes the region machinery
|
||||
@@ -54,8 +62,9 @@ public static class SdkAggregateOracle
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Runs the Part 13 oracle for <paramref name="aggregate"/> over <paramref name="samples"/>: the baseline calculator,
|
||||
/// except PercentGood / DurationGood, which count Good regions only (see the remarks, #796/#820).
|
||||
/// Runs the Part 13 oracle for <paramref name="aggregate"/> over <paramref name="samples"/>: the live calculator
|
||||
/// (<see cref="RunLive"/>) — the baseline, except PercentGood / DurationGood, which count Good regions only, and
|
||||
/// TimeAverage / Total, which leave the span before an Uncertain first value out (see the remarks, #796/#820/#809/#821).
|
||||
/// </summary>
|
||||
/// <param name="aggregate">The aggregate.</param>
|
||||
/// <param name="start">Request start (later than <paramref name="end"/> for a reverse read).</param>
|
||||
@@ -65,16 +74,31 @@ public static class SdkAggregateOracle
|
||||
/// <returns>The oracle's processed values, in the order it returns them.</returns>
|
||||
public static IReadOnlyList<Result> Run(HistoryAggregateType aggregate, DateTime start, DateTime end, TimeSpan interval,
|
||||
IEnumerable<(DateTime Time, HistorianValue Value, uint Status)> samples) =>
|
||||
aggregate is HistoryAggregateType.PercentGood or HistoryAggregateType.DurationGood
|
||||
? RunCore(new LegacyCalculatorAdapter(new Part13GoodCalculator(AggregateId(aggregate),
|
||||
start, end, interval.TotalMilliseconds, false, DefaultConfiguration, Telemetry,
|
||||
usePercent: aggregate == HistoryAggregateType.PercentGood)), start, end, samples)
|
||||
: RunSdkAsIs(aggregate, start, end, interval, samples);
|
||||
RunLive(aggregate, start, end, interval, samples);
|
||||
|
||||
/// <summary>
|
||||
/// Runs the baseline (1.5.378.182) calculator for <paramref name="aggregate"/> exactly as it computes it, under
|
||||
/// <see cref="DefaultConfiguration"/> — including its PercentGood rule (Uncertain counted as Good) that
|
||||
/// <see cref="Run"/> deliberately does not use (#796).
|
||||
/// Runs exactly the calculator the server registers for live AggregateFilter monitored items
|
||||
/// (<c>CompatibilityAggregateFactory.Create</c>, the #809 Good-only PercentGood / DurationGood and the #821
|
||||
/// covered-only TimeAverage / Total included).
|
||||
/// </summary>
|
||||
/// <param name="aggregate">The aggregate.</param>
|
||||
/// <param name="start">Request start (later than <paramref name="end"/> for a reverse read).</param>
|
||||
/// <param name="end">Request end.</param>
|
||||
/// <param name="interval">Processing interval.</param>
|
||||
/// <param name="samples">Every stored sample of the tag, any order (sorted here).</param>
|
||||
/// <param name="configuration">The configuration (default <see cref="DefaultConfiguration"/>).</param>
|
||||
/// <returns>The calculator's processed values, in the order it returns them.</returns>
|
||||
public static IReadOnlyList<Result> RunLive(HistoryAggregateType aggregate, DateTime start, DateTime end,
|
||||
TimeSpan interval, IEnumerable<(DateTime Time, HistorianValue Value, uint Status)> samples,
|
||||
AggregateConfiguration? configuration = null) =>
|
||||
RunCore(CompatibilityAggregateFactory.Create(AggregateId(aggregate), start, end, interval.TotalMilliseconds, false,
|
||||
configuration ?? DefaultConfiguration, Telemetry), start, end, samples);
|
||||
|
||||
/// <summary>
|
||||
/// Runs the untouched baseline (1.5.378.182) calculator for <paramref name="aggregate"/> exactly as it computes it,
|
||||
/// under <see cref="DefaultConfiguration"/> — including its PercentGood / DurationGood rule (Uncertain counted as
|
||||
/// Good) and its TimeAverage / Total ramp before an Uncertain first value, which neither HistoryRead nor, since
|
||||
/// #809 / #821, the live calculators use.
|
||||
/// </summary>
|
||||
/// <param name="aggregate">The aggregate.</param>
|
||||
/// <param name="start">Request start (later than <paramref name="end"/> for a reverse read).</param>
|
||||
@@ -86,8 +110,9 @@ public static class SdkAggregateOracle
|
||||
public static IReadOnlyList<Result> RunSdkAsIs(HistoryAggregateType aggregate, DateTime start, DateTime end,
|
||||
TimeSpan interval, IEnumerable<(DateTime Time, HistorianValue Value, uint Status)> samples,
|
||||
AggregateConfiguration? configuration = null) =>
|
||||
RunCore(CompatibilityAggregateFactory.Create(AggregateId(aggregate), start, end, interval.TotalMilliseconds, false,
|
||||
configuration ?? DefaultConfiguration, Telemetry), start, end, samples);
|
||||
RunCore(new LegacyCalculatorAdapter(BaselineAggregators.CreateStandardCalculator(AggregateId(aggregate), start, end,
|
||||
interval.TotalMilliseconds, false, configuration ?? DefaultConfiguration, Telemetry)
|
||||
?? throw new ArgumentOutOfRangeException(nameof(aggregate))), start, end, samples);
|
||||
|
||||
private static IReadOnlyList<Result> RunCore(IAggregateCalculator calc, DateTime start, DateTime end,
|
||||
IEnumerable<(DateTime Time, HistorianValue Value, uint Status)> samples)
|
||||
@@ -155,42 +180,7 @@ public static class SdkAggregateOracle
|
||||
problems.ShouldBeEmpty($"{label} differs from the SDK:\n" + string.Join("\n", problems));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The baseline <see cref="BaselineStatusAggregateCalculator"/> with PercentGood / DurationGood counting <b>Good</b>
|
||||
/// regions only — Part 13 §5.4.3.31/§5.4.3.33, and the 1.5.378.106 rule. Everything else (bounds, regions, Partial,
|
||||
/// status, the millisecond Duration unit) is the baseline's own code, run under the same configuration as
|
||||
/// PercentBad / DurationBad.
|
||||
/// </summary>
|
||||
private sealed class Part13GoodCalculator(NodeId aggregateId, DateTime startTime, DateTime endTime,
|
||||
double processingInterval, bool stepped, AggregateConfiguration configuration, ITelemetryContext telemetry,
|
||||
bool usePercent)
|
||||
: BaselineStatusAggregateCalculator(aggregateId, startTime, endTime, processingInterval, stepped, configuration, telemetry)
|
||||
{
|
||||
protected override LegacyDataValue ComputeValue(TimeSlice slice)
|
||||
{
|
||||
var values = GetValuesWithSimpleBounds(slice);
|
||||
if (values == null || values.Count == 0) return GetNoDataValue(slice);
|
||||
|
||||
var regions = GetRegionsInValueSet(values, false, true);
|
||||
double good = 0, total = 0;
|
||||
foreach (var region in regions)
|
||||
{
|
||||
total += region.Duration;
|
||||
if (StatusCode.IsGood(region.StatusCode)) good += region.Duration;
|
||||
}
|
||||
|
||||
var value = new LegacyDataValue
|
||||
{
|
||||
WrappedValue = ZB.MOM.WW.OtOpcUa.OpcUaServer.SdkValueBoundary.FromClr(usePercent ? good / total * 100 : good),
|
||||
SourceTimestamp = GetTimestamp(slice),
|
||||
ServerTimestamp = GetTimestamp(slice),
|
||||
};
|
||||
value.StatusCode = value.StatusCode.WithAggregateBits(AggregateBits.Calculated);
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The <see cref="CompatibilityAggregateFactory"/> adapter, for a calculator built here.</summary>
|
||||
/// <summary>The <see cref="CompatibilityAggregateFactory"/> adapter, for a baseline calculator built here.</summary>
|
||||
private sealed class LegacyCalculatorAdapter(BaselineAggregateCalculator calculator) : IAggregateCalculator
|
||||
{
|
||||
public NodeId AggregateId => calculator.AggregateId;
|
||||
|
||||
+84
-18
@@ -15,33 +15,99 @@ public sealed class GatewayGalaxyPlatformSourceTests
|
||||
return o;
|
||||
}
|
||||
|
||||
/// <summary>Every request carries the platform template filter, no attributes, and the page size; replies are re-filtered.</summary>
|
||||
[Fact]
|
||||
public async Task Requests_filter_on_platform_templates_without_attributes_and_replies_are_refiltered()
|
||||
/// <summary>
|
||||
/// A fake gateway that applies the request filters with the SAME semantics as the shared
|
||||
/// <c>GalaxyHierarchyProjector.MatchesFilters</c>: every <c>template_chain_contains</c> token must
|
||||
/// match (AND, case-insensitive substring over the chain), <c>category_ids</c> is an OR set. Pages
|
||||
/// the filtered result at the request's page size.
|
||||
/// </summary>
|
||||
private sealed class FilteringGateway(params GalaxyObject[] galaxy)
|
||||
{
|
||||
var requests = new List<DiscoverHierarchyRequest>();
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
public List<DiscoverHierarchyRequest> Requests { get; } = [];
|
||||
|
||||
public Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
requests.Add(req);
|
||||
Requests.Add(req.Clone());
|
||||
var matched = galaxy.Where(o => Matches(o, req)).ToList();
|
||||
var offset = req.PageToken == "" ? 0 : int.Parse(req.PageToken.Split(':')[1]);
|
||||
var reply = new DiscoverHierarchyReply();
|
||||
if (req.PageToken == "")
|
||||
reply.Objects.Add(matched.Skip(offset).Take(req.PageSize));
|
||||
if (offset + req.PageSize < matched.Count)
|
||||
{
|
||||
reply.Objects.Add([Obj("Plat01", "$WinPlatform"), Obj("Tank01", "$UserDefined")]);
|
||||
reply.NextPageToken = "p2";
|
||||
}
|
||||
else
|
||||
{
|
||||
reply.Objects.Add([Obj("Eng01", "$AppEngine", "$WinPlatform"), Obj("Plat01", "$WinPlatform"), Obj("", "$WinPlatform")]);
|
||||
reply.NextPageToken = $"{string.Join(",", req.TemplateChainContains)}:{offset + req.PageSize}";
|
||||
}
|
||||
return Task.FromResult(reply);
|
||||
}
|
||||
|
||||
var names = await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 50, CancellationToken.None);
|
||||
private static bool Matches(GalaxyObject obj, DiscoverHierarchyRequest req)
|
||||
{
|
||||
if (req.CategoryIds.Count > 0 && !req.CategoryIds.Contains(obj.CategoryId)) return false;
|
||||
foreach (var filter in req.TemplateChainContains)
|
||||
{
|
||||
if (!obj.TemplateChain.Any(t => t.Contains(filter, StringComparison.OrdinalIgnoreCase))) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
names.ShouldBe(["Plat01", "Eng01"]);
|
||||
requests.Count.ShouldBe(2);
|
||||
requests.ShouldAllBe(r => r.PageSize == 50 && r.IncludeAttributes == false);
|
||||
requests.ShouldAllBe(r => r.TemplateChainContains.Contains("$WinPlatform") && r.TemplateChainContains.Contains("$AppEngine"));
|
||||
/// <summary>
|
||||
/// #864: against a gateway that ANDs template filters (as the real one does), every deployed
|
||||
/// <c>$WinPlatform</c> AND every <c>$AppEngine</c> is found — the old single request carrying both
|
||||
/// tokens matched none. Shape of the VD03 DEV galaxy: 3 platforms, 2 engines.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Finds_every_platform_and_engine_against_an_AND_filtering_gateway()
|
||||
{
|
||||
var gw = new FilteringGateway(
|
||||
Obj("CTRX_DEV", "$WinPlatform"),
|
||||
Obj("DevWinPlatform", "$WinPlatform"),
|
||||
Obj("JoePlatform", "$JoePlatformTemplate", "$WinPlatform"),
|
||||
Obj("DevAppEngine", "$AppEngine"),
|
||||
Obj("JoeAppEngine", "$AppEngine"),
|
||||
Obj("Tank01", "$UserDefined"),
|
||||
Obj("Area01", "$Area"));
|
||||
|
||||
var names = await GatewayGalaxyPlatformSource.QueryAsync(gw.Discover, pageSize: 50, CancellationToken.None);
|
||||
|
||||
names.ShouldBe(["CTRX_DEV", "DevWinPlatform", "JoePlatform", "DevAppEngine", "JoeAppEngine"]);
|
||||
gw.Requests.ShouldAllBe(r => r.TemplateChainContains.Count == 1 && r.PageSize == 50 && r.IncludeAttributes == false);
|
||||
gw.Requests.Select(r => r.TemplateChainContains[0]).Distinct().ShouldBe(["$WinPlatform", "$AppEngine"]);
|
||||
}
|
||||
|
||||
/// <summary>Each per-template sweep pages to completion independently; tag names are de-duplicated across sweeps.</summary>
|
||||
[Fact]
|
||||
public async Task Each_template_sweep_pages_independently_and_names_are_deduplicated()
|
||||
{
|
||||
var gw = new FilteringGateway(
|
||||
Obj("Plat01", "$WinPlatform"),
|
||||
Obj("Plat02", "$WinPlatform"),
|
||||
Obj("Plat03", "$WinPlatform"),
|
||||
// An object whose chain carries both templates is reported once.
|
||||
Obj("Hybrid01", "$AppEngine", "$WinPlatform"),
|
||||
Obj("Eng01", "$AppEngine"),
|
||||
Obj("Eng02", "$AppEngine"),
|
||||
Obj("", "$WinPlatform"));
|
||||
|
||||
var names = await GatewayGalaxyPlatformSource.QueryAsync(gw.Discover, pageSize: 2, CancellationToken.None);
|
||||
|
||||
names.ShouldBe(["Plat01", "Plat02", "Plat03", "Hybrid01", "Eng01", "Eng02"]);
|
||||
// $WinPlatform: 5 matches (incl. the empty tag) => 3 pages; $AppEngine: 3 matches => 2 pages.
|
||||
gw.Requests.Count(r => r.TemplateChainContains[0] == "$WinPlatform").ShouldBe(3);
|
||||
gw.Requests.Count(r => r.TemplateChainContains[0] == "$AppEngine").ShouldBe(2);
|
||||
}
|
||||
|
||||
/// <summary>A gateway that ignores the filter still yields only platforms (replies are re-filtered on the chain).</summary>
|
||||
[Fact]
|
||||
public async Task Replies_are_refiltered_on_the_template_chain()
|
||||
{
|
||||
Task<DiscoverHierarchyReply> Discover(DiscoverHierarchyRequest req, CancellationToken ct)
|
||||
{
|
||||
var reply = new DiscoverHierarchyReply();
|
||||
reply.Objects.Add([Obj("Plat01", "$WinPlatform"), Obj("Tank01", "$UserDefined"), Obj("Eng01", "$AppEngine")]);
|
||||
return Task.FromResult(reply);
|
||||
}
|
||||
|
||||
(await GatewayGalaxyPlatformSource.QueryAsync(Discover, pageSize: 50, CancellationToken.None)).ShouldBe(["Plat01", "Eng01"]);
|
||||
}
|
||||
|
||||
/// <summary>A gateway that strips the template chain is trusted to have filtered server-side.</summary>
|
||||
|
||||
@@ -844,6 +844,7 @@ public sealed class EarlyEventWindowTests
|
||||
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber)
|
||||
{
|
||||
RecreateOverrideForTests = _ => Task.CompletedTask,
|
||||
StreamRequestWait = TimeSpan.FromMilliseconds(100), // advises in the reopen window on purpose (#865 bound)
|
||||
};
|
||||
var delivered = new ConcurrentQueue<(string Ref, double Value)>();
|
||||
driver.OnDataChange += (_, e) => delivered.Enqueue((e.FullReference, (double)e.Snapshot.Value!));
|
||||
@@ -897,6 +898,31 @@ public sealed class EarlyEventWindowTests
|
||||
driver.SubscriptionsForTests.ResolveSubscribers(77).ShouldBeEmpty();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #865 stress finding: the dispatch fast path looked the handle up, then checked for an open window —
|
||||
/// not atomically. A subscribe that registered the handle AND closed its window in between made the
|
||||
/// event "stale" and its initial value was dropped for good. It must resolve.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_registration_landing_between_the_fast_path_lookup_and_the_window_check_still_resolves()
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
var session = registry.CurrentSession;
|
||||
var window = registry.BeginPendingSubscribe();
|
||||
registry.AfterFastPathMissForTests = () =>
|
||||
{
|
||||
registry.AfterFastPathMissForTests = null;
|
||||
registry.TryRegister(registry.NextSubscriptionId(), [new TagBinding("Tank.A", 7)], session).ShouldBeTrue();
|
||||
window.Dispose(); // the subscribe's window closes right after its registration
|
||||
};
|
||||
|
||||
registry.ResolveOrPark(DataChange(7, 1), session, out var subscribers, out _).ShouldBe(EarlyEventOutcome.Resolved);
|
||||
subscribers.Single().FullReference.ShouldBe("Tank.A");
|
||||
|
||||
// Positive control for the Stale branch: an unknown handle with no window open is still dropped.
|
||||
registry.ResolveOrPark(DataChange(8, 1), session, out _, out _).ShouldBe(EarlyEventOutcome.Stale);
|
||||
}
|
||||
|
||||
/// <summary>I-B: a purge that completes after a reconnect does not delete the new session's parked values.</summary>
|
||||
[Fact]
|
||||
public void A_purge_for_an_older_session_is_a_no_op()
|
||||
@@ -1027,6 +1053,9 @@ public sealed class EarlyEventWindowTests
|
||||
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber)
|
||||
{
|
||||
RecreateOverrideForTests = _ => Task.CompletedTask,
|
||||
// These tests advise in the reopen → replay window on purpose (fencing, not values). #865 makes
|
||||
// such an advise wait for the replay's pump, bounded; keep the bound short so they don't idle.
|
||||
StreamRequestWait = TimeSpan.FromMilliseconds(100),
|
||||
};
|
||||
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
||||
return driver;
|
||||
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// #865: <see cref="EventPump.StreamRequested"/> — what advise paths wait on so an initial value is
|
||||
/// never pushed before the stream is requested. It must complete once the RPC is started (before any
|
||||
/// event), and must never strand a waiter on a pump whose loop died before getting that far.
|
||||
/// </summary>
|
||||
public sealed class EventPumpStreamRequestedTests
|
||||
{
|
||||
private sealed class ScriptedSubscriber(Func<CancellationToken, IAsyncEnumerable<MxEvent>> stream) : IGalaxySubscriber
|
||||
{
|
||||
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<SubscribeResult>>([]);
|
||||
|
||||
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public IAsyncEnumerable<MxEvent> StreamEventsAsync(CancellationToken cancellationToken) => stream(cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>A stream that is requested and then never yields: StreamRequested completes; the RPC started first.</summary>
|
||||
[Fact]
|
||||
public async Task Completes_once_the_stream_is_requested_before_any_event()
|
||||
{
|
||||
var rpcStarted = 0;
|
||||
async IAsyncEnumerable<MxEvent> Never([EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
Interlocked.Exchange(ref rpcStarted, 1); // the synchronous first step — where a real call is created
|
||||
await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
|
||||
yield break;
|
||||
}
|
||||
|
||||
await using var pump = new EventPump(new ScriptedSubscriber(Never), new SubscriptionRegistry());
|
||||
pump.StreamRequested.IsCompleted.ShouldBeFalse("not before Start");
|
||||
pump.Start();
|
||||
|
||||
await TestWait.ForAsync(pump.StreamRequested, "StreamRequested must complete once the stream call is started");
|
||||
Volatile.Read(ref rpcStarted).ShouldBe(1, "StreamRequested must not precede the stream call");
|
||||
}
|
||||
|
||||
/// <summary>A subscriber that throws before reaching the RPC still completes StreamRequested (no stranded waiter).</summary>
|
||||
[Fact]
|
||||
public async Task Completes_when_the_stream_fails_before_the_rpc()
|
||||
{
|
||||
var faulted = new TaskCompletionSource<Exception>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
await using var pump = new EventPump(
|
||||
new ScriptedSubscriber(_ => throw new InvalidOperationException("not connected")),
|
||||
new SubscriptionRegistry(),
|
||||
onStreamFault: ex => faulted.TrySetResult(ex));
|
||||
pump.Start();
|
||||
|
||||
await TestWait.ForAsync(pump.StreamRequested, "a dead pump must not strand a StreamRequested waiter");
|
||||
(await TestWait.ForAsync(faulted.Task, "positive control: the failure is still reported as a stream fault"))
|
||||
.ShouldBeOfType<InvalidOperationException>();
|
||||
}
|
||||
}
|
||||
+11
-2
@@ -272,7 +272,10 @@ public sealed class GalaxyDriverFixRound1Tests
|
||||
[Fact]
|
||||
public async Task Production_path_queries_the_platform_source_at_init()
|
||||
{
|
||||
var platforms = new ScriptedPlatformSource();
|
||||
// An empty galaxy: since #865 a probe advise starts the event stream first, and this owned runtime's
|
||||
// gRPC channel points at no real gateway — its stream would fault and (correctly) degrade health.
|
||||
// The query itself is what this test pins.
|
||||
var platforms = new ScriptedPlatformSource { Script = _ => [] };
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, platformSource: platforms)
|
||||
{
|
||||
SessionOpenOverrideForTests = _ => Task.CompletedTask,
|
||||
@@ -303,6 +306,7 @@ public sealed class GalaxyDriverFixRound1Tests
|
||||
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
driver.HasEventPumpForTests.ShouldBeTrue("positive control: subscribe works on a live runtime");
|
||||
var oldPump = driver.EventPumpInstanceForTests;
|
||||
|
||||
sub.ParkUnsubscribe = true;
|
||||
var shutdown = driver.ShutdownAsync(CancellationToken.None);
|
||||
@@ -330,7 +334,12 @@ public sealed class GalaxyDriverFixRound1Tests
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(0);
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
driver.HasEventPumpForTests.ShouldBeFalse("the next runtime starts with no inherited pump");
|
||||
// #865: the new runtime's background probe seed starts a pump of its own (its advises need a live
|
||||
// stream) — but it must be a FRESH one, never the dead subscriber's.
|
||||
await WaitUntilAsync(() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == 1,
|
||||
"positive control: the new runtime's background seed must advise its platform");
|
||||
driver.EventPumpInstanceForTests.ShouldNotBeNull("the seed's probe advise must have started a pump")
|
||||
.ShouldNotBeSameAs(oldPump, "the next runtime must not inherit the old pump");
|
||||
}
|
||||
|
||||
// ───────────────────────────── IMPORTANT 3: bounded gateway close ─────────────────────────────
|
||||
|
||||
+419
@@ -0,0 +1,419 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// #865: the platform ScanState probes' initial value. The gateway pushes an advise's current value
|
||||
/// once, as its first event, and a FRESH <c>StreamEvents</c> (<c>after_worker_sequence = 0</c>) never
|
||||
/// replays — so a probe advised before the driver's event stream is attached never gets a value, and
|
||||
/// ScanState rarely changes again. The event pump used to start only on the first Read/Subscribe,
|
||||
/// so a driver with no authored tags (or any driver before its first read) had no host statuses.
|
||||
/// </summary>
|
||||
public sealed class ProbeInitialValueTests
|
||||
{
|
||||
private static readonly string[] Platforms = ["CTRX_DEV", "DevWinPlatform", "DevAppEngine"];
|
||||
|
||||
private static GalaxyDriverOptions Opts() => new(
|
||||
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
||||
new GalaxyMxAccessOptions("ProbeInitialValue"),
|
||||
new GalaxyRepositoryOptions(WatchDeployEvents: false),
|
||||
new GalaxyReconnectOptions());
|
||||
|
||||
private sealed class FixedPlatformSource(params string[] platforms) : IGalaxyPlatformSource
|
||||
{
|
||||
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
||||
=> Task.FromResult<IReadOnlyList<string>>(platforms);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Models the gateway's event semantics: every advise pushes its current value exactly once, and
|
||||
/// only to a stream that is attached at that moment (no replay for a fresh stream). Records the
|
||||
/// order in which the stream was requested and the advises were made.
|
||||
/// </summary>
|
||||
private sealed class GatewayLikeSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
private Channel<MxEvent>? _attached;
|
||||
private int _nextHandle;
|
||||
|
||||
/// <summary>"stream" per StreamEvents call, "subscribe:<refs>" per SubscribeBulk, in call order.</summary>
|
||||
public List<string> Calls { get; } = [];
|
||||
|
||||
/// <summary>Initial values pushed while no stream was attached — lost, as on the real gateway.</summary>
|
||||
public int DroppedInitialValues;
|
||||
|
||||
private int _streams;
|
||||
|
||||
/// <summary>Streams beyond this count end immediately after attaching (a gateway refusing/ending them).</summary>
|
||||
public int StreamsThatStayOpen = int.MaxValue;
|
||||
|
||||
/// <summary>The ScanState value the next advise reports.</summary>
|
||||
public volatile bool ScanStateRunning = true;
|
||||
|
||||
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
{
|
||||
var results = new List<SubscribeResult>();
|
||||
lock (_gate)
|
||||
{
|
||||
Calls.Add("subscribe:" + string.Join(",", fullReferences));
|
||||
foreach (var r in fullReferences)
|
||||
{
|
||||
var h = ++_nextHandle;
|
||||
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
|
||||
var initial = new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = h,
|
||||
Value = new MxValue { BoolValue = ScanStateRunning },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
};
|
||||
if (_attached is null || !_attached.Writer.TryWrite(initial)) Interlocked.Increment(ref DroppedInitialValues);
|
||||
}
|
||||
}
|
||||
return Task.FromResult<IReadOnlyList<SubscribeResult>>(results);
|
||||
}
|
||||
|
||||
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
// Attached synchronously on the first MoveNext — the point a real gRPC call is issued.
|
||||
var mine = Channel.CreateUnbounded<MxEvent>();
|
||||
lock (_gate)
|
||||
{
|
||||
Calls.Add("stream");
|
||||
_attached = mine;
|
||||
}
|
||||
if (Interlocked.Increment(ref _streams) > StreamsThatStayOpen)
|
||||
{
|
||||
lock (_gate) { if (ReferenceEquals(_attached, mine)) _attached = null; }
|
||||
yield break; // the gateway completed this stream at once
|
||||
}
|
||||
try
|
||||
{
|
||||
await foreach (var ev in mine.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (ReferenceEquals(_attached, mine)) _attached = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fault the currently attached stream (the gateway dropping it).</summary>
|
||||
public void FaultAttachedStream(Exception cause)
|
||||
{
|
||||
lock (_gate) _attached.ShouldNotBeNull("a stream must be attached").Writer.TryComplete(cause);
|
||||
}
|
||||
|
||||
public string[] CallsSnapshot()
|
||||
{
|
||||
lock (_gate) return [.. Calls];
|
||||
}
|
||||
}
|
||||
|
||||
private static HostState? StateOf(GalaxyDriver driver, string host) =>
|
||||
driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
|
||||
|
||||
private static string Describe(GalaxyDriver driver, GatewayLikeSubscriber sub) =>
|
||||
$"hosts=[{string.Join(", ", driver.GetHostStatuses().Select(s => $"{s.HostName}={s.State}"))}] " +
|
||||
$"calls=[{string.Join(" | ", sub.CallsSnapshot())}] dropped={sub.DroppedInitialValues} pump={driver.HasEventPumpForTests}";
|
||||
|
||||
/// <summary>
|
||||
/// A driver with NO authored tags — nothing ever Reads or Subscribes — still reports every seeded
|
||||
/// platform with its initial ScanState. Before #865 the pump never started, every initial value
|
||||
/// was pushed to no stream, and only the transport entry ever appeared.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Driver_with_no_authored_tags_reports_every_platform_from_its_initial_ScanState()
|
||||
{
|
||||
var sub = new GatewayLikeSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
||||
platformSource: new FixedPlatformSource(Platforms));
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
await TestWait.UntilAsync(
|
||||
() => Platforms.All(p => StateOf(driver, p) == HostState.Running),
|
||||
"every seeded platform must surface Running from its initial ScanState, with no Read/Subscribe",
|
||||
() => Describe(driver, sub));
|
||||
sub.DroppedInitialValues.ShouldBe(0, "no probe's initial value may be pushed before the stream is attached");
|
||||
driver.TrackedSubscriptionCountForTests.ShouldBe(1, "only the probe entry — no data subscription was ever made");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ordering: the event stream is requested BEFORE the first probe SubscribeBulk. On the old code
|
||||
/// the probe advise went out with no stream at all.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Probe_subscribe_is_issued_only_after_the_event_stream_is_requested()
|
||||
{
|
||||
var sub = new GatewayLikeSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
||||
platformSource: new FixedPlatformSource(Platforms));
|
||||
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await TestWait.UntilAsync(
|
||||
() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == Platforms.Length,
|
||||
"positive control: the background seed must advise every platform",
|
||||
() => Describe(driver, sub));
|
||||
|
||||
var calls = sub.CallsSnapshot();
|
||||
var firstSubscribe = Array.FindIndex(calls, c => c.StartsWith("subscribe:", StringComparison.Ordinal));
|
||||
var firstStream = Array.IndexOf(calls, "stream");
|
||||
firstSubscribe.ShouldBeGreaterThanOrEqualTo(0);
|
||||
firstStream.ShouldBeGreaterThanOrEqualTo(0, $"the probe seed must start the event stream; calls=[{string.Join(" | ", calls)}]");
|
||||
firstStream.ShouldBeLessThan(firstSubscribe, $"the stream must be requested before the probe advise; calls=[{string.Join(" | ", calls)}]");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A reconnect replay re-advises the probes on the recreated session; their new initial values must
|
||||
/// arrive too (the replay restarts the pump first). The value flips to Stopped so the presence wait
|
||||
/// can only pass on a post-replay event.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Reconnect_replay_delivers_the_re_advised_probes_initial_values()
|
||||
{
|
||||
var sub = new GatewayLikeSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
||||
platformSource: new FixedPlatformSource(Platforms));
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await TestWait.UntilAsync(
|
||||
() => Platforms.All(p => StateOf(driver, p) == HostState.Running),
|
||||
"positive control: the initial seed reports Running",
|
||||
() => Describe(driver, sub));
|
||||
|
||||
sub.ScanStateRunning = false;
|
||||
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
||||
|
||||
await TestWait.UntilAsync(
|
||||
() => Platforms.All(p => StateOf(driver, p) == HostState.Stopped),
|
||||
"the re-advised probes' initial values (Stopped) must reach the host statuses after a replay",
|
||||
() => Describe(driver, sub));
|
||||
sub.DroppedInitialValues.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Review finding: a probe advised in the reopen → replay window (the session was swapped, the old
|
||||
/// pump still streams the dead session) must still get its initial value. The advise waits for the
|
||||
/// replay's restarted pump instead of advising onto the stale one (its value would route under the
|
||||
/// old session and be lost, and the replay skips a current-session handle) or starting a competing
|
||||
/// stream (the gateway's default single-subscriber mode refuses a second attach).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Probe_advised_between_reopen_and_replay_still_gets_its_initial_value()
|
||||
{
|
||||
var sub = new GatewayLikeSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
||||
platformSource: new FixedPlatformSource(Platforms))
|
||||
{
|
||||
RecreateOverrideForTests = _ => Task.CompletedTask,
|
||||
// Generous: the window advise must still be waiting when the replay below starts the pump, even
|
||||
// on a stalled agent; the wait ends the moment that pump exists.
|
||||
StreamRequestWait = TimeSpan.FromSeconds(20),
|
||||
};
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await TestWait.UntilAsync(
|
||||
() => Platforms.All(p => StateOf(driver, p) == HostState.Running),
|
||||
"positive control: the initial seed reports Running",
|
||||
() => Describe(driver, sub));
|
||||
var oldPump = driver.EventPumpInstanceForTests;
|
||||
|
||||
await driver.InvokeReopenForTestAsync(CancellationToken.None); // session swapped; no replay yet
|
||||
sub.ScanStateRunning = false;
|
||||
var callsBeforeWindow = sub.CallsSnapshot().Length;
|
||||
// The window advise must WAIT for the replay's current-session pump rather than advise onto the
|
||||
// stale one (or start a competing stream).
|
||||
var sync = driver.ProbeWatcherForTests.ShouldNotBeNull()
|
||||
.SyncPlatformsAsync([.. Platforms, "LatePlatform"], CancellationToken.None);
|
||||
|
||||
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
||||
await TestWait.ForAsync(sync, "the window advise completes once the replay restarted the pump");
|
||||
|
||||
await TestWait.UntilAsync(
|
||||
() => StateOf(driver, "LatePlatform") == HostState.Stopped
|
||||
&& Platforms.All(p => StateOf(driver, p) == HostState.Stopped),
|
||||
"the window advise and the replay's re-advises all get their initial values",
|
||||
() => Describe(driver, sub));
|
||||
driver.EventPumpInstanceForTests.ShouldNotBeSameAs(oldPump, "the replay restarted the stale pump");
|
||||
var windowCalls = sub.CallsSnapshot()[callsBeforeWindow..];
|
||||
var lateAdvise = Array.FindIndex(windowCalls, c => c.Contains("LatePlatform", StringComparison.Ordinal));
|
||||
var newStream = Array.IndexOf(windowCalls, "stream");
|
||||
newStream.ShouldBeGreaterThanOrEqualTo(0, $"the replay must start a new stream; calls=[{string.Join(" | ", windowCalls)}]");
|
||||
lateAdvise.ShouldBeGreaterThan(newStream, $"the window advise must follow the new stream; calls=[{string.Join(" | ", windowCalls)}]");
|
||||
windowCalls.Count(c => c == "stream").ShouldBe(1, "no competing stream may be started for the window advise");
|
||||
sub.DroppedInitialValues.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The data paths too: a first SubscribeAsync and a first ReadAsync on a driver whose pump was never
|
||||
/// started request the stream BEFORE advising, so their initial values arrive (a read of a value that
|
||||
/// never changes again would otherwise time out).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task First_subscribe_and_read_advise_only_after_the_stream_is_requested()
|
||||
{
|
||||
var sub = new GatewayLikeSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
|
||||
{
|
||||
ReadViaSubscribeDeadline = TimeSpan.FromSeconds(10),
|
||||
};
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
var delivered = new TaskCompletionSource<DataChangeEventArgs>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
driver.OnDataChange += (_, e) => delivered.TrySetResult(e);
|
||||
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
var calls = sub.CallsSnapshot();
|
||||
Array.IndexOf(calls, "stream").ShouldBe(0, $"the stream must be requested before the advise; calls=[{string.Join(" | ", calls)}]");
|
||||
(await TestWait.ForAsync(delivered.Task, "the subscribe's initial value must arrive", () => Describe(driver, sub)))
|
||||
.FullReference.ShouldBe("Tank01.Level");
|
||||
|
||||
sub.DroppedInitialValues.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>A first ReadAsync on a fresh driver (no pump yet) requests the stream before its advise.</summary>
|
||||
[Fact]
|
||||
public async Task First_read_advises_only_after_the_stream_is_requested()
|
||||
{
|
||||
var sub = new GatewayLikeSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
|
||||
{
|
||||
ReadViaSubscribeDeadline = TimeSpan.FromSeconds(10),
|
||||
};
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
driver.HasEventPumpForTests.ShouldBeFalse("precondition: the read is the first advise");
|
||||
|
||||
var read = await driver.ReadAsync(["Tank02.Level"], CancellationToken.None);
|
||||
|
||||
var calls = sub.CallsSnapshot();
|
||||
Array.IndexOf(calls, "stream").ShouldBe(0, $"the stream must be requested before the read's advise; calls=[{string.Join(" | ", calls)}]");
|
||||
read[0].StatusCode.ShouldBe(0u, "the read's initial value must arrive (not BadTimeout)");
|
||||
sub.DroppedInitialValues.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The replay's re-advise of DATA subscriptions (which has no per-advise hook) waits for the restarted
|
||||
/// pump's stream, so each re-advised tag's new initial value arrives. The value flips so only a
|
||||
/// post-replay event can satisfy the wait.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Reconnect_replay_delivers_re_advised_data_subscriptions_initial_values()
|
||||
{
|
||||
var sub = new GatewayLikeSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
var last = new System.Collections.Concurrent.ConcurrentDictionary<string, object?>();
|
||||
driver.OnDataChange += (_, e) => last[e.FullReference] = e.Snapshot.Value;
|
||||
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
||||
await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is true,
|
||||
"positive control: the subscribe's initial value arrives", () => Describe(driver, sub));
|
||||
|
||||
sub.ScanStateRunning = false; // the fake's next initial value
|
||||
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
||||
|
||||
await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is false,
|
||||
"the replay's re-advise must deliver its new initial value",
|
||||
() => Describe(driver, sub) + $" last=[{string.Join(",", last.Select(kv => $"{kv.Key}={kv.Value}"))}] parked={driver.ParkedEarlyEventCountForTests} h2=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(2).Select(b => b.FullReference))}] h1=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(1).Select(b => b.FullReference))}]");
|
||||
sub.DroppedInitialValues.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A replay whose restarted pump's stream ends at once must FAIL (the supervisor then backs off and
|
||||
/// stays Degraded) rather than succeed onto a dead stream — succeeding went Healthy, and the next
|
||||
/// advise's re-report started a new cycle with no backoff: a tight reconnect loop. Positive control:
|
||||
/// the first replay, whose stream stays open, succeeds.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Replay_fails_when_the_restarted_stream_ends_at_once()
|
||||
{
|
||||
var sub = new GatewayLikeSubscriber { StreamsThatStayOpen = 2 };
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // stream 1
|
||||
|
||||
await driver.InvokeReplayForTestAsync(CancellationToken.None); // stream 2 stays open — succeeds
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => driver.InvokeReplayForTestAsync(CancellationToken.None), // stream 3 ends at once
|
||||
"a replay must not report success over a dead event stream");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #865 review: a stream fault is escalated only from the LIVE pump. A fault from a pump superseded
|
||||
/// by a reopen (its session is no longer current) is ignored — it would fail a good replay — while
|
||||
/// the live pump's faults (before the reopen, and the replay's new pump after it) still escalate.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Only_the_live_pumps_stream_faults_are_escalated()
|
||||
{
|
||||
var sub = new GatewayLikeSubscriber();
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
|
||||
{
|
||||
RecreateOverrideForTests = _ => Task.CompletedTask,
|
||||
};
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // pump 1
|
||||
|
||||
// Positive control: the live pump's fault escalates.
|
||||
sub.FaultAttachedStream(new IOException("drop 1"));
|
||||
await TestWait.UntilAsync(() => driver.EscalatedStreamFaultsForTests == 1, "the live pump's fault must escalate");
|
||||
|
||||
// Recover onto pump 2 (the replay restarts the faulted pump), then reopen: pump 2 is now superseded.
|
||||
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
||||
var pump2 = driver.EventPumpInstanceForTests.ShouldNotBeNull();
|
||||
await driver.InvokeReopenForTestAsync(CancellationToken.None);
|
||||
sub.FaultAttachedStream(new IOException("late fault from the superseded pump"));
|
||||
await TestWait.UntilAsync(() => ((EventPump)pump2).HasEnded, "the superseded pump's stream must have faulted");
|
||||
|
||||
// Positive control after the absence: the replay's new pump faults and that DOES escalate — so the
|
||||
// count can only be 2 if the superseded pump's fault was ignored.
|
||||
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
||||
var pump3 = (EventPump)driver.EventPumpInstanceForTests.ShouldNotBeNull();
|
||||
pump3.ShouldNotBeSameAs(pump2);
|
||||
sub.FaultAttachedStream(new IOException("drop 3"));
|
||||
// HasEnded is set only after the fault callback ran, so once both pumps have ended every escalation
|
||||
// that will ever happen has happened.
|
||||
await TestWait.UntilAsync(() => pump3.HasEnded, "the new live pump's stream must have faulted");
|
||||
driver.EscalatedStreamFaultsForTests.ShouldBe(2, "the new live pump's fault escalates; the superseded pump's did not");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A probe sync on a driver whose pump was never started (no platform source, e.g. the
|
||||
/// DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just
|
||||
/// the init path.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Probe_sync_without_a_platform_source_still_starts_the_stream_first()
|
||||
{
|
||||
var sub = new GatewayLikeSubscriber();
|
||||
// No platform source: init starts no pump; the probe set arrives through SyncPlatformsAsync.
|
||||
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
driver.HasEventPumpForTests.ShouldBeFalse("precondition: nothing has started the pump yet");
|
||||
|
||||
await driver.ProbeWatcherForTests.ShouldNotBeNull().SyncPlatformsAsync(Platforms, CancellationToken.None);
|
||||
|
||||
await TestWait.UntilAsync(
|
||||
() => Platforms.All(p => StateOf(driver, p) == HostState.Running),
|
||||
"the watcher must start the stream before advising, so the initial values arrive",
|
||||
() => Describe(driver, sub));
|
||||
sub.DroppedInitialValues.ShouldBe(0);
|
||||
}
|
||||
}
|
||||
+79
@@ -15,6 +15,85 @@ public sealed class ReconnectSupervisorTests
|
||||
new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5),
|
||||
MaxBackoffOverride: TimeSpan.FromMilliseconds(20));
|
||||
|
||||
/// <summary>
|
||||
/// #865: AttemptInProgress is true while a reopen (through the replay) runs and FALSE during the backoff
|
||||
/// between attempts — when the state still reads Reopening. Advises wait for a replay only while one is
|
||||
/// actually running, not for a whole backoff.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task AttemptInProgress_is_true_during_an_attempt_and_false_during_the_backoff()
|
||||
{
|
||||
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var releaseReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var reopens = 0;
|
||||
using var sup = new ReconnectSupervisor(
|
||||
reopen: async _ =>
|
||||
{
|
||||
if (Interlocked.Increment(ref reopens) == 1)
|
||||
{
|
||||
reopenEntered.TrySetResult();
|
||||
await releaseReopen.Task;
|
||||
throw new IOException("gateway still down");
|
||||
}
|
||||
},
|
||||
replay: _ => Task.CompletedTask,
|
||||
backoffDelay: (_, _, _) => TimeSpan.FromMinutes(10)); // park in the backoff
|
||||
|
||||
sup.AttemptInProgress.ShouldBeFalse("Healthy: no attempt");
|
||||
sup.ReportTransportFailure(new IOException("drop"));
|
||||
await TestWait.ForAsync(reopenEntered.Task, "the first reopen must start");
|
||||
sup.AttemptInProgress.ShouldBeTrue("a reopen is running");
|
||||
|
||||
releaseReopen.SetResult(); // the reopen fails -> backoff of 10 min
|
||||
await TestWait.UntilAsync(() => !sup.AttemptInProgress, "the failed attempt must clear the flag");
|
||||
sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Reopening, "the backoff still reads Reopening — why the flag exists");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #865 review: a transport failure reported WHILE the replay runs (e.g. the restarted event stream is
|
||||
/// refused a round trip later) must fail that attempt — not go Healthy over a dead transport. The
|
||||
/// supervisor retries (with backoff) and goes Healthy only after a clean replay. Without the fix the
|
||||
/// first replay's success went Healthy and replay ran once.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_failure_reported_during_the_replay_fails_that_attempt()
|
||||
{
|
||||
ReconnectSupervisor? self = null;
|
||||
var replays = 0;
|
||||
using var sup = new ReconnectSupervisor(
|
||||
reopen: _ => Task.CompletedTask,
|
||||
replay: _ =>
|
||||
{
|
||||
if (Interlocked.Increment(ref replays) == 1)
|
||||
{
|
||||
self!.ReportTransportFailure(new IOException("stream refused during replay"));
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
},
|
||||
options: FastBackoff());
|
||||
self = sup;
|
||||
|
||||
sup.ReportTransportFailure(new IOException("drop"));
|
||||
await TestWait.HealthyAsync(sup, "the second, clean replay must recover");
|
||||
Volatile.Read(ref replays).ShouldBe(2, "the faulted replay must not count as recovered");
|
||||
}
|
||||
|
||||
/// <summary>#865: a transport failure reported after StopAsync never starts a recovery loop (it throws ODE).</summary>
|
||||
[Fact]
|
||||
public async Task ReportTransportFailure_after_stop_never_starts_a_loop()
|
||||
{
|
||||
var reopens = 0;
|
||||
var sup = new ReconnectSupervisor(
|
||||
reopen: _ => { Interlocked.Increment(ref reopens); return Task.CompletedTask; },
|
||||
replay: _ => Task.CompletedTask,
|
||||
options: FastBackoff());
|
||||
await sup.StopAsync();
|
||||
|
||||
Should.Throw<ObjectDisposedException>(() => sup.ReportTransportFailure(new IOException("late")));
|
||||
sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
|
||||
Volatile.Read(ref reopens).ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the supervisor starts in a healthy state with no errors.</summary>
|
||||
[Fact]
|
||||
public void InitialState_IsHealthy()
|
||||
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Text;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Heidenhain.Tests.Simulator;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Heidenhain.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#867: the published value's RUNTIME type must be the CLR type of the leaf's declared data type.
|
||||
/// Shouldly equality cannot catch this (7.0 equals 7L), so every assertion here compares <c>GetType()</c>.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class HeidenhainLeafRuntimeTypeTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static Type ClrType(DriverDataType type, bool isArray)
|
||||
{
|
||||
var scalar = type switch
|
||||
{
|
||||
DriverDataType.Boolean => typeof(bool),
|
||||
DriverDataType.Int16 => typeof(short),
|
||||
DriverDataType.Int32 => typeof(int),
|
||||
DriverDataType.Int64 => typeof(long),
|
||||
DriverDataType.Float64 => typeof(double),
|
||||
DriverDataType.String => typeof(string),
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(type), type, null),
|
||||
};
|
||||
return isArray ? scalar.MakeArrayType() : scalar;
|
||||
}
|
||||
|
||||
/// <summary>Every fixed value leaf (capability strings excluded), with its declared type.</summary>
|
||||
public static TheoryData<string> CatalogValueLeaves()
|
||||
{
|
||||
var data = new TheoryData<string>();
|
||||
foreach (var e in HeidenhainCatalog.Entries.Where(e => !e.IsCapability)) data.Add(e.Address);
|
||||
return data;
|
||||
}
|
||||
|
||||
private static SimulatedControl FullyPopulatedControl()
|
||||
{
|
||||
var c = new SimulatedControl();
|
||||
c.Runtime[22] = () => SimReply.Ok("S_RI", [0, 1, .. Encoding.ASCII.GetBytes("1.5\0X\0")]);
|
||||
c.Runtime[23] = () => SimReply.Ok("S_RI", [0, 4]);
|
||||
c.Runtime[26] = () => SimReply.Ok("S_RI", [0, 7]);
|
||||
c.Runtime[24] = () => SimReply.Ok("S_RI", [0, 0, 0, 5, .. "A\0B\0"u8.ToArray()]);
|
||||
c.Runtime[25] = () => SimReply.Ok("S_RI", new byte[12]);
|
||||
var alarm = new byte[10];
|
||||
BinaryPrimitives.WriteInt16BigEndian(alarm, 3);
|
||||
BinaryPrimitives.WriteInt16BigEndian(alarm.AsSpan(2), 4);
|
||||
BinaryPrimitives.WriteInt32BigEndian(alarm.AsSpan(4), 12);
|
||||
alarm[8] = 65;
|
||||
c.Runtime[27] = () => SimReply.Ok("S_RI", alarm);
|
||||
c.Runtime[28] = () => SimReply.Refuse(57);
|
||||
// 24-byte tool reply: number 49, index 1, axis 2, length 100.5, radius 3.25 (float64le).
|
||||
var tool = new byte[24];
|
||||
BinaryPrimitives.WriteUInt32BigEndian(tool, 49);
|
||||
BinaryPrimitives.WriteUInt16BigEndian(tool.AsSpan(4), 1);
|
||||
BinaryPrimitives.WriteUInt16BigEndian(tool.AsSpan(6), 2);
|
||||
BinaryPrimitives.WriteDoubleLittleEndian(tool.AsSpan(8), 100.5);
|
||||
BinaryPrimitives.WriteDoubleLittleEndian(tool.AsSpan(16), 3.25);
|
||||
c.Runtime[51] = () => SimReply.Ok("S_RI", tool);
|
||||
for (var s = 30; s <= 34; s++) c.Runtime[s] = () => SimReply.Ok("S_RI", [0, 0, 1, 0]);
|
||||
return c;
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(CatalogValueLeaves))]
|
||||
public async Task Catalog_leaf_publishes_the_clr_type_of_its_declared_data_type(string address)
|
||||
{
|
||||
HeidenhainCatalog.TryGet(address, out var entry).ShouldBeTrue();
|
||||
var control = FullyPopulatedControl();
|
||||
await using var sim = Lsv2Simulator.WithControl(control);
|
||||
var client = new Lsv2.Lsv2Client("127.0.0.1", sim.Port, TimeSpan.FromSeconds(1));
|
||||
var tag = TestTags.Def(address);
|
||||
var settings = new HeidenhainSessionSettings(true, true, false, 64, true, false, false);
|
||||
await using var session = new HeidenhainSession(client, settings, (_, _) => ValueTask.FromResult(""));
|
||||
await client.OpenAsync(Ct);
|
||||
await session.InitializeAsync(Ct);
|
||||
|
||||
var sample = (await session.ReadBatchAsync([tag], Ct))[0];
|
||||
|
||||
sample.Status.ShouldBe(HeidenhainStatus.Good, address);
|
||||
sample.Value.ShouldNotBeNull(address);
|
||||
sample.Value.GetType().ShouldBe(ClrType(entry!.DataType, entry.IsArray), address);
|
||||
control.Violations.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_integer_catalog_leaf_is_covered()
|
||||
{
|
||||
// Pins the theory's reach: the integer leaves #867 broke are in the data set.
|
||||
var leaves = CatalogValueLeaves().Select(r => r.Data).ToList();
|
||||
leaves.ShouldContain("Tool/Number");
|
||||
leaves.ShouldContain("Tool/Index");
|
||||
leaves.ShouldContain("Tool/AxisCode");
|
||||
leaves.ShouldContain("Alarms/Count");
|
||||
leaves.ShouldContain("Program/Line");
|
||||
leaves.ShouldContain("Status/ExecutionState");
|
||||
leaves.Count(l => l.StartsWith("Timers/", StringComparison.Ordinal)).ShouldBe(HeidenhainCatalog.TimerNames.Count);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(DriverDataType.Int16, 42)]
|
||||
[InlineData(DriverDataType.Int32, 42)]
|
||||
[InlineData(DriverDataType.Int64, 42)]
|
||||
[InlineData(DriverDataType.Int16, 42L)]
|
||||
[InlineData(DriverDataType.Int32, 42L)]
|
||||
[InlineData(DriverDataType.Int64, 42L)]
|
||||
[InlineData(DriverDataType.Int16, 42.0)]
|
||||
[InlineData(DriverDataType.Int32, 42.0)]
|
||||
[InlineData(DriverDataType.Int64, 42.0)]
|
||||
[InlineData(DriverDataType.Boolean, true)]
|
||||
[InlineData(DriverDataType.Boolean, false)]
|
||||
public void Binding_coercion_publishes_the_clr_type_of_the_declared_data_type(DriverDataType type, object native)
|
||||
{
|
||||
HeidenhainBindingCodec.Coerce(native, type).GetType().ShouldBe(ClrType(type, false));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HeidenhainPlcArea.Byte, new byte[] { 7 }, DriverDataType.Int16)]
|
||||
[InlineData(HeidenhainPlcArea.Byte, new byte[] { 7 }, DriverDataType.Int32)]
|
||||
[InlineData(HeidenhainPlcArea.Word, new byte[] { 7, 0 }, DriverDataType.Int64)]
|
||||
[InlineData(HeidenhainPlcArea.Dword, new byte[] { 7, 0, 0, 0 }, DriverDataType.Int32)]
|
||||
[InlineData(HeidenhainPlcArea.InputWord, new byte[] { 7, 0 }, DriverDataType.Int32)]
|
||||
[InlineData(HeidenhainPlcArea.Marker, new byte[] { 1 }, DriverDataType.Boolean)]
|
||||
public void Plc_binding_decode_publishes_the_clr_type_of_the_declared_data_type(HeidenhainPlcArea area, byte[] bytes, DriverDataType type)
|
||||
{
|
||||
HeidenhainBindingCodec.DecodePlc(area, bytes, type).GetType().ShouldBe(ClrType(type, false));
|
||||
}
|
||||
}
|
||||
+170
-12
@@ -414,9 +414,10 @@ public sealed class SdkAggregateParityTests
|
||||
public void PercentGood_keeps_Uncertain_out_of_Good_per_Part_13_where_the_baseline_SDK_counts_it_as_Good(
|
||||
double from, double to)
|
||||
{
|
||||
// #796, documented divergence (SdkAggregateOracle remarks). Part 13 §5.4.3.31 / §5.4.3.33: PercentGood is the Good
|
||||
// share of the interval. The 1.5.378.182 StatusAggregateCalculator, under the server's TreatUncertainAsBad = false,
|
||||
// adds Uncertain regions to it (1.5.378.106 did not). Hand-computed over 4 × 5 s regions G / U / B / G:
|
||||
// #796, documented divergence of the untouched baseline (SdkAggregateOracle remarks; live follows Part 13 since
|
||||
// #809). Part 13 §5.4.3.31 / §5.4.3.33: PercentGood is the Good share of the interval. The 1.5.378.182
|
||||
// StatusAggregateCalculator, under the server's TreatUncertainAsBad = false, adds Uncertain regions to it
|
||||
// (1.5.378.106 did not). Hand-computed over 4 × 5 s regions G / U / B / G:
|
||||
// Good 10 s ⇒ 50 %, Bad 5 s ⇒ 25 %, Uncertain 5 s in neither; the baseline says 75 % Good.
|
||||
Pct(AsIs(HistoryAggregateType.PercentGood, from, to, GoodUncertainBadGood))
|
||||
.ShouldBe(75d, 1e-9, "the baseline counts the Uncertain 5 s as Good — if this moves, revisit the oracle (#796)");
|
||||
@@ -524,16 +525,19 @@ public sealed class SdkAggregateParityTests
|
||||
public void An_Uncertain_first_sample_in_a_late_start_interval_makes_the_SDK_integrate_the_time_before_the_data(
|
||||
double from, double to)
|
||||
{
|
||||
// #821, pinned divergence (pending the #809/#821 owner decision — this test records, it does not choose).
|
||||
// Part 13 §5.4.3.3/§5.4.3.8: the interpolated-bounding signal is defined from the first sample only — 10 → 11 over
|
||||
// [5,12) (73.5) and 11 → 11 + 8/13 over [12,20) (4·(22 + 8/13)) ⇒ Total 163.96…, TimeAverage = Total ÷ 15 s.
|
||||
// #821, decided 2026-11-11: live follows Part 13. Part 13 §5.4.3.6/§5.4.3.8 ("No Start Bound: Calculate Partial
|
||||
// Interval"): the interpolated-bounding signal is defined from the first sample only — 10 → 11 over [5,12) (73.5)
|
||||
// and 11 → 11 + 8/13 over [12,20) (4·(22 + 8/13)) ⇒ Total 163.96…, TimeAverage = Total ÷ 15 s.
|
||||
const double part13Total = 73.5 + 88 + 32d / 13;
|
||||
Pct(Ours(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total, 1e-9);
|
||||
Pct(Ours(HistoryAggregateType.TimeAverage, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total / 15, 1e-9);
|
||||
// The live AggregateFilter calculators (CompatibilityAggregateFactory — CoveredTimeAverageAggregateCalculator since
|
||||
// #821; Sdk2AggregateCompatibilityTests proves the runtime manager serves the same code) answer the same.
|
||||
Pct(Sdk(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total, 1e-9);
|
||||
Pct(Sdk(HistoryAggregateType.TimeAverage, from, to, 0, UncertainFirstLateStart)).ShouldBe(part13Total / 15, 1e-9);
|
||||
|
||||
// The frozen 1.5.378.182 calculators — exactly what the server registers for live AggregateFilter items
|
||||
// (CompatibilityAggregateFactory; Sdk2AggregateCompatibilityTests proves the runtime manager serves the same code)
|
||||
// — integrate the uncovered [0,5) as a ramp 0 → 10 (+25) and divide TimeAverage by the WHOLE 20 s interval.
|
||||
// The untouched frozen 1.5.378.182 calculators (live before #821) integrate the uncovered [0,5) as a ramp 0 → 10
|
||||
// (+25) and divide TimeAverage by the WHOLE 20 s interval.
|
||||
Pct(AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart)).ShouldBe(part13Total + 25, 1e-9);
|
||||
Pct(AsIs(HistoryAggregateType.TimeAverage, from, to, UncertainFirstLateStart)).ShouldBe((part13Total + 25) / 20, 1e-9);
|
||||
// SDK 2.0.1's own stock calculators answer the same: the upgrade did not change this.
|
||||
@@ -544,29 +548,183 @@ public sealed class SdkAggregateParityTests
|
||||
// Status agrees everywhere (Uncertain — an Uncertain value is used and the interval is not fully covered).
|
||||
(AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart).Single().StatusCode & StatusBits.CodeBitsMask)
|
||||
.ShouldBe(HistorianStatusCodes.UncertainDataSubNormal);
|
||||
Sdk(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart).Single().StatusCode
|
||||
.ShouldBe(AsIs(HistoryAggregateType.Total, from, to, UncertainFirstLateStart).Single().StatusCode);
|
||||
(Ours(HistoryAggregateType.Total, from, to, 0, UncertainFirstLateStart).Single().StatusCode & StatusBits.CodeBitsMask)
|
||||
.ShouldBe(HistorianStatusCodes.UncertainDataSubNormal);
|
||||
|
||||
// Control: the SAME data with a Good first sample — all three agree on the Part 13 answer, so the trigger is the
|
||||
// Control: the SAME data with a Good first sample — all agree on the Part 13 answer, so the trigger is the
|
||||
// Uncertain first sample, not the late start.
|
||||
foreach (var agg in new[] { HistoryAggregateType.Total, HistoryAggregateType.TimeAverage })
|
||||
{
|
||||
var expected = agg == HistoryAggregateType.Total ? part13Total : part13Total / 15;
|
||||
Pct(Ours(agg, from, to, 0, GoodFirstLateStart)).ShouldBe(expected, 1e-9);
|
||||
Pct(Sdk(agg, from, to, 0, GoodFirstLateStart)).ShouldBe(expected, 1e-9);
|
||||
Pct(AsIs(agg, from, to, GoodFirstLateStart)).ShouldBe(expected, 1e-9);
|
||||
Stock201(agg, from, to, GoodFirstLateStart).Single().Value!.Value.ShouldBe(expected, 1e-9);
|
||||
}
|
||||
|
||||
// #809 on the same input: DurationGood / PercentGood — Part 13 (ours) and stock SDK 2.0.1 count Good regions only
|
||||
// ([12,20) = 8 s); the frozen live calculators also count the Uncertain [5,12) as Good (15 s).
|
||||
// #809 on the same input: DurationGood / PercentGood — Part 13 (ours), stock SDK 2.0.1 and, since #809, the live
|
||||
// calculators count Good regions only ([12,20) = 8 s); the untouched baseline (live before #809) also counted the
|
||||
// Uncertain [5,12) as Good (15 s).
|
||||
Pct(Ours(HistoryAggregateType.DurationGood, from, to, 0, UncertainFirstLateStart)).ShouldBe(8_000d, 1e-9);
|
||||
Stock201(HistoryAggregateType.DurationGood, from, to, UncertainFirstLateStart).Single().Value!.Value.ShouldBe(8_000d, 1e-9);
|
||||
Pct(Sdk(HistoryAggregateType.DurationGood, from, to, 0, UncertainFirstLateStart)).ShouldBe(8_000d, 1e-9);
|
||||
Pct(AsIs(HistoryAggregateType.DurationGood, from, to, UncertainFirstLateStart)).ShouldBe(15_000d, 1e-9);
|
||||
Pct(Ours(HistoryAggregateType.PercentGood, from, to, 0, UncertainFirstLateStart)).ShouldBe(40d, 1e-9);
|
||||
Stock201(HistoryAggregateType.PercentGood, from, to, UncertainFirstLateStart).Single().Value!.Value.ShouldBe(40d, 1e-9);
|
||||
Pct(Sdk(HistoryAggregateType.PercentGood, from, to, 0, UncertainFirstLateStart)).ShouldBe(40d, 1e-9);
|
||||
Pct(AsIs(HistoryAggregateType.PercentGood, from, to, UncertainFirstLateStart)).ShouldBe(75d, 1e-9);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #809: the #810 QualityMix tag Q — Good / Uncertain / Bad samples and an outage marker, Uncertain regions starting
|
||||
/// at a sample, through a sloped bound and held into an outage, with Good samples before (−10) and after (+90) every
|
||||
/// window so no interval holds the start or the end of the data.
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] UncertainMix =
|
||||
[
|
||||
S(-10, HistorianValue.FromDouble(10), Good),
|
||||
S(0, HistorianValue.FromDouble(10), Good),
|
||||
S(4, HistorianValue.FromDouble(12), Uncertain),
|
||||
S(7, HistorianValue.FromDouble(13), Bad),
|
||||
S(10, HistorianValue.FromDouble(14), Good),
|
||||
S(13, HistorianValue.FromDouble(15), Uncertain),
|
||||
S(20, HistorianValue.FromDouble(16), Uncertain),
|
||||
S(25, HistorianValue.FromDouble(17), Uncertain),
|
||||
S(33, HistorianValue.FromDouble(18), Uncertain),
|
||||
S(40, HistorianValue.FromDouble(20), Good),
|
||||
S(45, HistorianValue.Empty, BadNoCommunication),
|
||||
S(50, HistorianValue.FromDouble(22), Uncertain),
|
||||
S(56, HistorianValue.FromDouble(23), Good),
|
||||
S(64, HistorianValue.FromDouble(25), Uncertain),
|
||||
S(67, HistorianValue.FromDouble(26), Good),
|
||||
S(75, HistorianValue.FromDouble(27), Bad),
|
||||
S(78, HistorianValue.FromDouble(28), Good),
|
||||
S(90, HistorianValue.FromDouble(30), Good),
|
||||
];
|
||||
|
||||
/// <summary>Windows of whole intervals only (no truncated interval — ours flags it Partial, a documented divergence).</summary>
|
||||
public static TheoryData<double, double, double> WholeIntervalWindows => new()
|
||||
{
|
||||
{ 0, 80, 20 }, { 80, 0, 20 },
|
||||
{ 0, 80, 10 }, { 80, 0, 10 },
|
||||
{ 5, 80, 15 }, { 80, 5, 15 },
|
||||
{ 0, 60, 0 }, { 60, 0, 0 },
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(WholeIntervalWindows))]
|
||||
public void Live_AggregateFilter_PercentGood_and_DurationGood_equal_HistoryRead_over_Uncertain_samples(
|
||||
double from, double to, double intervalS)
|
||||
{
|
||||
// #809: a live AggregateFilter monitored item and a HistoryRead-Processed (SqlServer / QuestDB share
|
||||
// TimeWeightedAggregator) answer PercentGood / DurationGood identically over Uncertain data — same values,
|
||||
// timestamps, status and Partial / Calculated bits, with NO documented-divergence mask. RunLive is exactly
|
||||
// CompatibilityAggregateFactory.Create, the calculator the server registers (Sdk2AggregateCompatibilityTests
|
||||
// proves the booted runtime manager serves the same code).
|
||||
var interval = intervalS > 0 ? TimeSpan.FromSeconds(intervalS) : (At(to) - At(from)).Duration();
|
||||
var failures = new List<string>();
|
||||
var discriminated = 0;
|
||||
foreach (var agg in new[] { HistoryAggregateType.PercentGood, HistoryAggregateType.DurationGood })
|
||||
foreach (var (name, data) in new[] { ("UncertainMix", UncertainMix), ("DenseUncertain", DenseUncertain) })
|
||||
{
|
||||
var rows = data.Select(x => (x.TimeUtc, x.Value, x.Status)).ToArray();
|
||||
var live = SdkAggregateOracle.RunLive(agg, At(from), At(to), interval, rows);
|
||||
try
|
||||
{
|
||||
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), live, $"live {agg} {name} {from}→{to}/{intervalS}s");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
failures.Add(ex.Message);
|
||||
}
|
||||
// The data must discriminate: the pre-#809 live calculator (the untouched baseline) counts the Uncertain
|
||||
// time as Good, so it disagrees somewhere — or this parity would pass against the old behaviour too.
|
||||
var before = SdkAggregateOracle.RunSdkAsIs(agg, At(from), At(to), interval, rows);
|
||||
if (before.Zip(live).Any(p => Math.Abs(Convert.ToDouble(p.First.Value) - Convert.ToDouble(p.Second.Value)) > 1e-9))
|
||||
discriminated++;
|
||||
}
|
||||
failures.ShouldBeEmpty(string.Join("\n", failures));
|
||||
discriminated.ShouldBe(4, "every aggregate × data set must differ from the pre-#809 live answer somewhere");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Live_AggregateFilter_PercentGood_and_DurationGood_are_the_hand_computed_Part_13_values()
|
||||
{
|
||||
// #809, hand-computed over UncertainMix's eight 10 s intervals (QualityMixConformance derives them region by
|
||||
// region): Good seconds 4, 3, 0, 0, 5, 4, 3, 2. The pre-#809 live answer added the Uncertain time
|
||||
// (7, 10, 10, 10, 5, 10, 10, 7 s).
|
||||
double[] goodSeconds = [4, 3, 0, 0, 5, 4, 3, 2];
|
||||
var rows = UncertainMix.Select(x => (x.TimeUtc, x.Value, x.Status)).ToArray();
|
||||
var ten = TimeSpan.FromSeconds(10);
|
||||
SdkAggregateOracle.RunLive(HistoryAggregateType.DurationGood, At(0), At(80), ten, rows)
|
||||
.Select(r => Convert.ToDouble(r.Value)).ShouldBe(goodSeconds.Select(g => g * 1000), tolerance: 1e-9);
|
||||
SdkAggregateOracle.RunLive(HistoryAggregateType.PercentGood, At(0), At(80), ten, rows)
|
||||
.Select(r => Convert.ToDouble(r.Value)).ShouldBe(goodSeconds.Select(g => g * 10), tolerance: 1e-9);
|
||||
SdkAggregateOracle.RunSdkAsIs(HistoryAggregateType.DurationGood, At(0), At(80), ten, rows)
|
||||
.Select(r => Convert.ToDouble(r.Value)).ShouldBe([7000d, 10000, 10000, 10000, 5000, 10000, 10000, 7000], tolerance: 1e-9);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #821: late-start data whose first usable sample is Uncertain (+7, after every window's start), with Good / Uncertain / Bad samples after it
|
||||
/// and a Good sample past every window's end (+90), so only the start of the data is in question. No sample sits on
|
||||
/// an interval edge of <see cref="WholeIntervalWindows"/> (the reverse on-edge divergence, class remarks).
|
||||
/// </summary>
|
||||
private static readonly StoredSample[] UncertainFirstLate =
|
||||
[
|
||||
S(7, HistorianValue.FromDouble(10), Uncertain),
|
||||
S(12, HistorianValue.FromDouble(11), Good),
|
||||
S(25, HistorianValue.FromDouble(12), Uncertain),
|
||||
S(33, HistorianValue.FromDouble(14), Good),
|
||||
S(47, HistorianValue.FromDouble(13), Bad),
|
||||
S(52, HistorianValue.FromDouble(15), Good),
|
||||
S(90, HistorianValue.FromDouble(16), Good),
|
||||
];
|
||||
|
||||
/// <summary>#821: the same, preceded by Bad samples only (an outage before the data — still no start bound).</summary>
|
||||
private static readonly StoredSample[] BadThenUncertainFirstLate =
|
||||
[
|
||||
S(-10, HistorianValue.Empty, BadNoCommunication),
|
||||
S(2, HistorianValue.FromDouble(7), Bad),
|
||||
.. UncertainFirstLate,
|
||||
];
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(WholeIntervalWindows))]
|
||||
public void Live_AggregateFilter_TimeAverage_and_Total_equal_HistoryRead_over_an_Uncertain_first_late_start(
|
||||
double from, double to, double intervalS)
|
||||
{
|
||||
// #821: a live AggregateFilter monitored item and a HistoryRead-Processed (SqlServer / QuestDB share
|
||||
// TimeWeightedAggregator) answer TimeAverage / Total identically when the data starts inside an interval with an
|
||||
// Uncertain value — same values, timestamps, status and Partial / Calculated bits, NO documented-divergence mask.
|
||||
// RunLive is exactly CompatibilityAggregateFactory.Create, the calculator the server registers.
|
||||
var interval = intervalS > 0 ? TimeSpan.FromSeconds(intervalS) : (At(to) - At(from)).Duration();
|
||||
var failures = new List<string>();
|
||||
var discriminated = 0;
|
||||
foreach (var agg in new[] { HistoryAggregateType.TimeAverage, HistoryAggregateType.Total })
|
||||
foreach (var (name, data) in new[] { ("UncertainFirstLate", UncertainFirstLate), ("BadThenUncertainFirstLate", BadThenUncertainFirstLate) })
|
||||
{
|
||||
var rows = data.Select(x => (x.TimeUtc, x.Value, x.Status)).ToArray();
|
||||
var live = SdkAggregateOracle.RunLive(agg, At(from), At(to), interval, rows);
|
||||
try
|
||||
{
|
||||
SdkAggregateOracle.ShouldMatch(Ours(agg, from, to, intervalS, data), live, $"live {agg} {name} {from}→{to}/{intervalS}s");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
failures.Add(ex.Message);
|
||||
}
|
||||
// The data must discriminate: the pre-#821 live calculator (the untouched baseline) integrates the span before
|
||||
// the data, so it disagrees on the late-start interval — or this parity would pass against the old behaviour too.
|
||||
var before = SdkAggregateOracle.RunSdkAsIs(agg, At(from), At(to), interval, rows);
|
||||
if (before.Zip(live).Any(p => Math.Abs(Convert.ToDouble(p.First.Value) - Convert.ToDouble(p.Second.Value)) > 1e-9))
|
||||
discriminated++;
|
||||
}
|
||||
failures.ShouldBeEmpty(string.Join("\n", failures));
|
||||
discriminated.ShouldBe(4, "every aggregate × data set must differ from the pre-#821 live answer somewhere");
|
||||
}
|
||||
|
||||
/// <summary>#684: Good 10 / 30 with an Uncertain value BELOW the Good minimum (+4) and one ABOVE the maximum (+6).</summary>
|
||||
private static readonly StoredSample[] UncertainBeyond =
|
||||
[
|
||||
|
||||
+91
-5
@@ -320,21 +320,107 @@ public sealed class MTConnectObservationIndexTests
|
||||
.ShouldBe(new DateTime(2026, 7, 24, 12, 0, 0, 500, DateTimeKind.Utc));
|
||||
}
|
||||
|
||||
/// <summary>An observation whose text is empty is "the Agent supplied no value" — never a Good empty string.</summary>
|
||||
/// <summary>
|
||||
/// lmxopcua#824 (Java parity): an empty element on a string-typed item is a Good empty string, not an
|
||||
/// absence. Whitespace-only text is the same (the parser trims it).
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
public void Empty_observation_value_maps_to_BadNoCommunication(string value)
|
||||
[InlineData("", DriverDataType.String)]
|
||||
[InlineData(" ", DriverDataType.String)]
|
||||
[InlineData("", DriverDataType.Reference)]
|
||||
public void Empty_observation_value_on_a_string_item_is_a_Good_empty_string(string value, DriverDataType type)
|
||||
{
|
||||
var idx = new MTConnectObservationIndex([Tag("x", DriverDataType.String)]);
|
||||
var idx = new MTConnectObservationIndex([Tag("x", type)]);
|
||||
idx.Apply(Synthetic("x", value));
|
||||
|
||||
var snap = idx.Get("x");
|
||||
|
||||
snap.StatusCode.ShouldBe(Good);
|
||||
snap.Value.ShouldBe(string.Empty);
|
||||
}
|
||||
|
||||
/// <summary>An unauthored item coerces as String, so its empty value is a Good empty string too (#824).</summary>
|
||||
[Fact]
|
||||
public void Empty_observation_value_on_an_unauthored_item_is_a_Good_empty_string()
|
||||
{
|
||||
var idx = new MTConnectObservationIndex();
|
||||
idx.Apply(Synthetic("x", ""));
|
||||
|
||||
var snap = idx.Get("x");
|
||||
|
||||
snap.StatusCode.ShouldBe(Good);
|
||||
snap.Value.ShouldBe(string.Empty);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #824: on a non-string item "" cannot be a value, so it stays BadNoCommunication (the Agent supplied
|
||||
/// no value) — not BadTypeMismatch, which would blame the tag's authored type.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(DriverDataType.Float64)]
|
||||
[InlineData(DriverDataType.Float32)]
|
||||
[InlineData(DriverDataType.Int32)]
|
||||
[InlineData(DriverDataType.Int64)]
|
||||
[InlineData(DriverDataType.Boolean)]
|
||||
[InlineData(DriverDataType.DateTime)]
|
||||
public void Empty_observation_value_on_a_non_string_item_stays_BadNoCommunication(DriverDataType type)
|
||||
{
|
||||
var idx = new MTConnectObservationIndex([Tag("x", type)]);
|
||||
idx.Apply(Synthetic("x", " "));
|
||||
|
||||
var snap = idx.Get("x");
|
||||
|
||||
snap.StatusCode.ShouldBe(BadNoCommunication);
|
||||
snap.Value.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #824 control: the UNAVAILABLE sentinel is unaffected — still BadNoCommunication on a String item, where an
|
||||
/// empty value is now Good.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Unavailable_on_a_string_item_is_still_BadNoCommunication()
|
||||
{
|
||||
var idx = new MTConnectObservationIndex([Tag("x", DriverDataType.String)]);
|
||||
idx.Apply(Synthetic("x", "UNAVAILABLE"));
|
||||
|
||||
var snap = idx.Get("x");
|
||||
|
||||
snap.StatusCode.ShouldBe(BadNoCommunication);
|
||||
snap.Value.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>#824 end to end through the parser: an empty, a self-closing and a blank element are all Good "".</summary>
|
||||
[Theory]
|
||||
[InlineData("""<Program dataItemId="p" timestamp="2026-07-24T12:00:00Z" sequence="1"></Program>""")]
|
||||
[InlineData("""<Program dataItemId="p" timestamp="2026-07-24T12:00:00Z" sequence="1"/>""")]
|
||||
[InlineData("""<Program dataItemId="p" timestamp="2026-07-24T12:00:00Z" sequence="1"> </Program>""")]
|
||||
public void A_parsed_empty_event_on_a_string_item_is_a_Good_empty_string(string element)
|
||||
{
|
||||
var idx = new MTConnectObservationIndex([Tag("p", DriverDataType.String)]);
|
||||
idx.Apply(MTConnectStreamsParser.Parse($"""
|
||||
<MTConnectStreams>
|
||||
<Header instanceId="1" nextSequence="2" firstSequence="1"/>
|
||||
<Streams><DeviceStream><ComponentStream><Events>{element}</Events></ComponentStream></DeviceStream></Streams>
|
||||
</MTConnectStreams>
|
||||
"""));
|
||||
|
||||
var snap = idx.Get("p");
|
||||
|
||||
snap.StatusCode.ShouldBe(Good);
|
||||
snap.Value.ShouldBe(string.Empty);
|
||||
}
|
||||
|
||||
/// <summary>A structured observation with no text stays BadNotSupported — it never becomes a Good "" (#824).</summary>
|
||||
[Fact]
|
||||
public void A_structured_observation_with_no_text_stays_BadNotSupported()
|
||||
{
|
||||
var idx = new MTConnectObservationIndex([Tag("x", DriverDataType.String)]);
|
||||
idx.Apply(Synthetic("x", "", isStructured: true));
|
||||
|
||||
idx.Get("x").StatusCode.ShouldBe(BadNotSupported);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------- unknown lookups
|
||||
|
||||
/// <summary>A dataItemId that is neither authored nor ever observed is unknown to this driver.</summary>
|
||||
|
||||
@@ -1,22 +1,17 @@
|
||||
// Unit tests for TwinCATDriverProbe — the degrade-guarded ADS ReadState Test-Connect probe.
|
||||
// Unit tests for TwinCATDriverProbe — the ADS ReadState Test-Connect probe (#863).
|
||||
//
|
||||
// Coverage here is the set of outcomes that are *deterministic on a CI/dev box with no TwinCAT
|
||||
// runtime*: invalid JSON, missing host/port, an unreachable TCP target, and — the crux — the
|
||||
// DEGRADE path. On this box there is no AMS router, so once the TCP preflight succeeds against a
|
||||
// reachable listener the managed ADS client cannot initialise (it throws a "Check for a running
|
||||
// TwinCAT router instance!" server exception, NOT an ADS device rejection). The probe MUST treat
|
||||
// that as "TCP reachability only" (Ok=true) so it never regresses below the old TCP-only probe.
|
||||
//
|
||||
// The two paths that REQUIRE a live ADS target are LIVE-VERIFY DEFERRED (no TwinCAT runtime on the
|
||||
// rig): (a) the happy "ADS state: Run/Config/Stop" path (Ok=true with the AdsState name), and
|
||||
// (b) the route/auth-reject RED path (a reachable router that refuses the route, surfaced as an
|
||||
// AdsErrorException carrying a TargetPortNotFound / route AdsErrorCode → Ok=false with the
|
||||
// "check the target's ADS route table" message). Those are exercised against a real TwinCAT XAR /
|
||||
// remote runtime, out of scope for this unit suite.
|
||||
// The probe takes exactly the driver's path: AdsClient.Connect(netId, port) through the LOCAL AMS
|
||||
// router, then one ReadStateAsync. It must never open a socket of its own, and never to a host derived
|
||||
// from the AMS Net ID (the old first-phase check dialled the Net ID's first four octets on the AMS
|
||||
// port — an unrelated host, often a public one, on a port ADS does not use — so Test Connect timed out
|
||||
// against every real runtime). The classification and budget tests drive the handshake through the
|
||||
// internal seam; the no-dial and no-router tests use the real AdsClient on a box with no AMS router.
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Shouldly;
|
||||
using TwinCAT.Ads;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
|
||||
@@ -24,199 +19,162 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class TwinCATDriverProbeTests
|
||||
{
|
||||
private static readonly TwinCATDriverProbe Probe = new();
|
||||
private const string Z35474Like = "{\"devices\":[{\"hostAddress\":\"ads://5.118.56.93.1.1:851\"}]}";
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// 1. Invalid JSON
|
||||
// -------------------------------------------------------------------------
|
||||
private static TwinCATDriverProbe ProbeWith(Func<TwinCATAmsAddress, TimeSpan, CancellationToken, Task<AdsState>> handshake)
|
||||
=> new(handshake);
|
||||
|
||||
// ---- config validation ----------------------------------------------------------------
|
||||
|
||||
/// <summary>Invalid JSON returns Ok=false with a message containing "invalid".</summary>
|
||||
[Fact]
|
||||
public async Task InvalidJson_Returns_OkFalse_WithInvalidMessage()
|
||||
{
|
||||
var result = await Probe.ProbeAsync(
|
||||
"not-valid-json{{{",
|
||||
TimeSpan.FromSeconds(3),
|
||||
TestContext.Current.CancellationToken);
|
||||
var result = await new TwinCATDriverProbe().ProbeAsync(
|
||||
"not-valid-json{{{", TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken);
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
result.Message.ShouldNotBeNull();
|
||||
result.Message!.ToLowerInvariant().ShouldContain("invalid");
|
||||
result.Latency.ShouldBeNull();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// 2. Config with no device / no host
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/// <summary>An empty Devices list returns Ok=false with a "no host/port" message.</summary>
|
||||
[Fact]
|
||||
public async Task NoDevices_Returns_OkFalse_WithNoHostPortMessage()
|
||||
[Theory]
|
||||
[InlineData("{\"devices\":[]}")]
|
||||
[InlineData("{\"devices\":[{\"hostAddress\":\"not-an-ads-url\"}]}")]
|
||||
public async Task NoUsableDevice_Returns_OkFalse_WithNoHostPortMessage(string json)
|
||||
{
|
||||
var result = await Probe.ProbeAsync(
|
||||
"{\"devices\":[]}",
|
||||
TimeSpan.FromSeconds(3),
|
||||
TestContext.Current.CancellationToken);
|
||||
var called = false;
|
||||
var result = await ProbeWith((_, _, _) => { called = true; return Task.FromResult(AdsState.Run); })
|
||||
.ProbeAsync(json, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken);
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
result.Message.ShouldNotBeNull();
|
||||
result.Message!.ShouldContain("no host/port");
|
||||
result.Latency.ShouldBeNull();
|
||||
called.ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A first device with a malformed (non-<c>ads://</c>) host address returns Ok=false with a
|
||||
/// "no host/port" message because <see cref="TwinCATAmsAddress.TryParse"/> returns null.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task MalformedHostAddress_Returns_OkFalse_WithNoHostPortMessage()
|
||||
{
|
||||
var result = await Probe.ProbeAsync(
|
||||
"{\"devices\":[{\"hostAddress\":\"not-an-ads-url\"}]}",
|
||||
TimeSpan.FromSeconds(3),
|
||||
TestContext.Current.CancellationToken);
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
result.Message.ShouldNotBeNull();
|
||||
result.Message!.ShouldContain("no host/port");
|
||||
result.Latency.ShouldBeNull();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// 3. Unreachable target — TCP preflight fails first
|
||||
// -------------------------------------------------------------------------
|
||||
// ---- #863: never dial a host derived from the AMS Net ID --------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// A closed port causes the TCP preflight <c>ConnectAsync</c> to throw
|
||||
/// <see cref="SocketException"/>; the probe returns Ok=false with "Connect failed".
|
||||
/// The port is reserved then released so the OS refuses the connection immediately.
|
||||
/// Regression for #863. A listener sits on exactly the endpoint the old probe dialled
|
||||
/// (Net ID octets 1–4 as IPv4, AMS port as TCP port). The real probe must never connect to
|
||||
/// it — the old TCP pre-check connected here first, so this fails on the old behaviour.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ClosedPort_Returns_OkFalse_WithConnectFailedMessage()
|
||||
public async Task Probe_NeverConnects_ToAmsNetIdDerivedEndpoint()
|
||||
{
|
||||
// Reserve an ephemeral port then release it so the port is definitely closed.
|
||||
var reserved = new TcpListener(IPAddress.Loopback, 0);
|
||||
reserved.Start();
|
||||
var port = ((IPEndPoint)reserved.LocalEndpoint).Port;
|
||||
reserved.Stop();
|
||||
|
||||
// 127.0.0.1 as the first four AMS-Net-ID octets → the probe's TCP target is 127.0.0.1.
|
||||
var configJson = $"{{\"devices\":[{{\"hostAddress\":\"ads://127.0.0.1.1.1:{port}\"}}]}}";
|
||||
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(
|
||||
TestContext.Current.CancellationToken);
|
||||
cts.CancelAfter(TimeSpan.FromSeconds(5));
|
||||
|
||||
var result = await Probe.ProbeAsync(
|
||||
configJson,
|
||||
TimeSpan.FromSeconds(3),
|
||||
cts.Token);
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
result.Message.ShouldNotBeNull();
|
||||
// Either a fast SocketException ("Connect failed") or, if the OS stalls, a timeout.
|
||||
var lower = result.Message!.ToLowerInvariant();
|
||||
(lower.Contains("connect failed") || lower.Contains("timed out")).ShouldBeTrue(
|
||||
$"Expected 'Connect failed' or 'timed out' but got: {result.Message}");
|
||||
result.Latency.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A target on a non-routable IP (192.0.2.1 — TEST-NET-1, RFC 5737) with a short
|
||||
/// cancellation window times out (or fails fast on EHOSTUNREACH) at the TCP preflight.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task BlackHoleTarget_Returns_OkFalse_WithTimedOutOrConnectFailed()
|
||||
{
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(
|
||||
TestContext.Current.CancellationToken);
|
||||
cts.CancelAfter(TimeSpan.FromSeconds(1));
|
||||
|
||||
var configJson = "{\"devices\":[{\"hostAddress\":\"ads://192.0.2.1.1.1:851\"}]}";
|
||||
var result = await Probe.ProbeAsync(
|
||||
configJson,
|
||||
TimeSpan.FromSeconds(1),
|
||||
cts.Token);
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
result.Message.ShouldNotBeNull();
|
||||
var lower = result.Message!.ToLowerInvariant();
|
||||
(lower.Contains("timed out") || lower.Contains("connect failed")).ShouldBeTrue(
|
||||
$"Expected 'timed out' or 'Connect failed' but got: {result.Message}");
|
||||
result.Latency.ShouldBeNull();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// 4. DEGRADE path — TCP reachable, but the ADS handshake can't be attempted here
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// The crux. A reachable TCP listener satisfies the preflight, but on this box (no TwinCAT
|
||||
/// AMS router) the managed ADS client cannot establish a session — <c>AdsClient.Connect</c>
|
||||
/// throws an environment/router exception (NOT an ADS device rejection). The probe MUST
|
||||
/// DEGRADE: <c>Ok=true</c> with the "TCP reachability only" note, never a false RED. This
|
||||
/// guarantees the new probe never reports worse than the old TCP-only probe on a host with
|
||||
/// no ADS runtime.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ReachableTcp_NoAdsRuntime_Degrades_To_OkTrue_WithTcpOnlyNote()
|
||||
{
|
||||
// Stand up a real TCP listener and accept (then drop) connections so the preflight
|
||||
// ConnectAsync succeeds. We never speak ADS — the ADS handshake is what must degrade.
|
||||
var listener = new TcpListener(IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
|
||||
using var acceptCts = CancellationTokenSource.CreateLinkedTokenSource(
|
||||
TestContext.Current.CancellationToken);
|
||||
var acceptLoop = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var client = await listener.AcceptTcpClientAsync(acceptCts.Token);
|
||||
// Hold the socket briefly so the preflight sees a clean accept, then close.
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
await Task.Delay(200);
|
||||
client.Dispose();
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) { /* accept cancelled — expected on teardown */ }
|
||||
catch (ObjectDisposedException) { /* listener stopped — expected on teardown */ }
|
||||
catch (SocketException) { /* listener stopped — expected on teardown */ }
|
||||
}, TestContext.Current.CancellationToken);
|
||||
|
||||
try
|
||||
{
|
||||
var configJson = $"{{\"devices\":[{{\"hostAddress\":\"ads://127.0.0.1.1.1:{port}\"}}]}}";
|
||||
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(
|
||||
TestContext.Current.CancellationToken);
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken);
|
||||
cts.CancelAfter(TimeSpan.FromSeconds(10));
|
||||
await new TwinCATDriverProbe().ProbeAsync(
|
||||
$"{{\"devices\":[{{\"hostAddress\":\"ads://127.0.0.1.1.1:{port}\"}}]}}",
|
||||
TimeSpan.FromSeconds(3), cts.Token);
|
||||
|
||||
var result = await Probe.ProbeAsync(
|
||||
configJson,
|
||||
TimeSpan.FromSeconds(3),
|
||||
cts.Token);
|
||||
|
||||
// DEGRADE guard: must be Ok=true (never worse than TCP-only), with the explicit note
|
||||
// and a latency. If on some box AdsClient instead threw synchronously at construction,
|
||||
// the probe still degrades the same way — this assertion holds either way.
|
||||
result.Ok.ShouldBeTrue(
|
||||
$"Degrade guard violated — probe returned a worse-than-TCP result: {result.Message}");
|
||||
result.Message.ShouldNotBeNull();
|
||||
result.Message!.ShouldContain("ADS handshake unavailable on this host");
|
||||
result.Message!.ShouldContain("TCP reachability only");
|
||||
result.Latency.ShouldNotBeNull();
|
||||
listener.Pending().ShouldBeFalse(
|
||||
"the probe opened a TCP connection to the AmsNetId-derived address — it must only use the local AMS router");
|
||||
}
|
||||
finally
|
||||
{
|
||||
await acceptCts.CancelAsync();
|
||||
listener.Stop();
|
||||
try { await acceptLoop; } catch { /* ignore teardown races */ }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The handshake receives the AMS address unchanged — the probe resolves no host itself.</summary>
|
||||
[Fact]
|
||||
public async Task Handshake_IsHandedTheAmsAddress_Unchanged()
|
||||
{
|
||||
TwinCATAmsAddress? seen = null;
|
||||
var result = await ProbeWith((t, _, _) => { seen = t; return Task.FromResult(AdsState.Run); })
|
||||
.ProbeAsync(Z35474Like, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken);
|
||||
|
||||
result.Ok.ShouldBeTrue();
|
||||
seen.ShouldBe(new TwinCATAmsAddress("5.118.56.93.1.1", 851));
|
||||
}
|
||||
|
||||
// ---- outcome classification -------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public async Task ReadStateOk_Returns_OkTrue_WithStateAndLatency()
|
||||
{
|
||||
var result = await ProbeWith((_, _, _) => Task.FromResult(AdsState.Run))
|
||||
.ProbeAsync(Z35474Like, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken);
|
||||
|
||||
result.Ok.ShouldBeTrue();
|
||||
result.Message.ShouldBe("ADS state: Run");
|
||||
result.Latency.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// On a box with no AMS router the real AdsClient cannot connect. That is a RED naming the
|
||||
/// missing router — no longer a green "TCP reachability only", since nothing was contacted.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task NoLocalRouter_RealClient_Returns_OkFalse_NamingTheRouter()
|
||||
{
|
||||
var result = await new TwinCATDriverProbe().ProbeAsync(
|
||||
Z35474Like, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken);
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
result.Message!.ShouldContain("No local AMS router");
|
||||
result.Message!.ShouldContain("not contacted");
|
||||
result.Latency.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AmsServerException_Classified_AsNoLocalRouter()
|
||||
{
|
||||
var result = await ProbeWith((_, _, _) => throw new global::TwinCAT.Ams.AmsServerException("Connect to Server failed!"))
|
||||
.ProbeAsync(Z35474Like, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken);
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
result.Message!.ShouldContain("No local AMS router");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(AdsErrorCode.ClientPortNotOpen, "No local AMS router")]
|
||||
[InlineData(AdsErrorCode.TargetMachineNotFound, "No route to 5.118.56.93.1.1:851 or target unreachable")]
|
||||
[InlineData(AdsErrorCode.TargetPortNotFound, "No route to 5.118.56.93.1.1:851 or target unreachable")]
|
||||
[InlineData(AdsErrorCode.WSA_HostUnreachable, "No route to 5.118.56.93.1.1:851 or target unreachable")]
|
||||
[InlineData(AdsErrorCode.ClientSyncTimeOut, "timed out")]
|
||||
[InlineData(AdsErrorCode.DeviceInvalidGroup, "ADS handshake with 5.118.56.93.1.1:851 failed")]
|
||||
public async Task AdsErrorCodes_AreClassified(AdsErrorCode code, string expected)
|
||||
{
|
||||
var result = await ProbeWith((_, _, _) => throw new AdsErrorException("x", code))
|
||||
.ProbeAsync(Z35474Like, TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken);
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
result.Message!.ShouldContain(expected);
|
||||
result.Message!.ShouldContain(code.ToString());
|
||||
result.Latency.ShouldBeNull();
|
||||
}
|
||||
|
||||
// ---- time budget --------------------------------------------------------------------------
|
||||
|
||||
/// <summary>A handshake that never returns (and ignores its token) is cut off at the budget.</summary>
|
||||
[Fact]
|
||||
public async Task HangingHandshake_IsBoundedByTimeout()
|
||||
{
|
||||
var never = new TaskCompletionSource<AdsState>();
|
||||
var sw = Stopwatch.StartNew();
|
||||
var result = await ProbeWith((_, _, _) => never.Task)
|
||||
.ProbeAsync(Z35474Like, TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken);
|
||||
sw.Stop();
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
result.Message!.ShouldContain("timed out after 1s");
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CallerCancellation_Returns_TimedOut()
|
||||
{
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(200));
|
||||
var result = await ProbeWith(async (_, _, ct) => { await Task.Delay(Timeout.Infinite, ct); return AdsState.Run; })
|
||||
.ProbeAsync(Z35474Like, TimeSpan.FromSeconds(10), cts.Token);
|
||||
|
||||
result.Ok.ShouldBeFalse();
|
||||
result.Message!.ShouldContain("Probe timed out after 10s");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
using System.Text;
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Clients;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// #786: the <c>/fleet/diagnostics</c> "Faulted devices" tile counts DISTINCT faulted devices across a
|
||||
/// cluster's nodes (a device faulted on both nodes of a redundant pair counts once), Faulted only.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class FleetDiagnosticsAggregationTests
|
||||
{
|
||||
private static readonly DateTime Now = DateTime.UtcNow;
|
||||
|
||||
private static DriverInstanceDiagnostics Driver(
|
||||
string name, int faulted, IReadOnlyList<string>? faultedHosts, int stopped = 0) =>
|
||||
new(Guid.NewGuid(), name, "Degraded", 0, faulted, Now, stopped, faultedHosts);
|
||||
|
||||
private static NodeDiagnosticsSnapshot Node(string nodeId, params DriverInstanceDiagnostics[] drivers) =>
|
||||
new(NodeId.Parse(nodeId), null, drivers, Now);
|
||||
|
||||
[Fact]
|
||||
public void The_same_device_faulted_on_both_nodes_of_a_pair_counts_once()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-modbus", 2, new[] { "plc-1:502", "plc-2:502" }));
|
||||
var b = Node("node-b:4053", Driver("drv-modbus", 1, new[] { "PLC-1:502" })); // host names compare case-insensitively
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Different_devices_faulted_on_each_node_are_unioned()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }));
|
||||
var b = Node("node-b:4053", Driver("drv-modbus", 1, new[] { "plc-2:502" }));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_same_host_name_under_two_drivers_is_two_devices()
|
||||
{
|
||||
var a = Node("node-a:4053",
|
||||
Driver("drv-modbus", 1, new[] { "10.0.0.5" }),
|
||||
Driver("drv-s7", 1, new[] { "10.0.0.5" }));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a }).ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_faulted_single_host_driver_on_both_nodes_counts_once()
|
||||
{
|
||||
// No IHostConnectivityProbe: the driver itself is the device and carries no host name.
|
||||
var a = Node("node-a:4053", Driver("drv-galaxy", 1, Array.Empty<string>()));
|
||||
var b = Node("node-b:4053", Driver("drv-galaxy", 1, Array.Empty<string>()));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, b }).ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stopped_devices_are_not_counted()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-galaxy", 0, Array.Empty<string>(), stopped: 3));
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a }).ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_pre_786_sender_without_host_names_is_combined_by_max_not_sum()
|
||||
{
|
||||
var oldSender = Node("node-a:4053", Driver("drv-modbus", 3, faultedHosts: null));
|
||||
var newSender = Node("node-b:4053", Driver("drv-modbus", 2, new[] { "plc-1:502", "plc-2:502" }));
|
||||
|
||||
// Unnamed 3 (max over nodes) + named 2: the old sender's devices cannot be matched by name, so the
|
||||
// figure is the best the wire can prove, never a doubled sum of the same unnamed devices.
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { oldSender, oldSender }).ShouldBe(3);
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { oldSender, newSender }).ShouldBe(5);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_unreachable_node_empty_snapshot_contributes_nothing()
|
||||
{
|
||||
var a = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }));
|
||||
var unreachable = Node("node-b:4053");
|
||||
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(new[] { a, unreachable }).ShouldBe(1);
|
||||
FleetDiagnosticsAggregation.DistinctFaultedDevices(Array.Empty<NodeDiagnosticsSnapshot>()).ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cluster_node_ids_are_the_enabled_rows_of_the_selected_nodes_cluster_self_first()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
||||
.UseInMemoryDatabase(Guid.NewGuid().ToString("N")).Options;
|
||||
await using (var seed = new OtOpcUaConfigDbContext(options))
|
||||
{
|
||||
seed.ClusterNodes.AddRange(
|
||||
Row("site-a-2:4053", "SITE-A"),
|
||||
Row("site-a-1:4053", "SITE-A"),
|
||||
Row("site-a-3:4053", "SITE-A", enabled: false),
|
||||
Row("site-b-1:4053", "SITE-B"));
|
||||
await seed.SaveChangesAsync(TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
await using var db = new OtOpcUaConfigDbContext(options);
|
||||
var (clusterId, nodeIds) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(
|
||||
db, "site-a-2:4053", TestContext.Current.CancellationToken);
|
||||
clusterId.ShouldBe("SITE-A");
|
||||
nodeIds.ShouldBe(new[] { "site-a-2:4053", "site-a-1:4053" });
|
||||
|
||||
var (unknownCluster, alone) = await FleetDiagnosticsAggregation.LoadClusterNodeIdsAsync(
|
||||
db, "stray:4053", TestContext.Current.CancellationToken);
|
||||
unknownCluster.ShouldBeNull();
|
||||
alone.ShouldBe(new[] { "stray:4053" });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_snapshot_from_a_pre_786_sender_still_deserializes_over_the_akka_wire()
|
||||
{
|
||||
// The snapshot crosses Akka remoting with the default (Newtonsoft JSON) serializer. Strip the #786
|
||||
// members from a serialized snapshot to reproduce an older sender's bytes.
|
||||
using var system = ActorSystem.Create("diag-compat");
|
||||
try
|
||||
{
|
||||
var snapshot = Node("node-a:4053", Driver("drv-modbus", 1, new[] { "plc-1:502" }, stopped: 2));
|
||||
var serializer = system.Serialization.FindSerializerFor(snapshot);
|
||||
var json = Encoding.UTF8.GetString(serializer.ToBinary(snapshot));
|
||||
json.ShouldContain("StoppedDevices");
|
||||
|
||||
var tree = JObject.Parse(json);
|
||||
foreach (var prop in tree.Descendants().OfType<JProperty>()
|
||||
.Where(p => p.Name is "StoppedDevices" or "FaultedHostNames").ToList())
|
||||
prop.Remove();
|
||||
var oldJson = tree.ToString(Formatting.None);
|
||||
oldJson.ShouldNotContain("StoppedDevices");
|
||||
oldJson.ShouldNotContain("FaultedHostNames");
|
||||
|
||||
var back = (NodeDiagnosticsSnapshot)serializer.FromBinary(Encoding.UTF8.GetBytes(oldJson), typeof(NodeDiagnosticsSnapshot));
|
||||
var driver = back.Drivers.ShouldHaveSingleItem();
|
||||
driver.FaultedDevices.ShouldBe(1);
|
||||
driver.StoppedDevices.ShouldBe(0);
|
||||
driver.FaultedHostNames.ShouldBeNull();
|
||||
}
|
||||
finally
|
||||
{
|
||||
await system.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
private static ClusterNode Row(string nodeId, string clusterId, bool enabled = true) => new()
|
||||
{
|
||||
NodeId = nodeId, ClusterId = clusterId, Host = nodeId.Split(':')[0],
|
||||
ApplicationUri = "urn:" + nodeId, CreatedBy = "test", Enabled = enabled,
|
||||
};
|
||||
}
|
||||
+28
@@ -208,6 +208,34 @@ public sealed class OpcUaClientDriverPageFormSerializationTests
|
||||
reparsed.CertificateValidation.EffectiveValidateServerHostName.ShouldBe(value ?? false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #837: the host-name check stays opt-in (unset = off) and sits in the Security panel with help text naming
|
||||
/// the NAT/IP, IPv6-zone and loose-first-label caveats. Source scan (no bUnit in this repo).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void The_host_name_check_is_an_opt_in_in_the_security_panel_with_its_caveats()
|
||||
{
|
||||
var dir = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "ZB.MOM.WW.OtOpcUa.slnx")))
|
||||
dir = dir.Parent;
|
||||
dir.ShouldNotBeNull("could not locate the repository root");
|
||||
var razor = File.ReadAllText(Path.Combine(dir!.FullName, "src", "Server", "ZB.MOM.WW.OtOpcUa.AdminUI",
|
||||
"Components", "Shared", "Drivers", "Forms", "OpcUaClientDriverForm.razor"));
|
||||
|
||||
razor.ShouldContain("<option value=\"\">Default (off)</option>");
|
||||
|
||||
var security = razor.IndexOf("<div class=\"panel-head\">Security</div>", StringComparison.Ordinal);
|
||||
var next = razor.IndexOf("<div class=\"panel-head\">", security + 1, StringComparison.Ordinal);
|
||||
security.ShouldBeGreaterThan(-1);
|
||||
var panel = razor[security..next];
|
||||
panel.ShouldContain("_form.OpcUa.ValidateServerHostName");
|
||||
panel.ShouldContain("Opt-in");
|
||||
panel.ShouldContain("NAT");
|
||||
panel.ShouldContain("IPv6 zone ids");
|
||||
panel.ShouldContain("Loose match");
|
||||
razor.Split("_form.OpcUa.ValidateServerHostName").Length.ShouldBe(2, "the toggle must be rendered once");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #648: stored DriverConfig JSON from before <c>targetNamespaceKind</c>/<c>unsMappingTable</c> were removed
|
||||
/// must still load into the form (the form's options use <c>UnmappedMemberHandling.Skip</c>) and the legacy
|
||||
|
||||
+620
@@ -0,0 +1,620 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#868: the coordinator's recovery read used to run synchronously in <c>PreStart</c>. With the
|
||||
/// ConfigDb unreachable when the singleton started, the actor died on initialization, was never re-created,
|
||||
/// and every deployment hung. It now loads asynchronously with backoff, queues what arrives meanwhile and
|
||||
/// reports <see cref="ConfigPublishCoordinatorState.Loading"/> until the load succeeds.
|
||||
/// </summary>
|
||||
public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTestBase
|
||||
{
|
||||
private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('d', 64));
|
||||
private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5);
|
||||
private static readonly TimeSpan RetryMin = TimeSpan.FromMilliseconds(50);
|
||||
private static readonly TimeSpan RetryMax = TimeSpan.FromMilliseconds(200);
|
||||
private const string Node = "site-a-1:4053";
|
||||
|
||||
[Fact]
|
||||
public void Db_unreachable_at_start_does_not_kill_the_coordinator_and_a_deploy_sent_during_the_outage_completes_once_it_returns()
|
||||
{
|
||||
var dbName = Guid.NewGuid().ToString("N");
|
||||
var direct = NewInMemoryDbFactory(dbName);
|
||||
var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
|
||||
SeedNodes(direct, Node);
|
||||
var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
|
||||
var readiness = new ConfigPublishCoordinatorReadiness();
|
||||
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
|
||||
outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
|
||||
Watch(coordinator);
|
||||
|
||||
// Several failed loads: the actor keeps retrying (it is alive) and reports Loading with the error.
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
outage.Attempts.ShouldBeGreaterThanOrEqualTo(3);
|
||||
readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Loading);
|
||||
readiness.Current.FailedLoadAttempts.ShouldBeGreaterThanOrEqualTo(2);
|
||||
readiness.Current.LastError.ShouldNotBeNull();
|
||||
readiness.Current.LastError.ShouldContain("unreachable");
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
// A deployment dispatched during the outage is queued, not processed and not dropped.
|
||||
coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
|
||||
using (var db = direct.CreateDbContext())
|
||||
{
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching);
|
||||
db.NodeDeploymentStates.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
// The DB comes back: the load succeeds, the queued dispatch runs, and the node's ack seals it.
|
||||
outage.Down = false;
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
|
||||
db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { Node });
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); // never Terminated
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void In_flight_deployment_is_recovered_after_the_outage_and_an_ack_queued_during_it_is_not_lost()
|
||||
{
|
||||
var dbName = Guid.NewGuid().ToString("N");
|
||||
var direct = NewInMemoryDbFactory(dbName);
|
||||
var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
|
||||
SeedNodes(direct, Node, "site-a-2:4053");
|
||||
var deploymentId = SeedDeployment(direct, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow);
|
||||
using (var db = direct.CreateDbContext())
|
||||
{
|
||||
db.NodeDeploymentStates.Add(new NodeDeploymentState
|
||||
{ NodeId = Node, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying });
|
||||
db.NodeDeploymentStates.Add(new NodeDeploymentState
|
||||
{ NodeId = "site-a-2:4053", DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied, AppliedAtUtc = DateTime.UtcNow });
|
||||
db.SaveChanges();
|
||||
}
|
||||
|
||||
var readiness = new ConfigPublishCoordinatorReadiness();
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
|
||||
outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
|
||||
AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(2),
|
||||
duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
// The last outstanding ack arrives while the coordinator is still loading: it is queued, then applied
|
||||
// against the recovered expected set (site-a-2 already Applied in the DB) and seals the deployment.
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
outage.Down = false;
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Review finding 1: a recovered deployment whose every expected ack is already in the DB as a FINAL outcome
|
||||
/// (a seal whose SaveChanges failed, or a fused node's own row — which <c>DriverHostActor</c> writes Applied only
|
||||
/// once the address-space outcome is known; a pre-outcome row stays Applying, second review M1) must be completed — inside the deadline it used to wait for
|
||||
/// acks that had already landed, past it the deadline's "race" guard returned without sealing; either way it
|
||||
/// sat in AwaitingApplyAcks forever and AdminOps refused every later deploy.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(false, false, DeploymentStatus.Sealed)]
|
||||
[InlineData(true, false, DeploymentStatus.Sealed)]
|
||||
[InlineData(false, true, DeploymentStatus.PartiallyFailed)]
|
||||
[InlineData(true, true, DeploymentStatus.PartiallyFailed)]
|
||||
public void Recovered_deployment_with_every_ack_already_persisted_is_completed(bool pastDeadline, bool oneFailed, DeploymentStatus expected)
|
||||
{
|
||||
var dbFactory = NewInMemoryDbFactory();
|
||||
SeedNodes(dbFactory, Node, "site-a-2:4053");
|
||||
var deadline = TimeSpan.FromMinutes(1);
|
||||
var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks,
|
||||
pastDeadline ? DateTime.UtcNow.AddMinutes(-10) : DateTime.UtcNow);
|
||||
SeedNodeState(dbFactory, deploymentId, Node, NodeDeploymentStatus.Applied);
|
||||
SeedNodeState(dbFactory, deploymentId, "site-a-2:4053", oneFailed ? NodeDeploymentStatus.Failed : NodeDeploymentStatus.Applied);
|
||||
|
||||
Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, deadline));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(expected);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Review finding 2: a recovered deployment that is already past its deadline must first apply the acks that
|
||||
/// were queued while the load was failing (they arrived in time); the deadline used to run inline before the
|
||||
/// replay, so they were discarded as stale and the deployment was timed out.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Ack_queued_during_the_outage_is_applied_before_a_past_deadline_recovery_times_out()
|
||||
{
|
||||
var dbName = Guid.NewGuid().ToString("N");
|
||||
var direct = NewInMemoryDbFactory(dbName);
|
||||
var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
|
||||
SeedNodes(direct, Node);
|
||||
var deploymentId = SeedDeployment(direct, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow.AddMinutes(-10));
|
||||
SeedNodeState(direct, deploymentId, Node, NodeDeploymentStatus.Applying);
|
||||
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
|
||||
outage, TimeSpan.FromMinutes(1), loadRetryMin: RetryMin, loadRetryMax: RetryMax));
|
||||
AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(2),
|
||||
duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
outage.Down = false;
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
db.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Review finding 3: an ack whose persist fails (ConfigDb blip while Ready) is retried, not dropped. It used to
|
||||
/// throw, restart the actor and lose the ack — the node never resends and never writes its own row.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Ack_whose_persist_fails_is_retried_not_dropped()
|
||||
{
|
||||
var dbName = Guid.NewGuid().ToString("N");
|
||||
var direct = NewInMemoryDbFactory(dbName);
|
||||
var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName));
|
||||
SeedNodes(direct, Node);
|
||||
var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
|
||||
var readiness = new ConfigPublishCoordinatorReadiness();
|
||||
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
|
||||
outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
|
||||
coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
outage.Down = true;
|
||||
var before = outage.Attempts;
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(before + 2),
|
||||
duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
outage.Down = false;
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
db.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Second review M2: an ack that arrived but whose persist is still failing at the deadline must not be timed
|
||||
/// out as "no ack from" its node. The deadline is deferred until the backlog persists, then the deployment seals.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Deadline_does_not_time_out_a_received_but_unpersisted_ack()
|
||||
{
|
||||
var (direct, outage, deploymentId, coordinator, _) = DispatchedSingleNode(TimeSpan.FromSeconds(2));
|
||||
|
||||
outage.Down = true;
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
// Hold the DB down past the 2 s apply deadline.
|
||||
Thread.Sleep(TimeSpan.FromSeconds(3.5));
|
||||
outage.Down = false;
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
db.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Second review M2: an unpersisted ack must survive an actor restart (its retry timer does not). Here a second
|
||||
/// dispatch's guard read throws while the DB is down, restarting the coordinator.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Unpersisted_ack_survives_a_coordinator_restart()
|
||||
{
|
||||
var (direct, outage, deploymentId, coordinator, _) = DispatchedSingleNode(NoTimeout);
|
||||
|
||||
outage.Down = true;
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
var before = outage.Attempts;
|
||||
AwaitAssert(() => outage.Attempts.ShouldBeGreaterThan(before),
|
||||
duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
// Its guard read throws (DB down) => the coordinator restarts.
|
||||
coordinator.Tell(new DispatchDeployment(DeploymentId.NewId(), TestRevision, CorrelationId.NewId()));
|
||||
Thread.Sleep(TimeSpan.FromMilliseconds(500));
|
||||
outage.Down = false;
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single(d => d.DeploymentId == deploymentId.Value).Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Second review L3: a retry of a node's OLDER unpersisted ack must not overwrite the newer ack that persisted
|
||||
/// after it (it used to flip the node to Failed and turn a clean deploy PartiallyFailed).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_stale_ack_retry_does_not_overwrite_a_newer_persisted_ack()
|
||||
{
|
||||
var dbName = Guid.NewGuid().ToString("N");
|
||||
var direct = NewInMemoryDbFactory(dbName);
|
||||
var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName));
|
||||
SeedNodes(direct, Node, "site-a-2:4053");
|
||||
var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
|
||||
outage, NoTimeout, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
|
||||
coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
// Node 1's first answer (Failed) cannot be persisted; its corrected answer (Applied) can.
|
||||
outage.FailNext(1);
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Failed, "first try", CorrelationId.NewId()));
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
Thread.Sleep(TimeSpan.FromMilliseconds(600)); // several retry periods
|
||||
using (var db = direct.CreateDbContext())
|
||||
db.NodeDeploymentStates.Single(s => s.NodeId == Node).Status.ShouldBe(NodeDeploymentStatus.Applied);
|
||||
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-2:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Second review L4: a dispatch QUEUED while the load failed for longer than the apply deadline was never sent;
|
||||
/// it must be dispatched (fresh deadline) once the load succeeds, not closed "Dispatch was lost".
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Dispatch_queued_through_an_outage_longer_than_the_deadline_is_sent_not_closed_as_lost()
|
||||
{
|
||||
var dbName = Guid.NewGuid().ToString("N");
|
||||
var direct = NewInMemoryDbFactory(dbName);
|
||||
var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
|
||||
SeedNodes(direct, Node);
|
||||
var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
|
||||
var deadline = TimeSpan.FromSeconds(3);
|
||||
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
|
||||
outage, deadline, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
|
||||
coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
|
||||
Thread.Sleep(deadline + TimeSpan.FromSeconds(1));
|
||||
outage.Down = false;
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
|
||||
db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { Node });
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Second review L5: an in-flight row recovery cannot interpret (blank RevisionHash on a row it must re-dispatch)
|
||||
/// used to throw on every load and keep the coordinator Loading forever. It is closed with a reason, and the
|
||||
/// coordinator serves the next deployment.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Malformed_in_flight_row_is_closed_with_a_reason_and_the_coordinator_carries_on()
|
||||
{
|
||||
var dbFactory = NewInMemoryDbFactory();
|
||||
SeedNodes(dbFactory, Node);
|
||||
var bad = DeploymentId.NewId();
|
||||
using (var db = dbFactory.CreateDbContext())
|
||||
{
|
||||
db.Deployments.Add(new Deployment
|
||||
{
|
||||
DeploymentId = bad.Value, RevisionHash = " ", Status = DeploymentStatus.Dispatching,
|
||||
CreatedBy = "test", CreatedAtUtc = DateTime.UtcNow,
|
||||
});
|
||||
db.SaveChanges();
|
||||
}
|
||||
var readiness = new ConfigPublishCoordinatorReadiness();
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
|
||||
dbFactory, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
var row = db.Deployments.Single(d => d.DeploymentId == bad.Value);
|
||||
row.Status.ShouldBe(DeploymentStatus.PartiallyFailed);
|
||||
row.FailureReason.ShouldNotBeNull();
|
||||
row.FailureReason.ShouldContain("RevisionHash");
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
var next = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(1));
|
||||
coordinator.Tell(new DispatchDeployment(next, TestRevision, CorrelationId.NewId()));
|
||||
coordinator.Tell(new ApplyAck(next, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.Deployments.Single(d => d.DeploymentId == next.Value).Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Second review L6: an ack-persist backlog is visible on readiness while Ready, and clears.</summary>
|
||||
[Fact]
|
||||
public void Readiness_reports_the_ack_persist_backlog()
|
||||
{
|
||||
var (_, outage, deploymentId, coordinator, readiness) = DispatchedSingleNode(NoTimeout);
|
||||
|
||||
outage.Down = true;
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
|
||||
readiness.Current.PendingAckPersists.ShouldBe(1);
|
||||
readiness.Current.LastPersistError.ShouldNotBeNull();
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
outage.Down = false;
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
readiness.Current.PendingAckPersists.ShouldBe(0);
|
||||
readiness.Current.LastPersistError.ShouldBeNull();
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Third review #1: a Failed ack's reason (node free text) was written unbounded; on SQL Server the overflow
|
||||
/// refuses the SaveChanges forever and the deployment wedged. It is truncated to the column.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void An_oversized_failure_reason_is_truncated_not_retried_forever()
|
||||
{
|
||||
var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(NoTimeout);
|
||||
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Failed,
|
||||
new string('x', 5000), CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var ctx = db.CreateDbContext();
|
||||
ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.PartiallyFailed);
|
||||
var row = ctx.NodeDeploymentStates.Single();
|
||||
row.Status.ShouldBe(NodeDeploymentStatus.Failed);
|
||||
row.FailureReason!.Length.ShouldBe(FailureReasons.MaxLength);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Third review #1: a write the DB REJECTS (non-transient DbUpdateException) is not retried forever — the ack is
|
||||
/// counted from memory and the deployment completes.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_rejected_ack_write_is_counted_from_memory_not_retried_forever()
|
||||
{
|
||||
var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(NoTimeout);
|
||||
db.PoisonNodeStateUpdates = true;
|
||||
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var ctx = db.CreateDbContext();
|
||||
ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Third review #1: a write that keeps failing transiently defers the deadline only up to a cap
|
||||
/// (10 × the retry ceiling); past it the outcome is decided from the acks held in memory.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Deadline_deferral_on_unpersisted_acks_is_bounded()
|
||||
{
|
||||
var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(TimeSpan.FromSeconds(1));
|
||||
db.FailNodeStateUpdatesTransiently = true;
|
||||
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
|
||||
// Deadline 1 s + cap 10 × 200 ms: decided within a few seconds, never left in flight.
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var ctx = db.CreateDbContext();
|
||||
ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Third review #3: a lost dispatch recovered with only a sliver of its deadline left used to be re-driven with
|
||||
/// that sliver — the nodes applied and the deployment timed out moments later. It gets a full deadline.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_lost_dispatch_redriven_near_its_deadline_gets_a_full_deadline()
|
||||
{
|
||||
var dbFactory = NewInMemoryDbFactory();
|
||||
SeedNodes(dbFactory, Node);
|
||||
var deadline = TimeSpan.FromSeconds(3);
|
||||
var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow - TimeSpan.FromSeconds(2.5));
|
||||
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, deadline));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
// Well past the 0.5 s that was left, still inside a fresh 3 s deadline.
|
||||
Thread.Sleep(TimeSpan.FromSeconds(1.5));
|
||||
using (var db = dbFactory.CreateDbContext())
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
|
||||
|
||||
coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
private (ConstraintEnforcingDbFactory Db, DeploymentId Id, IActorRef Coordinator) DispatchedOnConstraintDb(TimeSpan applyDeadline)
|
||||
{
|
||||
var db = new ConstraintEnforcingDbFactory(Guid.NewGuid().ToString("N"));
|
||||
SeedNodes(db, Node);
|
||||
var deploymentId = SeedDeployment(db, DeploymentStatus.Dispatching, DateTime.UtcNow);
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
|
||||
db, applyDeadline, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
|
||||
coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var ctx = db.CreateDbContext();
|
||||
ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
return (db, deploymentId, coordinator);
|
||||
}
|
||||
|
||||
/// <summary>A single-node deployment dispatched and AwaitingApplyAcks through an outage-capable factory.</summary>
|
||||
private (IDbContextFactory<OtOpcUaConfigDbContext> Direct, OutageDbFactory Outage, DeploymentId Id, IActorRef Coordinator,
|
||||
ConfigPublishCoordinatorReadiness Readiness) DispatchedSingleNode(TimeSpan applyDeadline)
|
||||
{
|
||||
var dbName = Guid.NewGuid().ToString("N");
|
||||
var direct = NewInMemoryDbFactory(dbName);
|
||||
var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName));
|
||||
SeedNodes(direct, Node);
|
||||
var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
|
||||
var readiness = new ConfigPublishCoordinatorReadiness();
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
|
||||
outage, applyDeadline, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
|
||||
coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var db = direct.CreateDbContext();
|
||||
db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
|
||||
}, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
return (direct, outage, deploymentId, coordinator, readiness);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Readiness_is_NotHosted_after_the_coordinator_stops()
|
||||
{
|
||||
var readiness = new ConfigPublishCoordinatorReadiness();
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(NewInMemoryDbFactory(), NoTimeout, readiness: readiness));
|
||||
AwaitAssert(() => readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready),
|
||||
duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
|
||||
Watch(coordinator);
|
||||
Sys.Stop(coordinator);
|
||||
ExpectTerminated(coordinator, cancellationToken: TestContext.Current.CancellationToken);
|
||||
readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.NotHosted);
|
||||
}
|
||||
|
||||
private static DeploymentId SeedDeployment(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentStatus status, DateTime createdAtUtc)
|
||||
{
|
||||
var id = DeploymentId.NewId();
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.Deployments.Add(new Deployment
|
||||
{
|
||||
DeploymentId = id.Value,
|
||||
RevisionHash = TestRevision.Value,
|
||||
Status = status,
|
||||
CreatedBy = "test",
|
||||
CreatedAtUtc = createdAtUtc,
|
||||
});
|
||||
db.SaveChanges();
|
||||
return id;
|
||||
}
|
||||
|
||||
private static void SeedNodeState(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentId deploymentId, string nodeId, NodeDeploymentStatus status)
|
||||
{
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.NodeDeploymentStates.Add(new NodeDeploymentState
|
||||
{
|
||||
NodeId = nodeId, DeploymentId = deploymentId.Value, Status = status,
|
||||
AppliedAtUtc = status == NodeDeploymentStatus.Applying ? null : DateTime.UtcNow,
|
||||
});
|
||||
db.SaveChanges();
|
||||
}
|
||||
|
||||
private static void SeedNodes(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, params string[] nodeIds)
|
||||
{
|
||||
using var db = dbFactory.CreateDbContext();
|
||||
db.ServerClusters.Add(new ServerCluster
|
||||
{
|
||||
ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a",
|
||||
NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
|
||||
});
|
||||
foreach (var nodeId in nodeIds)
|
||||
{
|
||||
db.ClusterNodes.Add(new ClusterNode
|
||||
{
|
||||
NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0],
|
||||
ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test",
|
||||
});
|
||||
}
|
||||
db.SaveChanges();
|
||||
}
|
||||
|
||||
/// <summary>A factory that throws while <see cref="Down"/> — a ConfigDb that is unreachable.</summary>
|
||||
private sealed class OutageDbFactory(IDbContextFactory<OtOpcUaConfigDbContext> inner) : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
private volatile bool _down;
|
||||
private int _attempts;
|
||||
private int _failNext;
|
||||
|
||||
public bool Down { get => _down; set => _down = value; }
|
||||
|
||||
public int Attempts => Volatile.Read(ref _attempts);
|
||||
|
||||
/// <summary>The next <paramref name="count"/> context creations throw, then the DB is reachable again.</summary>
|
||||
public void FailNext(int count) => Volatile.Write(ref _failNext, count);
|
||||
|
||||
public OtOpcUaConfigDbContext CreateDbContext()
|
||||
{
|
||||
Interlocked.Increment(ref _attempts);
|
||||
if (_down) throw new InvalidOperationException("ConfigDb unreachable (simulated outage)");
|
||||
if (Volatile.Read(ref _failNext) > 0 && Interlocked.Decrement(ref _failNext) >= 0)
|
||||
throw new InvalidOperationException("ConfigDb unreachable (simulated blip)");
|
||||
return inner.CreateDbContext();
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -136,9 +136,9 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes
|
||||
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout));
|
||||
var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty<IDriverProbe>()));
|
||||
|
||||
// Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and
|
||||
// the actor would be stopped, not restarted). Identify is processed after creation, so a
|
||||
// resolved selection proves PreStart completed.
|
||||
// Arm the fault AFTER PreStart has run. Since lmxopcua#868 the recovery read is asynchronous and
|
||||
// retried, so a fault that lands on it instead of on HandleDispatch is also recovered (the dispatch is
|
||||
// queued until the retried load succeeds) — either way the deployment must complete.
|
||||
await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget, TestContext.Current.CancellationToken);
|
||||
faulting.FailNext();
|
||||
admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Diagnostics;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// An in-memory ConfigDb that behaves like SQL Server where the InMemory provider does not (lmxopcua#868 third
|
||||
/// review): a <c>FailureReason</c> longer than the column refuses the whole <c>SaveChanges</c> with a
|
||||
/// <see cref="DbUpdateException"/>. It can also reject (<see cref="PoisonNodeStateUpdates"/>, a non-transient
|
||||
/// <see cref="DbUpdateException"/>) or fail transiently (<see cref="FailNodeStateUpdatesTransiently"/>) every
|
||||
/// update of an existing <c>NodeDeploymentState</c> row.
|
||||
/// </summary>
|
||||
public sealed class ConstraintEnforcingDbFactory(string dbName) : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
private readonly Interceptor _interceptor = new();
|
||||
|
||||
/// <summary>Every update of an existing node row is rejected as a constraint violation.</summary>
|
||||
public bool PoisonNodeStateUpdates { get => _interceptor.Poison; set => _interceptor.Poison = value; }
|
||||
|
||||
/// <summary>Every update of an existing node row fails as if the connection dropped.</summary>
|
||||
public bool FailNodeStateUpdatesTransiently { get => _interceptor.Transient; set => _interceptor.Transient = value; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public OtOpcUaConfigDbContext CreateDbContext() => new(new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
||||
.UseInMemoryDatabase(dbName)
|
||||
.AddInterceptors(_interceptor)
|
||||
.Options);
|
||||
|
||||
private sealed class Interceptor : SaveChangesInterceptor
|
||||
{
|
||||
public volatile bool Poison;
|
||||
public volatile bool Transient;
|
||||
|
||||
public override InterceptionResult<int> SavingChanges(DbContextEventData eventData, InterceptionResult<int> result)
|
||||
{
|
||||
Check(eventData.Context!);
|
||||
return result;
|
||||
}
|
||||
|
||||
public override ValueTask<InterceptionResult<int>> SavingChangesAsync(
|
||||
DbContextEventData eventData, InterceptionResult<int> result, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Check(eventData.Context!);
|
||||
return ValueTask.FromResult(result);
|
||||
}
|
||||
|
||||
private void Check(DbContext ctx)
|
||||
{
|
||||
foreach (var e in ctx.ChangeTracker.Entries())
|
||||
{
|
||||
var reason = e.Entity switch
|
||||
{
|
||||
NodeDeploymentState n => n.FailureReason,
|
||||
Deployment d => d.FailureReason,
|
||||
_ => null,
|
||||
};
|
||||
if (reason is { Length: > FailureReasons.MaxLength })
|
||||
throw new DbUpdateException("String or binary data would be truncated in column 'FailureReason'.");
|
||||
if (e.Entity is NodeDeploymentState && e.State == EntityState.Modified)
|
||||
{
|
||||
if (Poison) throw new DbUpdateException("The UPDATE statement conflicted with a CHECK constraint.");
|
||||
if (Transient) throw new InvalidOperationException("A network-related error occurred (simulated).");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Health;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Health;
|
||||
|
||||
/// <summary>lmxopcua#868: the deploy coordinator reports Degraded (never Unhealthy) while it cannot load from the ConfigDb.</summary>
|
||||
public sealed class ConfigPublishCoordinatorHealthCheckTests
|
||||
{
|
||||
private static async Task<HealthCheckResult> Check(ConfigPublishCoordinatorReadiness? readiness) =>
|
||||
await new ConfigPublishCoordinatorHealthCheck(readiness).CheckHealthAsync(new HealthCheckContext(), CancellationToken.None);
|
||||
|
||||
[Fact]
|
||||
public async Task Loading_after_failed_attempts_is_degraded_and_names_the_error()
|
||||
{
|
||||
var r = new ConfigPublishCoordinatorReadiness();
|
||||
r.MarkLoading(3, "connection refused");
|
||||
var result = await Check(r);
|
||||
result.Status.ShouldBe(HealthStatus.Degraded);
|
||||
result.Description.ShouldNotBeNull();
|
||||
result.Description.ShouldContain("connection refused");
|
||||
result.Description.ShouldContain("queued");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ready_is_healthy()
|
||||
{
|
||||
var r = new ConfigPublishCoordinatorReadiness();
|
||||
r.MarkLoading(2, "x");
|
||||
r.MarkReady();
|
||||
(await Check(r)).Status.ShouldBe(HealthStatus.Healthy);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ready_with_an_ack_persist_backlog_is_degraded_and_clears()
|
||||
{
|
||||
var r = new ConfigPublishCoordinatorReadiness();
|
||||
r.MarkReady();
|
||||
r.MarkAckPersistBacklog(2, "timeout");
|
||||
var result = await Check(r);
|
||||
result.Status.ShouldBe(HealthStatus.Degraded);
|
||||
result.Description.ShouldNotBeNull();
|
||||
result.Description.ShouldContain("timeout");
|
||||
|
||||
r.MarkAckPersistBacklog(0, null);
|
||||
(await Check(r)).Status.ShouldBe(HealthStatus.Healthy);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Not_hosted_and_untracked_are_healthy()
|
||||
{
|
||||
(await Check(new ConfigPublishCoordinatorReadiness())).Status.ShouldBe(HealthStatus.Healthy);
|
||||
(await Check(null)).Status.ShouldBe(HealthStatus.Healthy);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true, true)]
|
||||
[InlineData(false, false)]
|
||||
public void It_is_registered_on_the_ready_tier_only_on_admin_nodes(bool hasAdmin, bool registered)
|
||||
{
|
||||
var registrations = new ServiceCollection()
|
||||
.AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: true)
|
||||
.BuildServiceProvider()
|
||||
.GetRequiredService<IOptions<HealthCheckServiceOptions>>()
|
||||
.Value.Registrations
|
||||
.Where(x => x.Name == ConfigPublishCoordinatorHealthCheck.Name)
|
||||
.ToList();
|
||||
if (!registered)
|
||||
{
|
||||
registrations.ShouldBeEmpty();
|
||||
return;
|
||||
}
|
||||
registrations.ShouldHaveSingleItem().Tags.ShouldBe([ZbHealthTags.Ready]);
|
||||
}
|
||||
}
|
||||
+15
-55
@@ -24,16 +24,18 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
/// every other subscriber still receives them, and the refresh never waits for the stalled drain.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The overload is produced deterministically: the folder item's server-side <c>MonitoredItem</c> lock is held, so
|
||||
/// the SDK's per-node delivery task parks in <c>QueueEvent</c>, its 4096-slot channel fills, and the condition's
|
||||
/// later events are deferred — without holding the node manager's <c>Lock</c>, which the refresh needs.
|
||||
/// The overload is produced deterministically: delivery is stalled at the EqA folder (<see cref="NotifierStall"/>), so the
|
||||
/// condition's walk suspends there and its later events are deferred — without holding the node manager's <c>Lock</c>,
|
||||
/// which the refresh needs. (Before Gitea #855 this held the folder item's SDK lock so the folder's 4096-slot channel
|
||||
/// filled; folder items are now queued per item by the reporter, and no notifier of ours defers on its own.) The
|
||||
/// folder-item twin of these tests is <see cref="FolderConditionRefreshWireTests"/>.
|
||||
/// </remarks>
|
||||
public sealed class ConditionRefreshDeferralWireTests
|
||||
{
|
||||
private const string EqA = ServerEventRig.EqA;
|
||||
|
||||
// > 4096 (channel) + 1 (parked in the delivery task) + 1 (the in-flight deferred write), < 1024 pending: nothing coalesces.
|
||||
private const int FloodSize = 5000;
|
||||
// Every event after the stalled one is deferred; < 1024 pending, so nothing coalesces.
|
||||
private const int FloodSize = 500;
|
||||
|
||||
/// <param name="enforced">Per-folder / Server-object ACL on or off (off is the production default).</param>
|
||||
/// <param name="finalRetained">
|
||||
@@ -84,7 +86,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
|
||||
await folderSubscription.SetPublishingModeAsync(false, ct);
|
||||
|
||||
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
|
||||
@@ -200,7 +202,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
serverEvents.Has(onServer, "seed").ShouldBeFalse("the key holds no grant yet");
|
||||
await folderSubscription.SetPublishingModeAsync(false, ct);
|
||||
|
||||
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Project(native: true, Active("final"));
|
||||
@@ -271,7 +273,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
rig.Sink.WriteAlarmCondition(Free, Active("x2"), DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
};
|
||||
|
||||
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the cut path runs only while something is deferred");
|
||||
@@ -345,7 +347,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
proceed.Wait(TimeSpan.FromSeconds(30));
|
||||
};
|
||||
|
||||
using (ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
||||
using (NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Project(native: true, Active("final"));
|
||||
@@ -416,7 +418,6 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
rig.Project(native: true, Active("s0"));
|
||||
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "s0") && serverEvents.Has(onServer, "s0"));
|
||||
await folderSubscription.SetPublishingModeAsync(false, ct);
|
||||
var item = FindServerItem(rig.Nm, onA.Status.Id);
|
||||
|
||||
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
var counter = 0;
|
||||
@@ -440,7 +441,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
{
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
using (ItemLockStall.Hold(item))
|
||||
using (NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
Thread.Sleep(400);
|
||||
if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen);
|
||||
@@ -568,7 +569,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
|
||||
await folderSubscription.SetPublishingModeAsync(false, ct);
|
||||
|
||||
var stall = deferring ? ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)) : null;
|
||||
var stall = deferring ? NotifierStall.Hold(rig.Nm, EqA) : null;
|
||||
try
|
||||
{
|
||||
if (deferring)
|
||||
@@ -622,7 +623,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
}
|
||||
|
||||
/// <summary>The server-side Server-object (all-events) item behind a client item.</summary>
|
||||
private static IEventMonitoredItem FindAllEventsItem(OtOpcUaNodeManager nm, uint id)
|
||||
internal static IEventMonitoredItem FindAllEventsItem(OtOpcUaNodeManager nm, uint id)
|
||||
{
|
||||
var map = (IDictionary)typeof(OtOpcUaNodeManager)
|
||||
.GetField("_allEventsItems", BindingFlags.Instance | BindingFlags.NonPublic)!
|
||||
@@ -636,7 +637,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
}
|
||||
|
||||
/// <summary>The server-side event item behind a client item, from the node manager's direct-item map.</summary>
|
||||
private static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id)
|
||||
internal static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id)
|
||||
{
|
||||
var map = (IDictionary)typeof(OtOpcUaNodeManager)
|
||||
.GetField("_eventItems", BindingFlags.Instance | BindingFlags.NonPublic)!
|
||||
@@ -664,45 +665,4 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
ScopeId = $"eq-{equipment}",
|
||||
PermissionFlags = NodePermissions.ReadOnly,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Holds a server-side monitored item's private lock on a dedicated thread, so the SDK's per-node delivery task
|
||||
/// parks in <c>QueueEvent</c> for that item (a stalled subscriber) without the node manager's <c>Lock</c> held.
|
||||
/// </summary>
|
||||
private sealed class ItemLockStall : IDisposable
|
||||
{
|
||||
private readonly ManualResetEventSlim _release = new();
|
||||
private readonly Thread _thread;
|
||||
|
||||
private ItemLockStall(IEventMonitoredItem item)
|
||||
{
|
||||
var gate = (Lock)typeof(Opc.Ua.Server.MonitoredItem)
|
||||
.GetField("m_lock", BindingFlags.Instance | BindingFlags.NonPublic)!
|
||||
.GetValue(item)!;
|
||||
using var held = new ManualResetEventSlim();
|
||||
_thread = new Thread(() =>
|
||||
{
|
||||
gate.Enter();
|
||||
try
|
||||
{
|
||||
held.Set();
|
||||
_release.Wait();
|
||||
}
|
||||
finally { gate.Exit(); }
|
||||
}) { IsBackground = true };
|
||||
_thread.Start();
|
||||
held.Wait();
|
||||
}
|
||||
|
||||
public static ItemLockStall Hold(IEventMonitoredItem item) => new(item);
|
||||
|
||||
public void Release() => _release.Set();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_release.Set();
|
||||
_thread.Join();
|
||||
_release.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+18
-13
@@ -27,13 +27,16 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
/// <item>no ACL-denied event reaches an item during the overflow, including deferred events whose grant was
|
||||
/// revoked while they waited.</item>
|
||||
/// </list>
|
||||
/// Since Gitea #855 the folder items are served per item by the reporter (as the Server-object items always were),
|
||||
/// so the flood no longer defers at all — the first test pins that — and the revocation test stalls the folder's
|
||||
/// delivery (<see cref="NotifierStall"/>) to get a deferred backlog.
|
||||
/// </summary>
|
||||
public sealed class EventQueueOverflowAclWireTests
|
||||
{
|
||||
private const string EqA = ServerEventRig.EqA;
|
||||
private const string EqB = ServerEventRig.EqB;
|
||||
|
||||
// Native events per flood: past the SDK's 4096-slot per-notifier channel, so the native condition defers.
|
||||
// Native events per flood: past the SDK's 4096-slot per-notifier channel (which deferred them before #855).
|
||||
private const int PerCondition = 6000;
|
||||
|
||||
/// <param name="singleHold">
|
||||
@@ -150,13 +153,12 @@ public sealed class EventQueueOverflowAclWireTests
|
||||
readLatencies.Max().ShouldBeLessThan(TimeSpan.FromSeconds(5), "a Read must not queue behind the whole flood");
|
||||
(await TimedReadAsync(anonymous, eqA, ct)).ShouldBeLessThan(TimeSpan.FromSeconds(2), "the node manager is free afterwards");
|
||||
|
||||
// Memory: the node manager's own overflow store stayed within its per-condition budget (queued + in flight)…
|
||||
Volatile.Read(ref peakDeferred).ShouldBeLessThanOrEqualTo(
|
||||
2 * (OtOpcUaNodeManager.MaxDeferredEventsPerCondition + 1), "deferral is bounded per condition");
|
||||
if (singleHold) rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the native tail overflowed and coalesced");
|
||||
// Memory: the node manager held nothing of its own for the overflow (#855: no notifier of ours defers, so the
|
||||
// item queues are the only store, bounded below)…
|
||||
Volatile.Read(ref peakDeferred).ShouldBe(0, "folder items are served per item by the reporter: nothing defers");
|
||||
rig.Nm.DiscardedEventReports.ShouldBe(0);
|
||||
|
||||
// …let the SDK channel finish draining into the (still unpublished) item queues, then publish.
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), ct);
|
||||
// …then publish the (still unpublished) item queues.
|
||||
await keySub.SetPublishingModeAsync(true, ct);
|
||||
var lastNative = $"n{PerCondition - 1}";
|
||||
await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(onB, lastNative) && keyEvents.Has(keyServer, lastNative));
|
||||
@@ -165,10 +167,9 @@ public sealed class EventQueueOverflowAclWireTests
|
||||
foreach (var item in new[] { onB, keyServer })
|
||||
{
|
||||
var received = keyEvents.Of(item);
|
||||
// ACL: not one event of the denied (EqA-owned) scripted alarm, overflow or not. (Under a single hold the
|
||||
// deferred native tail drains last and evicts every synchronously-queued event from the 1000-slot queue, so
|
||||
// there this check cannot see a leak; the "after-s" leg below is the falsifier for that mode — verified by
|
||||
// running with enforcement off: chunked fails here, single-hold fails at "after-s".)
|
||||
// ACL: not one event of the denied (EqA-owned) scripted alarm, overflow or not. (The 1000-slot queue keeps
|
||||
// only the last events of the flood, which interleave n/s, so a leak shows here in both modes; the
|
||||
// "after-s" leg below is a second falsifier.)
|
||||
received.Where(f => RecordedEvents.Message(f)?.StartsWith('s') == true)
|
||||
.ShouldBeEmpty($"{item.DisplayName}: an ACL-denied event leaked during the overflow");
|
||||
|
||||
@@ -211,7 +212,7 @@ public sealed class EventQueueOverflowAclWireTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A grant revoked while a deferred backlog waits behind a full notifier channel: the deferred events are decided
|
||||
/// A grant revoked while a deferred backlog waits behind a stalled notifier: the deferred events are decided
|
||||
/// at delivery against the CURRENT generation, so none of them — nor any event reported after the revocation —
|
||||
/// reaches the revoked session's folder or Server-object item, while a still-granted session sharing the same
|
||||
/// folder channel receives the whole tail (positive control).
|
||||
@@ -249,6 +250,9 @@ public sealed class EventQueueOverflowAclWireTests
|
||||
await anonSub.ApplyChangesAsync(ct);
|
||||
anonA.Status.Error.ShouldBeNull();
|
||||
|
||||
// The EqA folder's delivery stalls (#855: nothing of ours defers on its own any more), so the native condition's
|
||||
// walk suspends there and the rest of the flood is deferred behind it.
|
||||
using var stall = NotifierStall.Hold(rig.Nm, EqA);
|
||||
var flood = Task.Factory.StartNew(() =>
|
||||
{
|
||||
lock (rig.Nm.Lock)
|
||||
@@ -276,11 +280,12 @@ public sealed class EventQueueOverflowAclWireTests
|
||||
throw new ShouldAssertException("revoking a grant under Lock while events were deferred stalled (deadlock)");
|
||||
}
|
||||
|
||||
stall.Release();
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), ct);
|
||||
await keySub.SetPublishingModeAsync(true, ct);
|
||||
|
||||
// Positive control: the still-granted session on the SAME folder channel gets the whole deferred tail.
|
||||
// Positive control: the still-granted session on the SAME folder gets the whole deferred tail.
|
||||
var lastNative = $"n{PerCondition - 1}";
|
||||
await RecordedEvents.WaitUntilAsync(() =>
|
||||
anonEvents.Has(anonA, lastNative) && anonEvents.Has(anonA, "after-revoke")
|
||||
|
||||
+841
@@ -0,0 +1,841 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Client;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
||||
using MonitoredItem = Opc.Ua.Client.MonitoredItem;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #855: a <c>ConditionRefresh</c> of an event item on an equipment / raw device FOLDER gets the guarantee the
|
||||
/// Server object got in #854 — no condition event older than what the refresh replayed reaches the item after
|
||||
/// RefreshEnd, per-condition order is kept, the per-folder ACL (#826) decides every event and every replayed condition,
|
||||
/// and neither a reporter nor the refresh ever waits on a stalled delivery.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Folder items are now served per item by the node manager (no SDK channel), so in production no notifier defers
|
||||
/// at all; <see cref="Production_overload_never_defers_and_never_overtakes_a_folder_refresh"/> pins that under a
|
||||
/// flood with concurrent refreshes. Every other test produces the deferral the cut exists for by stalling delivery
|
||||
/// at a notifier (<see cref="NotifierStall"/>), the way a full SDK channel used to.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The rig's native condition is wired into EqA and EqB and hangs under the raw device folder, so its walk is
|
||||
/// condition → EqA → … → device folder; a stall at EqA suspends it before EqA's items and every later hop.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class FolderConditionRefreshWireTests
|
||||
{
|
||||
private const string EqA = ServerEventRig.EqA;
|
||||
|
||||
// Every event after the stalled one is deferred; < 1024 pending, so nothing coalesces.
|
||||
private const int FloodSize = 500;
|
||||
|
||||
/// <param name="enforced">Per-folder ACL on or off (off is the production default).</param>
|
||||
/// <param name="finalRetained">Whether the condition is still Retain at refresh time (re-sent) or cleared (omitted).</param>
|
||||
/// <param name="refresh2">ConditionRefresh2 on the folder item instead of ConditionRefresh on its subscription.</param>
|
||||
/// <param name="modify">
|
||||
/// The folder item is modified (ModifyMonitoredItems re-subscribes it to its notifier) between the refresh and the
|
||||
/// drain: its cut must survive, as the Server object's does across an unsubscribe + resubscribe.
|
||||
/// </param>
|
||||
[Theory]
|
||||
[InlineData(false, true, false, false)]
|
||||
[InlineData(true, true, false, false)]
|
||||
[InlineData(false, false, false, false)]
|
||||
[InlineData(true, false, false, false)]
|
||||
[InlineData(false, true, true, false)]
|
||||
[InlineData(true, false, true, false)]
|
||||
[InlineData(false, true, false, true)]
|
||||
[InlineData(true, false, false, true)]
|
||||
public async Task Deferred_events_older_than_a_refresh_never_reach_the_refreshed_folder_item_after_RefreshEnd(
|
||||
bool enforced, bool finalRetained, bool refresh2, bool modify)
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
// A second, non-refreshing subscriber on the SAME folder: it must keep the whole deferred tail.
|
||||
using var otherSession = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var otherEvents = new RecordedEvents();
|
||||
var otherSubscription = await RecordedEvents.SubscribeAsync(otherSession, otherEvents, ct);
|
||||
var otherOnA = RecordedEvents.AddEventItem(otherSubscription, ServerEventRig.UnsNodeId(otherSession, EqA));
|
||||
await otherSubscription.ApplyChangesAsync(ct);
|
||||
otherOnA.Status.Error.ShouldBeNull();
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
onA.Status.Error.ShouldBeNull();
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed") && otherEvents.Has(otherOnA, "seed"));
|
||||
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the stall must actually have deferred events");
|
||||
rig.Project(native: true, finalRetained ? Active("final") : Cleared("final"));
|
||||
|
||||
var refresh = Stopwatch.StartNew();
|
||||
await RefreshAsync(session, subscription, onA, refresh2, ct);
|
||||
refresh.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), "ConditionRefresh must not wait for the stalled delivery");
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the deferred events were still waiting at RefreshEnd");
|
||||
rig.Nm.ActiveRefreshCuts.ShouldBe(1, "only the refreshed folder item holds a cut");
|
||||
|
||||
if (modify)
|
||||
{
|
||||
onA.QueueSize = 1001;
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
onA.Status.Error.ShouldBeNull();
|
||||
}
|
||||
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.ActiveRefreshCuts == 0);
|
||||
rig.Project(native: true, Active("after"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after") && otherEvents.Has(otherOnA, "after"));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
|
||||
|
||||
var (replayed, afterEnd) = Window(events.Of(onA));
|
||||
if (finalRetained)
|
||||
{
|
||||
replayed.ShouldBe(["final"], "the refresh re-sends the condition's current state");
|
||||
afterEnd.ShouldBe(["after"], "nothing older than the refreshed state may follow RefreshEnd");
|
||||
}
|
||||
else
|
||||
{
|
||||
replayed.ShouldBeEmpty("a cleared (Retain=false) condition is not part of a refresh");
|
||||
afterEnd.ShouldBe(["final", "after"], "stale Retain=true events would re-add the alarm; the Retain=false one agrees");
|
||||
events.Of(onA).Last(f => RecordedEvents.Message(f) == "final")[RecordedEvents.RetainField].GetBoolean().ShouldBeFalse();
|
||||
}
|
||||
|
||||
rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0);
|
||||
|
||||
// The other subscriber on the same folder did not refresh: it keeps every deferred event, in order.
|
||||
var flood = Indices(otherEvents.Of(otherOnA), 'f');
|
||||
flood.ShouldBe(Enumerable.Range(0, FloodSize).ToList(), "the non-refreshing folder subscriber keeps the whole tail, in order");
|
||||
otherEvents.Has(otherOnA, "final").ShouldBeTrue();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The race the item gate closes, for a folder item: a deferred event is between its supersession check and its
|
||||
/// queueing to the folder item when the refresh arrives. The refresh must wait for that one delivery, so it lands
|
||||
/// before the replay rather than after RefreshEnd.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_folder_refresh_waits_for_a_deferred_delivery_already_past_its_check()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Gate", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed"));
|
||||
|
||||
using var entered = new ManualResetEventSlim();
|
||||
using var proceed = new ManualResetEventSlim();
|
||||
var probed = 0;
|
||||
rig.Nm.DeferredFolderDeliveryProbe = () =>
|
||||
{
|
||||
if (Interlocked.Increment(ref probed) != 1) return;
|
||||
entered.Set();
|
||||
proceed.Wait(TimeSpan.FromSeconds(30));
|
||||
};
|
||||
|
||||
using (NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Project(native: true, Active("final"));
|
||||
}
|
||||
|
||||
// The released walk parks on its first deferred delivery to the folder item, past the check, inside the gate.
|
||||
entered.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue("the drain must reach the folder item");
|
||||
var refresh = session.CallAsync(
|
||||
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), ct);
|
||||
events.HasType(onA, ObjectTypeIds.RefreshEndEventType)
|
||||
.ShouldBeFalse("the refresh must wait for the delivery already inside the item's gate");
|
||||
proceed.Set();
|
||||
await refresh.WaitAsync(TimeSpan.FromSeconds(10), ct);
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
|
||||
rig.Nm.DeferredFolderDeliveryProbe = null;
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
rig.Project(native: true, Active("after"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after"));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
|
||||
|
||||
var (window, afterEnd) = Window(events.Of(onA));
|
||||
window.Count.ShouldBe(2, "the parked delivery, then the replay");
|
||||
window[0]!.ShouldStartWith("f");
|
||||
window[1].ShouldBe("final");
|
||||
afterEnd.ShouldBe(["after"]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The cut path queues its frozen replay OFF <c>Lock</c>. An event newer than the cut that reaches the folder item
|
||||
/// in that window (a condition delivered synchronously, reported from the probe between the cut and the queueing)
|
||||
/// must not be followed by its condition's older frozen replay entry.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task An_event_newer_than_the_folder_cut_is_never_followed_by_its_older_frozen_replay()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Pending", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
// A second condition under the device folder only: its walk never reaches the stalled EqA, so it stays synchronous.
|
||||
const string Free = ServerEventRig.RawDeviceFolder + "/free";
|
||||
rig.Sink.MaterialiseAlarmCondition(Free, ServerEventRig.RawDeviceFolder, "free", "OffNormalAlarm", 700, AddressSpaceRealm.Raw, isNative: true);
|
||||
rig.Sink.WriteAlarmCondition(Free, Active("x1"), DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
|
||||
// EqA needs an event item of its own to be a notifier fan-out the stall can apply to.
|
||||
using var eqASession = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var eqAEvents = new RecordedEvents();
|
||||
var eqASubscription = await RecordedEvents.SubscribeAsync(eqASession, eqAEvents, ct);
|
||||
RecordedEvents.AddEventItem(eqASubscription, ServerEventRig.UnsNodeId(eqASession, EqA));
|
||||
await eqASubscription.ApplyChangesAsync(ct);
|
||||
|
||||
// The refreshed item is on the raw device folder, which both conditions report through.
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onDevice = RecordedEvents.AddEventItem(subscription, RawNodeId(session, ServerEventRig.RawDeviceFolder));
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
onDevice.Status.Error.ShouldBeNull();
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onDevice, "seed"));
|
||||
|
||||
var injected = 0;
|
||||
rig.Nm.DirectRefreshProbe = (cut, _) =>
|
||||
{
|
||||
if (cut && Interlocked.Exchange(ref injected, 1) == 0)
|
||||
rig.Sink.WriteAlarmCondition(Free, Active("x2"), DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
};
|
||||
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the cut path runs only while something is deferred");
|
||||
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(onDevice, ObjectTypeIds.RefreshEndEventType));
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
rig.Nm.DirectRefreshProbe = null;
|
||||
Volatile.Read(ref injected).ShouldBe(1, "the folder refresh took the cut path");
|
||||
var stream = events.Of(onDevice).ToList();
|
||||
var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
|
||||
var free = stream.Take(end).Select(RecordedEvents.Message).Where(m => m is "x1" or "x2").ToList();
|
||||
free.ShouldContain("x2");
|
||||
free[^1].ShouldBe("x2", "the older frozen replay of the condition must not follow its newer event");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ACL on: a condition the folder refresh omitted because the item may not see it there (its reference into EqA was
|
||||
/// dropped by the generation) is not withheld afterwards — the reference returns before the drain, and the item must
|
||||
/// end on the condition's newest state. And while omitted, the replay itself carried nothing of it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Deferred_events_of_a_condition_the_folder_refresh_omitted_by_the_acl_are_not_withheld()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Acl", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
NodeAcl[] grants = [KeyEquipmentGrant(EqA)];
|
||||
var state = rig.Enforce(true, [ServerEventRig.Key()], grants);
|
||||
Deploy(state, "rev-2", grants, ServerEventRig.EqB); // the native alarm is no longer projected into EqA
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
onA.Status.Error.ShouldBeNull();
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
|
||||
events.Has(onA, "seed").ShouldBeFalse("the generation does not project the alarm into EqA");
|
||||
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Project(native: true, Active("final"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
|
||||
|
||||
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
|
||||
|
||||
// The reference returns while the condition's events still wait.
|
||||
Deploy(state, "rev-3", grants, EqA, ServerEventRig.EqB);
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "final"));
|
||||
var (replayed, after) = Window(events.Of(onA));
|
||||
replayed.ShouldBeEmpty("the refresh omitted the condition: not visible through EqA at the time");
|
||||
after.ShouldNotBeEmpty();
|
||||
after[^1].ShouldBe("final", "the item ends on the newest state of a condition it was never refreshed");
|
||||
after.Count(m => m!.StartsWith('f')).ShouldBeGreaterThan(0, "its deferred events were delivered, not withheld");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ACL on: a grant revoked while a refreshed folder item's deferred events wait. Neither the replay (taken while
|
||||
/// granted) nor anything after it leaks once revoked; a still-granted subscriber on the same folder gets the tail.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_grant_revoked_after_a_folder_refresh_withholds_every_deferred_event()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Revoke", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
var anonGrant = ServerEventRig.EquipmentGrant(EqA, NodePermissions.ReadOnly);
|
||||
var state = rig.Enforce(true, [ServerEventRig.Key()], KeyEquipmentGrant(EqA), anonGrant);
|
||||
|
||||
using var keyed = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var keyEvents = new RecordedEvents();
|
||||
var keySub = await RecordedEvents.SubscribeAsync(keyed, keyEvents, ct);
|
||||
var keyA = RecordedEvents.AddEventItem(keySub, ServerEventRig.UnsNodeId(keyed, EqA));
|
||||
await keySub.ApplyChangesAsync(ct);
|
||||
|
||||
using var anonymous = await rig.ConnectAsync(anonymous: true, ct);
|
||||
var anonEvents = new RecordedEvents();
|
||||
var anonSub = await RecordedEvents.SubscribeAsync(anonymous, anonEvents, ct);
|
||||
var anonA = RecordedEvents.AddEventItem(anonSub, ServerEventRig.UnsNodeId(anonymous, EqA));
|
||||
await anonSub.ApplyChangesAsync(ct);
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(keyA, "seed") && anonEvents.Has(anonA, "seed"));
|
||||
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
await keyed.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, keySub.Id);
|
||||
await RecordedEvents.WaitUntilAsync(() => keyEvents.HasType(keyA, ObjectTypeIds.RefreshEndEventType));
|
||||
|
||||
state.Apply(
|
||||
ServerEventRig.Cluster, "rev-2", [ServerEventRig.Key()], [anonGrant], ServerEventRig.ScopePaths(),
|
||||
ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(),
|
||||
ServerEventRig.TagReferences());
|
||||
rig.Project(native: true, Active("after-revoke"));
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
await RecordedEvents.WaitUntilAsync(() =>
|
||||
anonEvents.Has(anonA, $"f{FloodSize - 1}") && anonEvents.Has(anonA, "after-revoke"));
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), ct);
|
||||
|
||||
var (_, after) = Window(keyEvents.Of(keyA));
|
||||
after.ShouldBeEmpty("after RefreshEnd the revoked session gets nothing: older events are superseded, later ones denied");
|
||||
keyEvents.Has(keyA, "after-revoke").ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #855 review: the cut path decides its replay before taking <c>Lock</c>; a grant revoked between that decision
|
||||
/// and the queueing of the frozen replay must still withhold the replay (it is re-decided right before each entry).
|
||||
/// The revocation lands from the refresh probe, which runs after the cut is installed and before the replay is queued.
|
||||
/// </summary>
|
||||
/// <param name="serverObject">Refresh a Server-object item instead of a folder item.</param>
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task A_grant_revoked_between_the_cut_and_its_replay_withholds_the_replay(bool serverObject)
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855RevokeCut", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
var anonGrant = ServerEventRig.EquipmentGrant(EqA, NodePermissions.ReadOnly);
|
||||
var state = rig.Enforce(true, [ServerEventRig.Key()], KeyEquipmentGrant(EqA), anonGrant);
|
||||
|
||||
// An anonymous subscriber keeps EqA a notifier fan-out the stall applies to (and stays granted throughout).
|
||||
using var anonymous = await rig.ConnectAsync(anonymous: true, ct);
|
||||
var anonEvents = new RecordedEvents();
|
||||
var anonSub = await RecordedEvents.SubscribeAsync(anonymous, anonEvents, ct);
|
||||
RecordedEvents.AddEventItem(anonSub, ServerEventRig.UnsNodeId(anonymous, EqA));
|
||||
await anonSub.ApplyChangesAsync(ct);
|
||||
|
||||
using var keyed = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var keyEvents = new RecordedEvents();
|
||||
var keySub = await RecordedEvents.SubscribeAsync(keyed, keyEvents, ct);
|
||||
var item = RecordedEvents.AddEventItem(keySub, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(keyed, EqA));
|
||||
await keySub.ApplyChangesAsync(ct);
|
||||
item.Status.Error.ShouldBeNull();
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(item, "seed"));
|
||||
|
||||
var revoked = 0;
|
||||
void Revoke(bool cut, bool _)
|
||||
{
|
||||
if (!cut || Interlocked.Exchange(ref revoked, 1) != 0) return;
|
||||
state.Apply(
|
||||
ServerEventRig.Cluster, "rev-2", [ServerEventRig.Key()], [anonGrant], ServerEventRig.ScopePaths(),
|
||||
ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(),
|
||||
ServerEventRig.TagReferences());
|
||||
}
|
||||
|
||||
if (serverObject) rig.Nm.ServerObjectRefreshProbe = Revoke;
|
||||
else rig.Nm.DirectRefreshProbe = Revoke;
|
||||
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
|
||||
await keyed.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, keySub.Id);
|
||||
await RecordedEvents.WaitUntilAsync(() => keyEvents.HasType(item, ObjectTypeIds.RefreshEndEventType));
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
rig.Nm.ServerObjectRefreshProbe = null;
|
||||
rig.Nm.DirectRefreshProbe = null;
|
||||
Volatile.Read(ref revoked).ShouldBe(1, "the refresh took the cut path");
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
|
||||
|
||||
var (replayed, after) = Window(keyEvents.Of(item));
|
||||
replayed.ShouldBeEmpty("the grant was revoked before the replay was queued");
|
||||
after.ShouldBeEmpty("and nothing reaches the item after RefreshEnd either");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #855 review 2: a condition removed by a deploy between the cut and the queueing of its frozen replay must not be
|
||||
/// replayed — its frozen Retain=true state would be a ghost no Retain=false ever follows. ACL off, so nothing but the
|
||||
/// liveness check stands in the way.
|
||||
/// </summary>
|
||||
/// <param name="serverObject">Refresh a Server-object item instead of a folder item.</param>
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task A_condition_removed_between_the_cut_and_its_replay_is_not_replayed(bool serverObject)
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855RemoveCut", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
// A second subscriber keeps EqA a notifier fan-out the stall applies to.
|
||||
using var other = await rig.ConnectAsync(anonymous: true, ct);
|
||||
var otherEvents = new RecordedEvents();
|
||||
var otherSub = await RecordedEvents.SubscribeAsync(other, otherEvents, ct);
|
||||
RecordedEvents.AddEventItem(otherSub, ServerEventRig.UnsNodeId(other, EqA));
|
||||
await otherSub.ApplyChangesAsync(ct);
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var item = RecordedEvents.AddEventItem(subscription, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(session, EqA));
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(item, "seed"));
|
||||
|
||||
var removed = 0;
|
||||
void Remove(bool cut, bool _)
|
||||
{
|
||||
if (cut && Interlocked.Exchange(ref removed, 1) == 0)
|
||||
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
|
||||
}
|
||||
|
||||
if (serverObject) rig.Nm.ServerObjectRefreshProbe = Remove;
|
||||
else rig.Nm.DirectRefreshProbe = Remove;
|
||||
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
|
||||
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType));
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
rig.Nm.ServerObjectRefreshProbe = null;
|
||||
rig.Nm.DirectRefreshProbe = null;
|
||||
Volatile.Read(ref removed).ShouldBe(1, "the refresh took the cut path");
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
|
||||
|
||||
var (replayed, after) = Window(events.Of(item));
|
||||
replayed.ShouldBeEmpty("the condition was removed before its frozen replay was queued");
|
||||
after.ShouldBeEmpty("and none of its deferred events follows");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #855 review 3: the plain refresh path (nothing deferred — the only path production takes) collects live
|
||||
/// conditions off <c>Lock</c> and queues them afterwards; a condition removed in between must not be replayed.
|
||||
/// </summary>
|
||||
/// <param name="serverObject">Refresh a Server-object item instead of a folder item.</param>
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task A_condition_removed_during_a_plain_refresh_is_not_replayed(bool serverObject)
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855RemovePlain", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var item = RecordedEvents.AddEventItem(subscription, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(session, EqA));
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(item, "seed"));
|
||||
|
||||
var removed = 0;
|
||||
void Remove(bool cut, bool _)
|
||||
{
|
||||
if (!cut && Interlocked.Exchange(ref removed, 1) == 0)
|
||||
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
|
||||
}
|
||||
|
||||
if (serverObject) rig.Nm.ServerObjectRefreshProbe = Remove;
|
||||
else rig.Nm.DirectRefreshProbe = Remove;
|
||||
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType));
|
||||
rig.Nm.ServerObjectRefreshProbe = null;
|
||||
rig.Nm.DirectRefreshProbe = null;
|
||||
|
||||
Volatile.Read(ref removed).ShouldBe(1, "the plain path ran");
|
||||
Window(events.Of(item)).Replayed.ShouldBeEmpty("the condition was removed before the replay was queued");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #855 review: a delivery that fails inline (here: a hop that throws) is over, not suspended — it must not be booked
|
||||
/// as a deferral episode (which moved the episode a concurrent plain refresh checks, and cleared cuts under the
|
||||
/// event-report gate while <c>Lock</c> was held). The Server-object item, served before the failing hop, still gets
|
||||
/// the event, and the next event is delivered normally.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_delivery_that_throws_is_not_booked_as_a_deferral()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Fault", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
|
||||
var onServer = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
|
||||
var episodes = rig.Nm.DeferralEpisodes;
|
||||
rig.Nm.NotifierDeliveryStall = node =>
|
||||
node.NodeId.IdentifierAsString == EqA ? throw new InvalidOperationException("hop fault (test)") : null;
|
||||
rig.Project(native: true, Active("faulted"));
|
||||
rig.Nm.NotifierDeliveryStall = null;
|
||||
|
||||
rig.Nm.DeferredEventReports.ShouldBe(0);
|
||||
rig.Nm.DeferralEpisodes.ShouldBe(episodes, "a failed delivery is not a deferral episode");
|
||||
rig.Project(native: true, Active("next"));
|
||||
await RecordedEvents.WaitUntilAsync(() =>
|
||||
events.Has(onServer, "faulted") && events.Has(onServer, "next") && events.Has(onA, "next"));
|
||||
events.Has(onA, "faulted").ShouldBeFalse("the faulted hop delivered nothing");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Production shape, no stall: bursts reported under one <c>Lock</c> hold (an apply pass) and single reports, against
|
||||
/// folder items that refresh repeatedly while their client is slow to publish. Folder items are served per item, so
|
||||
/// nothing is ever deferred; each refresh's replay is never older than an event the item already received, nothing
|
||||
/// after RefreshEnd goes backwards, the stream converges on the newest state, and no replay is queued under <c>Lock</c>.
|
||||
/// </summary>
|
||||
/// <param name="enforced">Per-folder ACL on or off.</param>
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Production_overload_never_defers_and_never_overtakes_a_folder_refresh(bool enforced)
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Load", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
|
||||
onA.QueueSize = 100_000; // keep every event so the ordering check sees the whole stream
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
|
||||
rig.Project(native: true, Active("s0"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s0"));
|
||||
|
||||
var probes = new System.Collections.Concurrent.ConcurrentQueue<(bool Cut, bool LockHeld)>();
|
||||
rig.Nm.DirectRefreshProbe = (cut, held) => probes.Enqueue((cut, held));
|
||||
|
||||
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
var counter = 0;
|
||||
var peakDeferred = 0;
|
||||
var reporter = Task.Factory.StartNew(() =>
|
||||
{
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
lock (rig.Nm.Lock)
|
||||
{
|
||||
for (var i = 0; i < 1000; i++)
|
||||
{
|
||||
rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
|
||||
// Sampled inside Lock, after each report: a deferral cannot open and drain unseen (#855 review).
|
||||
peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports);
|
||||
}
|
||||
}
|
||||
|
||||
rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
|
||||
Thread.Sleep(2);
|
||||
}
|
||||
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
|
||||
|
||||
var refusedAsInProgress = 0;
|
||||
for (var r = 0; r < 30; r++)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RefreshAsync(session, subscription, onA, refresh2: r % 3 == 2, ct).WaitAsync(TimeSpan.FromSeconds(20), ct);
|
||||
}
|
||||
catch (ServiceResultException ex) when (ex.StatusCode == StatusCodes.BadRefreshInProgress)
|
||||
{
|
||||
refusedAsInProgress++;
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(80), ct);
|
||||
}
|
||||
|
||||
await stop.CancelAsync();
|
||||
await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
|
||||
rig.Nm.DirectRefreshProbe = null;
|
||||
|
||||
peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item by the reporter");
|
||||
rig.Nm.DiscardedEventReports.ShouldBe(0);
|
||||
var newest = $"s{Volatile.Read(ref counter)}";
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, newest));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
|
||||
|
||||
CheckStream(events.Of(onA)).ShouldBeEmpty();
|
||||
refusedAsInProgress.ShouldBeLessThan(30, "most refreshes must actually have run");
|
||||
probes.ShouldNotBeEmpty();
|
||||
probes.ShouldAllBe(p => !p.Cut && !p.LockHeld);
|
||||
RecordedEvents.Message(events.Of(onA).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)))
|
||||
.ShouldBe(newest, "the stream converges on the newest state");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Race / stress with deferral: reports (bursts under <c>Lock</c> and single), repeated refreshes of the folder item
|
||||
/// and an intermittent stall of its folder's delivery all run at once. Outside a refresh the folder item's stream
|
||||
/// never goes backwards, each refresh ends on the newest state the item has seen, and it converges. Nothing
|
||||
/// deadlocks, and neither refresh path queues its replay under <c>Lock</c>.
|
||||
/// </summary>
|
||||
/// <param name="enforced">Per-folder ACL on or off.</param>
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Concurrent_reports_folder_refreshes_and_stalls_never_deliver_a_pre_refresh_state_after_RefreshEnd(bool enforced)
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Stress", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
|
||||
onA.QueueSize = 100_000;
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
|
||||
rig.Project(native: true, Active("s0"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s0"));
|
||||
|
||||
var probes = new System.Collections.Concurrent.ConcurrentQueue<(bool Cut, bool LockHeld)>();
|
||||
rig.Nm.DirectRefreshProbe = (cut, held) => probes.Enqueue((cut, held));
|
||||
|
||||
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
var counter = 0;
|
||||
var reporter = Task.Factory.StartNew(() =>
|
||||
{
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
lock (rig.Nm.Lock)
|
||||
{
|
||||
for (var i = 0; i < 300; i++) rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
|
||||
}
|
||||
|
||||
rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
|
||||
Thread.Sleep(2);
|
||||
}
|
||||
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
|
||||
|
||||
var deferredSeen = 0;
|
||||
var staller = Task.Factory.StartNew(() =>
|
||||
{
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
using (NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
Thread.Sleep(300);
|
||||
if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen);
|
||||
}
|
||||
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
|
||||
|
||||
var refusedAsInProgress = 0;
|
||||
var stressDeadline = DateTime.UtcNow + TimeSpan.FromSeconds(90);
|
||||
for (var r = 0; r < 40 || (Volatile.Read(ref deferredSeen) <= 1 && DateTime.UtcNow < stressDeadline); r++)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RefreshAsync(session, subscription, onA, refresh2: r % 3 == 2, ct).WaitAsync(TimeSpan.FromSeconds(20), ct);
|
||||
}
|
||||
catch (ServiceResultException ex) when (ex.StatusCode == StatusCodes.BadRefreshInProgress)
|
||||
{
|
||||
refusedAsInProgress++;
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(80), ct);
|
||||
}
|
||||
|
||||
await stop.CancelAsync();
|
||||
await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
|
||||
await staller.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
|
||||
rig.Nm.DirectRefreshProbe = null;
|
||||
Volatile.Read(ref deferredSeen).ShouldBeGreaterThan(1, "the stress must actually have run with deferred events");
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
var newest = $"s{Volatile.Read(ref counter)}";
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, newest));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
|
||||
|
||||
CheckStream(events.Of(onA)).ShouldBeEmpty();
|
||||
refusedAsInProgress.ShouldBeLessThan(40, "most refreshes must actually have run");
|
||||
rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0, "some folder refresh must actually have overtaken deferred events");
|
||||
probes.ShouldContain(p => p.Cut, "some folder refresh must have taken the cut path");
|
||||
probes.ShouldAllBe(p => !p.LockHeld);
|
||||
RecordedEvents.Message(events.Of(onA).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)))
|
||||
.ShouldBe(newest, "the stream converges on the newest state");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Walks one condition's stream: outside a refresh its states never go backwards, and each refresh ends on the newest
|
||||
/// state the item has seen (its replay is never older than an event already delivered, and nothing older follows).
|
||||
/// </summary>
|
||||
private static List<string> CheckStream(IReadOnlyList<ArrayOf<Variant>> stream)
|
||||
{
|
||||
var max = -1;
|
||||
var last = -1;
|
||||
var replaying = false;
|
||||
var violations = new List<string>();
|
||||
foreach (var f in stream)
|
||||
{
|
||||
var type = f[RecordedEvents.EventTypeField].GetNodeId();
|
||||
if (type == ObjectTypeIds.RefreshStartEventType)
|
||||
{
|
||||
replaying = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (type == ObjectTypeIds.RefreshEndEventType)
|
||||
{
|
||||
replaying = false;
|
||||
if (last < max) violations.Add($"a refresh ended on s{last} after s{max} had been delivered");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!f[RecordedEvents.AckedField].TryGetValue(out bool _)) continue;
|
||||
var n = int.Parse(RecordedEvents.Message(f)![1..], CultureInfo.InvariantCulture);
|
||||
if (!replaying && n < max) violations.Add($"s{n} delivered after s{max}");
|
||||
max = Math.Max(max, n);
|
||||
last = n;
|
||||
}
|
||||
|
||||
return violations;
|
||||
}
|
||||
|
||||
private static Task RefreshAsync(ISession session, Subscription subscription, MonitoredItem item, bool refresh2, CancellationToken ct) =>
|
||||
refresh2
|
||||
? session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct, subscription.Id, item.Status.Id)
|
||||
: session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
|
||||
|
||||
/// <summary>The condition-event messages inside the item's LAST RefreshStart..RefreshEnd, and those after it.</summary>
|
||||
private static (List<string?> Replayed, List<string?> AfterEnd) Window(IReadOnlyList<ArrayOf<Variant>> stream)
|
||||
{
|
||||
var list = stream.ToList();
|
||||
var start = list.FindLastIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
|
||||
var end = list.FindLastIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
|
||||
start.ShouldBeGreaterThanOrEqualTo(0);
|
||||
end.ShouldBeGreaterThan(start);
|
||||
static bool IsCondition(ArrayOf<Variant> f) => f[RecordedEvents.AckedField].TryGetValue(out bool _);
|
||||
return (
|
||||
list.Skip(start + 1).Take(end - start - 1).Where(IsCondition).Select(RecordedEvents.Message).ToList(),
|
||||
list.Skip(end + 1).Where(IsCondition).Select(RecordedEvents.Message).ToList());
|
||||
}
|
||||
|
||||
private static List<int> Indices(IReadOnlyList<ArrayOf<Variant>> received, char prefix) =>
|
||||
received
|
||||
.Select(RecordedEvents.Message)
|
||||
.Where(m => m is { Length: > 1 } && m[0] == prefix && char.IsAsciiDigit(m[1]))
|
||||
.Select(m => int.Parse(m![1..], CultureInfo.InvariantCulture))
|
||||
.ToList();
|
||||
|
||||
private static NodeId RawNodeId(ISession session, string id) =>
|
||||
new(id, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri));
|
||||
|
||||
private static void Deploy(DataPlaneAuthorityState state, string revision, NodeAcl[] grants, params string[] referencing) =>
|
||||
state.Apply(
|
||||
ServerEventRig.Cluster, revision, [ServerEventRig.Key()], grants, ServerEventRig.ScopePaths(),
|
||||
ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(),
|
||||
new Dictionary<string, IReadOnlySet<string>> { [ServerEventRig.RawAlarmPath] = referencing.ToHashSet() });
|
||||
|
||||
private static AlarmConditionSnapshot Active(string message) =>
|
||||
new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message);
|
||||
|
||||
private static AlarmConditionSnapshot Cleared(string message) =>
|
||||
new(false, true, true, true, AlarmShelvingKind.Unshelved, 700, message);
|
||||
|
||||
private static NodeAcl KeyEquipmentGrant(string equipment) => new()
|
||||
{
|
||||
NodeAclId = $"acl-key-{equipment}",
|
||||
ClusterId = ServerEventRig.Cluster,
|
||||
Principal = ServerEventRig.KeyPrincipal,
|
||||
ScopeKind = NodeAclScopeKind.Equipment,
|
||||
ScopeId = $"eq-{equipment}",
|
||||
PermissionFlags = NodePermissions.ReadOnly,
|
||||
};
|
||||
}
|
||||
+350
@@ -0,0 +1,350 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Client;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
||||
using MonitoredItem = Opc.Ua.Client.MonitoredItem;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #855 cost measurement: since #855 a condition event is queued to every event item on an equipment folder by
|
||||
/// the reporter itself, under the node manager's <c>Lock</c> (per-item verdict, <c>QueueEvent</c> with the SDK's
|
||||
/// filter evaluation, duplicate scan over the last 1000 queued events and a <c>List.RemoveAt(0)</c> at a full queue).
|
||||
/// Before #855 that work ran on the SDK's per-node channel consumer, off <c>Lock</c>. This harness reports, per alarm
|
||||
/// report, the <c>Lock</c> hold time and the end-to-end delivery latency, for one equipment folder carrying N event
|
||||
/// items, ACL on / off, with the folder's subscribers publishing or stalled (publishing disabled, item queues full and
|
||||
/// discarding oldest) — and the latency of a Server-object (<c>i=2253</c>) subscriber while the folder's subscribers
|
||||
/// are stalled (the ScadaBridge shape).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>Measurement only: nothing is asserted beyond the harness's own sanity checks. Opt-in and never part of a
|
||||
/// default run: every case skips unless <c>OTOPCUA_EVENT_PERF=1</c>.</para>
|
||||
/// <para>Run: <c>OTOPCUA_EVENT_PERF=1 dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests -c Release
|
||||
/// --filter "FullyQualifiedName~FolderEventDeliveryPerfTests" --logger "console;verbosity=detailed"</c>.
|
||||
/// <c>OTOPCUA_EVENT_PERF_REPORTS</c> sets the measured reports per case (default 1000) and
|
||||
/// <c>OTOPCUA_EVENT_PERF_OUT</c> appends each result row to a file.</para>
|
||||
/// <para>The rig is the same one <see cref="FolderEventFloodWireTests"/> uses and existed before #855, so this file
|
||||
/// compiles against the pre-#855 tree for a before/after comparison.</para>
|
||||
/// </remarks>
|
||||
[Trait("Category", "Perf")]
|
||||
public sealed class FolderEventDeliveryPerfTests
|
||||
{
|
||||
private const string OptInVariable = "OTOPCUA_EVENT_PERF";
|
||||
private const string EqA = ServerEventRig.EqA;
|
||||
private const uint DefaultQueueSize = 1000;
|
||||
private const double PublishingIntervalMs = 50;
|
||||
|
||||
private static int Reports =>
|
||||
int.TryParse(Environment.GetEnvironmentVariable("OTOPCUA_EVENT_PERF_REPORTS"), out var r) && r > 0 ? r : 1000;
|
||||
|
||||
private static void RequireOptIn()
|
||||
{
|
||||
if (Environment.GetEnvironmentVariable(OptInVariable) != "1")
|
||||
Assert.Skip($"perf harness: set {OptInVariable}=1 to run (not part of a default run)");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lock hold per report and (publishing only) the time until the LAST of the N folder items received the event.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(1, false, false)]
|
||||
[InlineData(10, false, false)]
|
||||
[InlineData(50, false, false)]
|
||||
[InlineData(200, false, false)]
|
||||
[InlineData(1, true, false)]
|
||||
[InlineData(10, true, false)]
|
||||
[InlineData(50, true, false)]
|
||||
[InlineData(200, true, false)]
|
||||
[InlineData(1, false, true)]
|
||||
[InlineData(10, false, true)]
|
||||
[InlineData(50, false, true)]
|
||||
[InlineData(200, false, true)]
|
||||
[InlineData(1, true, true)]
|
||||
[InlineData(10, true, true)]
|
||||
[InlineData(50, true, true)]
|
||||
[InlineData(200, true, true)]
|
||||
public Task Folder_items(int n, bool acl, bool stalled) => RunAsync(n, acl, stalled, serverObjectProbe: false);
|
||||
|
||||
/// <summary>
|
||||
/// A publishing Server-object (<c>i=2253</c>) subscriber on its own session, while N folder items on another
|
||||
/// session are stalled with full queues. N = 0 is the baseline.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(0, false)]
|
||||
[InlineData(50, false)]
|
||||
[InlineData(200, false)]
|
||||
[InlineData(0, true)]
|
||||
[InlineData(50, true)]
|
||||
[InlineData(200, true)]
|
||||
public Task Server_object_subscriber_while_folder_items_stall(int n, bool acl) =>
|
||||
RunAsync(n, acl, stalled: true, serverObjectProbe: true);
|
||||
|
||||
/// <summary>
|
||||
/// Full-queue sensitivity: N = 50 stalled items at different <c>QueueSize</c>s. At a full queue every
|
||||
/// <c>QueueEvent</c> scans the last min(QueueSize, 1000) queued events for a duplicate and shifts the whole queue
|
||||
/// (<c>List.RemoveAt(0)</c>), so the per-item cost grows with the queue, not with N.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(10u)]
|
||||
[InlineData(100u)]
|
||||
[InlineData(1000u)]
|
||||
[InlineData(10000u)]
|
||||
public Task Stalled_items_queue_size_sensitivity(uint queueSize) =>
|
||||
RunAsync(50, acl: false, stalled: true, serverObjectProbe: false, queueSize);
|
||||
|
||||
private static async Task RunAsync(
|
||||
int n, bool acl, bool stalled, bool serverObjectProbe, uint queueSize = DefaultQueueSize)
|
||||
{
|
||||
RequireOptIn();
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync($"Perf855-{n}", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(acl, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
var reports = Reports;
|
||||
var reportedAt = new long[reports];
|
||||
var receipts = new Receipts(reports);
|
||||
|
||||
ISession? folderSession = null;
|
||||
ISession? serverSession = null;
|
||||
try
|
||||
{
|
||||
Subscription? folderSubscription = null;
|
||||
var folderHandles = new HashSet<uint>();
|
||||
if (n > 0)
|
||||
{
|
||||
folderSession = await rig.ConnectAsync(anonymous: false, ct);
|
||||
folderSubscription = await SubscribeAsync(folderSession, receipts, isServer: false, ct);
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
var item = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA));
|
||||
item.QueueSize = queueSize;
|
||||
folderHandles.Add(item.ClientHandle);
|
||||
}
|
||||
|
||||
await folderSubscription.ApplyChangesAsync(ct);
|
||||
folderSubscription.MonitoredItems.ShouldAllBe(m => m.Status.Error == null);
|
||||
// The server's revision (MaxEventQueueSize) — what the measured queues actually hold.
|
||||
queueSize = folderSubscription.MonitoredItems.First().Status.QueueSize;
|
||||
}
|
||||
|
||||
receipts.FolderHandles = folderHandles;
|
||||
|
||||
MonitoredItem? onServer = null;
|
||||
if (serverObjectProbe)
|
||||
{
|
||||
serverSession = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var serverSubscription = await SubscribeAsync(serverSession, receipts, isServer: true, ct);
|
||||
onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server);
|
||||
await serverSubscription.ApplyChangesAsync(ct);
|
||||
onServer.Status.Error.ShouldBeNull();
|
||||
receipts.ServerHandle = onServer.ClientHandle;
|
||||
}
|
||||
|
||||
if (stalled && folderSubscription is not null)
|
||||
{
|
||||
await folderSubscription.SetPublishingModeAsync(false, ct);
|
||||
|
||||
// Fill every folder item's queue to QueueSize (and past it, so the measured reports run at a full queue,
|
||||
// discarding oldest), then let any off-Lock consumer (pre-#855: the SDK's channel task) catch up.
|
||||
for (var i = 0; i < queueSize + 50; i++) rig.Project(native: true, State($"w{i}"));
|
||||
await SettleAsync(rig, n, ct);
|
||||
}
|
||||
|
||||
// Warm-up (JIT, first-verdict caches) — not measured.
|
||||
for (var i = 0; i < 50; i++) rig.Project(native: true, State($"x{i}"));
|
||||
await SettleAsync(rig, n, ct);
|
||||
|
||||
var holds = new double[reports];
|
||||
for (var i = 0; i < reports; i++)
|
||||
{
|
||||
var state = State($"p{i}");
|
||||
long t0, t1;
|
||||
lock (rig.Nm.Lock)
|
||||
{
|
||||
t0 = Stopwatch.GetTimestamp();
|
||||
reportedAt[i] = t0;
|
||||
rig.Project(native: true, state);
|
||||
t1 = Stopwatch.GetTimestamp();
|
||||
}
|
||||
|
||||
holds[i] = Stopwatch.GetElapsedTime(t0, t1).TotalMicroseconds;
|
||||
// ~1 kHz report rate: an alarm storm, not a single burst, so a consumer that keeps up is not penalised.
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
|
||||
// Wait for the tail of every probe that is expected to receive.
|
||||
var expectFolder = n > 0 && !stalled;
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
while (DateTime.UtcNow < deadline
|
||||
&& ((expectFolder && !receipts.FolderComplete(reports - 1, n))
|
||||
|| (serverObjectProbe && !receipts.ServerHas(reports - 1))))
|
||||
{
|
||||
await Task.Delay(50, ct);
|
||||
}
|
||||
|
||||
var deferredPeak = rig.Nm.DeferredEventReports;
|
||||
var row = new StringBuilder();
|
||||
row.Append(CultureInfo.InvariantCulture,
|
||||
$"{(serverObjectProbe ? "i=2253" : "folder"),-6} N={n,-3} acl={(acl ? "on " : "off")} {(stalled ? "stalled " : "publishing")} q={queueSize,-5} | ");
|
||||
row.Append(CultureInfo.InvariantCulture, $"lock hold us p50 {Pct(holds, 50),8:F1} p90 {Pct(holds, 90),8:F1} p99 {Pct(holds, 99),8:F1} max {holds.Max(),9:F1} mean {holds.Average(),8:F1}");
|
||||
row.Append(CultureInfo.InvariantCulture, $" | per-item mean {(n > 0 ? holds.Average() / n : 0),6:F2} us");
|
||||
if (expectFolder)
|
||||
{
|
||||
var lat = receipts.FolderLatenciesMs(reportedAt, n);
|
||||
row.Append(CultureInfo.InvariantCulture,
|
||||
$" | folder all-N latency ms p50 {Pct(lat, 50),6:F1} p99 {Pct(lat, 99),6:F1} max {Max(lat),7:F1} (got {lat.Length}/{reports})");
|
||||
}
|
||||
|
||||
if (serverObjectProbe)
|
||||
{
|
||||
var lat = receipts.ServerLatenciesMs(reportedAt);
|
||||
row.Append(CultureInfo.InvariantCulture,
|
||||
$" | i=2253 latency ms p50 {Pct(lat, 50),6:F1} p99 {Pct(lat, 99),6:F1} max {Max(lat),7:F1} (got {lat.Length}/{reports})");
|
||||
}
|
||||
|
||||
row.Append(CultureInfo.InvariantCulture, $" | deferred-now {deferredPeak}");
|
||||
Report(row.ToString());
|
||||
}
|
||||
finally
|
||||
{
|
||||
folderSession?.Dispose();
|
||||
serverSession?.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gives an off-<c>Lock</c> consumer (pre-#855) time to drain what was just reported; post-#855 nothing is pending.
|
||||
/// </summary>
|
||||
private static async Task SettleAsync(ServerEventRig rig, int n, CancellationToken ct)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(60);
|
||||
while (rig.Nm.DeferredEventReports > 0 && DateTime.UtcNow < deadline) await Task.Delay(50, ct);
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500 + (10 * n)), ct);
|
||||
}
|
||||
|
||||
private static async Task<Subscription> SubscribeAsync(ISession session, Receipts receipts, bool isServer, CancellationToken ct)
|
||||
{
|
||||
var subscription = new Subscription(session.DefaultSubscription)
|
||||
{
|
||||
PublishingInterval = (int)PublishingIntervalMs,
|
||||
MaxNotificationsPerPublish = 0,
|
||||
};
|
||||
subscription.FastEventCallback = receipts.OnEvents;
|
||||
session.AddSubscription(subscription);
|
||||
await subscription.CreateAsync(ct);
|
||||
return subscription;
|
||||
}
|
||||
|
||||
private static double Pct(double[] values, double p)
|
||||
{
|
||||
if (values.Length == 0) return double.NaN;
|
||||
var sorted = values.Order().ToArray();
|
||||
var idx = (int)Math.Ceiling(p / 100.0 * sorted.Length) - 1;
|
||||
return sorted[Math.Clamp(idx, 0, sorted.Length - 1)];
|
||||
}
|
||||
|
||||
private static double Max(double[] values) => values.Length == 0 ? double.NaN : values.Max();
|
||||
|
||||
private static void Report(string text)
|
||||
{
|
||||
TestContext.Current.TestOutputHelper?.WriteLine(text);
|
||||
if (Environment.GetEnvironmentVariable("OTOPCUA_EVENT_PERF_OUT") is { Length: > 0 } path)
|
||||
File.AppendAllText(path, text + Environment.NewLine);
|
||||
}
|
||||
|
||||
private static AlarmConditionSnapshot State(string message) =>
|
||||
new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message);
|
||||
|
||||
private static NodeAcl KeyEquipmentGrant(string equipment) => new()
|
||||
{
|
||||
NodeAclId = $"acl-key-{equipment}",
|
||||
ClusterId = ServerEventRig.Cluster,
|
||||
Principal = ServerEventRig.KeyPrincipal,
|
||||
ScopeKind = NodeAclScopeKind.Equipment,
|
||||
ScopeId = $"eq-{equipment}",
|
||||
PermissionFlags = NodePermissions.ReadOnly,
|
||||
};
|
||||
|
||||
/// <summary>Client-side receipt times of the measured (<c>p{i}</c>) events, per probe.</summary>
|
||||
private sealed class Receipts(int reports)
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private readonly int[] _folderCount = new int[reports];
|
||||
private readonly long[] _folderLast = new long[reports];
|
||||
private readonly long[] _serverFirst = new long[reports];
|
||||
|
||||
public IReadOnlySet<uint> FolderHandles { get; set; } = new HashSet<uint>();
|
||||
|
||||
public uint? ServerHandle { get; set; }
|
||||
|
||||
public void OnEvents(Subscription subscription, EventNotificationList notification, ArrayOf<string> stringTable)
|
||||
{
|
||||
var now = Stopwatch.GetTimestamp();
|
||||
lock (_gate)
|
||||
{
|
||||
foreach (var e in notification.Events)
|
||||
{
|
||||
var message = RecordedEvents.Message(e.EventFields);
|
||||
if (message is not { Length: > 1 } || message[0] != 'p') continue;
|
||||
if (!int.TryParse(message.AsSpan(1), NumberStyles.Integer, CultureInfo.InvariantCulture, out var i)
|
||||
|| i < 0 || i >= _folderCount.Length)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ServerHandle == e.ClientHandle)
|
||||
{
|
||||
if (_serverFirst[i] == 0) _serverFirst[i] = now;
|
||||
}
|
||||
else if (FolderHandles.Contains(e.ClientHandle))
|
||||
{
|
||||
_folderCount[i]++;
|
||||
_folderLast[i] = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public bool FolderComplete(int i, int n)
|
||||
{
|
||||
lock (_gate) return _folderCount[i] >= n;
|
||||
}
|
||||
|
||||
public bool ServerHas(int i)
|
||||
{
|
||||
lock (_gate) return _serverFirst[i] != 0;
|
||||
}
|
||||
|
||||
public double[] FolderLatenciesMs(long[] reportedAt, int n)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return Enumerable.Range(0, reportedAt.Length)
|
||||
.Where(i => _folderCount[i] >= n)
|
||||
.Select(i => Stopwatch.GetElapsedTime(reportedAt[i], _folderLast[i]).TotalMilliseconds)
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
public double[] ServerLatenciesMs(long[] reportedAt)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return Enumerable.Range(0, reportedAt.Length)
|
||||
.Where(i => _serverFirst[i] != 0)
|
||||
.Select(i => Stopwatch.GetElapsedTime(reportedAt[i], _serverFirst[i]).TotalMilliseconds)
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+36
-19
@@ -19,6 +19,13 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
/// consumer — a deadlock that also freezes every other operation needing <c>Lock</c>. Publishing is switched off
|
||||
/// on the client so nothing downstream drains either (the slow / absent consumer case).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Since Gitea #855 condition events no longer go through that channel at all: the node manager queues them to a
|
||||
/// folder's items itself, per item, on the reporter's thread (as it always did for Server-object items), so the same
|
||||
/// flood now completes without deferring anything. The deadlock test therefore also pins that per-item delivery is in
|
||||
/// force (it would defer again if the bypass stopped recognising the SDK's fan-out); the ghost test produces its
|
||||
/// deferral with <see cref="NotifierStall"/>.
|
||||
/// </remarks>
|
||||
public sealed class FolderEventFloodWireTests
|
||||
{
|
||||
private const string EqA = ServerEventRig.EqA;
|
||||
@@ -56,8 +63,12 @@ public sealed class FolderEventFloodWireTests
|
||||
// No publishing: the server-side item queue (QueueSize 1000) is the only sink, as with a stalled client.
|
||||
await subscription.SetPublishingModeAsync(false, ct);
|
||||
|
||||
// #855: the folder's SDK fan-out is the one the node manager recognises and serves per item itself.
|
||||
rig.Nm.TryGetFolder(EqA)!.OnReportEventAsync?.Target.ShouldBeOfType<Opc.Ua.Server.MonitoredNode2>();
|
||||
|
||||
// The whole burst under ONE hold of the node manager's Lock — the shape of any multi-event apply/projection
|
||||
// pass — starting from an uncached verdict (the item is new), so the delivery task needs Lock for event 1.
|
||||
var peakDeferred = 0;
|
||||
var flood = Task.Factory.StartNew(() =>
|
||||
{
|
||||
lock (rig.Nm.Lock)
|
||||
@@ -65,6 +76,7 @@ public sealed class FolderEventFloodWireTests
|
||||
for (var i = 0; i < FloodSize; i++)
|
||||
{
|
||||
rig.Project(native: true, State($"f{i}"));
|
||||
peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports);
|
||||
Volatile.Write(ref reported, i + 1);
|
||||
}
|
||||
}
|
||||
@@ -83,11 +95,9 @@ public sealed class FolderEventFloodWireTests
|
||||
+ "on Lock (deadlock)");
|
||||
}
|
||||
|
||||
// The overflow was deferred rather than blocking, and drains once Lock is free.
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
// …and the SDK's channel (up to 4096 accepted events) finishes draining into the item queue, still with
|
||||
// publishing off, so the count below measures the item queue and not events in transit.
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), ct);
|
||||
// #855: nothing was deferred at all — every event was queued to the folder item by the reporter itself.
|
||||
peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item, not through the SDK channel");
|
||||
rig.Nm.DeferredEventReports.ShouldBe(0);
|
||||
|
||||
// Delivery is bounded by the item's queue (QueueSize 1000, oldest discarded), not by the flood, and the
|
||||
// events that survive arrive in report order, ending with the last one reported.
|
||||
@@ -104,8 +114,7 @@ public sealed class FolderEventFloodWireTests
|
||||
flooded.Count.ShouldBeLessThanOrEqualTo((int)onA.Status.QueueSize, "delivery is bounded by the item's queue, not the flood");
|
||||
flooded.ShouldBe(flooded.Order().ToList(), "events leave in the order they were reported");
|
||||
flooded[^1].ShouldBe(FloodSize - 1);
|
||||
rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0,
|
||||
"past the per-condition budget the oldest waiting events give way to the newest (coalesced)");
|
||||
rig.Nm.DiscardedEventReports.ShouldBe(0, "nothing was deferred, so nothing was coalesced (the item queue is the bound)");
|
||||
|
||||
// And the node manager is live: a later event is decided by the per-folder ACL and delivered.
|
||||
rig.Project(native: true, State("after"));
|
||||
@@ -127,7 +136,8 @@ public sealed class FolderEventFloodWireTests
|
||||
/// #838 review — events deferred behind a full channel must not outlive their condition. A deploy that removes
|
||||
/// the condition while its events wait used to let them reach Server-object (<c>i=2253</c>) subscribers after the
|
||||
/// NodeDeleted — a ghost Retain alarm for ScadaBridge — and the folder subscribers too. Every hop of a deferred
|
||||
/// report now re-checks that the condition is still the live instance.
|
||||
/// report now re-checks that the condition is still the live instance (#855: also after a stall at a hop, before
|
||||
/// the folder's items are served).
|
||||
/// </summary>
|
||||
/// <param name="enforced">Per-folder / Server-object ACL on or off (off is the production default).</param>
|
||||
[Theory]
|
||||
@@ -151,16 +161,20 @@ public sealed class FolderEventFloodWireTests
|
||||
onServer.Status.Error.ShouldBeNull();
|
||||
await subscription.SetPublishingModeAsync(false, ct);
|
||||
|
||||
// One Lock hold: flood past the channel (the tail is deferred), then the deploy removes the condition.
|
||||
await Task.Factory.StartNew(() =>
|
||||
// One Lock hold, delivery stalled at the folder: flood (the tail is deferred), then the deploy removes the condition.
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
lock (rig.Nm.Lock)
|
||||
await Task.Factory.StartNew(() =>
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
|
||||
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
|
||||
}
|
||||
}, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct);
|
||||
lock (rig.Nm.Lock)
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
|
||||
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
|
||||
}
|
||||
}, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct);
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the waiting events were dropped, not delivered");
|
||||
@@ -174,12 +188,15 @@ public sealed class FolderEventFloodWireTests
|
||||
.Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
|
||||
.ToArray();
|
||||
|
||||
// Positive control: the events reported before the channel filled did arrive (Server object: synchronously at
|
||||
// the condition's own hop).
|
||||
// Positive control: the event reported before the stall did arrive at the Server object (synchronously, at the
|
||||
// condition's own hop, before the walk reached the stalled folder).
|
||||
Flooded(onServer).ShouldNotBeEmpty();
|
||||
// None of the deferred tail — which was still waiting when the condition was removed — reached anyone.
|
||||
Flooded(onServer).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the Server-object subscriber");
|
||||
Flooded(onA).DefaultIfEmpty(-1).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the folder subscriber");
|
||||
// Only the in-flight f0 ever reached the stalled EqA hop, and the condition was gone when the stall released: the
|
||||
// post-stall liveness re-check must have withheld it (#855 review 4 — "< FloodSize - 1" passed without that check).
|
||||
Flooded(onA).ShouldBeEmpty("no ghost reached the folder subscriber");
|
||||
Flooded(onServer).ShouldBe([0], "only the event reported before the stall reached the Server object");
|
||||
}
|
||||
|
||||
private static AlarmConditionSnapshot State(string message) =>
|
||||
|
||||
+4
-4
@@ -97,10 +97,10 @@ public sealed class FolderEventVerdictInvalidationTests
|
||||
rig.Project(native: false, State("s1"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s1"));
|
||||
|
||||
// Holding the node-manager lock parks EqA's SDK delivery task: the native event "n1" is queued first and its
|
||||
// verdict (a cache miss after the first invalidation) needs the lock in the inherited role check — its node is
|
||||
// never removed, so that check always takes it. The OLD scripted instance's "s2", queued behind it on the same
|
||||
// single-reader queue, is therefore decided only after the replace and the apply's invalidation.
|
||||
// Before Gitea #855, holding the node-manager lock parked EqA's SDK delivery task, so the OLD scripted instance's
|
||||
// "s2" was decided (and its verdict cached) only after the replace and the apply's invalidation. Condition events
|
||||
// are now decided per item at report time, uncached — "s2" is delivered while its instance is live — and this
|
||||
// pins that the replace still leaves the new instance's events flowing either way.
|
||||
lock (rig.Nm.Lock)
|
||||
{
|
||||
rig.Nm.CompleteAddressSpaceApply(); // forget the cached allows, so both verdicts must be computed
|
||||
|
||||
+14
-10
@@ -6,9 +6,11 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// #821 / #809 facts, through the BOOTED server's runtime <c>AggregateManager</c> — the calculators a live
|
||||
/// AggregateFilter monitored item gets, under the server's own default configuration. This pins current behaviour for
|
||||
/// the owner decision; it does not choose. The HistoryRead (Part 13) answers and SDK 2.0.1's stock answers for the
|
||||
/// same input are pinned beside it in <c>SdkAggregateParityTests</c>.
|
||||
/// AggregateFilter monitored item gets, under the server's own default configuration. Both are decided (owner,
|
||||
/// 2026-11-11): live PercentGood / DurationGood count Good time only (#809), and live Total / TimeAverage integrate only
|
||||
/// the covered part of an interval the data starts inside, even with an Uncertain first sample (#821) — equal to
|
||||
/// HistoryRead. The HistoryRead (Part 13) answers, the untouched baseline's and SDK 2.0.1's stock answers for the same
|
||||
/// input are pinned beside it in <c>SdkAggregateParityTests</c>.
|
||||
/// </summary>
|
||||
public sealed class LiveAggregateLateStartFactsTests
|
||||
{
|
||||
@@ -40,14 +42,16 @@ public sealed class LiveAggregateLateStartFactsTests
|
||||
return d;
|
||||
}
|
||||
|
||||
// Part 13 (HistoryRead on SqlServer/QuestDB): Total = 73.5 + 88 + 32/13 = 163.96…, TimeAverage = Total ÷ 15 s.
|
||||
// The live calculators integrate the uncovered [0,5) as a ramp from 0 (+25) and divide by the whole 20 s.
|
||||
// #821: Part 13 (HistoryRead on SqlServer/QuestDB): Total = 73.5 + 88 + 32/13 = 163.96…, TimeAverage = Total ÷ 15 s
|
||||
// — live as in HistoryRead (before #821 the live calculator integrated the uncovered [0,5) as a ramp from 0 (+25)
|
||||
// and divided by the whole 20 s: 188.96… / 9.448…).
|
||||
const double part13Total = 73.5 + 88 + 32d / 13;
|
||||
One(ObjectIds.AggregateFunction_Total).ShouldBe(part13Total + 25, 1e-9); // 188.96…
|
||||
One(ObjectIds.AggregateFunction_TimeAverage).ShouldBe((part13Total + 25) / 20, 1e-9); // 9.448… (Part 13: 10.93…)
|
||||
// #809: the Uncertain [5,12) counts as Good live (HistoryRead: 8 000 ms / 40 %).
|
||||
One(ObjectIds.AggregateFunction_DurationGood).ShouldBe(15_000d, 1e-9);
|
||||
One(ObjectIds.AggregateFunction_PercentGood).ShouldBe(75d, 1e-9);
|
||||
One(ObjectIds.AggregateFunction_Total).ShouldBe(part13Total, 1e-9); // 163.96…
|
||||
One(ObjectIds.AggregateFunction_TimeAverage).ShouldBe(part13Total / 15, 1e-9); // 10.93…
|
||||
// #809: the Uncertain [5,12) is not Good — only [12,20) is, live as in HistoryRead (8 000 ms / 40 %; before #809
|
||||
// the live calculator counted the Uncertain time too: 15 000 ms / 75 %).
|
||||
One(ObjectIds.AggregateFunction_DurationGood).ShouldBe(8_000d, 1e-9);
|
||||
One(ObjectIds.AggregateFunction_PercentGood).ShouldBe(40d, 1e-9);
|
||||
// DurationBad / PercentBad agree with HistoryRead: the 5 s before the data is Bad.
|
||||
One(ObjectIds.AggregateFunction_DurationBad).ShouldBe(5_000d, 1e-9);
|
||||
One(ObjectIds.AggregateFunction_PercentBad).ShouldBe(25d, 1e-9);
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
using Opc.Ua;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Stalls condition-event delivery at one of the node manager's notifiers (Gitea #855): every condition event whose walk
|
||||
/// reaches that notifier suspends there until <see cref="Release"/>, without blocking the reporter or holding the node
|
||||
/// manager's <c>Lock</c> — so the condition's later events are deferred, as they were behind a full SDK channel before
|
||||
/// folder items were delivered per item.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The pre-#855 tests produced deferral by holding a folder item's server-side <c>MonitoredItem</c> lock, which parked
|
||||
/// the SDK's per-node delivery task. Folder items are now queued synchronously by the reporter (as Server-object items
|
||||
/// always were), so holding an item lock would block the reporter instead; this goes through the node manager's test
|
||||
/// probe (<see cref="OtOpcUaNodeManager.NotifierDeliveryStall"/>). The probe is consulted only at a notifier that has
|
||||
/// event items (the hop the per-item delivery serves), so the stalled notifier must carry at least one.
|
||||
/// </remarks>
|
||||
internal sealed class NotifierStall : IDisposable
|
||||
{
|
||||
private readonly OtOpcUaNodeManager _nm;
|
||||
private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly Func<NodeState, Task?> _probe;
|
||||
|
||||
private NotifierStall(OtOpcUaNodeManager nm, string notifierId)
|
||||
{
|
||||
_nm = nm;
|
||||
_probe = node => !_released.Task.IsCompleted && node.NodeId.IdentifierAsString == notifierId ? _released.Task : null;
|
||||
nm.NotifierDeliveryStall = _probe;
|
||||
}
|
||||
|
||||
/// <summary>Starts stalling delivery at the notifier whose NodeId identifier is <paramref name="notifierId"/>.</summary>
|
||||
public static NotifierStall Hold(OtOpcUaNodeManager nm, string notifierId) => new(nm, notifierId);
|
||||
|
||||
/// <summary>Lets every stalled walk continue (on pool threads, off <c>Lock</c>).</summary>
|
||||
public void Release() => _released.TrySetResult();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Release();
|
||||
if (ReferenceEquals(_nm.NotifierDeliveryStall, _probe)) _nm.NotifierDeliveryStall = null;
|
||||
}
|
||||
}
|
||||
+276
-45
@@ -9,8 +9,33 @@ using ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates;
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>Compares all preserved aggregate factories with independently generated SDK 1.5.378.182 vectors.</summary>
|
||||
/// <remarks>
|
||||
/// #809 (owner-approved 2026-11-11): PercentGood / DurationGood no longer count Uncertain time as Good. The baseline
|
||||
/// vectors that changes are exempted EXACTLY — a PercentGood / DurationGood vector whose output differs from the same
|
||||
/// case computed by the baseline SDK with <c>TreatUncertainAsBad = true</c> (its "twin": there 1.5.378.182 already
|
||||
/// counted Good regions only) — and each is pinned to that twin instead. The twin is baseline evidence, not output of
|
||||
/// the application's calculator, so the pin never regenerates expectations from the implementation.
|
||||
/// <para>
|
||||
/// #821 (owner-approved 2026-11-11): TimeAverage / Total no longer integrate the span before an Uncertain first value
|
||||
/// when there is no start bound. No fixture vector changes (every data set has a usable value at −10 s, before every
|
||||
/// window, so no fixture interval lacks a start bound) — all 384 TimeAverage / Total vectors stay held to their own
|
||||
/// output. The change is pinned instead over late-start variants of every data set against a baseline-derived oracle
|
||||
/// (<see cref="Issue821_late_start_with_an_uncertain_first_sample_integrates_only_the_covered_part"/>) and by hand.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class Sdk2AggregateCompatibilityTests
|
||||
{
|
||||
private const string PercentGood = "AggregateFunction_PercentGood";
|
||||
private const string DurationGood = "AggregateFunction_DurationGood";
|
||||
private const int Issue809PercentGoodExemptions = 38;
|
||||
private const int Issue809DurationGoodExemptions = 26;
|
||||
private const string TimeAverage = "AggregateFunction_TimeAverage";
|
||||
private const string Total = "AggregateFunction_Total";
|
||||
/// <summary>#821: fixture vectors changed — none (every data set has a usable value before the window; class remarks).</summary>
|
||||
private const int Issue821FixtureExemptions = 0;
|
||||
private const int Issue821Cut5Changed = 150;
|
||||
private const int Issue821Cut25Changed = 142;
|
||||
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
@@ -19,57 +44,248 @@ public sealed class Sdk2AggregateCompatibilityTests
|
||||
await using var rig = throughRuntimeManager
|
||||
? await ServerEventRig.StartAsync("AggregateBaseline", encrypted: false, TestContext.Current.CancellationToken)
|
||||
: null;
|
||||
var baseline = JsonSerializer.Deserialize<Baseline>(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Fixtures", "opcua-1.5.378.182-aggregates.json")), new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
|
||||
var telemetry = DefaultTelemetry.Create(_ => { });
|
||||
var epoch = new DateTime(baseline.EpochTicks, DateTimeKind.Utc);
|
||||
var baseline = LoadBaseline();
|
||||
var exemptions = Issue809Exemptions(baseline);
|
||||
var failures = new List<string>();
|
||||
foreach (var vector in baseline.Vectors)
|
||||
{
|
||||
var id = (NodeId)typeof(ObjectIds).GetProperty(vector.Aggregate)!.GetValue(null)!;
|
||||
var start = epoch.AddSeconds(vector.Reverse ? 70 : 0); var end = epoch.AddSeconds(vector.Reverse ? 0 : 70);
|
||||
var profile = vector.Configuration;
|
||||
var config = new AggregateConfiguration
|
||||
{
|
||||
TreatUncertainAsBad = profile?.TreatUncertainAsBad ?? vector.UncertainAsBad,
|
||||
PercentDataBad = profile?.PercentDataBad ?? 100, PercentDataGood = profile?.PercentDataGood ?? 100,
|
||||
UseSlopedExtrapolation = profile?.UseSlopedExtrapolation ?? false,
|
||||
UseServerCapabilitiesDefaults = profile?.UseServerCapabilitiesDefaults ?? false,
|
||||
};
|
||||
var calculator = throughRuntimeManager
|
||||
? rig!.Server.CurrentInstance.AggregateManager.CreateCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config)
|
||||
: CompatibilityAggregateFactory.Create(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry);
|
||||
calculator.ShouldNotBeNull($"Runtime registration must serve {vector.Aggregate}.");
|
||||
var output = new List<Value>(); var accepted = new List<bool>(); string? error = null;
|
||||
try
|
||||
{
|
||||
var samples = baseline.Data[vector.DataSet].OrderBy(x => x.Seconds);
|
||||
foreach (var sample in vector.Reverse ? samples.Reverse() : samples)
|
||||
{
|
||||
var time = epoch.AddSeconds(sample.Seconds);
|
||||
accepted.Add(calculator.QueueRawValue(new DataValue(SdkValueBoundary.FromClr(Payload(sample)), sample.Status, time, time.AddTicks(sample.ServerOffsetTicks), sample.SourcePicoseconds, sample.ServerPicoseconds)));
|
||||
if (vector.Incremental) Drain(false);
|
||||
}
|
||||
Drain(true);
|
||||
}
|
||||
catch (Exception ex) { error = ex.GetType().FullName; }
|
||||
if (error != vector.Exception || !output.SequenceEqual(vector.Output) || (vector.QueueAccepted is { } expectedAccepted && !accepted.SequenceEqual(expectedAccepted)))
|
||||
{
|
||||
failures.Add($"{vector.Aggregate}/{vector.DataSet}/rev={vector.Reverse}/step={vector.Stepped}/uncertainBad={vector.UncertainAsBad}/incremental={vector.Incremental}: error {vector.Exception} vs {error}; expected {JsonSerializer.Serialize(vector.Output)} actual {JsonSerializer.Serialize(output)}");
|
||||
}
|
||||
void Drain(bool partial)
|
||||
{
|
||||
for (var guard = 0; guard < 100; guard++)
|
||||
{
|
||||
if (!calculator.TryGetProcessedValue(partial, out var value)) return;
|
||||
var payload = SdkValueBoundary.ToClr(value.WrappedValue);
|
||||
output.Add(new(partial, payload?.GetType().Name, Format(payload), value.StatusCode.Code, LegacyTimestampTicks(value.SourceTimestamp), LegacyTimestampTicks(value.ServerTimestamp), value.SourcePicoseconds, value.ServerPicoseconds));
|
||||
}
|
||||
throw new InvalidOperationException("Baseline calculator did not terminate.");
|
||||
}
|
||||
// #809: an exempted case is held to its baseline uncertain-as-bad twin; every other case to its own vector.
|
||||
var expected = exemptions.TryGetValue(vector, out var twin) ? twin.Output : vector.Output;
|
||||
if (Check(baseline, vector, expected, rig) is { } failure) failures.Add(failure);
|
||||
}
|
||||
baseline.Vectors.Length.ShouldBe(6144);
|
||||
baseline.Vectors.Select(x => x.Aggregate).Distinct().Count().ShouldBe(37);
|
||||
failures.ShouldBeEmpty("The application must retain baseline values, CLR types, complete status bits, timestamps and incremental processing.");
|
||||
exemptions.Keys.Count(x => x.Aggregate == PercentGood).ShouldBe(Issue809PercentGoodExemptions);
|
||||
exemptions.Keys.Count(x => x.Aggregate == DurationGood).ShouldBe(Issue809DurationGoodExemptions);
|
||||
// #821: no TimeAverage / Total fixture vector is exempt — all 384 are held to their own baseline output above.
|
||||
exemptions.Keys.Count(x => x.Aggregate is TimeAverage or Total).ShouldBe(Issue821FixtureExemptions);
|
||||
failures.ShouldBeEmpty("The application must retain baseline values, CLR types, complete status bits, timestamps and incremental processing (#809 exemptions: the uncertain-as-bad twin).");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #809 pin: the exemptions are exactly the PercentGood / DurationGood cases that counted Uncertain as Good, and
|
||||
/// every PercentGood / DurationGood case — exempt or not, whatever its configuration — now answers its baseline
|
||||
/// uncertain-as-bad twin (Good regions only).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Issue809_percent_and_duration_good_count_only_good_time()
|
||||
{
|
||||
var baseline = LoadBaseline();
|
||||
var exemptions = Issue809Exemptions(baseline);
|
||||
var failures = new List<string>();
|
||||
var checkedCount = 0;
|
||||
var twinned = 0;
|
||||
foreach (var vector in baseline.Vectors.Where(v => v.Aggregate is PercentGood or DurationGood))
|
||||
{
|
||||
// A data set with no uncertain-as-bad twin (the numeric-width edge sets) has no Uncertain region to move.
|
||||
var twin = UncertainAsBadTwin(baseline, vector);
|
||||
if (exemptions.ContainsKey(vector))
|
||||
EffectiveTreatUncertainAsBad(vector).ShouldBeFalse($"{Describe(vector)}: only an Uncertain-as-Good case may change.");
|
||||
if (Check(baseline, vector, twin?.Output ?? vector.Output, rig: null) is { } failure) failures.Add(failure);
|
||||
checkedCount++;
|
||||
if (twin is not null) twinned++;
|
||||
}
|
||||
checkedCount.ShouldBe(352);
|
||||
twinned.ShouldBeGreaterThan(exemptions.Count);
|
||||
exemptions.Count.ShouldBe(Issue809PercentGoodExemptions + Issue809DurationGoodExemptions);
|
||||
failures.ShouldBeEmpty("PercentGood / DurationGood must count Good regions only (Part 13 §5.4.3.31/§5.4.3.33, #809).");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #809 hand-computed pin, independent of the expected outputs in any fixture: the "uncertain" data set (0 s Good,
|
||||
/// 5 s Uncertain, 15 s Good, 25 s Bad, 35 s Good, 50 s Good) over 20 s intervals from 0 to 70 s, sloped, forward.
|
||||
/// [0,20): Good 0–5 and 15–20 = 10 s (the baseline said 20 s); [20,40): the bound at 20 slopes into the Bad sample
|
||||
/// (Uncertain), 25–35 Bad, 35–40 Good = 5 s (baseline 10 s); [40,60): 20 s; [60,70): all 10 s of the partial
|
||||
/// interval = 100 %.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(DurationGood, new[] { 10000d, 5000d, 20000d, 10000d })]
|
||||
[InlineData(PercentGood, new[] { 50d, 25d, 100d, 100d })]
|
||||
public void Issue809_uncertain_time_is_not_good_hand_computed(string aggregate, double[] expected)
|
||||
{
|
||||
var baseline = LoadBaseline();
|
||||
var vector = baseline.Vectors.Single(v => v.Aggregate == aggregate && v.DataSet == "uncertain" && !v.Reverse && !v.Stepped
|
||||
&& !v.UncertainAsBad && !v.Incremental && v.Configuration is null);
|
||||
var output = Run(baseline, vector, rig: null, out var error);
|
||||
error.ShouldBeNull();
|
||||
output.ShouldAllBe(x => x.Type == "Double");
|
||||
output.Select(x => double.Parse(x.Payload!, CultureInfo.InvariantCulture)).ShouldBe(expected);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #821 pin over late-start variants of every fixture data set (the fixture itself has none — see the class remarks):
|
||||
/// each TimeAverage / Total vector's data set cut at <paramref name="cutSeconds"/> (no usable value before the data
|
||||
/// starts, so no start bound) with its first remaining non-Bad sample made Uncertain, run through the application's
|
||||
/// calculator under the vector's own direction, stepping, configuration and incremental draining.
|
||||
/// <para>
|
||||
/// The oracle is the untouched BASELINE, never the application's calculator: under TreatUncertainAsBad = false
|
||||
/// the baseline treats an Uncertain value exactly like a Good one except for statuses, and with a Good first
|
||||
/// value it already leaves the uncovered span out (the #821 control). So the expected value of each interval is
|
||||
/// the baseline's on the same variant with that first sample Good, and the expected status, timestamps and bits
|
||||
/// the baseline's on the variant itself (the region before the data is non-Good either way) — except an
|
||||
/// interval the Good twin answers Bad (nothing covered: only the uncovered span ever fed it), which is the twin
|
||||
/// whole. Stepped variables and TreatUncertainAsBad = true never reached the baseline's defect and must be the
|
||||
/// baseline byte for byte.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(5, Issue821Cut5Changed)]
|
||||
[InlineData(25, Issue821Cut25Changed)]
|
||||
public void Issue821_late_start_with_an_uncertain_first_sample_integrates_only_the_covered_part(double cutSeconds, int expectedChanged)
|
||||
{
|
||||
var baseline = LoadBaseline();
|
||||
var failures = new List<string>();
|
||||
var changed = 0;
|
||||
// Why the fixture needs no #821 exemption: every data set with a usable value has one before the window (−10 s), so
|
||||
// no fixture interval lacks a start bound; the variants below remove it.
|
||||
baseline.Data.Values.Where(d => d.Any(s => !StatusCode.IsBad(s.Status)))
|
||||
.ShouldAllBe(d => d.Any(s => s.Seconds < 0 && !StatusCode.IsBad(s.Status)));
|
||||
var vectors = baseline.Vectors.Where(v => v.Aggregate is TimeAverage or Total).ToList();
|
||||
foreach (var vector in vectors)
|
||||
{
|
||||
var (variant, goodTwin) = LateStartVariant(baseline.Data[vector.DataSet], cutSeconds);
|
||||
var asIs = Run(baseline, vector, rig: null, out var asIsError, variant, baselineAsIs: true);
|
||||
var twin = Run(baseline, vector, rig: null, out var twinError, goodTwin, baselineAsIs: true);
|
||||
var actual = Run(baseline, vector, rig: null, out var error, variant);
|
||||
asIsError.ShouldBeNull(Describe(vector));
|
||||
twinError.ShouldBeNull(Describe(vector));
|
||||
twin.Count.ShouldBe(asIs.Count, Describe(vector));
|
||||
|
||||
Value[] expected = EffectiveTreatUncertainAsBad(vector) || vector.Stepped
|
||||
? [.. asIs]
|
||||
: [.. asIs.Zip(twin, (a, g) =>
|
||||
{
|
||||
g.SourceTicks.ShouldBe(a.SourceTicks, Describe(vector));
|
||||
return StatusCode.IsBad(g.Status) ? g : a with { Type = g.Type, Payload = g.Payload };
|
||||
})];
|
||||
if (!expected.SequenceEqual(asIs)) changed++;
|
||||
if (error is not null || !actual.SequenceEqual(expected))
|
||||
failures.Add($"{Describe(vector)} cut {cutSeconds}: error {error}; expected {JsonSerializer.Serialize(expected)} actual {JsonSerializer.Serialize(actual.ToArray())}");
|
||||
}
|
||||
vectors.Count.ShouldBe(384);
|
||||
failures.ShouldBeEmpty("Live TimeAverage / Total must leave the span before an Uncertain first value out (Part 13 'No Start Bound: Calculate Partial Interval', #821).");
|
||||
changed.ShouldBe(expectedChanged, "the late-start variants the #821 override changes from the untouched baseline");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #821 hand-computed pin, independent of every calculator: the "uncertain" data set from +5 (Uncertain 100 at 5,
|
||||
/// Good 18 at 15, Bad at 25, Good 19 at 35, …), forward, sloped, [0,20). Covered [5,20): 100 → 18 over 10 s = 590,
|
||||
/// then 18 → the bound at 20 (sloped 18 → 19 across the Bad sample: 18.25) over 5 s = 90.625 ⇒ Total 680.625,
|
||||
/// TimeAverage 680.625 ÷ 15 = 45.375. The baseline added the ramp 0 → 100 over [0,5) (+250): 930.625 and 46.53125.
|
||||
/// Uncertain either way (an Uncertain value is used, a Bad one skipped, the interval not covered).
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(Total, 680.625, 930.625)]
|
||||
[InlineData(TimeAverage, 45.375, 46.53125)]
|
||||
public void Issue821_late_start_hand_computed(string aggregate, double part13, double baselineValue)
|
||||
{
|
||||
var baseline = LoadBaseline();
|
||||
var vector = baseline.Vectors.Single(v => v.Aggregate == aggregate && v.DataSet == "uncertain" && !v.Reverse && !v.Stepped
|
||||
&& !v.UncertainAsBad && !v.Incremental && v.Configuration is null);
|
||||
var (variant, _) = LateStartVariant(baseline.Data["uncertain"], 5);
|
||||
variant[0].Status.ShouldBe(0x40A40000u, "the data set's own Uncertain 100 at +5");
|
||||
var output = Run(baseline, vector, rig: null, out var error, variant);
|
||||
var asIs = Run(baseline, vector, rig: null, out _, variant, baselineAsIs: true);
|
||||
error.ShouldBeNull();
|
||||
double.Parse(output[0].Payload!, CultureInfo.InvariantCulture).ShouldBe(part13, 1e-9);
|
||||
double.Parse(asIs[0].Payload!, CultureInfo.InvariantCulture).ShouldBe(baselineValue, 1e-9);
|
||||
(output[0].Status & 0xFFFF0000u).ShouldBe(StatusCodes.UncertainDataSubNormal.Code);
|
||||
output[0].Status.ShouldBe(asIs[0].Status);
|
||||
output.Skip(1).ShouldBe(asIs.Skip(1), "only the late-start interval changes");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A data set cut at <paramref name="cutSeconds"/> with its first non-Bad sample Uncertain (kept Uncertain if it already
|
||||
/// is), and the same with that sample Good.
|
||||
/// </summary>
|
||||
private static (Sample[] Variant, Sample[] GoodTwin) LateStartVariant(Sample[] data, double cutSeconds)
|
||||
{
|
||||
var cut = data.Where(s => s.Seconds >= cutSeconds).OrderBy(s => s.Seconds).ToArray();
|
||||
var first = Array.FindIndex(cut, s => !StatusCode.IsBad(s.Status));
|
||||
if (first < 0) return (cut, cut);
|
||||
var variant = cut.ToArray();
|
||||
var good = cut.ToArray();
|
||||
variant[first] = cut[first] with { Status = StatusCode.IsUncertain(cut[first].Status) ? cut[first].Status : 0x40A40000u };
|
||||
good[first] = cut[first] with { Status = 0u };
|
||||
return (variant, good);
|
||||
}
|
||||
|
||||
private static Baseline LoadBaseline() =>
|
||||
JsonSerializer.Deserialize<Baseline>(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Fixtures", "opcua-1.5.378.182-aggregates.json")), new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
|
||||
|
||||
private static bool EffectiveTreatUncertainAsBad(Vector vector) => vector.Configuration?.TreatUncertainAsBad ?? vector.UncertainAsBad;
|
||||
|
||||
/// <summary>The same case computed by the baseline SDK with <c>TreatUncertainAsBad = true</c> and no explicit profile.</summary>
|
||||
private static Vector? UncertainAsBadTwin(Baseline baseline, Vector vector) =>
|
||||
baseline.Vectors.SingleOrDefault(v => v.Aggregate == vector.Aggregate && v.DataSet == vector.DataSet && v.Reverse == vector.Reverse
|
||||
&& v.Stepped == vector.Stepped && v.Incremental == vector.Incremental && v.UncertainAsBad && v.Configuration is null);
|
||||
|
||||
/// <summary>#809: the PercentGood / DurationGood vectors whose baseline output differs from their uncertain-as-bad twin.</summary>
|
||||
private static Dictionary<Vector, Vector> Issue809Exemptions(Baseline baseline) =>
|
||||
baseline.Vectors
|
||||
.Where(v => v.Aggregate is PercentGood or DurationGood)
|
||||
.Select(v => (Vector: v, Twin: UncertainAsBadTwin(baseline, v)))
|
||||
.Where(x => x.Twin is not null && !x.Vector.Output.SequenceEqual(x.Twin.Output))
|
||||
.ToDictionary<(Vector Vector, Vector? Twin), Vector, Vector>(x => x.Vector, x => x.Twin!, System.Collections.Generic.ReferenceEqualityComparer.Instance);
|
||||
|
||||
private static string Describe(Vector vector) =>
|
||||
$"{vector.Aggregate}/{vector.DataSet}/rev={vector.Reverse}/step={vector.Stepped}/uncertainBad={vector.UncertainAsBad}/incremental={vector.Incremental}/config={JsonSerializer.Serialize(vector.Configuration)}";
|
||||
|
||||
private static string? Check(Baseline baseline, Vector vector, Value[] expected, ServerEventRig? rig)
|
||||
{
|
||||
var output = Run(baseline, vector, rig, out var error);
|
||||
return error != vector.Exception || !output.SequenceEqual(expected) || (vector.QueueAccepted is { } expectedAccepted && !output.Accepted.SequenceEqual(expectedAccepted))
|
||||
? $"{Describe(vector)}: error {vector.Exception} vs {error}; expected {JsonSerializer.Serialize(expected)} actual {JsonSerializer.Serialize(output.ToArray())}"
|
||||
: null;
|
||||
}
|
||||
|
||||
private static RunOutput Run(Baseline baseline, Vector vector, ServerEventRig? rig, out string? error,
|
||||
Sample[]? data = null, bool baselineAsIs = false)
|
||||
{
|
||||
var telemetry = DefaultTelemetry.Create(_ => { });
|
||||
var epoch = new DateTime(baseline.EpochTicks, DateTimeKind.Utc);
|
||||
var id = (NodeId)typeof(ObjectIds).GetProperty(vector.Aggregate)!.GetValue(null)!;
|
||||
var start = epoch.AddSeconds(vector.Reverse ? 70 : 0); var end = epoch.AddSeconds(vector.Reverse ? 0 : 70);
|
||||
var profile = vector.Configuration;
|
||||
var config = new AggregateConfiguration
|
||||
{
|
||||
TreatUncertainAsBad = profile?.TreatUncertainAsBad ?? vector.UncertainAsBad,
|
||||
PercentDataBad = profile?.PercentDataBad ?? 100, PercentDataGood = profile?.PercentDataGood ?? 100,
|
||||
UseSlopedExtrapolation = profile?.UseSlopedExtrapolation ?? false,
|
||||
UseServerCapabilitiesDefaults = profile?.UseServerCapabilitiesDefaults ?? false,
|
||||
};
|
||||
var calculator = rig is not null
|
||||
? rig.Server.CurrentInstance.AggregateManager.CreateCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config)
|
||||
: baselineAsIs
|
||||
? new BaselineAdapter(Aggregators.CreateStandardCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry)!)
|
||||
: CompatibilityAggregateFactory.Create(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry);
|
||||
calculator.ShouldNotBeNull($"Runtime registration must serve {vector.Aggregate}.");
|
||||
var output = new RunOutput();
|
||||
error = null;
|
||||
try
|
||||
{
|
||||
var samples = (data ?? baseline.Data[vector.DataSet]).OrderBy(x => x.Seconds);
|
||||
foreach (var sample in vector.Reverse ? samples.Reverse() : samples)
|
||||
{
|
||||
var time = epoch.AddSeconds(sample.Seconds);
|
||||
output.Accepted.Add(calculator.QueueRawValue(new DataValue(SdkValueBoundary.FromClr(Payload(sample)), sample.Status, time, time.AddTicks(sample.ServerOffsetTicks), sample.SourcePicoseconds, sample.ServerPicoseconds)));
|
||||
if (vector.Incremental) Drain(false);
|
||||
}
|
||||
Drain(true);
|
||||
}
|
||||
catch (Exception ex) { error = ex.GetType().FullName; }
|
||||
return output;
|
||||
|
||||
void Drain(bool partial)
|
||||
{
|
||||
for (var guard = 0; guard < 100; guard++)
|
||||
{
|
||||
if (!calculator.TryGetProcessedValue(partial, out var value)) return;
|
||||
var payload = SdkValueBoundary.ToClr(value.WrappedValue);
|
||||
output.Add(new(partial, payload?.GetType().Name, Format(payload), value.StatusCode.Code, LegacyTimestampTicks(value.SourceTimestamp), LegacyTimestampTicks(value.ServerTimestamp), value.SourcePicoseconds, value.ServerPicoseconds));
|
||||
}
|
||||
throw new InvalidOperationException("Baseline calculator did not terminate.");
|
||||
}
|
||||
}
|
||||
|
||||
// UA FILETIME zero is the old CLR unset sentinel, not a different historical instant.
|
||||
@@ -84,6 +300,21 @@ public sealed class Sdk2AggregateCompatibilityTests
|
||||
"Single" => float.Parse(sample.Payload!, CultureInfo.InvariantCulture),
|
||||
"Double" => double.Parse(sample.Payload!, CultureInfo.InvariantCulture), "Int64" => long.Parse(sample.Payload!, CultureInfo.InvariantCulture), "Boolean" => bool.Parse(sample.Payload!), _ => null };
|
||||
static string? Format(object? value) => value switch { null => null, double number => number.ToString("R", CultureInfo.InvariantCulture), float number => number.ToString("R", CultureInfo.InvariantCulture), IFormattable f => f.ToString(null, CultureInfo.InvariantCulture), _ => value.ToString() };
|
||||
private sealed class RunOutput : List<Value> { public List<bool> Accepted { get; } = []; }
|
||||
|
||||
/// <summary>The untouched baseline calculator behind the SDK 2 protocol (what live served before #809 / #821).</summary>
|
||||
private sealed class BaselineAdapter(AggregateCalculator calculator) : Opc.Ua.Server.IAggregateCalculator
|
||||
{
|
||||
public NodeId AggregateId => calculator.AggregateId;
|
||||
public bool QueueRawValue(DataValue value) => calculator.QueueRawValue(LegacyDataValue.FromSdk(value));
|
||||
public bool HasEndTimePassed(DateTimeUtc currentTime) => calculator.HasEndTimePassed((DateTime)currentTime);
|
||||
public bool TryGetProcessedValue(bool returnPartial, out DataValue value)
|
||||
{
|
||||
var processed = calculator.GetProcessedValue(returnPartial);
|
||||
value = processed is null ? default : processed.ToSdk();
|
||||
return processed is not null;
|
||||
}
|
||||
}
|
||||
record Baseline(long EpochTicks, int IntervalMilliseconds, Dictionary<string,Sample[]> Data, Vector[] Vectors);
|
||||
record Sample(double Seconds,string Type,string? Payload,uint Status,ushort SourcePicoseconds = 0,ushort ServerPicoseconds = 0,long ServerOffsetTicks = 0);
|
||||
record Value(bool PartialRequested,string? Type,string? Payload,uint Status,long SourceTicks,long ServerTicks,ushort SourcePicoseconds,ushort ServerPicoseconds);
|
||||
|
||||
+25
@@ -56,6 +56,31 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
|
||||
cache.UnkeyedReads.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#868 third review #2: the cache is written before the address-space outcome, so a cache-booted
|
||||
/// revision may never have been acked. A re-dispatch of it must RE-APPLY (here: fail honestly, since the
|
||||
/// ConfigDb is unreachable) — never short-circuit a clean Applied for an outcome nobody confirmed.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_cache_booted_revision_is_unconfirmed_so_a_redispatch_reapplies_instead_of_acking_clean()
|
||||
{
|
||||
var cached = new CachedDeploymentArtifact(
|
||||
DeploymentId.NewId().ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow.AddMinutes(-1));
|
||||
var coordinator = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, ackRouter: coordinator.Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: new StubArtifactCache(cached)));
|
||||
AskDiagnostics(actor).CurrentRevision.ShouldBe(CachedRev);
|
||||
|
||||
actor.Tell(new ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy.DispatchDeployment(
|
||||
DeploymentId.NewId(), CachedRev, CorrelationId.NewId()));
|
||||
|
||||
var ack = coordinator.ExpectMsg<ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy.ApplyAck>(
|
||||
PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
ack.Outcome.ShouldNotBe(ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy.ApplyAckOutcome.Applied);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BootFromCache_HandsTheCachedArtifactToTheAddressSpaceRebuild()
|
||||
{
|
||||
|
||||
+92
-3
@@ -174,12 +174,49 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase
|
||||
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_local_row_stays_Applying_until_the_address_space_outcome_is_known()
|
||||
{
|
||||
// lmxopcua#868 second review M1: the node used to write its own row Applied/null the moment the revision
|
||||
// advanced, up to ~105 s before the outcome. ConfigPublishCoordinator's recovery counts a non-Applying row
|
||||
// as an ack, so a coordinator restart in that window sealed the deployment CLEAN while the node was about
|
||||
// to report degraded. The row must carry only the final outcome.
|
||||
var (db, deploymentId) = SeedOneTagDeployment();
|
||||
var coordinator = CreateTestProbe();
|
||||
var publish = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
|
||||
opcUaPublishActor: publish.Ref));
|
||||
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
|
||||
|
||||
// The ack is held: the row must still read Applying (the apply's synchronous writes are done by now — the
|
||||
// rebuild Tell above is issued after them on the actor thread).
|
||||
using (var ctx = db.CreateDbContext())
|
||||
ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value)
|
||||
.Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applying);
|
||||
|
||||
publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, false, null));
|
||||
coordinator.ExpectMsg<ApplyAck>(PresenceBudget, cancellationToken: TestContext.Current.CancellationToken).Degraded.ShouldBeFalse();
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var ctx = db.CreateDbContext();
|
||||
var row = ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value);
|
||||
row.Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applied);
|
||||
row.FailureReason.ShouldBeNull();
|
||||
row.AppliedAtUtc.ShouldNotBeNull();
|
||||
}, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_degraded_apply_records_its_reason_on_the_local_NodeDeploymentState_row()
|
||||
{
|
||||
// #601 review — a node WITH a ConfigDb writes its own row Applied/null before the outcome is known.
|
||||
// Central later adds the reason from the ack, but only when it accepts that ack (a stale/timed-out
|
||||
// deployment's ack is discarded), so the node must not leave its own row reading clean.
|
||||
// #601 review — a node WITH a ConfigDb records its own row. Central later adds the reason from the ack,
|
||||
// but only when it accepts that ack (a stale/timed-out deployment's ack is discarded), so the node must
|
||||
// not leave its own row reading clean.
|
||||
var (db, deploymentId) = SeedOneTagDeployment();
|
||||
var coordinator = CreateTestProbe();
|
||||
var publish = CreateTestProbe();
|
||||
@@ -201,6 +238,58 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase
|
||||
}, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_oversized_degraded_reason_is_truncated_so_the_local_row_still_records_the_outcome()
|
||||
{
|
||||
// lmxopcua#868 third review #1: the row's FailureReason is nvarchar(2048). An unbounded reason made SQL Server
|
||||
// refuse the write (logged and swallowed), so the row stayed Applying forever. A DB that enforces the column.
|
||||
var db = new LengthEnforcingDbFactory(Guid.NewGuid().ToString("N"));
|
||||
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(db, RevA,
|
||||
rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false) },
|
||||
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
|
||||
var coordinator = CreateTestProbe();
|
||||
var publish = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
|
||||
opcUaPublishActor: publish.Ref));
|
||||
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, true, new string('f', 5000)));
|
||||
coordinator.ExpectMsg<ApplyAck>(PresenceBudget, cancellationToken: TestContext.Current.CancellationToken).Degraded.ShouldBeTrue();
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var ctx = db.CreateDbContext();
|
||||
var row = ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value);
|
||||
row.Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applied);
|
||||
row.FailureReason!.Length.ShouldBe(FailureReasons.MaxLength);
|
||||
row.FailureReason.ShouldStartWith(ApplyAck.DegradedReasonPrefix);
|
||||
}, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>In-memory ConfigDb that refuses an over-length FailureReason, as SQL Server does.</summary>
|
||||
private sealed class LengthEnforcingDbFactory(string dbName) : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
public OtOpcUaConfigDbContext CreateDbContext() => new(new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
||||
.UseInMemoryDatabase(dbName)
|
||||
.AddInterceptors(new Enforcer())
|
||||
.Options);
|
||||
|
||||
private sealed class Enforcer : Microsoft.EntityFrameworkCore.Diagnostics.SaveChangesInterceptor
|
||||
{
|
||||
public override Microsoft.EntityFrameworkCore.Diagnostics.InterceptionResult<int> SavingChanges(
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.DbContextEventData eventData,
|
||||
Microsoft.EntityFrameworkCore.Diagnostics.InterceptionResult<int> result)
|
||||
{
|
||||
foreach (var e in eventData.Context!.ChangeTracker.Entries<ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeDeploymentState>())
|
||||
if (e.Entity.FailureReason is { Length: > FailureReasons.MaxLength })
|
||||
throw new DbUpdateException("String or binary data would be truncated in column 'FailureReason'.");
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_default_outcome_timeout_is_derived_from_centrals_apply_deadline_minus_a_margin()
|
||||
{
|
||||
|
||||
+43
@@ -4,6 +4,8 @@ using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
@@ -43,6 +45,7 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
|
||||
driver.State.ShouldBe("Healthy");
|
||||
driver.ConnectedDevices.ShouldBe(1);
|
||||
driver.FaultedDevices.ShouldBe(0);
|
||||
driver.StoppedDevices.ShouldBe(0);
|
||||
driver.LastChangeUtc.ShouldBeGreaterThan(before);
|
||||
driver.LastChangeUtc.ShouldBeLessThanOrEqualTo(DateTime.UtcNow);
|
||||
}
|
||||
@@ -66,6 +69,46 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
|
||||
driver.FaultedDevices.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Faulted_counts_only_Faulted_hosts_and_Stopped_is_reported_separately()
|
||||
{
|
||||
// #786: a Stopped host used to be counted as faulted.
|
||||
var now = DateTime.UtcNow;
|
||||
var health = new ChildHealthRecord(
|
||||
DriverState.Degraded, now, null,
|
||||
new[]
|
||||
{
|
||||
new HostConnectivityStatus("plc-a:502", HostState.Running, now),
|
||||
new HostConnectivityStatus("plc-b:502", HostState.Faulted, now),
|
||||
new HostConnectivityStatus("plc-c:502", HostState.Stopped, now),
|
||||
new HostConnectivityStatus("plc-d:502", HostState.Stopped, now),
|
||||
new HostConnectivityStatus("plc-e:502", HostState.Unknown, now),
|
||||
},
|
||||
now);
|
||||
|
||||
var row = DriverHostActor.DiagnosticsFromHealth(Guid.NewGuid(), DriverId, health);
|
||||
|
||||
row.ConnectedDevices.ShouldBe(1);
|
||||
row.FaultedDevices.ShouldBe(1);
|
||||
row.StoppedDevices.ShouldBe(2);
|
||||
row.FaultedHostNames.ShouldBe(new[] { "plc-b:502" });
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(DriverState.Faulted, 0, 1)]
|
||||
[InlineData(DriverState.Healthy, 1, 0)]
|
||||
[InlineData(DriverState.Degraded, 1, 0)]
|
||||
public void A_single_host_driver_is_one_device_with_no_host_name(DriverState state, int connected, int faulted)
|
||||
{
|
||||
var row = DriverHostActor.DiagnosticsFromHealth(
|
||||
Guid.NewGuid(), DriverId, new ChildHealthRecord(state, null, null, null, DateTime.UtcNow));
|
||||
|
||||
row.ConnectedDevices.ShouldBe(connected);
|
||||
row.FaultedDevices.ShouldBe(faulted);
|
||||
row.StoppedDevices.ShouldBe(0);
|
||||
row.FaultedHostNames.ShouldNotBeNull().ShouldBeEmpty();
|
||||
}
|
||||
|
||||
private async Task<NodeDiagnosticsSnapshot> AwaitDiagnosticsAsync(IActorRef host, Func<NodeDiagnosticsSnapshot, bool> ready)
|
||||
{
|
||||
NodeDiagnosticsSnapshot? snapshot = null;
|
||||
|
||||
@@ -32,6 +32,12 @@ public sealed class ScriptRunnerPerfTests
|
||||
{
|
||||
private const string OptInVariable = "OTOPCUA_SCRIPT_PERF";
|
||||
private const int AcceptedGateEvalsPerSecond = 5_000;
|
||||
|
||||
/// <summary>
|
||||
/// §8.2 "Worker idle RSS ≤ 200 MB + scripts", per shard (revised by the owner 2026-11-11, lmxopcua#789: compiles
|
||||
/// run in the worker, so Roslyn stays resident; the original 80 MB [A-3] is only met by a warm-PE-cache worker).
|
||||
/// </summary>
|
||||
private const int WorkerIdleRssBudgetMb = 200;
|
||||
private const int Keys = 2_000;
|
||||
private const int DistinctSources = 50;
|
||||
|
||||
@@ -481,7 +487,7 @@ public sealed class ScriptRunnerPerfTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// §8.2 "Shard restart → first result, 1,000 cached scripts ≤ 2 s", "Worker idle RSS ≤ 80 MB + scripts" and
|
||||
/// §8.2 "Shard restart → first result, 1,000 cached scripts ≤ 2 s", "Worker idle RSS ≤ 200 MB + scripts" and
|
||||
/// "Kill latency after T+G ≤ 100 ms". Report-only.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
@@ -559,14 +565,20 @@ public sealed class ScriptRunnerPerfTests
|
||||
}
|
||||
}
|
||||
|
||||
static string RssCell(long bytes)
|
||||
{
|
||||
var mb = bytes / 1048576.0;
|
||||
return $"{mb:N0} MB {(mb <= WorkerIdleRssBudgetMb ? "PASS" : "MISS")}";
|
||||
}
|
||||
|
||||
report.AppendLine("| Measure | Result | §8.2 budget |")
|
||||
.AppendLine("|---|---:|---|")
|
||||
.AppendLine($"| {scripts} scripts, empty cache, all first results (compile + write) | {cold.TotalMilliseconds:N0} ms | — |")
|
||||
.AppendLine($"| {scripts} scripts, fresh runner on a warm cache, all first results | {warmFresh.TotalMilliseconds:N0} ms | ≤ 2,000 ms |")
|
||||
.AppendLine($"| kill both shards → every script's next correct result | {killRestart.TotalMilliseconds:N0} ms | ≤ 2,000 ms |")
|
||||
.AppendLine($"| worker RSS after first eval, compiled (max of 2 shards) | {rssIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |")
|
||||
.AppendLine($"| worker RSS after first eval, compiled (max of 2 shards) | {RssCell(rssIdle)} | ≤ {WorkerIdleRssBudgetMb} MB + scripts |")
|
||||
.AppendLine($"| worker RSS with {scripts / 2} scripts compiled per shard (max) | {rssLoaded / 1048576.0:N0} MB | — |")
|
||||
.AppendLine($"| worker RSS after first eval, warm PE cache, no compile (max) | {rssWarmIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |")
|
||||
.AppendLine($"| worker RSS after first eval, warm PE cache, no compile (max) | {RssCell(rssWarmIdle)} | ≤ {WorkerIdleRssBudgetMb} MB + scripts |")
|
||||
.AppendLine($"| worker RSS with {scripts / 2} scripts loaded from the cache per shard (max) | {rssWarmLoaded / 1048576.0:N0} MB | — |")
|
||||
.AppendLine($"| kill latency after T+G (5 runs, ms) | {string.Join(" / ", killLatencies.Select(l => l.ToString("N0")))} | ≤ 100 ms |");
|
||||
Report(report);
|
||||
|
||||
Reference in New Issue
Block a user