From f826af8e8a7879a760ec72424d9c026e65d678c8 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Fri, 13 Nov 2026 04:36:57 -0500 Subject: [PATCH] docs(717): T13 live gate evidence on an isolated Warm pair Every applicable T13 leg passes on 0c19f26c + c569f806. The leg list covers: - metadata and ServiceLevel bands; - standby projection; - alarm suppression with Standby ConditionRefresh and the promotion refresh; - the three D4 demotion timings, run live; - standby write and Part 9 refusal (BadInvalidState); - graceful and crash failover, with no failback; - subscription transfer; - the Mode=Unset regression. The two crash-path defects the gate found are fixed in c569f806. Evidence, rig files and the raw-client harness are in docs/plans/evidence/2026-11-13-717-t13/. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi --- .../2026-11-11-warm-redundancy-717-plan.md | 6 +- .../2026-11-13-717-t13/L1-L2-meta-warm.txt | 16 + .../2026-11-13-717-t13/L3-L10-probe-warm.txt | 32 ++ .../evidence/2026-11-13-717-t13/RESULTS.md | 86 ++++ .../2026-11-13-717-t13/harness/Program.cs | 445 ++++++++++++++++++ .../harness/T13Harness.csproj | 13 + .../2026-11-13-717-t13/rig/Dockerfile | 5 + .../2026-11-13-717-t13/rig/compose.t13.yml | 103 ++++ .../evidence/2026-11-13-717-t13/rig/fixeq.sql | 8 + .../evidence/2026-11-13-717-t13/rig/seed.sql | 29 ++ 10 files changed, 742 insertions(+), 1 deletion(-) create mode 100644 docs/plans/evidence/2026-11-13-717-t13/L1-L2-meta-warm.txt create mode 100644 docs/plans/evidence/2026-11-13-717-t13/L3-L10-probe-warm.txt create mode 100644 docs/plans/evidence/2026-11-13-717-t13/RESULTS.md create mode 100644 docs/plans/evidence/2026-11-13-717-t13/harness/Program.cs create mode 100644 docs/plans/evidence/2026-11-13-717-t13/harness/T13Harness.csproj create mode 100644 docs/plans/evidence/2026-11-13-717-t13/rig/Dockerfile create mode 100644 docs/plans/evidence/2026-11-13-717-t13/rig/compose.t13.yml create mode 100644 docs/plans/evidence/2026-11-13-717-t13/rig/fixeq.sql create mode 100644 docs/plans/evidence/2026-11-13-717-t13/rig/seed.sql diff --git a/docs/plans/2026-11-11-warm-redundancy-717-plan.md b/docs/plans/2026-11-11-warm-redundancy-717-plan.md index de06eb4ee..63cf86089 100644 --- a/docs/plans/2026-11-11-warm-redundancy-717-plan.md +++ b/docs/plans/2026-11-11-warm-redundancy-717-plan.md @@ -301,7 +301,11 @@ failover also works against `Unset` servers. T1–T10 and T12 are implemented on `master`, default-off (`OpcUa:Redundancy:Mode` defaults to `Unset`, pinned by `WarmRedundancyWireTests.Unset_is_todays_server_exactly` and `OpcUaPublishActorWarmTests.Unset_never_drives_the_projection_and_keeps_todays_levels`). -T11 is ScadaBridge#138 (separate). T13 has not run: `docker-dev/docker-compose.warm.yml` is ready (see T13). +T11 is ScadaBridge#138 (separate). **T13 ran 2026-11-13** on an isolated two-node pair (not the shared docker-dev rig): +every applicable leg passes after `c569f806`, which fixes two crash-failover defects the gate found. The survivor no +longer demotes when auto-down removes the Primary, and the verdict reads its own cluster events instead of the lagging +read view. Evidence: `docs/plans/evidence/2026-11-13-717-t13/RESULTS.md`. L8 (None) was not run, and L12 is not +applicable (durable subscriptions are off). G8/G9 (#615) stay open. Test homes: `WarmStandbyUnitTests` (OpcUaServer.Tests: marker, rewrite, decision record, the SDK-field regression pin, option rules), `WarmRedundancyWireTests` (OpcUaServer.IntegrationTests: metadata per mode, read projection, write and diff --git a/docs/plans/evidence/2026-11-13-717-t13/L1-L2-meta-warm.txt b/docs/plans/evidence/2026-11-13-717-t13/L1-L2-meta-warm.txt new file mode 100644 index 000000000..fc071005a --- /dev/null +++ b/docs/plans/evidence/2026-11-13-717-t13/L1-L2-meta-warm.txt @@ -0,0 +1,16 @@ +##### :14840 +endpoint opc.tcp://localhost:14840/OtOpcUa server=urn:OtOpcUa:central-1 +RedundancySupport i=3709: 2 [Good] +ServerUriArray i=11314: [urn:OtOpcUa:central-1, urn:OtOpcUa:central-2] [Good] +ServiceLevel i=2267: 250 [Good] +ServerArray i=2254: [urn:OtOpcUa:central-1, urn:OtOpcUa:central-2] [Good] +ServerRedundancy i=2296 TypeDefinition: NonTransparentRedundancyType i=2039 +Write ServerUriArray: BadNotWritable +##### :14841 +endpoint opc.tcp://localhost:14841/OtOpcUa server=urn:OtOpcUa:central-2 +RedundancySupport i=3709: 2 [Good] +ServerUriArray i=11314: [urn:OtOpcUa:central-2, urn:OtOpcUa:central-1] [Good] +ServiceLevel i=2267: 90 [Good] +ServerArray i=2254: [urn:OtOpcUa:central-2, urn:OtOpcUa:central-1] [Good] +ServerRedundancy i=2296 TypeDefinition: NonTransparentRedundancyType i=2039 +Write ServerUriArray: BadNotWritable diff --git a/docs/plans/evidence/2026-11-13-717-t13/L3-L10-probe-warm.txt b/docs/plans/evidence/2026-11-13-717-t13/L3-L10-probe-warm.txt new file mode 100644 index 000000000..369712071 --- /dev/null +++ b/docs/plans/evidence/2026-11-13-717-t13/L3-L10-probe-warm.txt @@ -0,0 +1,32 @@ +##### Standby central-2 :14841 +09:02:27.873 probe opc.tcp://localhost:14841/OtOpcUa ServiceLevel=90 + read raw calc-seconds: null [BadOutOfService] src=09:02:27.892 + read uns calc-seconds: null [BadOutOfService] src=09:02:27.892 + read uns vt-second: null [BadOutOfService] src=09:02:27.893 + read raw calc-writable: null [BadOutOfService] src=09:02:27.893 + write raw calc-writable=7: BadInvalidState + write uns calc-writable=7: BadInvalidState + HistoryRead raw calc-seconds: service=Good result=BadHistoryOperationUnsupported +09:02:27.889 create item h=10 i=2253: Good id=2 + ConditionRefresh call: Good + 09:02:28.146 event h=10(ev Server) type=RefreshStart msg="Condition refresh started for subscription 906693065." cond= retain= active= acked= time=09:02:28.059 + 09:02:28.146 event h=10(ev Server) type=i=10637 msg="T13 even minute" cond=ns=3;s=AL-t13-minute retain=True active=True acked=False time=09:02:08.355 + 09:02:28.146 event h=10(ev Server) type=RefreshEnd msg="Condition refresh completed for subscription 906693065." cond= retain= active= acked= time=09:02:28.059 + Acknowledge on ns=3;s=AL-t13-minute: BadInvalidState + AddComment on ns=3;s=AL-t13-minute: BadInvalidState +##### Active central-1 :14840 +09:02:31.598 probe opc.tcp://localhost:14840/OtOpcUa ServiceLevel=250 + read raw calc-seconds: 32551 [Good] src=09:02:31.364 + read uns calc-seconds: 32551 [Good] src=09:02:31.364 + read uns vt-second: 31 [Good] src=09:02:31.376 + read raw calc-writable: 0 [Good] src=09:02:05.071 + write raw calc-writable=7: Good + write uns calc-writable=7: Good + HistoryRead raw calc-seconds: service=Good result=BadHistoryOperationUnsupported +09:02:31.615 create item h=10 i=2253: Good id=2 + ConditionRefresh call: Good + 09:02:31.898 event h=10(ev Server) type=RefreshStart msg="Condition refresh started for subscription 3744078708." cond= retain= active= acked= time=09:02:31.817 + 09:02:31.898 event h=10(ev Server) type=i=10637 msg="T13 even minute" cond=ns=3;s=AL-t13-minute retain=True active=True acked=False time=09:02:08.351 + 09:02:31.898 event h=10(ev Server) type=RefreshEnd msg="Condition refresh completed for subscription 3744078708." cond= retain= active= acked= time=09:02:31.817 + Acknowledge on ns=3;s=AL-t13-minute: Good + AddComment on ns=3;s=AL-t13-minute: Good diff --git a/docs/plans/evidence/2026-11-13-717-t13/RESULTS.md b/docs/plans/evidence/2026-11-13-717-t13/RESULTS.md new file mode 100644 index 000000000..5e9380165 --- /dev/null +++ b/docs/plans/evidence/2026-11-13-717-t13/RESULTS.md @@ -0,0 +1,86 @@ +# Gitea #717 T13 live gate: Warm redundancy on an isolated docker pair (2026-11-13) + +**Result: every leg that applies passes on the fixed build. The gate found two product defects on the crash-failover +path, both fixed in `c569f806` with tests (mutation-checked) and re-verified live.** + +## Rig + +An isolated two-node pair on the Mac, separate from the shared `otopcua-dev` rig, which was not touched. + +| Item | Value | +|---|---| +| Build | `origin/master` `0c19f26c` (runs 1 and 2), then `0c19f26c` + fix `c569f806` (runs 2b, 3 and 4). Published on the host and copied into a runtime-only `aspnet:10.0` image (`rig/Dockerfile`). | +| Images | `otopcua-host:t13-0c19f26c` and `otopcua-host:t13-0c19f26c-fix`. Both were removed at teardown. | +| Compose | project `otopcua-t13` (`rig/compose.t13.yml`) with its own network, its own SQL Server 2022 and its own volumes (`t13-mssql-data`, `t13-c1`, `t13-c2`). Ports: OPC UA `14840`/`14841`, HTTP `19210`/`19211`, SQL `14339`. | +| Topology | `central-1` + `central-2`, roles `admin,driver,cluster-MAIN`, self-first seeds, per-node `ApplicationUri` with the partner in `PeerApplicationUris` (as in docker-dev). `OpcUa__Redundancy__Mode=Warm` on both nodes, and `Unset` for the last run. `PromoteWhilePrimaryUnreachable` is at its default (`true`). | +| External contact | None. LDAP points at loopback (AdminUI login is disabled), the historian is off, and no driver reaches a device. | +| Schema | `dotnet ef database update` from the host against `:14339`. | +| Config | `rig/seed.sql` plus `rig/fixeq.sql`, then `POST /api/deployments`. The deploy was rejected once with `EquipmentIdNotDerived`; `fixeq.sql` renames the equipment to the canonical `EQ-3326c7a4728e`. The config contains:
• a Calculation driver `t13-calc/Device1`, with `calc-seconds` (Int32, Read, a 1 s timer, value = UTC seconds-of-day) and `calc-writable` (ReadWrite);
• UNS `t13/warm/t13-equipment`, with references to both tags;
• VirtualTag `vt-second`;
• scripted alarm `t13-even-minute` (OffNormal, Retain), active on even UTC minutes;
• one API key `t13-operator` (principal `t13-op`) with a Cluster-scope grant (WriteOperate, AlarmAck and the rest).
Deployment `5c90c82f`, sealed with acks from both nodes. | +| Client | `harness/` is a raw OPC UA client on SDK 2.0.1. It uses raw Publish/Republish/TransferSubscriptions so sequence numbers are visible. `meta`/`probe` sessions are anonymous or API-key; `watch`, `timing*` and `transfer` sessions use the API-key `UserName` identity. | + +Logs: `logs/run1` (master, the first graceful + crash run), `logs/run2` (the first fix, crash only), `logs/run3` +(final fix: graceful, crash, timings and transfer), `logs/run4-unset`. `logs/actions.log` holds the operator timeline. +The `server-*.log` files are the containers' stdout. From run 2 onwards they carry millisecond timestamps. + +## Legs + +| Leg | Result | Evidence | +|---|---|---| +| L1 metadata | **PASS** | `L1-L2-meta-warm.txt`. Both nodes: `RedundancySupport=2` (Warm), `i=2296` type definition `NonTransparentRedundancyType` (`i=2039`). `ServerUriArray` lists self first: `[central-1, central-2]` on :14840 and `[central-2, central-1]` on :14841. A Write to it returns `BadNotWritable`. `ServerArray` is unchanged and lists the same URIs. | +| L2 bands | **PASS** | Steady state: Primary 250, Standby 90. Boot reads on a rejoin were 1 (run 3) or 50 (run 1, the Standby-degraded band). | +| L3 standby data | **PASS** | `L3-L10-probe-warm.txt`. Standby reads of the raw tag, its UNS reference, the VirtualTag and the writable tag all return `BadOutOfService`/null, while the Primary reads Good at the same moment. Standby subscriptions deliver one `BadOutOfService` per item and then silence. Acquisition continues: on promotion the current value arrives Good without a new driver publish (`run1/watch-c2.log` 09:06:00.371). | +| L4 standby alarms | **PASS** | `run1/watch-c*.log` 09:02:40–09:05:54: the Primary delivered 3 transitions (09:03, 09:04, 09:05) on both the Server-object item and the equipment-folder item; the Standby delivered 0. Reconfirmed after the crash (run 3, 09:26:00). A ConditionRefresh on the Standby returns RefreshStart, the retained `T13 even minute` condition, RefreshEnd. | +| Unsolicited promotion refresh | **PASS** | On every promotion, each live event item received RefreshStart, the retained condition, RefreshEnd, followed by new events (run 1 09:06:00.371; run 3 09:23:28.650 and 09:10:59/09:24:49 crash). | +| L5 graceful failover (`docker stop` of the Primary, i.e. CoordinatedShutdown → cluster Leave) | **PASS** (see note A) | Run 3, final build: the old Primary demoted at 09:23:28.332, the same millisecond it was marked Leaving. Its watcher saw ServiceLevel 0, then `BadOutOfService` on all three tags, with the last Good at src .213 and no Good after the drop. The survivor promoted at .601–.609 in the spec order: RefreshStart/retained/RefreshEnd, then SL 250, then Good data. There was no dual-Active window (~270 ms with no Active node). | +| Three demotion timings (first delivery after demotion) | **PASS** | `run3/timing1..3.txt`, each across a live graceful demotion (detected 96–127 ms after `docker stop`). **T1:** 4 Good samples queued with publishing disabled, then the client handle changed 1→99, then demotion. The first Publish carried 5× `BadOutOfService`/null for h99, while `i=2267` (h2) stayed Good, so it was not rewritten. Republish of the same sequence number returned the same projected payload. **T2:** `maxNotificationsPerPublish=1` with prebuilt messages (`MoreNotifications=true`). Message *a* (seq 2) was Good while Active. After demotion, Republish of the prebuilt never-sent seq 3 returned `BadMessageNotAvailable`, and Publish then delivered seq 3 projected; its Republish was also projected. **T3:** Good was delivered while Active. After demotion, Republish returned the original Good; after the ack it returned `BadMessageNotAvailable`. | +| L10 writes / Part 9 methods on the Standby | **PASS** | `L3-L10-probe-warm.txt` (API-key session with WriteOperate and AlarmAck). Standby: Write on the raw and UNS `calc-writable` returns **`BadInvalidState`**, and so do Acknowledge and AddComment on the condition. Active, same session shape: the Write is Good (the optimistic path; the Calculation driver later reverts it), Acknowledge is Good and AddComment is Good. HistoryRead returns `BadHistoryOperationUnsupported` on **both** nodes because the rig has no historian, so it is not refused by role. | +| L6 crash failover (`docker kill` of the Primary, `PromoteWhilePrimaryUnreachable` on) | **FAIL on `0c19f26c` → PASS after `c569f806`** (findings 1 and 2) | Run 3, final build: kill at 09:24:37.877; unreachable at 09:24:49.263; the survivor promoted at **09:24:49.275** (`unreachable-primary`, 12 ms after its own event). The measured gap from kill to Active is **11.4 s**, all of it the failure detector. The survivor stayed Active through auto-down's removal (09:25:05) with Good data throughout. ServiceLevel went 250 → 200 (Active-stale, because no singleton heartbeat arrived while the Primary was unreachable) → 250 once its own singleton published (09:25:20). | +| L7 rejoin / no failback | **PASS** | After the graceful stop and after the crash, the restarted node came back Standby: the boot read was 1 or 50, then 90, and 250/255 never appeared. The Active node stayed at 250 throughout (`run1`, `run3` 09:24:00, 09:25:48). | +| L11 transfer | **PASS** | `run3/transfer-c1.log`. Session A (API key) created a subscription on the **Standby**, and its items read `BadOutOfService`. Session B, with the same identity, called `TransferSubscriptions(sendInitialValues=true)` and got Good; A received `GoodSubscriptionTransferred`. B's initial values were projected (`BadOutOfService`), and B received no Good until promotion. The Primary was then stopped and the node promoted: B got SL 250 and Good at 09:29:16.621. The other node was restarted and this node was stopped: B got SL 0 and `BadOutOfService` for both tags (09:30:09.514) before the session ended. | +| L9 `Mode=Unset` regression | **PASS** | `run4-unset/`, same pair and same fixed build, recreated with `OpcUa__Redundancy__Mode=Unset`. The levels are 250/240. `RedundancySupport=0` and `i=2296` is `ServerRedundancyType`, as the SDK default. The orphan `i=11314` reads Good `[]`. Both nodes serve Good data, the secondary accepts writes and AddComment (today's path), and the secondary delivers alarm events (09:33:00) with 0 `BadOutOfService` in 190 Good notifications. | +| L8 `Mode=None` standalone | not run | It was not in this gate's scope. It is covered by `WarmRedundancyWireTests` (None metadata). | +| L12 durable restore | not applicable | Durable subscriptions are not enabled (SDK default off), as recorded in #717. | + +## Findings + +**Finding 1: the crash survivor demoted when auto-down removed the Primary (fixed).** On `0c19f26c` (`run1`, +09:10:47 kill), the survivor promoted at 09:10:59 because the Primary was unreachable. At 09:11:15 auto-down removed the +Primary. The snapshot's Primary then left the unreachable set, so rule (3′) no longer held, and the survivor **demoted +to Standby** (`reason=snapshot`, SL 50, `BadOutOfService` to every client). It was Active again only at 09:11:30, when +its own singleton finished the hand-over retries and published. The result was a ~15 s outage immediately after a +successful crash failover. **Fix:** a Primary that is Down, or that this node saw removed (and not back since), counts as +gone (`WarmRoleAuthority.SnapshotPrimaryGone`, removal tracked from `MemberRemoved` in `OpcUaPublishActor`). A member +that is only Leaving/Exiting, or that the node never saw, does not count. Re-verified in run 2 (09:17:53) and run 3 +(09:24:37): no demotion through the removal. + +**Finding 2: the Warm verdict read Akka's read view, which lags the event being handled (fixed).** `OpcUaPublishActor` +re-evaluated on each member/reachability event, but it read `Cluster.State`/`SelfMember`. Those come from +`ClusterReadView`, which is a *separate* subscriber to the same events. In run 2 the survivor stayed Standby **4.3 s** +after its own `UnreachableMember` event (09:18:05.722 → 09:18:09.988); it promoted only on the next health tick. The +same lag shows in run 1's graceful handover. The old Primary demoted only at `Exiting` (SL src 09:06:00.552), while the +survivor was already Active from .316. That left a ~240 ms dual-Active window, in which the old Primary delivered a Good +sample at .400. **Fix:** the Warm verdict uses a membership view built from the actor's own subscription, falling back to +the read view only before the first event. Re-verified: promotion now comes 12 ms after the unreachable event (run 3), +and the graceful demotion lands at the same millisecond as Leaving, with no overlap (run 3, 09:23:28.332). + +Tests (Runtime.Tests, 1058 passed, 3 skipped): + +- `WarmRoleAuthorityTests.A_snapshot_Primary_that_auto_down_removed_is_still_gone_and_the_survivor_stays_Active`, a + truth table that includes the Leaving/Exiting/never-seen negatives; +- `OpcUaPublishActorWarmTests.The_crash_survivor_promotes_on_its_own_unreachable_event_and_stays_Active_through_the_removal`, + which uses synthetic events that the read view never sees, with a positive control: the Primary comes back and + demotes the node. + +Mutations: a read-view-only verdict fails the actor test at promotion; turning off the removed arm fails both tests. + +## Notes + +- **A. The old Primary's level on a graceful leave is 0, not 90.** `ServiceLevelCalculator.ComputeWarm` returns 0 for a + member that is not Up/Joining, so a leaving Primary drops 250 → 0. The plan's L5 text says "drops to 90". 0 is below + every band, and it is what the code specifies for a non-Up member. This is a wording difference, not a defect. +- **B. The Active-stale band after a crash.** The survivor sits at 200 (Active, stale snapshot) for ~10–20 s until its + own singleton publishes. Data stays Good. This is expected from `ComputeWarm`, and clients still see it as the + higher-level node. +- **C. The manual-failover button** (AdminUI) was not used. `docker stop` drives the same mechanism + (CoordinatedShutdown → cluster Leave), and it can be scripted with millisecond timing. +- Both nodes ran the same build in every run. The pre-fix image was used only for runs 1 and 2. diff --git a/docs/plans/evidence/2026-11-13-717-t13/harness/Program.cs b/docs/plans/evidence/2026-11-13-717-t13/harness/Program.cs new file mode 100644 index 000000000..39e33cc14 --- /dev/null +++ b/docs/plans/evidence/2026-11-13-717-t13/harness/Program.cs @@ -0,0 +1,445 @@ +// #717 T13 live-gate harness: a raw OPC UA client (SDK 2.0.1) against the isolated otopcua-t13 pair. +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using Opc.Ua; +using Opc.Ua.Client; + +const string RawNs = "https://zb.com/otopcua/raw"; +const string UnsNs = "https://zb.com/otopcua/uns"; +const string Equip = "EQ-3326c7a4728e"; +var apiKey = File.ReadAllText(Environment.GetEnvironmentVariable("T13_APIKEY_FILE") ?? "../apikey.txt").Trim(); + +var cmd = args[0]; +var url = args.Length > 1 ? args[1] : "opc.tcp://localhost:14840/OtOpcUa"; +using var cts = new CancellationTokenSource(); +Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); }; +var ct = cts.Token; + +switch (cmd) +{ + case "meta": await Meta(url); break; + case "browse": await Browse(url, args[2]); break; + case "watch": await Watch(url, args[2], args.Length > 3 && args[3] == "anon"); break; + case "probe": await Probe(url, args.Length > 2 && args[2] == "ack"); break; + case "timing1": await Timing1(url, args[2]); break; + case "timing2": await Timing2(url, args[2]); break; + case "timing3": await Timing3(url, args[2]); break; + case "transfer": await Transfer(url, args[2]); break; + default: throw new ArgumentException(cmd); +} + +return; + +// ------------------------------------------------------------------ helpers + +static string Now() => DateTime.UtcNow.ToString("HH:mm:ss.fff"); +static RequestHeader H() => new() { Timestamp = DateTimeUtc.Now, TimeoutHint = 10_000 }; +static string Val(DataValue v) => $"{(v.WrappedValue.IsNull ? "null" : v.WrappedValue.AsBoxedObject(Variant.BoxingBehavior.Legacy)?.ToString())} [{StatusCode.LookupSymbolicId(v.StatusCode.Code) ?? v.StatusCode.ToString()}] src={((DateTime)v.SourceTimestamp):HH:mm:ss.fff}"; +static string Sc(StatusCode s) => StatusCode.LookupSymbolicId(s.Code) ?? s.ToString(); + +async Task Connect(string endpointUrl, bool anonymous = false) +{ + var telemetry = DefaultTelemetry.Create(_ => { }); + var pki = Path.Combine(Path.GetTempPath(), "t13-client-pki"); + var config = new ApplicationConfiguration(telemetry) + { + ApplicationName = "T13Harness", + ApplicationUri = $"urn:T13Harness:{Guid.NewGuid():N}", + ApplicationType = ApplicationType.Client, + SecurityConfiguration = new SecurityConfiguration + { + ApplicationCertificate = new CertificateIdentifier { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(pki, "own") }, + TrustedIssuerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(pki, "issuer") }, + TrustedPeerCertificates = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(pki, "trusted") }, + RejectedCertificateStore = new CertificateTrustList { StoreType = CertificateStoreType.Directory, StorePath = Path.Combine(pki, "rejected") }, + AutoAcceptUntrustedCertificates = true, + }, + ClientConfiguration = new ClientConfiguration { DefaultSessionTimeout = 60_000 }, + }; + await config.ValidateAsync(ApplicationType.Client, ct); + var endpoint = await CoreClientUtils.SelectEndpointAsync(config, endpointUrl, false, telemetry, ct); + var identity = anonymous ? new UserIdentity(new AnonymousIdentityToken()) : new UserIdentity("apikey", Encoding.UTF8.GetBytes(apiKey)); + return await new DefaultSessionFactory(telemetry).CreateAsync(config, + new ConfiguredEndpoint(null, endpoint!, EndpointConfiguration.Create(config)), + false, false, "T13Harness", 60_000, identity, default, ct); +} + +NodeId RawN(ISession s, string id) => new(id, (ushort)s.NamespaceUris.GetIndex(RawNs)); +NodeId UnsN(ISession s, string id) => new(id, (ushort)s.NamespaceUris.GetIndex(UnsNs)); + +async Task Read(ISession s, NodeId n, uint attr = Attributes.Value) +{ + var r = await ((SessionClient)s).ReadAsync(H(), 0, TimestampsToReturn.Both, new[] { new ReadValueId { NodeId = n, AttributeId = attr } }, ct); + return r.Results[0]; +} + +async Task ReadSl(ISession s) => Convert.ToByte((await Read(s, VariableIds.Server_ServiceLevel)).WrappedValue.AsBoxedObject(Variant.BoxingBehavior.Legacy)); + +static EventFilter EventFilter() +{ + var f = new EventFilter(); + f.AddSelectClause(new NodeId(ObjectTypes.BaseEventType), QualifiedName.From(BrowseNames.EventId)); // 0 + f.AddSelectClause(new NodeId(ObjectTypes.BaseEventType), QualifiedName.From(BrowseNames.EventType)); // 1 + f.AddSelectClause(new NodeId(ObjectTypes.BaseEventType), QualifiedName.From(BrowseNames.Message)); // 2 + f.AddSelectClause(new NodeId(ObjectTypes.BaseEventType), QualifiedName.From(BrowseNames.Time)); // 3 + f.SelectClauses += new SimpleAttributeOperand { TypeDefinitionId = ObjectTypeIds.ConditionType, AttributeId = Attributes.NodeId }; // 4 ConditionId + f.AddSelectClause(ObjectTypeIds.ConditionType, QualifiedName.From(BrowseNames.Retain)); // 5 + f.SelectClauses += new SimpleAttributeOperand(ObjectTypeIds.AlarmConditionType, [QualifiedName.From(BrowseNames.ActiveState), QualifiedName.From(BrowseNames.Id)]); // 6 + f.SelectClauses += new SimpleAttributeOperand(ObjectTypeIds.AcknowledgeableConditionType, [QualifiedName.From(BrowseNames.AckedState), QualifiedName.From(BrowseNames.Id)]); // 7 + f.AddSelectClause(new NodeId(ObjectTypes.BaseEventType), QualifiedName.From(BrowseNames.SourceName)); // 8 + return f; +} + +static string TypeName(NodeId t) => + t == ObjectTypeIds.RefreshStartEventType ? "RefreshStart" : + t == ObjectTypeIds.RefreshEndEventType ? "RefreshEnd" : + t == ObjectTypeIds.GeneralModelChangeEventType ? "ModelChange" : + t.IsNull ? "?" : t.ToString(); + +static string EventStr(ArrayOf f) +{ + var type = TypeName(f[1].GetNodeId()); + var msg = f[2].GetLocalizedText().Text; + var cond = f[4].IsNull ? "" : f[4].AsBoxedObject(Variant.BoxingBehavior.Legacy)?.ToString(); + return $"type={type} msg=\"{msg}\" cond={cond} retain={f[5].AsBoxedObject(Variant.BoxingBehavior.Legacy)} active={f[6].AsBoxedObject(Variant.BoxingBehavior.Legacy)} acked={f[7].AsBoxedObject(Variant.BoxingBehavior.Legacy)} time={(f[3].IsNull ? "" : ((DateTime)f[3].GetDateTime()).ToString("HH:mm:ss.fff"))}"; +} + +async Task AddItem(SessionClient c, uint sub, NodeId node, uint handle, uint queue = 10, MonitoringMode mode = MonitoringMode.Reporting, MonitoringFilter? filter = null, uint attr = Attributes.Value) +{ + var r = await c.CreateMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[] + { + new MonitoredItemCreateRequest + { + ItemToMonitor = new ReadValueId { NodeId = node, AttributeId = attr }, + MonitoringMode = mode, + RequestedParameters = new MonitoringParameters + { + ClientHandle = handle, SamplingInterval = attr == Attributes.EventNotifier ? 0 : 100, QueueSize = queue, DiscardOldest = true, + Filter = filter is null ? ExtensionObject.Null : new ExtensionObject(filter), + }, + }, + }, ct); + Console.WriteLine($"{Now()} create item h={handle} {node}: {Sc(r.Results[0].StatusCode)} id={r.Results[0].MonitoredItemId}"); + return r.Results[0].MonitoredItemId; +} + +static List Lines(NotificationMessage m, IReadOnlyDictionary names) +{ + var list = new List(); + for (var i = 0; i < m.NotificationData.Count; i++) + { + if (m.NotificationData[i].TryGetValue(out var d)) + for (var j = 0; j < d.MonitoredItems.Count; j++) + list.Add($"data h={d.MonitoredItems[j].ClientHandle}({names.GetValueOrDefault(d.MonitoredItems[j].ClientHandle, "?")}) {Val(d.MonitoredItems[j].Value)}"); + else if (m.NotificationData[i].TryGetValue(out var e)) + foreach (var ev in e.Events) list.Add($"event h={ev.ClientHandle}({names.GetValueOrDefault(ev.ClientHandle, "?")}) {EventStr(ev.EventFields)}"); + else if (m.NotificationData[i].TryGetValue(out var s)) + list.Add($"status {Sc(s.Status)}"); + } + + return list; +} + +static List<(uint Handle, DataValue Value)> DataChanges(NotificationMessage m) +{ + var list = new List<(uint, DataValue)>(); + for (var i = 0; i < m.NotificationData.Count; i++) + if (m.NotificationData[i].TryGetValue(out var d)) + for (var j = 0; j < d.MonitoredItems.Count; j++) list.Add((d.MonitoredItems[j].ClientHandle, d.MonitoredItems[j].Value)); + return list; +} + +async Task PublishData(SessionClient c, ArrayOf acks = default) +{ + for (var i = 0; i < 40; i++) + { + var r = await c.PublishAsync(H(), acks, ct); + acks = default; + if (r.NotificationMessage.NotificationData.Count > 0) return r; + } + + throw new InvalidOperationException("no data message"); +} + +// Stops the container in the background and returns once this session reads a non-Active ServiceLevel. +async Task<(Process Stop, DateTime Issued, DateTime Demoted, byte Sl)> StopAndWaitForDemotion(ISession s, string container) +{ + var issued = DateTime.UtcNow; + var p = Process.Start(new ProcessStartInfo("docker", $"stop -t 60 {container}") { RedirectStandardOutput = true, RedirectStandardError = true })!; + Console.WriteLine($"{Now()} docker stop {container} issued"); + while (true) + { + var sl = await ReadSl(s); + if (sl < 200) { var t = DateTime.UtcNow; Console.WriteLine($"{Now()} demoted: ServiceLevel={sl} after {(t - issued).TotalMilliseconds:F0} ms"); return (p, issued, t, sl); } + await Task.Delay(10, ct); + } +} + +// ------------------------------------------------------------------ commands + +async Task Meta(string u) +{ + using var s = await Connect(u, anonymous: true); + Console.WriteLine($"endpoint {s.Endpoint.EndpointUrl} server={s.Endpoint.Server.ApplicationUri}"); + foreach (var (name, n) in new[] { ("RedundancySupport i=3709", VariableIds.Server_ServerRedundancy_RedundancySupport), + ("ServerUriArray i=11314", VariableIds.Server_ServerRedundancy_ServerUriArray), + ("ServiceLevel i=2267", VariableIds.Server_ServiceLevel), + ("ServerArray i=2254", VariableIds.Server_ServerArray) }) + { + var v = await Read(s, n); + var o = v.WrappedValue.AsBoxedObject(Variant.BoxingBehavior.Legacy); + Console.WriteLine($"{name}: {(o is Array a ? "[" + string.Join(", ", a.Cast()) + "]" : o)} [{Sc(v.StatusCode)}]"); + } + + var b = await ((SessionClient)s).BrowseAsync(H(), null, 0, new[] + { + new BrowseDescription { NodeId = ObjectIds.Server_ServerRedundancy, BrowseDirection = BrowseDirection.Forward, + ReferenceTypeId = ReferenceTypeIds.HasTypeDefinition, IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All }, + }, ct); + Console.WriteLine($"ServerRedundancy i=2296 TypeDefinition: {string.Join(",", b.Results[0].References.ToArray()!.Select(r => r.BrowseName.Name + " " + r.NodeId))}"); + var w = await ((SessionClient)s).WriteAsync(H(), new[] + { + new WriteValue { NodeId = VariableIds.Server_ServerRedundancy_ServerUriArray, AttributeId = Attributes.Value, + Value = new DataValue(Variant.From(new[] { "urn:x" })) }, + }, ct); + Console.WriteLine($"Write ServerUriArray: {Sc(w.Results[0])}"); +} + +async Task Browse(string u, string node) +{ + using var s = await Connect(u, anonymous: true); + var start = node.StartsWith("raw:") ? RawN(s, node[4..]) : node.StartsWith("uns:") ? UnsN(s, node[4..]) : NodeId.Parse(node); + var b = await ((SessionClient)s).BrowseAsync(H(), null, 0, new[] + { + new BrowseDescription { NodeId = start, BrowseDirection = BrowseDirection.Forward, ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, + IncludeSubtypes = true, ResultMask = (uint)BrowseResultMask.All }, + }, ct); + Console.WriteLine($"browse {start}: {Sc(b.Results[0].StatusCode)}"); + foreach (var r in b.Results[0].References.ToArray()!) Console.WriteLine($" {r.BrowseName.Name} {r.NodeId} {r.NodeClass}"); +} + +async Task Watch(string u, string outFile, bool anonymous) +{ + var tag = Path.GetFileNameWithoutExtension(outFile); + await using var w = new StreamWriter(outFile, append: true) { AutoFlush = true }; + void Log(string line) { var l = $"{Now()} [{tag}] {line}"; w.WriteLine(l); Console.WriteLine(l); } + var names = new Dictionary { [1] = "SL", [2] = "raw calc-seconds", [3] = "uns calc-seconds", [4] = "uns vt-second", [10] = "ev Server", [11] = "ev equipment" }; + while (!ct.IsCancellationRequested) + { + ISession s; + try { s = await Connect(u, anonymous); } + catch (Exception ex) { Log($"connect failed: {ex.GetType().Name} {ex.Message.Split('\n')[0]}"); await Task.Delay(500, ct); continue; } + try + { + Log($"connected session={s.SessionId}; boot read ServiceLevel={await ReadSl(s)}"); + var c = (SessionClient)s; + var sub = (await c.CreateSubscriptionAsync(H(), 100, 600, 10, 0, true, 0, ct)).SubscriptionId; + await AddItem(c, sub, VariableIds.Server_ServiceLevel, 1); + await AddItem(c, sub, RawN(s, "t13-calc/Device1/calc-seconds"), 2); + await AddItem(c, sub, UnsN(s, "t13/warm/t13-equipment/calc-seconds"), 3); + await AddItem(c, sub, UnsN(s, $"{Equip}/vt-second"), 4); + await AddItem(c, sub, ObjectIds.Server, 10, 1000, filter: EventFilter(), attr: Attributes.EventNotifier); + await AddItem(c, sub, UnsN(s, Equip), 11, 1000, filter: EventFilter(), attr: Attributes.EventNotifier); + ArrayOf acks = default; + while (!ct.IsCancellationRequested) + { + var r = await c.PublishAsync(H(), acks, ct); + var m = r.NotificationMessage; + acks = m.NotificationData.Count > 0 ? new[] { new SubscriptionAcknowledgement { SubscriptionId = r.SubscriptionId, SequenceNumber = m.SequenceNumber } } : default; + foreach (var line in Lines(m, names)) Log($"seq={m.SequenceNumber} {line}"); + } + } + catch (Exception ex) when (!ct.IsCancellationRequested) + { + Log($"session lost: {ex.GetType().Name} {ex.Message.Split('\n')[0]}"); + } + finally { try { s.Dispose(); } catch { } } + await Task.Delay(250, CancellationToken.None); + } +} + +async Task Probe(string u, bool ack) +{ + using var s = await Connect(u); + var c = (SessionClient)s; + Console.WriteLine($"{Now()} probe {u} ServiceLevel={await ReadSl(s)}"); + foreach (var (name, n) in new[] { ("raw calc-seconds", RawN(s, "t13-calc/Device1/calc-seconds")), ("uns calc-seconds", UnsN(s, "t13/warm/t13-equipment/calc-seconds")), + ("uns vt-second", UnsN(s, $"{Equip}/vt-second")), ("raw calc-writable", RawN(s, "t13-calc/Device1/calc-writable")) }) + Console.WriteLine($" read {name}: {Val(await Read(s, n))}"); + + foreach (var (name, n) in new[] { ("raw calc-writable", RawN(s, "t13-calc/Device1/calc-writable")), ("uns calc-writable", UnsN(s, "t13/warm/t13-equipment/calc-writable")) }) + { + var w = await c.WriteAsync(H(), new[] { new WriteValue { NodeId = n, AttributeId = Attributes.Value, Value = new DataValue(Variant.From(7)) } }, ct); + Console.WriteLine($" write {name}=7: {Sc(w.Results[0])}"); + } + + var hr = await c.HistoryReadAsync(H(), new ExtensionObject(new ReadRawModifiedDetails + { + StartTime = DateTime.UtcNow.AddMinutes(-5), EndTime = DateTime.UtcNow, NumValuesPerNode = 10, ReturnBounds = false, + }), TimestampsToReturn.Both, false, new[] { new HistoryReadValueId { NodeId = RawN(s, "t13-calc/Device1/calc-seconds") } }, ct); + Console.WriteLine($" HistoryRead raw calc-seconds: service={Sc(hr.ResponseHeader.ServiceResult)} result={Sc(hr.Results[0].StatusCode)}"); + + // ConditionRefresh on a fresh subscription with a Server-object event item. + var sub = (await c.CreateSubscriptionAsync(H(), 100, 600, 10, 0, true, 0, ct)).SubscriptionId; + await AddItem(c, sub, ObjectIds.Server, 10, 1000, filter: EventFilter(), attr: Attributes.EventNotifier); + var names = new Dictionary { [10] = "ev Server" }; + await c.PublishAsync(H(), default, ct); // drain anything initial + var refresh = await c.CallAsync(H(), new[] { new CallMethodRequest { ObjectId = ObjectTypeIds.ConditionType, MethodId = MethodIds.ConditionType_ConditionRefresh, + InputArguments = new[] { Variant.From(sub) } } }, ct); + Console.WriteLine($" ConditionRefresh call: {Sc(refresh.Results[0].StatusCode)}"); + byte[]? eventId = null; NodeId? condId = null; + var until = DateTime.UtcNow.AddSeconds(3); + ArrayOf acks = default; + while (DateTime.UtcNow < until) + { + var r = await c.PublishAsync(H(), acks, ct); + acks = r.NotificationMessage.NotificationData.Count > 0 ? new[] { new SubscriptionAcknowledgement { SubscriptionId = sub, SequenceNumber = r.NotificationMessage.SequenceNumber } } : default; + foreach (var line in Lines(r.NotificationMessage, names)) Console.WriteLine($" {Now()} {line}"); + for (var i = 0; i < r.NotificationMessage.NotificationData.Count; i++) + if (r.NotificationMessage.NotificationData[i].TryGetValue(out var e)) + foreach (var ev in e.Events) + if (!ev.EventFields[4].IsNull) { eventId = ev.EventFields[0].GetByteString().ToArray(); condId = ev.EventFields[4].GetNodeId(); } + } + + if (condId is not null) + { + foreach (var (mname, method, inArgs) in new (string, NodeId, Variant[])[] + { + ("Acknowledge", MethodIds.AcknowledgeableConditionType_Acknowledge, [Variant.From(ByteString.From(eventId!)), Variant.From(new LocalizedText("t13"))]), + ("AddComment", MethodIds.ConditionType_AddComment, [Variant.From(ByteString.From(eventId!)), Variant.From(new LocalizedText("t13 note"))]), + }) + { + if (mname == "Acknowledge" && !ack) continue; + var call = await c.CallAsync(H(), new[] { new CallMethodRequest { ObjectId = condId.Value, MethodId = method, InputArguments = inArgs } }, ct); + Console.WriteLine($" {mname} on {condId}: {Sc(call.Results[0].StatusCode)}"); + } + } + else Console.WriteLine(" no condition seen in refresh"); +} + +async Task Timing1(string u, string container) +{ + using var s = await Connect(u); + var c = (SessionClient)s; + Console.WriteLine($"{Now()} T1 on {u}: ServiceLevel={await ReadSl(s)}"); + var sub = (await c.CreateSubscriptionAsync(H(), 50, 600, 10, 0, true, 0, ct)).SubscriptionId; + var tagItem = await AddItem(c, sub, RawN(s, "t13-calc/Device1/calc-seconds"), 1, 10); + await AddItem(c, sub, VariableIds.Server_ServiceLevel, 2, 5); + var first = DataChanges((await PublishData(c)).NotificationMessage); + Console.WriteLine($"{Now()} initial: {string.Join("; ", first.Select(x => $"h{x.Handle} {Val(x.Value)}"))}"); + await c.SetPublishingModeAsync(H(), false, new[] { sub }, ct); + await Task.Delay(3500, ct); // ~3 Good samples queue in the item while Active + await c.ModifyMonitoredItemsAsync(H(), sub, TimestampsToReturn.Both, new[] + { + new MonitoredItemModifyRequest { MonitoredItemId = tagItem, RequestedParameters = new MonitoringParameters { ClientHandle = 99, SamplingInterval = 100, QueueSize = 10, DiscardOldest = true } }, + }, ct); + Console.WriteLine($"{Now()} publishing disabled 3.5 s, client handle 1 -> 99"); + var (stop, _, _, _) = await StopAndWaitForDemotion(s, container); + await c.SetPublishingModeAsync(H(), true, new[] { sub }, ct); + var msg = await PublishData(c); + var changes = DataChanges(msg.NotificationMessage); + foreach (var x in changes) Console.WriteLine($"{Now()} first delivery seq={msg.NotificationMessage.SequenceNumber}: h{x.Handle} {Val(x.Value)}"); + var tagChanges = changes.Where(x => x.Handle is 1 or 99).ToList(); + var pass1 = tagChanges.Count > 0 && tagChanges.All(x => x.Value.StatusCode == StatusCodes.BadOutOfService && x.Value.WrappedValue.IsNull); + var re = await c.RepublishAsync(H(), sub, msg.NotificationMessage.SequenceNumber, ct); + var reTag = DataChanges(re.NotificationMessage).Where(x => x.Handle is 1 or 99).ToList(); + foreach (var x in DataChanges(re.NotificationMessage)) Console.WriteLine($"{Now()} republish seq={msg.NotificationMessage.SequenceNumber}: h{x.Handle} {Val(x.Value)}"); + var pass2 = reTag.Count == tagChanges.Count && reTag.All(x => x.Value.StatusCode == StatusCodes.BadOutOfService); + Console.WriteLine($"T1 RESULT first-delivery-projected={pass1} republish-matches={pass2} (queued Good samples={tagChanges.Count})"); + await stop.WaitForExitAsync(ct); + Console.WriteLine($"{Now()} docker stop returned"); +} + +async Task Timing2(string u, string container) +{ + using var s = await Connect(u); + var c = (SessionClient)s; + Console.WriteLine($"{Now()} T2 on {u}: ServiceLevel={await ReadSl(s)}"); + var sub = (await c.CreateSubscriptionAsync(H(), 50, 600, 10, 1, true, 0, ct)).SubscriptionId; // maxNotificationsPerPublish=1 + await AddItem(c, sub, RawN(s, "t13-calc/Device1/calc-seconds"), 1, 10); + Console.WriteLine($"{Now()} initial: {string.Join("; ", DataChanges((await PublishData(c)).NotificationMessage).Select(x => Val(x.Value)))}"); + await c.SetPublishingModeAsync(H(), false, new[] { sub }, ct); + await Task.Delay(4500, ct); + await c.SetPublishingModeAsync(H(), true, new[] { sub }, ct); + var a = await PublishData(c); + var aVal = DataChanges(a.NotificationMessage).Single(); + Console.WriteLine($"{Now()} a seq={a.NotificationMessage.SequenceNumber} {Val(aVal.Value)} available=[{string.Join(",", a.AvailableSequenceNumbers.ToArray()!)}] more={a.MoreNotifications}"); + var (stop, _, _, _) = await StopAndWaitForDemotion(s, container); + var next = a.NotificationMessage.SequenceNumber + 1; + string unsent; + try { await c.RepublishAsync(H(), sub, next, ct); unsent = "Good (unexpected)"; } + catch (ServiceResultException ex) { unsent = Sc(ex.StatusCode); } + Console.WriteLine($"{Now()} Republish(prebuilt never-sent seq={next}): {unsent}"); + var b = await PublishData(c); + foreach (var x in DataChanges(b.NotificationMessage)) Console.WriteLine($"{Now()} b seq={b.NotificationMessage.SequenceNumber}: {Val(x.Value)}"); + var bProjected = DataChanges(b.NotificationMessage).All(x => x.Value.StatusCode == StatusCodes.BadOutOfService); + var bRe = await c.RepublishAsync(H(), sub, b.NotificationMessage.SequenceNumber, ct); + var bReProjected = DataChanges(bRe.NotificationMessage).All(x => x.Value.StatusCode == StatusCodes.BadOutOfService); + Console.WriteLine($"T2 RESULT a-good={aVal.Value.StatusCode == StatusCodes.Good} unsent-republish={unsent} b-seq-is-next={b.NotificationMessage.SequenceNumber == next} b-projected={bProjected} b-republish-projected={bReProjected}"); + await stop.WaitForExitAsync(ct); + Console.WriteLine($"{Now()} docker stop returned"); +} + +async Task Timing3(string u, string container) +{ + using var s = await Connect(u); + var c = (SessionClient)s; + Console.WriteLine($"{Now()} T3 on {u}: ServiceLevel={await ReadSl(s)}"); + var sub = (await c.CreateSubscriptionAsync(H(), 50, 600, 10, 0, true, 0, ct)).SubscriptionId; + await AddItem(c, sub, RawN(s, "t13-calc/Device1/calc-seconds"), 1, 10); + var first = await PublishData(c); + foreach (var x in DataChanges(first.NotificationMessage)) Console.WriteLine($"{Now()} first seq={first.NotificationMessage.SequenceNumber}: {Val(x.Value)}"); + var firstGood = DataChanges(first.NotificationMessage).All(x => x.Value.StatusCode == StatusCodes.Good); + await c.SetPublishingModeAsync(H(), false, new[] { sub }, ct); // keep later sequence numbers out of the way + var (stop, _, _, _) = await StopAndWaitForDemotion(s, container); + var again = await c.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct); + foreach (var x in DataChanges(again.NotificationMessage)) Console.WriteLine($"{Now()} republish after demotion seq={first.NotificationMessage.SequenceNumber}: {Val(x.Value)}"); + var againGood = DataChanges(again.NotificationMessage).All(x => x.Value.StatusCode == StatusCodes.Good); + await c.SetPublishingModeAsync(H(), true, new[] { sub }, ct); + var acked = await c.PublishAsync(H(), new[] { new SubscriptionAcknowledgement { SubscriptionId = sub, SequenceNumber = first.NotificationMessage.SequenceNumber } }, ct); + Console.WriteLine($"{Now()} ack seq={first.NotificationMessage.SequenceNumber}: {Sc(acked.Results[0])}"); + string after; + try { await c.RepublishAsync(H(), sub, first.NotificationMessage.SequenceNumber, ct); after = "Good (unexpected)"; } + catch (ServiceResultException ex) { after = Sc(ex.StatusCode); } + Console.WriteLine($"T3 RESULT first-good={firstGood} republish-after-demotion-good={againGood} republish-after-ack={after}"); + await stop.WaitForExitAsync(ct); + Console.WriteLine($"{Now()} docker stop returned"); +} + +async Task Transfer(string u, string outFile) +{ + var tag = Path.GetFileNameWithoutExtension(outFile); + await using var w = new StreamWriter(outFile, append: true) { AutoFlush = true }; + void Log(string line) { var l = $"{Now()} [{tag}] {line}"; w.WriteLine(l); Console.WriteLine(l); } + var names = new Dictionary { [1] = "SL", [2] = "raw calc-seconds", [3] = "uns calc-seconds" }; + using var a = await Connect(u); + var ca = (SessionClient)a; + Log($"session A={a.SessionId} ServiceLevel={await ReadSl(a)}"); + var sub = (await ca.CreateSubscriptionAsync(H(), 100, 600, 10, 0, true, 0, ct)).SubscriptionId; + await AddItem(ca, sub, VariableIds.Server_ServiceLevel, 1); + await AddItem(ca, sub, RawN(a, "t13-calc/Device1/calc-seconds"), 2); + await AddItem(ca, sub, UnsN(a, "t13/warm/t13-equipment/calc-seconds"), 3); + var pre = await ca.PublishAsync(H(), default, ct); + foreach (var line in Lines(pre.NotificationMessage, names)) Log($"A seq={pre.NotificationMessage.SequenceNumber} {line}"); + + using var b = await Connect(u); + var cb = (SessionClient)b; + var t = await cb.TransferSubscriptionsAsync(H(), new[] { sub }, true, ct); + Log($"session B={b.SessionId} TransferSubscriptions({sub}, sendInitialValues=true): {Sc(t.Results[0].StatusCode)}"); + var aAfter = await ca.PublishAsync(H(), default, ct); + foreach (var line in Lines(aAfter.NotificationMessage, names)) Log($"A after transfer: {line}"); + + ArrayOf acks = default; + try + { + while (!ct.IsCancellationRequested) + { + var r = await cb.PublishAsync(H(), acks, ct); + var m = r.NotificationMessage; + acks = m.NotificationData.Count > 0 ? new[] { new SubscriptionAcknowledgement { SubscriptionId = r.SubscriptionId, SequenceNumber = m.SequenceNumber } } : default; + foreach (var line in Lines(m, names)) Log($"B seq={m.SequenceNumber} {line}"); + } + } + catch (Exception ex) when (!ct.IsCancellationRequested) { Log($"B session lost: {ex.GetType().Name} {ex.Message.Split('\n')[0]}"); } +} diff --git a/docs/plans/evidence/2026-11-13-717-t13/harness/T13Harness.csproj b/docs/plans/evidence/2026-11-13-717-t13/harness/T13Harness.csproj new file mode 100644 index 000000000..29edd2a8c --- /dev/null +++ b/docs/plans/evidence/2026-11-13-717-t13/harness/T13Harness.csproj @@ -0,0 +1,13 @@ + + + Exe + net10.0 + enable + enable + $(NoWarn);CS1591;CS0618 + + + + + + diff --git a/docs/plans/evidence/2026-11-13-717-t13/rig/Dockerfile b/docs/plans/evidence/2026-11-13-717-t13/rig/Dockerfile new file mode 100644 index 000000000..c8c2804bb --- /dev/null +++ b/docs/plans/evidence/2026-11-13-717-t13/rig/Dockerfile @@ -0,0 +1,5 @@ +FROM mcr.microsoft.com/dotnet/aspnet:10.0 +WORKDIR /app +COPY publish/ ./ +EXPOSE 9000 4053 4840 +ENTRYPOINT ["dotnet", "OtOpcUa.Host.dll"] diff --git a/docs/plans/evidence/2026-11-13-717-t13/rig/compose.t13.yml b/docs/plans/evidence/2026-11-13-717-t13/rig/compose.t13.yml new file mode 100644 index 000000000..f65b7eeba --- /dev/null +++ b/docs/plans/evidence/2026-11-13-717-t13/rig/compose.t13.yml @@ -0,0 +1,103 @@ +# Isolated #717 T13 pair. Project name otopcua-t13; own network, own SQL + volumes, own image tag. +name: otopcua-t13 +x-host-env: &host-env + OTOPCUA_ROLES: "admin,driver,cluster-MAIN" + ASPNETCORE_URLS: "http://+:9000" + ConnectionStrings__ConfigDb: "Server=sql,1433;Database=OtOpcUa;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;" + Secrets__SqlitePath: "/data/otopcua-secrets.db" + Secrets__Replication__Enabled: "true" + Secrets__Replication__AnnounceInterval: "00:00:05" + ZB_SECRETS_MASTER_KEY: "ZYGhIX0luS/XsevpCB2W18jYHMcqO6AjM9oXy+T6Zp4=" + Security__ApiKeys__Pepper: "t13-apikey-pepper-not-for-production" + # LDAP block is validated at boot; it is AdminUI-login only and login is disabled, so point it at loopback + # (nothing here contacts the shared GLAuth). + Security__Ldap__Enabled: "true" + Security__Ldap__DevStubMode: "false" + Security__Ldap__Server: "127.0.0.1" + Security__Ldap__Port: "3893" + Security__Ldap__Transport: "None" + Security__Ldap__AllowInsecure: "true" + Security__Ldap__SearchBase: "dc=zb,dc=local" + Security__Ldap__ServiceAccountDn: "cn=serviceaccount,dc=zb,dc=local" + Security__Ldap__ServiceAccountPassword: "unused" + Cluster__Hostname: "0.0.0.0" + Cluster__Port: "4053" + ConfigServe__GrpcListenPort: "4055" + ConfigServe__ApiKey: "configserve-t13-key" + ConfigSource__Mode: "Direct" + Telemetry__GrpcListenPort: "4056" + Telemetry__Mode: "Grpc" + Telemetry__ApiKey: "telemetry-t13-key" + TelemetryDial__Mode: "Grpc" + TelemetryDial__ApiKey: "telemetry-t13-key" + Cluster__Roles__0: "admin" + Cluster__Roles__1: "driver" + Cluster__Roles__2: "cluster-MAIN" + MeshTransport__Mode: "Grpc" + MeshTransport__GrpcPsk: "t13-mesh-psk-MAIN" + MeshTransport__CentralGrpcEndpoints__0: "http://central-1:4055" + MeshTransport__CentralGrpcEndpoints__1: "http://central-2:4055" + MeshTransport__ClusterPsks__MAIN: "t13-mesh-psk-MAIN" + Security__Jwt__SigningKey: "t13-signing-key-with-at-least-32-bytes-of-utf8-content-12345678" + Security__Jwt__Issuer: "otopcua-t13" + Security__Jwt__Audience: "otopcua-t13" + Security__DeployApiKey: "t13-deploy-key" + Security__Auth__DisableLogin: "true" + Security__Acl__Enabled: "false" + ServerHistorian__Enabled: "false" + ContinuousHistorization__Enabled: "false" + Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore: "Warning" + Serilog__MinimumLevel__Override__Microsoft.AspNetCore: "Warning" + LocalDb__Path: "/app/data/otopcua-localdb.db" + OpcUa__Redundancy__Mode: "${T13_MODE:-Warm}" + Serilog__WriteTo__0__Args__outputTemplate: "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}" + +services: + sql: + image: mcr.microsoft.com/mssql/server:2022-latest + environment: + ACCEPT_EULA: "Y" + SA_PASSWORD: "OtOpcUa!Dev123" + MSSQL_PID: Developer + ports: ["14339:1433"] + volumes: [t13-mssql-data:/var/opt/mssql] + healthcheck: + test: ["CMD-SHELL", "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'OtOpcUa!Dev123' -No -Q 'SELECT 1' || exit 1"] + interval: 10s + timeout: 5s + retries: 20 + + central-1: + image: otopcua-host:${T13_IMAGE:-t13-0c19f26c} + mem_limit: 2g + restart: unless-stopped + depends_on: { sql: { condition: service_healthy } } + environment: + <<: *host-env + Cluster__PublicHostname: "central-1" + OpcUa__ApplicationUri: "urn:OtOpcUa:central-1" + OpcUa__PeerApplicationUris__0: "urn:OtOpcUa:central-2" + Cluster__SeedNodes__0: "akka.tcp://otopcua@central-1:4053" + Cluster__SeedNodes__1: "akka.tcp://otopcua@central-2:4053" + ports: ["14840:4840", "19210:9000"] + volumes: [t13-c1:/app/data, t13-c1:/data] + + central-2: + image: otopcua-host:${T13_IMAGE:-t13-0c19f26c} + mem_limit: 2g + restart: unless-stopped + depends_on: { sql: { condition: service_healthy }, central-1: { condition: service_started } } + environment: + <<: *host-env + Cluster__PublicHostname: "central-2" + OpcUa__ApplicationUri: "urn:OtOpcUa:central-2" + OpcUa__PeerApplicationUris__0: "urn:OtOpcUa:central-1" + Cluster__SeedNodes__0: "akka.tcp://otopcua@central-2:4053" + Cluster__SeedNodes__1: "akka.tcp://otopcua@central-1:4053" + ports: ["14841:4840", "19211:9000"] + volumes: [t13-c2:/app/data, t13-c2:/data] + +volumes: + t13-mssql-data: + t13-c1: + t13-c2: diff --git a/docs/plans/evidence/2026-11-13-717-t13/rig/fixeq.sql b/docs/plans/evidence/2026-11-13-717-t13/rig/fixeq.sql new file mode 100644 index 000000000..a4591bd7f --- /dev/null +++ b/docs/plans/evidence/2026-11-13-717-t13/rig/fixeq.sql @@ -0,0 +1,8 @@ +SET XACT_ABORT ON; BEGIN TRAN; +ALTER TABLE dbo.VirtualTag NOCHECK CONSTRAINT ALL; ALTER TABLE dbo.ScriptedAlarm NOCHECK CONSTRAINT ALL; ALTER TABLE dbo.UnsTagReference NOCHECK CONSTRAINT ALL; +UPDATE dbo.Equipment SET EquipmentId='EQ-3326c7a4728e' WHERE EquipmentId='EQ-t13'; +UPDATE dbo.VirtualTag SET EquipmentId='EQ-3326c7a4728e' WHERE EquipmentId='EQ-t13'; +UPDATE dbo.ScriptedAlarm SET EquipmentId='EQ-3326c7a4728e' WHERE EquipmentId='EQ-t13'; +UPDATE dbo.UnsTagReference SET EquipmentId='EQ-3326c7a4728e' WHERE EquipmentId='EQ-t13'; +ALTER TABLE dbo.VirtualTag WITH CHECK CHECK CONSTRAINT ALL; ALTER TABLE dbo.ScriptedAlarm WITH CHECK CHECK CONSTRAINT ALL; ALTER TABLE dbo.UnsTagReference WITH CHECK CHECK CONSTRAINT ALL; +COMMIT; diff --git a/docs/plans/evidence/2026-11-13-717-t13/rig/seed.sql b/docs/plans/evidence/2026-11-13-717-t13/rig/seed.sql new file mode 100644 index 000000000..b6d45d04a --- /dev/null +++ b/docs/plans/evidence/2026-11-13-717-t13/rig/seed.sql @@ -0,0 +1,29 @@ + +SET NOCOUNT ON; SET XACT_ABORT ON; BEGIN TRAN; +INSERT dbo.ServerCluster (ClusterId,Name,Enterprise,Site,NodeCount,RedundancyMode,Enabled,Notes,CreatedBy) + VALUES ('MAIN','Main cluster','zb','t13',2,'Warm',1,'717 T13 isolated pair','t13-seed'); +INSERT dbo.ClusterNode (NodeId,ClusterId,Host,OpcUaPort,DashboardPort,AkkaPort,GrpcPort,ApplicationUri,ServiceLevelBase,Enabled,CreatedBy) + VALUES ('central-1:4053','MAIN','central-1',4840,8081,4053,4056,'urn:OtOpcUa:central-1',200,1,'t13-seed'), + ('central-2:4053','MAIN','central-2',4840,8081,4053,4056,'urn:OtOpcUa:central-2',150,1,'t13-seed'); +INSERT dbo.Script (ScriptId,Name,SourceCode,SourceHash,Language) VALUES + ('SC-t13-calc','T13-Calc',N'return (int)(ctx.Now.TimeOfDay.TotalSeconds);','1476d80c7a7a30dfe7c60c1b01c5703e09df49289f855507bb1047ae4ee03e06','CSharp'), + ('SC-t13-wr','T13-Writable',N'return 0;','6be2e46a3d0765fd4d08def164c35b75afcc481db6fd11cc5118c4e2c20b634d','CSharp'), + ('SC-t13-vt','T13-VT',N'return ctx.Now.Second;','0dede8751d015ae75a5b8b0c48a3d059bea7d3af38dd31833a0ebd09be72153f','CSharp'), + ('SC-t13-alarm','T13-AlarmPredicate',N'var v = ctx.GetTag("t13-calc/Device1/calc-seconds"); if (v.Value is null) return false; return (System.Convert.ToInt32(v.Value) / 60) % 2 == 0;','e1a76a504472219eda6bf2e41a60c3522c5c87fee3ef12f599a60cc7211b389b','CSharp'); +INSERT dbo.DriverInstance (DriverInstanceId,ClusterId,Name,DriverType,Enabled,DriverConfig) + VALUES ('DI-t13-calc','MAIN','t13-calc','Calculation',1,'{}'); +INSERT dbo.Device (DeviceId,DriverInstanceId,Name,Enabled,DeviceConfig) VALUES ('DEV-t13-1','DI-t13-calc','Device1',1,'{}'); +INSERT dbo.Tag (TagId,DeviceId,Name,DataType,AccessLevel,WriteIdempotent,TagConfig) VALUES + ('TAG-t13-sec','DEV-t13-1','calc-seconds','Int32','Read',0,'{"scriptId":"SC-t13-calc","changeTriggered":true,"timerIntervalMs":1000}'), + ('TAG-t13-wr','DEV-t13-1','calc-writable','Int32','ReadWrite',0,'{"scriptId":"SC-t13-wr","changeTriggered":true,"timerIntervalMs":5000}'); +INSERT dbo.UnsArea (UnsAreaId,ClusterId,Name) VALUES ('area-t13','MAIN','t13'); +INSERT dbo.UnsLine (UnsLineId,UnsAreaId,Name) VALUES ('line-t13','area-t13','warm'); +INSERT dbo.Equipment (EquipmentId,EquipmentUuid,UnsLineId,Name,MachineCode,Enabled) VALUES ('EQ-t13',NEWID(),'line-t13','t13-equipment','T13',1); +INSERT dbo.UnsTagReference (UnsTagReferenceId,EquipmentId,TagId,SortOrder) VALUES ('REF-t13-sec','EQ-t13','TAG-t13-sec',0),('REF-t13-wr','EQ-t13','TAG-t13-wr',1); +INSERT dbo.VirtualTag (VirtualTagId,EquipmentId,Name,DataType,ScriptId,ChangeTriggered,TimerIntervalMs,Historize,Enabled) + VALUES ('VT-t13-second','EQ-t13','vt-second','Int32','SC-t13-vt',1,1000,0,1); +INSERT dbo.ScriptedAlarm (ScriptedAlarmId,EquipmentId,Name,AlarmType,Severity,MessageTemplate,PredicateScriptId,HistorizeToAveva,Retain,Enabled) + VALUES ('AL-t13-minute','EQ-t13','t13-even-minute','OffNormalAlarm',600,'T13 even minute','SC-t13-alarm',0,1,1); +INSERT dbo.ApiKey (ApiKeyId,Name,SecretHash,Principal,Enabled,CreatedUtc,Notes) VALUES ('a1b2c3d4e5f60718','t13-operator','','t13-op',1,SYSUTCDATETIME(),'T13 harness'); +INSERT dbo.NodeAcl (NodeAclId,ClusterId,Principal,ScopeKind,ScopeId,PermissionFlags,Notes) VALUES ('ACL-t13-op','MAIN','t13-op','Cluster',NULL,8191,'T13 harness'); +COMMIT;