From 69f7c526d0b593df94dbffe4251766f46e6edd65 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 16 Jun 2026 17:13:14 -0400 Subject: [PATCH] docs: implementation plan for native-typed JSON List values + normalization 6 tasks (NJ-1..NJ-6): native codec + read-both decode; stamp override ElementDataType (#93/M3); idempotent central startup normalizer; site override-load normalization; normalize-on-import; integration + docs. --- docs/plans/2026-06-16-native-typed-json.md | 282 ++++++++++++++++++ ...2026-06-16-native-typed-json.md.tasks.json | 14 + 2 files changed, 296 insertions(+) create mode 100644 docs/plans/2026-06-16-native-typed-json.md create mode 100644 docs/plans/2026-06-16-native-typed-json.md.tasks.json diff --git a/docs/plans/2026-06-16-native-typed-json.md b/docs/plans/2026-06-16-native-typed-json.md new file mode 100644 index 00000000..4eb36023 --- /dev/null +++ b/docs/plans/2026-06-16-native-typed-json.md @@ -0,0 +1,282 @@ +# Native-Typed JSON for List Attributes — Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:executing-plans (or subagent-driven-development) to implement this plan task-by-task. + +**Goal:** Make `AttributeValueCodec` encode List values as native-typed JSON (`[10,20]`, `[true,false]`, ISO dates) while decoding both old (array-of-strings) and new forms, and normalize already-persisted data to the native form across MS SQL, site SQLite, and bundle import. + +**Architecture:** Native `Encode` + read-both `Decode` in the codec is the foundation. An idempotent central MS SQL startup normalizer rewrites old-form rows; the InstanceActor re-persists native form on override-load (site SQLite); the bundle importer re-encodes on import. To give the central normalizer a reliable element type for instance-override rows, the override-write path now stamps `ElementDataType` (folds in follow-up #93/M3). + +**Tech Stack:** C#/.NET 10, System.Text.Json, EF Core 10 (MS SQL + SQLite), Akka.NET, Transport bundles. + +**Design doc:** `docs/plans/2026-06-16-native-typed-json-design.md` (approved). +**Branch:** `feature/native-typed-json` (off main; design committed `91b1aa1`). + +**Conventions for every task:** TDD (failing test → fail → implement → pass → commit). Targeted builds/tests only — `dotnet build src//.csproj`, `dotnet test tests//.csproj --filter `. Full-solution build only in the final task. Do NOT create git worktrees. Commit with pathspec form (`git commit -m "…" -- `) and retry on `.git/index.lock` when implementers run concurrently. + +--- + +### Task NJ-1: Codec — native-typed Encode + read-both Decode + +**Classification:** high-risk +**Estimated implement time:** ~5 min +**Parallelizable with:** NJ-2 +**Blocked by:** none (foundation) + +**Files:** +- Modify: `src/ZB.MOM.WW.ScadaBridge.Commons/Types/AttributeValueCodec.cs` +- Modify: `tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Types/AttributeValueCodecTests.cs` + +This changes the canonical on-disk/on-wire form for List values. It is backward compatible (Decode reads both forms) and additive on the gRPC `string value` field (List is a new type with no external consumer of the old quoted form). + +**Step 1 — Update/extend tests (TDD).** Update the existing `Encode_DoubleList_IsInvariant` assertion to the native form, and add the native + backward-compat cases: + +```csharp +[Fact] +public void Encode_Int32List_ProducesNativeNumbers() => + Assert.Equal("[10,20,30]", AttributeValueCodec.Encode(new List { 10, 20, 30 })); + +[Fact] +public void Encode_BoolList_ProducesNativeBooleans() => + Assert.Equal("[true,false]", AttributeValueCodec.Encode(new List { true, false })); + +[Fact] +public void Encode_DoubleList_IsNativeAndInvariant() +{ + var original = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = new CultureInfo("de-DE"); + Assert.Equal("[1.5,2.5]", AttributeValueCodec.Encode(new List { 1.5, 2.5 })); + } + finally { CultureInfo.CurrentCulture = original; } +} + +[Fact] +public void Encode_StringList_StaysQuoted() => + Assert.Equal("[\"a\",\"b\"]", AttributeValueCodec.Encode(new List { "a", "b" })); + +[Fact] +public void Encode_DateTimeList_IsIso8601() +{ + var json = AttributeValueCodec.Encode(new List { new(2026, 6, 16, 0, 0, 0, DateTimeKind.Utc) }); + Assert.Contains("2026-06-16T00:00:00", json); // ISO, not "06/16/2026 00:00:00" +} + +[Fact] +public void Decode_NewNativeIntForm_Parses() +{ + var back = (IList)AttributeValueCodec.Decode("[10,20]", DataType.List, DataType.Int32)!; + Assert.Equal(new[] { 10, 20 }, back); +} + +[Fact] +public void Decode_OldStringIntForm_BackwardCompatible() +{ + var back = (IList)AttributeValueCodec.Decode("[\"10\",\"20\"]", DataType.List, DataType.Int32)!; + Assert.Equal(new[] { 10, 20 }, back); +} + +[Theory] +[InlineData("[true,false]")] // new native bools +[InlineData("[\"True\",\"False\"]")] // old string bools +public void Decode_BoolForms_BothParse(string json) +{ + var back = (IList)AttributeValueCodec.Decode(json, DataType.List, DataType.Boolean)!; + Assert.Equal(new[] { true, false }, back); +} +``` +Keep the existing round-trip tests (Int32/Float/Double/DateTime/Bool) — they assert round-trip, not exact form, so they continue to pass. Keep `Decode_MalformedJson_Throws`. + +**Step 2 — Run, expect FAIL** on the new native-form encode assertions. +Run: `dotnet test tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/ZB.MOM.WW.ScadaBridge.Commons.Tests.csproj --filter AttributeValueCodecTests` + +**Step 3 — Implement.** In `Encode`, replace ONLY the `IEnumerable` branch: + +```csharp +case IEnumerable e: + // Native-typed JSON: serialize the runtime collection so STJ emits numbers/bools + // unquoted, strings quoted, DateTime as ISO-8601. Boxed as object so STJ uses the + // runtime element type. STJ numbers/dates are culture-invariant by spec. + return JsonSerializer.Serialize(e, JsonOpts); +``` +(The `null`, `string`, and `IFormattable` scalar branches are unchanged.) + +In `Decode`, change the list deserialization to read `JsonElement[]` and derive a string per element: + +```csharp +JsonElement[] raw; +try { raw = JsonSerializer.Deserialize(value) ?? []; } +catch (JsonException ex) { throw new FormatException("Malformed list JSON.", ex); } + +var clrType = ElementClrType(elementType.Value); +var listType = typeof(List<>).MakeGenericType(clrType); +var result = (IList)Activator.CreateInstance(listType)!; +foreach (var el in raw) + result.Add(ParseScalar(JsonElementToString(el), elementType.Value)); +return result; +``` +Add the helper (handles both new native elements and old quoted-string elements): + +```csharp +private static string? JsonElementToString(JsonElement el) => el.ValueKind switch +{ + JsonValueKind.String => el.GetString(), // old form, or string-typed lists + JsonValueKind.Null => null, // ParseScalar throws "may not be null" + _ => el.GetRawText() // number/bool → "10" / "1.5" / "true" +}; +``` +`ParseScalar` is unchanged: it parses the string per element type (invariant), so `"10"` (from either form), `"true"`, and ISO/US-invariant dates all parse. Add `using System.Text.Json;` if not already present (it is). + +**Step 4 — Run, expect PASS** (all AttributeValueCodec tests). + +**Step 5 — Commit.** +```bash +git commit -m "feat(commons): native-typed JSON for List values; Decode reads both forms" -- \ + src/ZB.MOM.WW.ScadaBridge.Commons/Types/AttributeValueCodec.cs \ + tests/ZB.MOM.WW.ScadaBridge.Commons.Tests/Types/AttributeValueCodecTests.cs +``` + +**Acceptance:** numbers/bools encode unquoted, strings stay quoted, DateTime is ISO; both old and new forms decode; scalars unchanged; malformed still throws. + +--- + +### Task NJ-2: Stamp ElementDataType on instance-override writes + +**Classification:** small +**Estimated implement time:** ~3 min +**Parallelizable with:** NJ-1 +**Blocked by:** none + +**Files:** +- Modify: `src/ZB.MOM.WW.ScadaBridge.TemplateEngine/Services/InstanceService.cs:146-198` (`SetAttributeOverrideAsync`) +- Test: `tests/ZB.MOM.WW.ScadaBridge.TemplateEngine.Tests/Services/InstanceServiceTests.cs` (mirror existing style) + +`SetAttributeOverrideAsync` already loads the base `templateAttr` (line 159). Set `ElementDataType` from it on both the update and create branches so the persisted `InstanceAttributeOverride` row carries the element type — this is what the NJ-3 central normalizer reads for override rows (folds in follow-up #93/M3). + +**Step 1 — Failing test:** setting an override on a `List`/`String` template attribute persists an `InstanceAttributeOverride` whose `ElementDataType == DataType.String`; setting an override on a scalar attribute persists `ElementDataType == null`. + +**Step 2 — Run, expect FAIL.** + +**Step 3 — Implement:** in the update branch, `existingOverride.ElementDataType = templateAttr.ElementDataType;` before the repo update; in the create branch, set `ElementDataType = templateAttr.ElementDataType` in the `new InstanceAttributeOverride(...)` initializer. + +**Step 4 — Run, expect PASS.** + +**Step 5 — Commit:** `feat(template): stamp ElementDataType on instance attribute overrides`. + +--- + +### Task NJ-3: Central MS SQL idempotent startup normalizer + +**Classification:** high-risk +**Estimated implement time:** ~5 min +**Parallelizable with:** NJ-4, NJ-5 (after NJ-1) +**Blocked by:** NJ-1, NJ-2 + +**Files:** +- Create: `src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ListValueNormalizer.cs` +- Modify: `src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/MigrationHelper.cs` (call the normalizer after the migrate/validate step) +- Test: `tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/...` (use the existing test DbContext/SQLite-in-memory or test-double pattern the project uses — investigate the existing ConfigurationDatabase tests first) + +**Behavior:** A static `ListValueNormalizer.NormalizeAsync(ScadaBridgeDbContext db, ILogger? logger, CancellationToken ct)`: +- `TemplateAttributes` where `DataType == DataType.List`: for each, compute `var native = AttributeValueCodec.Encode(AttributeValueCodec.Decode(a.Value, DataType.List, a.ElementDataType));` (parse the row's own `ElementDataType` — it is a `DataType?`). If `native != a.Value`, set `a.Value = native`. +- `InstanceAttributeOverrides` whose `ElementDataType != null` (i.e. a List override stamped by NJ-2): same re-encode against `OverrideValue`. Rows with null `ElementDataType` are skipped with a debug log (legacy/scalar rows — none exist in practice since the feature has no deployed data). +- Wrap each row in try/catch(`FormatException`): on a malformed value, log a warning and SKIP that row — continue with the rest. The normalizer MUST NOT throw out of `NormalizeAsync` (never abort startup). +- `SaveChangesAsync` once at the end. Log a summary (`Normalized {n} list attribute value(s) to native JSON`). +- Idempotent: native→native yields identical bytes, so a second run makes no changes. + +**Wire-in (MigrationHelper):** after the `if (isDevelopment) { Migrate } else { validate }` block in `ApplyOrValidateMigrationsAsync`, call `await ListValueNormalizer.NormalizeAsync(dbContext, logger, cancellationToken);`. (Runs on every central startup; idempotent and safe even if both central nodes run it — concurrent UPDATEs set identical values.) + +**Steps (TDD):** failing tests (a TemplateAttributes row in old form `["10","20"]` with ElementDataType Int32 → rewritten to `[10,20]`; a native row → unchanged/no-op; a String-list `["a","b"]` → unchanged; a malformed row → skipped, sibling rows still normalized; an override row with ElementDataType set → normalized, with null → skipped) → run targeted → FAIL → implement `ListValueNormalizer` + wire-in → run targeted → PASS → build ConfigurationDatabase → commit `feat(db): idempotent startup normalizer rewriting List values to native JSON`. + +--- + +### Task NJ-4: Site SQLite active normalization on override load + +**Classification:** high-risk +**Estimated implement time:** ~4 min +**Parallelizable with:** NJ-3, NJ-5 (after NJ-1) +**Blocked by:** NJ-1 + +**Files:** +- Modify: `src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/InstanceActor.cs:932-960` (`HandleOverridesLoaded`) +- Test: `tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/InstanceActorTests.cs` + +In `HandleOverridesLoaded`, the List branch already decodes the stored override via `DecodeAttributeValue(resolved, kvp.Value)`. Extend it: when the decode succeeds and the re-encoded native form differs from the stored string (i.e. the stored value is old-form), re-persist the native form: + +```csharp +var decoded = DecodeAttributeValue(resolved, kvp.Value); +_attributes[kvp.Key] = decoded; +if (decoded is null && !string.IsNullOrEmpty(kvp.Value)) +{ + _attributeQualities[kvp.Key] = "Bad"; +} +else if (decoded is not null) +{ + var native = AttributeValueCodec.Encode(decoded); + if (native != kvp.Value) // stored value was old-form → normalize on disk + { + var key = kvp.Key; + _storage.SetStaticOverrideAsync(_instanceUniqueName, key, native!) + .ContinueWith(t => _logger.LogWarning(t.Exception?.GetBaseException(), + "Failed to normalize static override {Instance}.{Attr} to native JSON", _instanceUniqueName, key), + TaskContinuationOptions.OnlyOnFaulted); + } +} +``` +Idempotent (native→native ⇒ `native == kvp.Value` ⇒ no write). Scalars untouched. Never throws into the actor (decode already FormatException-safe; persist is fire-and-forget). + +**Steps (TDD):** failing tests (an old-form List static override on load → `SetStaticOverrideAsync` called with the native form; a native-form override → NOT re-persisted; a scalar override → unaffected; malformed → Bad quality + no crash, as today) → run targeted `--filter "FullyQualifiedName~InstanceActor"` → FAIL → implement → PASS → build SiteRuntime → commit `feat(siteruntime): normalize old-form List static overrides to native JSON on load`. + +--- + +### Task NJ-5: Normalize List values on bundle import + +**Classification:** standard +**Estimated implement time:** ~4 min +**Parallelizable with:** NJ-3, NJ-4 (after NJ-1) +**Blocked by:** NJ-1 + +**Files:** +- Modify: `src/ZB.MOM.WW.ScadaBridge.Transport/Import/BundleImporter.cs` (the 3 `TemplateAttribute` construction sites from MV-12: BuildTemplate + the two in SyncTemplateAttributesAsync) — and/or `src/ZB.MOM.WW.ScadaBridge.Transport/Serialization/EntitySerializer.cs` `FromBundleContent` if that is the single choke point for DTO→entity. +- Test: `tests/ZB.MOM.WW.ScadaBridge.Transport.Tests/...` + +**Behavior:** when importing a `TemplateAttributeDto` whose `DataType == DataType.List`, write the **native** Value: `var v = NormalizeListValue(dto.Value, dto.ElementDataType);` where `NormalizeListValue` does `AttributeValueCodec.Encode(AttributeValueCodec.Decode(value, DataType.List, elementType))` inside try/catch(FormatException) → on failure return the original value unchanged + log (don't fail the import; the MS SQL normalizer is a backstop). Prefer a single private helper used at all construction sites (DRY). Non-List attributes write `dto.Value` unchanged. + +**Steps (TDD):** failing test (import a bundle/DTO with an old-form List attribute `["10","20"]` + ElementDataType Int32 → the persisted TemplateAttribute.Value is `[10,20]`; a String list stays quoted; a malformed value imports unchanged) → run targeted → FAIL → implement the helper + apply at construction sites → PASS → build Transport → commit `feat(transport): normalize List attribute values to native JSON on import`. + +--- + +### Task NJ-6: Integration verification + docs + +**Classification:** standard +**Estimated implement time:** ~5 min +**Parallelizable with:** none +**Blocked by:** NJ-1 … NJ-5 + +**Files:** +- Modify: `docs/plans/2026-06-16-native-typed-json-design.md` (mark Status complete) +- Modify: `docs/requirements/Component-Commons.md` (update the `DataType`/codec note: List values are native-typed JSON; codec reads both forms) +- Modify: `docs/plans/2026-06-16-multivalue-attribute.md.tasks.json` follow-up note OR the task list — note that #93/M3 (override ElementDataType) was folded in here. + +**Steps:** +1. Full-solution build: `dotnet build ZB.MOM.WW.ScadaBridge.slnx` → 0/0. +2. Targeted test sweep across touched projects: Commons codec, TemplateEngine (InstanceService), ConfigurationDatabase (normalizer), SiteRuntime (InstanceActor), Transport (serializer/importer). +3. Update docs; mark design Status complete. +4. Commit: `docs: mark native-typed JSON feature complete; update Component-Commons codec note`. + +**Acceptance:** full build green; all touched-project targeted tests pass; docs synced. + +--- + +## Parallelization summary + +- **Wave 1:** NJ-1 (codec) ∥ NJ-2 (override stamp) — disjoint (Commons vs TemplateEngine). +- **Wave 2 (after NJ-1, +NJ-2 for NJ-3):** NJ-3 (normalizer), NJ-4 (site), NJ-5 (import) in parallel — disjoint (ConfigurationDatabase vs SiteRuntime vs Transport). +- **Wave 3:** NJ-6 (verify + docs). + +## Risk notes + +- **NJ-1** changes the canonical storage/wire form — high-risk, but backward-compatible (read-both) and additive on the proto field. +- **NJ-3** writes to the config DB at startup — must be idempotent and never abort startup (per-row skip+log). +- **NJ-4** touches the actor model — the normalization write is fire-and-forget and never throws into the actor. +- There is no deployed List data yet, so the normalizers are a safety net; correctness of NJ-1 (the format) is what matters most in practice. diff --git a/docs/plans/2026-06-16-native-typed-json.md.tasks.json b/docs/plans/2026-06-16-native-typed-json.md.tasks.json new file mode 100644 index 00000000..90dcdf72 --- /dev/null +++ b/docs/plans/2026-06-16-native-typed-json.md.tasks.json @@ -0,0 +1,14 @@ +{ + "planPath": "docs/plans/2026-06-16-native-typed-json.md", + "designDoc": "docs/plans/2026-06-16-native-typed-json-design.md", + "branch": "feature/native-typed-json", + "tasks": [ + {"id": 96, "ref": "NJ-1", "subject": "Codec — native-typed Encode + read-both Decode", "class": "high-risk", "status": "pending"}, + {"id": 97, "ref": "NJ-2", "subject": "Stamp ElementDataType on instance-override writes", "class": "small", "status": "pending"}, + {"id": 98, "ref": "NJ-3", "subject": "Central MS SQL idempotent startup normalizer", "class": "high-risk", "status": "pending", "blockedBy": [96, 97]}, + {"id": 99, "ref": "NJ-4", "subject": "Site SQLite active normalization on override load", "class": "high-risk", "status": "pending", "blockedBy": [96]}, + {"id": 100, "ref": "NJ-5", "subject": "Normalize List values on bundle import", "class": "standard", "status": "pending", "blockedBy": [96]}, + {"id": 101, "ref": "NJ-6", "subject": "Integration verification + docs", "class": "standard", "status": "pending", "blockedBy": [96, 97, 98, 99, 100]} + ], + "lastUpdated": "2026-06-16" +}