feat(db): idempotent startup normalizer rewriting List values to native JSON

2026-06-16 17:50:19 -04:00
parent e3d804a1a6
commit f4b101b532
3 changed files with 360 additions and 0 deletions
@@ -0,0 +1,123 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.ScadaBridge.Commons.Types;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+
+namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
+
+/// <summary>
+/// Idempotent central startup normalizer that rewrites already-persisted List attribute
+/// values from the old array-of-strings JSON form (<c>["10","20"]</c>) to the new
+/// native-typed form (<c>[10,20]</c>).
+///
+/// <para>
+/// This is a safety net: at the time of writing there is no deployed List attribute data,
+/// so in practice it finds nothing to rewrite. It runs on every central startup after
+/// migrations apply and MUST never abort startup for data reasons — every row's work is
+/// wrapped in a per-row try/catch that logs and skips on malformed data. A second run
+/// finds nothing to change (native → native re-encode is byte-identical).
+/// </para>
+/// </summary>
+public static class ListValueNormalizer
+{
+    /// <summary>
+    /// Rewrites old-form List attribute values to the native-typed JSON form across
+    /// <see cref="ScadaBridgeDbContext.TemplateAttributes"/> and
+    /// <see cref="ScadaBridgeDbContext.InstanceAttributeOverrides"/>. Idempotent and
+    /// best-effort: malformed rows are logged and skipped, never rethrown.
+    /// </summary>
+    /// <param name="db">The configuration database context.</param>
+    /// <param name="logger">Optional logger for diagnostics.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    public static async Task NormalizeAsync(
+        ScadaBridgeDbContext db,
+        ILogger? logger = null,
+        CancellationToken ct = default)
+    {
+        var rewritten = 0;
+
+        // TemplateAttributes: List rows carry the element type on the row itself.
+        var templateRows = await db.TemplateAttributes
+            .Where(a => a.DataType == DataType.List)
+            .ToListAsync(ct);
+
+        foreach (var a in templateRows)
+        {
+            try
+            {
+                var native = AttributeValueCodec.Encode(
+                    AttributeValueCodec.Decode(a.Value, DataType.List, a.ElementDataType));
+                if (native != a.Value)
+                {
+                    a.Value = native;
+                    rewritten++;
+                }
+            }
+            catch (FormatException ex)
+            {
+                logger?.LogWarning(ex,
+                    "List value normalizer: skipping unparseable list value for TemplateAttribute {Id}.",
+                    a.Id);
+            }
+        }
+
+        // InstanceAttributeOverrides: only rows that carry an element type are List rows.
+        // Rows with a null ElementDataType are scalar/legacy rows (no deployed List data
+        // exists, so none in practice) and are skipped.
+        var overrideRows = await db.InstanceAttributeOverrides
+            .Where(o => o.ElementDataType != null)
+            .ToListAsync(ct);
+
+        foreach (var o in overrideRows)
+        {
+            if (o.ElementDataType is null)
+            {
+                logger?.LogDebug(
+                    "List value normalizer: skipping InstanceAttributeOverride {Id} with no element type.",
+                    o.Id);
+                continue;
+            }
+
+            try
+            {
+                var native = AttributeValueCodec.Encode(
+                    AttributeValueCodec.Decode(o.OverrideValue, DataType.List, o.ElementDataType));
+                if (native != o.OverrideValue)
+                {
+                    o.OverrideValue = native;
+                    rewritten++;
+                }
+            }
+            catch (FormatException ex)
+            {
+                logger?.LogWarning(ex,
+                    "List value normalizer: skipping unparseable list value for InstanceAttributeOverride {Id}.",
+                    o.Id);
+            }
+        }
+
+        try
+        {
+            await db.SaveChangesAsync(ct);
+        }
+        catch (Exception ex)
+        {
+            // A catastrophic DB failure on SaveChanges may propagate, but log it first so
+            // startup diagnostics are not silent. Per-row data problems are already handled
+            // above and never reach here.
+            logger?.LogError(ex, "List value normalizer: SaveChanges failed.");
+            throw;
+        }
+
+        if (rewritten > 0)
+        {
+            logger?.LogInformation(
+                "List value normalizer: rewrote {n} attribute value(s) to native JSON.", rewritten);
+        }
+        else
+        {
+            logger?.LogDebug("List value normalizer: no attribute values required rewriting.");
+        }
+    }
+}
@@ -47,6 +47,12 @@ public static class MigrationHelper
                    "Apply migrations using 'dotnet ef database update' or the generated SQL scripts before starting in production mode.");
            }
        }
+
+        // Safety-net normalizer: rewrite any already-persisted List attribute values from the
+        // old array-of-strings JSON form to the new native-typed form. Idempotent and never
+        // aborts startup for data reasons (per-row skip + log). Safe even if both central
+        // nodes run it concurrently on startup.
+        await ListValueNormalizer.NormalizeAsync(dbContext, logger, cancellationToken);
    }

    private static async Task WaitForDatabaseReadyAsync(
@@ -0,0 +1,231 @@
+using Microsoft.EntityFrameworkCore;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
+
+namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests;
+
+/// <summary>
+/// Tests for <see cref="ListValueNormalizer"/> — the idempotent startup normalizer that
+/// rewrites already-persisted List attribute values from the old array-of-strings JSON
+/// form (<c>["10","20"]</c>) to the new native-typed form (<c>[10,20]</c>).
+/// </summary>
+public class ListValueNormalizerTests : IDisposable
+{
+    private readonly ScadaBridgeDbContext _context;
+
+    public ListValueNormalizerTests()
+    {
+        _context = SqliteTestHelper.CreateInMemoryContext();
+    }
+
+    public void Dispose()
+    {
+        _context.Database.CloseConnection();
+        _context.Dispose();
+    }
+
+    // Seeds a Template parent (satisfies the TemplateAttribute -> Template FK) and returns its Id.
+    private async Task<int> SeedTemplateAsync()
+    {
+        var template = new Template("T1");
+        _context.Templates.Add(template);
+        await _context.SaveChangesAsync();
+        return template.Id;
+    }
+
+    // Seeds Site + Template + Instance (satisfies the InstanceAttributeOverride -> Instance FK)
+    // and returns the Instance Id.
+    private async Task<int> SeedInstanceAsync()
+    {
+        var site = new Site("Site1", "S-001");
+        var template = new Template("T1");
+        _context.Sites.Add(site);
+        _context.Templates.Add(template);
+        await _context.SaveChangesAsync();
+
+        var instance = new Instance("Inst1") { SiteId = site.Id, TemplateId = template.Id };
+        _context.Instances.Add(instance);
+        await _context.SaveChangesAsync();
+        return instance.Id;
+    }
+
+    [Fact]
+    public async Task TemplateAttribute_OldStringForm_RewrittenToNative()
+    {
+        var templateId = await SeedTemplateAsync();
+        _context.TemplateAttributes.Add(new TemplateAttribute("intList")
+        {
+            TemplateId = templateId,
+            DataType = DataType.List,
+            ElementDataType = DataType.Int32,
+            Value = "[\"10\",\"20\"]",
+        });
+        await _context.SaveChangesAsync();
+
+        await ListValueNormalizer.NormalizeAsync(_context);
+
+        var reloaded = await _context.TemplateAttributes.AsNoTracking().SingleAsync();
+        Assert.Equal("[10,20]", reloaded.Value);
+    }
+
+    [Fact]
+    public async Task TemplateAttribute_AlreadyNative_IsNotRewritten()
+    {
+        var templateId = await SeedTemplateAsync();
+        _context.TemplateAttributes.Add(new TemplateAttribute("intList")
+        {
+            TemplateId = templateId,
+            DataType = DataType.List,
+            ElementDataType = DataType.Int32,
+            Value = "[10,20]",
+        });
+        await _context.SaveChangesAsync();
+
+        await ListValueNormalizer.NormalizeAsync(_context);
+
+        // No tracked entity should be marked Modified — idempotent no-op.
+        var tracked = _context.ChangeTracker.Entries<TemplateAttribute>()
+            .Where(e => e.State == EntityState.Modified)
+            .ToList();
+        Assert.Empty(tracked);
+
+        var reloaded = await _context.TemplateAttributes.AsNoTracking().SingleAsync();
+        Assert.Equal("[10,20]", reloaded.Value);
+    }
+
+    [Fact]
+    public async Task TemplateAttribute_StringList_IsUnchanged()
+    {
+        var templateId = await SeedTemplateAsync();
+        _context.TemplateAttributes.Add(new TemplateAttribute("stringList")
+        {
+            TemplateId = templateId,
+            DataType = DataType.List,
+            ElementDataType = DataType.String,
+            Value = "[\"a\",\"b\"]",
+        });
+        await _context.SaveChangesAsync();
+
+        await ListValueNormalizer.NormalizeAsync(_context);
+
+        var reloaded = await _context.TemplateAttributes.AsNoTracking().SingleAsync();
+        Assert.Equal("[\"a\",\"b\"]", reloaded.Value);
+    }
+
+    [Fact]
+    public async Task TemplateAttribute_Malformed_IsSkipped_AndSiblingStillNormalized()
+    {
+        var templateId = await SeedTemplateAsync();
+        _context.TemplateAttributes.Add(new TemplateAttribute("badList")
+        {
+            TemplateId = templateId,
+            DataType = DataType.List,
+            ElementDataType = DataType.Int32,
+            Value = "[\"a\"",      // malformed JSON
+        });
+        _context.TemplateAttributes.Add(new TemplateAttribute("goodList")
+        {
+            TemplateId = templateId,
+            DataType = DataType.List,
+            ElementDataType = DataType.Int32,
+            Value = "[\"5\"]",     // old form, valid
+        });
+        await _context.SaveChangesAsync();
+
+        // Must NOT throw despite the malformed row.
+        await ListValueNormalizer.NormalizeAsync(_context);
+
+        var bad = await _context.TemplateAttributes.AsNoTracking()
+            .SingleAsync(a => a.Name == "badList");
+        var good = await _context.TemplateAttributes.AsNoTracking()
+            .SingleAsync(a => a.Name == "goodList");
+
+        Assert.Equal("[\"a\"", bad.Value);   // skipped, untouched
+        Assert.Equal("[5]", good.Value);     // normalized
+    }
+
+    [Fact]
+    public async Task TemplateAttribute_NonListRow_IsUnchanged()
+    {
+        var templateId = await SeedTemplateAsync();
+        _context.TemplateAttributes.Add(new TemplateAttribute("scalar")
+        {
+            TemplateId = templateId,
+            DataType = DataType.Int32,
+            ElementDataType = null,
+            Value = "42",
+        });
+        await _context.SaveChangesAsync();
+
+        await ListValueNormalizer.NormalizeAsync(_context);
+
+        var reloaded = await _context.TemplateAttributes.AsNoTracking().SingleAsync();
+        Assert.Equal("42", reloaded.Value);
+    }
+
+    [Fact]
+    public async Task InstanceAttributeOverride_OldStringForm_RewrittenToNative()
+    {
+        var instanceId = await SeedInstanceAsync();
+        _context.InstanceAttributeOverrides.Add(new InstanceAttributeOverride("intList")
+        {
+            InstanceId = instanceId,
+            ElementDataType = DataType.Int32,
+            OverrideValue = "[\"5\"]",
+        });
+        await _context.SaveChangesAsync();
+
+        await ListValueNormalizer.NormalizeAsync(_context);
+
+        var reloaded = await _context.InstanceAttributeOverrides.AsNoTracking().SingleAsync();
+        Assert.Equal("[5]", reloaded.OverrideValue);
+    }
+
+    [Fact]
+    public async Task InstanceAttributeOverride_NullElementType_IsUntouched()
+    {
+        var instanceId = await SeedInstanceAsync();
+        _context.InstanceAttributeOverrides.Add(new InstanceAttributeOverride("scalar")
+        {
+            InstanceId = instanceId,
+            ElementDataType = null,
+            OverrideValue = "42",
+        });
+        await _context.SaveChangesAsync();
+
+        await ListValueNormalizer.NormalizeAsync(_context);
+
+        var reloaded = await _context.InstanceAttributeOverrides.AsNoTracking().SingleAsync();
+        Assert.Equal("42", reloaded.OverrideValue);
+    }
+
+    [Fact]
+    public async Task NormalizeAsync_IsIdempotent_SecondRunChangesNothing()
+    {
+        var templateId = await SeedTemplateAsync();
+        _context.TemplateAttributes.Add(new TemplateAttribute("intList")
+        {
+            TemplateId = templateId,
+            DataType = DataType.List,
+            ElementDataType = DataType.Int32,
+            Value = "[\"10\",\"20\"]",
+        });
+        await _context.SaveChangesAsync();
+
+        await ListValueNormalizer.NormalizeAsync(_context);
+        var afterFirst = await _context.TemplateAttributes.AsNoTracking().SingleAsync();
+        Assert.Equal("[10,20]", afterFirst.Value);
+
+        await ListValueNormalizer.NormalizeAsync(_context);
+        var afterSecond = await _context.TemplateAttributes.AsNoTracking().SingleAsync();
+        Assert.Equal("[10,20]", afterSecond.Value);
+
+        var tracked = _context.ChangeTracker.Entries<TemplateAttribute>()
+            .Where(e => e.State == EntityState.Modified)
+            .ToList();
+        Assert.Empty(tracked);
+    }
+}