fix(health-monitoring): resolve HealthMonitoring-001/002 — populate S&F buffer depth, make SiteHealthState immutable

tests/ScadaLink.HealthMonitoring.Tests/CentralHealthAggregatorTests.cs

@@ -161,7 +161,62 @@ public class CentralHealthAggregatorTests
         _aggregator.ProcessReport(report);
 
         var state = _aggregator.GetSiteState("site-1");
-        Assert.Equal(42, state!.LatestReport.ScriptErrorCount);
+        Assert.Equal(42, state!.LatestReport!.ScriptErrorCount);
+    }
+
+    /// <summary>
+    /// HealthMonitoring-002 regression: SiteHealthState is mutated from multiple
+    /// threads (ProcessReport, MarkHeartbeat, CheckForOfflineSites). With a mutable
+    /// class and unsynchronized field writes, a snapshot read could observe a torn
+    /// or half-applied state. The state must be immutable and every transition an
+    /// atomic reference swap, so a snapshot is always internally consistent and the
+    /// monotonic sequence-number guard is never subverted by a lost update.
+    /// </summary>
+    [Fact]
+    public async Task ProcessReport_ConcurrentUpdates_NeverLoseSequenceOrTearState()
+    {
+        const int iterations = 5_000;
+        // SiteHealthState must be an immutable record so handing the reference to
+        // UI callers (and reading it concurrently) is safe.
+        Assert.True(typeof(SiteHealthState).GetMethod("<Clone>$") != null,
+            "SiteHealthState must be an immutable record for safe concurrent reads.");
+
+        _aggregator.ProcessReport(MakeReport("site-1", 0));
+
+        var writer = Task.Run(() =>
+        {
+            for (long seq = 1; seq <= iterations; seq++)
+                _aggregator.ProcessReport(MakeReport("site-1", seq));
+        });
+
+        var heartbeater = Task.Run(() =>
+        {
+            for (int i = 0; i < iterations; i++)
+                _aggregator.MarkHeartbeat("site-1", _timeProvider.GetUtcNow());
+        });
+
+        long maxObserved = 0;
+        var reader = Task.Run(() =>
+        {
+            for (int i = 0; i < iterations; i++)
+            {
+                var state = _aggregator.GetSiteState("site-1");
+                if (state == null) continue;
+                // A consistent snapshot: the stored report's sequence number must
+                // always match the state's LastSequenceNumber (no half-applied update).
+                Assert.Equal(state.LastSequenceNumber, state.LatestReport!.SequenceNumber);
+                if (state.LastSequenceNumber > maxObserved)
+                    maxObserved = state.LastSequenceNumber;
+            }
+        });
+
+        await Task.WhenAll(writer, heartbeater, reader);
+
+        // The final state must reflect the highest sequence — no lost update.
+        var final = _aggregator.GetSiteState("site-1");
+        Assert.Equal(iterations, final!.LastSequenceNumber);
+        Assert.Equal(iterations, final.LatestReport!.SequenceNumber);
+        Assert.True(final.IsOnline);
     }
 
     [Fact]
@@ -173,11 +228,11 @@ public class CentralHealthAggregatorTests
         _aggregator.ProcessReport(MakeReport("site-1", 10));
         _aggregator.ProcessReport(MakeReport("site-1", 1));
 
-        var state = _aggregator.GetSiteState("site-1");
-        Assert.Equal(10, state!.LastSequenceNumber);
+        Assert.Equal(10, _aggregator.GetSiteState("site-1")!.LastSequenceNumber);
 
-        // Once it exceeds the old max, it works again
+        // Once it exceeds the old max, it works again. SiteHealthState is an
+        // immutable snapshot, so re-fetch to observe the new state.
         _aggregator.ProcessReport(MakeReport("site-1", 11));
-        Assert.Equal(11, state.LastSequenceNumber);
+        Assert.Equal(11, _aggregator.GetSiteState("site-1")!.LastSequenceNumber);
     }
 }

tests/ScadaLink.HealthMonitoring.Tests/HealthReportSenderTests.cs

@@ -1,7 +1,9 @@
+using Microsoft.Data.Sqlite;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
 using ScadaLink.Commons.Messages.Health;
 using ScadaLink.Commons.Types.Enums;
+using ScadaLink.StoreAndForward;
 
 namespace ScadaLink.HealthMonitoring.Tests;
 
@@ -136,6 +138,76 @@ public class HealthReportSenderTests
         }
     }
 
+    /// <summary>
+    /// HealthMonitoring-001 regression: the documented "store-and-forward buffer
+    /// depth" metric (pending messages by category) must actually be populated in
+    /// the emitted report. Previously SetStoreAndForwardDepths had no callers, so
+    /// StoreAndForwardBufferDepths was always empty. The sender must query the S&amp;F
+    /// engine's per-category depth API and include it alongside the parked count.
+    /// </summary>
+    [Fact]
+    public async Task ReportsIncludeStoreAndForwardBufferDepthsFromStorage()
+    {
+        var dbName = $"HealthSfDepth_{Guid.NewGuid():N}";
+        var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
+        // Keep one connection alive so the in-memory DB persists for the test.
+        using var keepAlive = new SqliteConnection(connStr);
+        keepAlive.Open();
+
+        var storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
+        await storage.InitializeAsync();
+
+        // Two pending ExternalSystem messages and one pending Notification message.
+        await storage.EnqueueAsync(MakePendingMessage("m1", StoreAndForwardCategory.ExternalSystem));
+        await storage.EnqueueAsync(MakePendingMessage("m2", StoreAndForwardCategory.ExternalSystem));
+        await storage.EnqueueAsync(MakePendingMessage("m3", StoreAndForwardCategory.Notification));
+
+        var transport = new FakeTransport();
+        var collector = new SiteHealthCollector();
+        collector.SetActiveNode(true);
+        var options = Options.Create(new HealthMonitoringOptions
+        {
+            ReportInterval = TimeSpan.FromMilliseconds(50)
+        });
+
+        var sender = new HealthReportSender(
+            collector,
+            transport,
+            options,
+            NullLogger<HealthReportSender>.Instance,
+            new FakeSiteIdentityProvider(),
+            sfStorage: storage);
+
+        using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(300));
+        try
+        {
+            await sender.StartAsync(cts.Token);
+            await Task.Delay(250, CancellationToken.None);
+            await sender.StopAsync(CancellationToken.None);
+        }
+        catch (OperationCanceledException) { }
+
+        Assert.True(transport.SentReports.Count >= 1);
+        var depths = transport.SentReports[^1].StoreAndForwardBufferDepths;
+        Assert.Equal(2, depths[nameof(StoreAndForwardCategory.ExternalSystem)]);
+        Assert.Equal(1, depths[nameof(StoreAndForwardCategory.Notification)]);
+        Assert.False(depths.ContainsKey(nameof(StoreAndForwardCategory.CachedDbWrite)));
+    }
+
+    private static StoreAndForwardMessage MakePendingMessage(string id, StoreAndForwardCategory category) =>
+        new()
+        {
+            Id = id,
+            Category = category,
+            Target = "target",
+            PayloadJson = "{}",
+            RetryCount = 0,
+            MaxRetries = 50,
+            RetryIntervalMs = 30_000,
+            CreatedAt = DateTimeOffset.UtcNow,
+            Status = StoreAndForwardMessageStatus.Pending
+        };
+
     [Fact]
     public void InitialSequenceNumberSeededWithUnixMs()
     {

tests/ScadaLink.HealthMonitoring.Tests/ScadaLink.HealthMonitoring.Tests.csproj

@@ -10,6 +10,7 @@
 
   <ItemGroup>
     <PackageReference Include="coverlet.collector" />
+    <PackageReference Include="Microsoft.Data.Sqlite" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
     <PackageReference Include="Microsoft.Extensions.Options" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />