using System.Data; using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Types; using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories; /// /// EF Core implementation of . See the /// interface for the monotonic-upsert contract; this class adds notes on the /// data-access strategy used by each method. /// public class SiteCallAuditRepository : ISiteCallAuditRepository { // SQL Server duplicate-key error numbers, identical to the AuditLogRepository // race-fix: 2601 = unique-index violation, 2627 = PK/unique-constraint // violation. The IF NOT EXISTS … INSERT pattern has a check-then-act window // and the loser surfaces as one of these; monotonic-upsert semantics demand // we swallow them. private const int SqlErrorUniqueIndexViolation = 2601; private const int SqlErrorPrimaryKeyViolation = 2627; // Monotonic status ordering: // Submitted < Forwarded < Attempted == Skipped < Delivered == Failed == Parked == Discarded. // A higher incoming rank always wins. WITHIN an equal NON-terminal rank // (Attempted/Skipped, rank 2 — and the transient Submitted/Forwarded ranks), // the newest UpdatedAtUtc wins so a retrying call's live RetryCount/LastError // no longer freezes at first-write. Equal terminal ranks (rank 3) // stay immutable — the freshness tiebreaker is deliberately scoped to // rank < 3, so a later terminal NEVER flips an earlier one (Delivered cannot // overwrite Parked). Still idempotent (equal stamps are inert) and still // regression-proof (a lower rank is always a no-op). private const int TerminalRank = 3; private static readonly Dictionary StatusRank = new(StringComparer.Ordinal) { ["Submitted"] = 0, ["Forwarded"] = 1, ["Attempted"] = 2, ["Skipped"] = 2, ["Delivered"] = 3, ["Failed"] = 3, ["Parked"] = 3, ["Discarded"] = 3, }; private readonly ScadaBridgeDbContext _context; private readonly ILogger _logger; /// /// Initializes a new instance of the class. /// /// The EF Core database context. /// Optional logger for diagnostic information. public SiteCallAuditRepository(ScadaBridgeDbContext context, ILogger? logger = null) { _context = context ?? throw new ArgumentNullException(nameof(context)); _logger = logger ?? NullLogger.Instance; } /// public async Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) { if (siteCall is null) { throw new ArgumentNullException(nameof(siteCall)); } var idText = siteCall.TrackedOperationId.Value.ToString("D"); var incomingRank = GetRankOrThrow(siteCall.Status); // ONE round trip, INSERT-first. Both statements ship in a single command // text, so the round-trip saving of the WP2.2 rewrite is preserved — but // the ORDER is back to insert-then-update, because UPDATE-first LOSES // DATA under a concurrent first-write. // // The UPDATE-first shape was: UPDATE; SET @updated = @@ROWCOUNT; // IF @updated = 0 AND NOT EXISTS(…) INSERT. Two writers racing the FIRST // packet of one TrackedOperationId — the cached dual-write and the // reconciliation pull, which routinely carry DIFFERENT lifecycle states — // both find no row, so both UPDATEs match nothing. The loser then either // fails its own NOT EXISTS re-check (READ COMMITTED, after the winner // committed) and skips the INSERT, or attempts it and eats a 2627 in the // catch below. EITHER WAY the loser's Status/RetryCount/HttpStatus/ // TerminalAtUtc are silently dropped: it never ran an UPDATE against the // winner's row. // // INSERT-first has no such hole. The loser's INSERT is skipped or faults, // and the monotonic UPDATE that FOLLOWS it applies its state to whichever // row won — so the newer lifecycle state survives regardless of // interleaving, and a stale one is still rejected by the rank guard. // // Monotonic update semantics are unchanged: mutate when EITHER the // incoming rank is strictly greater, OR the incoming rank equals the // stored rank AND that rank is non-terminal (< TerminalRank) AND the // incoming UpdatedAtUtc is strictly newer than the stored one — so a // retrying call's Attempted-phase RetryCount/LastError/HttpStatus stay // live instead of freezing at the first Attempted packet. Terminal ranks // are excluded from the tiebreaker, so a later terminal NEVER overwrites // an earlier one; equal stamps are inert (idempotent replay) and a lower // rank is always a no-op. That inertness is what makes the UPDATE // harmless immediately after this same call's own INSERT: equal rank, // equal UpdatedAtUtc, zero rows changed. // // Raw SQL with explicitly-typed parameters (rather than // ExecuteSqlInterpolated) so the monotonic UPDATE exists as ONE statement // text shared by the combined batch and the duplicate-key retry below — // the predicate is far too intricate to keep in two copies. Explicit // SqlDbType is load-bearing: an untyped DateTime parameter binds as // `datetime` (3.33 ms rounding), which would corrupt both the stored // datetime2 stamps and the `UpdatedAtUtc <` freshness tiebreaker. Sizes // are deliberately left at the value's own length so the server enforces // the column widths (see AuditLogRepository.AddParameter). // // SourceNode-stamping: the column is included in the INSERT column list / // VALUES so a fresh row carries the originating node name (node-a/node-b // for site rows). A null SourceNode (legacy hosts / unstamped reconciled // rows) writes NULL straight through. On the UPDATE leg SourceNode is // written via // COALESCE(@SourceNode, SourceNode). The operator returns @SourceNode // when it is non-null, otherwise the stored value — so the column // behaves protectively: a later packet that carries a null // SourceNode (e.g. a reconciliation pull from an unstamped node) // NEVER blanks out a value the first stamping packet set. A later // packet that DOES carry a non-null SourceNode replaces the previous // value — combined with the monotonic-rank guard this is // "last-non-null-wins on rank advance", which lets a missing // SourceNode be filled in later if Submit happened to be unstamped // and an Attempt/Resolve carries the node identity. Within one // lifecycle every packet should carry the same SourceNode value (one // execution, one node) so the "overwrite" path is in practice // idempotent. try { await _context.Database.ExecuteSqlRawAsync( InsertIfAbsentSql + "\n\n" + MonotonicUpdateSql, BuildUpsertParameters(siteCall, idText, incomingRank), ct) .ConfigureAwait(false); } catch (SqlException ex) when ( ex.Number == SqlErrorUniqueIndexViolation || ex.Number == SqlErrorPrimaryKeyViolation) { // Two concurrent sessions both found the row absent and both raced to // INSERT; the loser raises 2601/2627 against the TrackedOperationId // primary key. The winner's row IS the first-write, but it is NOT // necessarily this packet's lifecycle state — the reconciliation pull // and the cached dual-write feed this method with different states — // so the loser MUST still apply its monotonic UPDATE against the // winner's row. The batch aborted at the faulting INSERT, so re-run // the UPDATE alone here; it is idempotent and rank-guarded, so // re-running it is safe even if the batch did reach it. _logger.LogDebug( ex, "SiteCallAuditRepository.UpsertAsync swallowed duplicate-key violation (error {SqlErrorNumber}) for TrackedOperationId {TrackedOperationId}; re-running the monotonic update against the winning row.", ex.Number, idText); await _context.Database.ExecuteSqlRawAsync( MonotonicUpdateSql, BuildUpsertParameters(siteCall, idText, incomingRank), ct) .ConfigureAwait(false); } } // Leg 1 of the upsert: create the row when it does not exist yet. Runs FIRST // so a concurrent first-write loser still has a row to update (see UpsertAsync). private const string InsertIfAbsentSql = @" IF NOT EXISTS (SELECT 1 FROM dbo.SiteCalls WHERE TrackedOperationId = @Id) INSERT INTO dbo.SiteCalls (TrackedOperationId, Channel, Target, SourceSite, SourceNode, Status, RetryCount, LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc) VALUES (@Id, @Channel, @Target, @SourceSite, @SourceNode, @Status, @RetryCount, @LastError, @HttpStatus, @CreatedAtUtc, @UpdatedAtUtc, @TerminalAtUtc, @IngestedAtUtc);"; // Leg 2 of the upsert: the monotonic guard. Also re-run standalone from the // duplicate-key catch, which is why it lives in its own constant. private const string MonotonicUpdateSql = @" UPDATE dbo.SiteCalls SET Status = @Status, RetryCount = @RetryCount, LastError = @LastError, HttpStatus = @HttpStatus, UpdatedAtUtc = @UpdatedAtUtc, TerminalAtUtc = @TerminalAtUtc, IngestedAtUtc = @IngestedAtUtc, SourceNode = COALESCE(@SourceNode, SourceNode) WHERE TrackedOperationId = @Id AND ( @Rank > (CASE Status WHEN 'Submitted' THEN 0 WHEN 'Forwarded' THEN 1 WHEN 'Attempted' THEN 2 WHEN 'Skipped' THEN 2 WHEN 'Delivered' THEN 3 WHEN 'Failed' THEN 3 WHEN 'Parked' THEN 3 WHEN 'Discarded' THEN 3 ELSE -1 END) OR ( @Rank = (CASE Status WHEN 'Submitted' THEN 0 WHEN 'Forwarded' THEN 1 WHEN 'Attempted' THEN 2 WHEN 'Skipped' THEN 2 WHEN 'Delivered' THEN 3 WHEN 'Failed' THEN 3 WHEN 'Parked' THEN 3 WHEN 'Discarded' THEN 3 ELSE -1 END) AND @Rank < @TerminalRank AND UpdatedAtUtc < @UpdatedAtUtc ) );"; /// /// Builds one FRESH parameter set for the upsert statements. Fresh per call /// because a instance cannot be attached to two /// commands, and the duplicate-key retry issues a second command. /// private static object[] BuildUpsertParameters(SiteCall siteCall, string idText, int incomingRank) { return [ Param("@Id", SqlDbType.VarChar, idText), Param("@Channel", SqlDbType.VarChar, siteCall.Channel), Param("@Target", SqlDbType.VarChar, siteCall.Target), Param("@SourceSite", SqlDbType.VarChar, siteCall.SourceSite), Param("@SourceNode", SqlDbType.VarChar, siteCall.SourceNode), Param("@Status", SqlDbType.VarChar, siteCall.Status), Param("@RetryCount", SqlDbType.Int, siteCall.RetryCount), Param("@LastError", SqlDbType.NVarChar, siteCall.LastError), Param("@HttpStatus", SqlDbType.Int, siteCall.HttpStatus), Param("@CreatedAtUtc", SqlDbType.DateTime2, siteCall.CreatedAtUtc), Param("@UpdatedAtUtc", SqlDbType.DateTime2, siteCall.UpdatedAtUtc), Param("@TerminalAtUtc", SqlDbType.DateTime2, siteCall.TerminalAtUtc), Param("@IngestedAtUtc", SqlDbType.DateTime2, siteCall.IngestedAtUtc), Param("@Rank", SqlDbType.Int, incomingRank), Param("@TerminalRank", SqlDbType.Int, TerminalRank), ]; } /// /// Binds one explicitly-typed parameter, mapping a null CLR value to /// while KEEPING the declared type. Size is never set — /// SqlClient sizes from the value, so an over-long string is rejected by the /// server rather than truncated client-side. /// private static SqlParameter Param(string name, SqlDbType type, object? value) => new(name, type) { Value = value ?? DBNull.Value }; /// public async Task GetAsync(TrackedOperationId id, CancellationToken ct = default) { return await _context.Set().FindAsync(new object?[] { id }, ct); } /// public async Task> QueryAsync( SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) { if (filter is null) { throw new ArgumentNullException(nameof(filter)); } if (paging is null) { throw new ArgumentNullException(nameof(paging)); } // FormattableString interpolation parameterises every value (no concatenation) // so this is injection-safe. EF Core resolves the parameter values, the // composed sql is shaped to SQL Server's grammar and projected into the // SiteCall entity via FromSqlInterpolated. The CASE expressions wrap each // optional predicate so a null filter field degrades to a no-op (matches // every row) instead of branching at C# level into N variants. var afterCreated = paging.AfterCreatedAtUtc; var afterIdString = paging.AfterId?.Value.ToString("D"); var hasCursor = afterCreated is not null && afterIdString is not null; var fromUtc = filter.FromUtc; var toUtc = filter.ToUtc; var stuckCutoff = filter.StuckCutoffUtc; // The stuck predicate (TerminalAtUtc IS NULL AND CreatedAtUtc < cutoff) // is pushed into SQL here — both columns are plain (no value converter) // and compose with the keyset cursor, so a StuckOnly page is honest: // never under-filled with a non-null next cursor. Mirrors how // NotificationOutboxRepository.QueryAsync applies NotificationOutboxFilter.StuckCutoff. // // SELECT-list maintenance: EF Core's FromSqlInterpolated requires every // entity-tracked column to appear in the result set. Adding a new column // to the SiteCall entity means extending the list below too — otherwise // every read trips "The required column 'X' was not present" at runtime. FormattableString sql = $@" SELECT TOP ({paging.PageSize}) TrackedOperationId, Channel, Target, SourceSite, SourceNode, Status, RetryCount, LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc FROM dbo.SiteCalls WHERE ({filter.Channel} IS NULL OR Channel = {filter.Channel}) AND ({filter.SourceSite} IS NULL OR SourceSite = {filter.SourceSite}) AND ({filter.SourceNode} IS NULL OR SourceNode = {filter.SourceNode}) AND ({filter.Status} IS NULL OR Status = {filter.Status}) AND ({filter.Target} IS NULL OR Target = {filter.Target}) AND ({fromUtc} IS NULL OR CreatedAtUtc >= {fromUtc}) AND ({toUtc} IS NULL OR CreatedAtUtc <= {toUtc}) AND ({stuckCutoff} IS NULL OR (TerminalAtUtc IS NULL AND CreatedAtUtc < {stuckCutoff})) AND ({(hasCursor ? 1 : 0)} = 0 OR CreatedAtUtc < {afterCreated} OR (CreatedAtUtc = {afterCreated} AND TrackedOperationId < {afterIdString})) ORDER BY CreatedAtUtc DESC, TrackedOperationId DESC -- Every filter above is the (@p IS NULL OR col = @p) optional-parameter shape, so a -- single cached plan would be parameter-sniffed for whichever filters happened to be -- non-null on first compile. RECOMPILE lets the optimizer prune the dead (@p IS NULL) -- predicates per invocation and pick IX_SiteCalls_Status_Updated / IX_SiteCalls_NonTerminal -- for the filters actually supplied (arch-review 04, P5). Per-invocation compile cost is -- negligible at UI-page cadence. OPTION (RECOMPILE);"; var rows = await _context.Set() .FromSqlInterpolated(sql) .AsNoTracking() .ToListAsync(ct); return rows; } /// public async Task PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) { // Time-sliced batches (arch-review 04 round 2, R6 — the one maintenance DELETE // that missed round 1's batching pass): each DELETE covers at most one DAY of // terminal rows, capping the lock/log footprint per statement. Steady state // (daily purge, 365-day retention) is a single slice; only catch-up after an // outage runs several. One-day (not one-hour) slices are proportionate to // SiteCalls volume, which is far below KpiSample's. The MIN() anchor and the // DELETE predicate both seek IX_SiteCalls_Terminal (filtered IS NOT NULL). var total = 0; var floor = await _context.SiteCalls .Where(s => s.TerminalAtUtc != null && s.TerminalAtUtc < olderThanUtc) .MinAsync(s => s.TerminalAtUtc, ct); while (floor is not null && floor < olderThanUtc) { var ceiling = floor.Value.AddDays(1) < olderThanUtc ? floor.Value.AddDays(1) : olderThanUtc; total += await _context.Database.ExecuteSqlInterpolatedAsync( $"DELETE FROM dbo.SiteCalls WHERE TerminalAtUtc IS NOT NULL AND TerminalAtUtc < {ceiling};", ct); floor = await _context.SiteCalls .Where(s => s.TerminalAtUtc != null && s.TerminalAtUtc < olderThanUtc) .MinAsync(s => s.TerminalAtUtc, ct); } return total; } // Terminal status string literals for the interval-throughput KPIs. The // Status column is a plain varchar (no value converter), so these compare // directly in translated SQL. // // NOTE on the "buffered/non-terminal" definition: the SiteCalls operational // mirror stores AuditStatus-derived strings (Attempted/Delivered/Parked/ // Failed/...), NOT the tracking-lifecycle Pending/Retrying names the spec's // KPI section uses. There is therefore no Status string that means // "buffered". The schema-honest predicate for "non-terminal / buffered" is // TerminalAtUtc IS NULL — consistent with PurgeTerminalAsync's terminal // predicate and with the SiteCall entity's own contract ("TerminalAtUtc ... // null while still active"). All buffered / stuck / oldest-pending counts // below key off TerminalAtUtc, not Status. private const string StatusParked = "Parked"; private const string StatusDelivered = "Delivered"; private const string StatusFailed = "Failed"; /// public async Task ComputeKpisAsync( DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) { var now = DateTime.UtcNow; var bufferedCount = await _context.SiteCalls .CountAsync(s => s.TerminalAtUtc == null, ct); var parkedCount = await _context.SiteCalls .CountAsync(s => s.Status == StatusParked, ct); var failedLastInterval = await _context.SiteCalls .CountAsync(s => s.Status == StatusFailed && s.TerminalAtUtc != null && s.TerminalAtUtc >= intervalSince, ct); var deliveredLastInterval = await _context.SiteCalls .CountAsync(s => s.Status == StatusDelivered && s.TerminalAtUtc != null && s.TerminalAtUtc >= intervalSince, ct); var stuckCount = await _context.SiteCalls .CountAsync(s => s.TerminalAtUtc == null && s.CreatedAtUtc < stuckCutoff, ct); var nonTerminal = _context.SiteCalls.Where(s => s.TerminalAtUtc == null); TimeSpan? oldestPendingAge = null; if (await nonTerminal.AnyAsync(ct)) { var oldestCreatedAt = await nonTerminal.MinAsync(s => s.CreatedAtUtc, ct); oldestPendingAge = now - oldestCreatedAt; } return new SiteCallKpiSnapshot( BufferedCount: bufferedCount, ParkedCount: parkedCount, FailedLastInterval: failedLastInterval, DeliveredLastInterval: deliveredLastInterval, OldestPendingAge: oldestPendingAge, StuckCount: stuckCount); } /// public async Task> ComputePerSiteKpisAsync( DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) { var now = DateTime.UtcNow; var buffered = await CountBySiteAsync(s => s.TerminalAtUtc == null, ct); var parked = await CountBySiteAsync(s => s.Status == StatusParked, ct); var failed = await CountBySiteAsync( s => s.Status == StatusFailed && s.TerminalAtUtc != null && s.TerminalAtUtc >= intervalSince, ct); var delivered = await CountBySiteAsync( s => s.Status == StatusDelivered && s.TerminalAtUtc != null && s.TerminalAtUtc >= intervalSince, ct); var stuck = await CountBySiteAsync( s => s.TerminalAtUtc == null && s.CreatedAtUtc < stuckCutoff, ct); // Oldest non-terminal CreatedAtUtc per site — a server-side GROUP BY MIN. var oldest = (await _context.SiteCalls .Where(s => s.TerminalAtUtc == null) .GroupBy(s => s.SourceSite) .Select(g => new { Site = g.Key, Oldest = g.Min(s => s.CreatedAtUtc) }) .ToListAsync(ct)) .ToDictionary(x => x.Site, x => x.Oldest); var siteIds = buffered.Keys .Concat(parked.Keys).Concat(failed.Keys) .Concat(delivered.Keys).Concat(stuck.Keys) .Distinct() .OrderBy(s => s, StringComparer.Ordinal); return siteIds.Select(site => new SiteCallSiteKpiSnapshot( SourceSite: site, BufferedCount: buffered.GetValueOrDefault(site), ParkedCount: parked.GetValueOrDefault(site), FailedLastInterval: failed.GetValueOrDefault(site), DeliveredLastInterval: delivered.GetValueOrDefault(site), OldestPendingAge: oldest.TryGetValue(site, out var createdAt) ? now - createdAt : null, StuckCount: stuck.GetValueOrDefault(site))).ToList(); } /// public async Task> ComputePerNodeKpisAsync( DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) { var now = DateTime.UtcNow; // Exclude rows with NULL SourceNode — per-node KPIs are only meaningful // when the node identity is known. Each predicate guards n.SourceNode != null // so the GROUP BY key is always non-null. var buffered = await CountByNodeAsync( s => s.TerminalAtUtc == null && s.SourceNode != null, ct); var parked = await CountByNodeAsync( s => s.Status == StatusParked && s.SourceNode != null, ct); var failed = await CountByNodeAsync( s => s.Status == StatusFailed && s.TerminalAtUtc != null && s.TerminalAtUtc >= intervalSince && s.SourceNode != null, ct); var delivered = await CountByNodeAsync( s => s.Status == StatusDelivered && s.TerminalAtUtc != null && s.TerminalAtUtc >= intervalSince && s.SourceNode != null, ct); var stuck = await CountByNodeAsync( s => s.TerminalAtUtc == null && s.CreatedAtUtc < stuckCutoff && s.SourceNode != null, ct); // Oldest non-terminal CreatedAtUtc per node — server-side GROUP BY MIN. var oldest = (await _context.SiteCalls .Where(s => s.TerminalAtUtc == null && s.SourceNode != null) .GroupBy(s => s.SourceNode!) .Select(g => new { Node = g.Key, Oldest = g.Min(s => s.CreatedAtUtc) }) .ToListAsync(ct)) .ToDictionary(x => x.Node, x => x.Oldest); var nodeNames = buffered.Keys .Concat(parked.Keys).Concat(failed.Keys) .Concat(delivered.Keys).Concat(stuck.Keys) .Distinct() .OrderBy(n => n, StringComparer.Ordinal); return nodeNames.Select(node => new SiteCallNodeKpiSnapshot( SourceNode: node, BufferedCount: buffered.GetValueOrDefault(node), ParkedCount: parked.GetValueOrDefault(node), FailedLastInterval: failed.GetValueOrDefault(node), DeliveredLastInterval: delivered.GetValueOrDefault(node), OldestPendingAge: oldest.TryGetValue(node, out var createdAt) ? now - createdAt : null, StuckCount: stuck.GetValueOrDefault(node))).ToList(); } /// Counts SiteCalls rows matching , grouped by source site. private async Task> CountBySiteAsync( System.Linq.Expressions.Expression> predicate, CancellationToken ct) { return await _context.SiteCalls .Where(predicate) .GroupBy(s => s.SourceSite) .Select(g => new { Site = g.Key, Count = g.Count() }) .ToDictionaryAsync(x => x.Site, x => x.Count, ct); } /// /// Counts SiteCalls rows matching , grouped by source node. /// Only rows with a non-null SourceNode should be included; the predicate is /// responsible for enforcing that guard. /// private async Task> CountByNodeAsync( System.Linq.Expressions.Expression> predicate, CancellationToken ct) { return await _context.SiteCalls .Where(predicate) .GroupBy(s => s.SourceNode!) .Select(g => new { Node = g.Key, Count = g.Count() }) .ToDictionaryAsync(x => x.Node, x => x.Count, ct); } private static int GetRankOrThrow(string status) { if (!StatusRank.TryGetValue(status, out var rank)) { throw new ArgumentException( $"Unknown SiteCall status '{status}'. Expected one of: {string.Join(", ", StatusRank.Keys)}.", nameof(status)); } return rank; } }