diff --git a/ScadaLink.slnx b/ScadaLink.slnx index 21b4f3a..3a70a13 100644 --- a/ScadaLink.slnx +++ b/ScadaLink.slnx @@ -12,6 +12,7 @@ + @@ -35,6 +36,7 @@ + diff --git a/docs/plans/2026-05-20-auditlog-m3-cached-operations.md b/docs/plans/2026-05-20-auditlog-m3-cached-operations.md new file mode 100644 index 0000000..a9594a0 --- /dev/null +++ b/docs/plans/2026-05-20-auditlog-m3-cached-operations.md @@ -0,0 +1,212 @@ +# Audit Log #23 — M3 Cached Operations + Dual-Write Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:subagent-driven-development (bundled cadence per `feedback_subagent_cadence`). + +**Goal:** Cached external calls (`ExternalSystem.CachedCall`) and cached DB writes (`Database.CachedWrite`) each produce 4+ audit rows per operation (`CachedSubmit` → `ApiCallCached`/`DbWriteCached` × N attempts with statuses `Forwarded` then `Attempted` then `Delivered`/`Failed` → `CachedResolve` terminal) AND a `SiteCalls` row at central. Combined telemetry: site emits one packet per lifecycle event carrying both the AuditEvent and the SiteCalls upsert; central writes both in one MS SQL transaction. Audit-write failure never aborts the script. + +**Recommended-defaults applied:** +- Telemetry proto: **new top-level RPC `IngestCachedTelemetry(CachedTelemetryBatch) returns (IngestAck)`** (sitestream.proto), separate from the M2 `IngestAuditEvents` to keep payload shapes distinct. +- Forwarder: **separate `CachedCallTelemetryForwarder`** actor (or static dispatcher hooking into the existing `SiteAuditTelemetryActor`'s SQLite queue) — write the audit row + tracking row in one SQLite transaction, then let the existing telemetry actor drain both via the new RPC. Reuse the M2 Channel/SQLite hot-path infrastructure; do NOT introduce a parallel writer. +- Provenance: mirror M2's `ScriptRuntimeContext` wrapper pattern — ScriptRuntimeContext's cached-call helpers capture instance/script/site and feed the combined packet. +- IntegrationTests E2E: same component-level pattern as M2 Bundle H (`DirectActorSiteStreamAuditClient`), but extracted into `tests/ScadaLink.AuditLog.Tests/Integration/Infrastructure/` for reuse. + +**M2 realities baked in (from roadmap line 446-459):** +- Use M1 vocabulary: `AuditKind.CachedSubmit` (enqueue), `AuditKind.ApiCallCached` / `AuditKind.DbWriteCached` (each attempt + post-forward), `AuditKind.CachedResolve` (terminal). `AuditStatus.Submitted` → `Forwarded` → `Attempted` × N → `Delivered`/`Failed`/`Parked`/`Discarded`. NO `CachedEnqueued`/`CachedAttempt`/`CachedTerminal` strings appear in code (those are pre-M1 spec wording the roadmap text still mentions; honor the enum vocabulary). +- NoOpSiteStreamAuditClient still in production until M6; E2E tests use the M2 Bundle H pattern. +- AuditEventMapper duplication note from M2: M3 should move the mapper into Commons (or document the gRPC inline duplication) since M3 adds a SECOND gRPC handler with the same DTO→entity translation work. +- CachedCallTelemetry message creates from scratch (additive per Commons REQ-COM-5a) — NOT renamed to CachedOperationTelemetry. + +--- + +## Bundles + +- **Bundle A — Commons types + tracking store** (T1, T2, T3, T4): TrackedOperationId, OperationTrackingStore, Tracking.Status API, CachedCallTelemetry message. +- **Bundle B — SiteCalls table EF + migration + repo** (T5, T6, T7). +- **Bundle C — SiteCallAudit project + actor** (T8). +- **Bundle D — Proto + central dual-write transaction** (T9, T10). +- **Bundle E — ESG / DB-gateway / S&F emissions** (T11, T12, T13, T14). +- **Bundle F — Host registration** (T15). +- **Bundle G — Integration tests** (T16, T17, T18). + +Final cross-bundle reviewer + merge to main. + +--- + +## Bundle A — Commons types + tracking store + +### Task A1: TrackedOperationId strong-typed ID +File: `src/ScadaLink.Commons/Types/TrackedOperationId.cs` — `public readonly record struct TrackedOperationId(Guid Value)`. Static `New()`, `Parse(string)`, `ToString()` returns Value.ToString("D"). Implicit conversion from Guid via `From(Guid)` (no operator implicit because record struct doesn't allow). Tests in `tests/ScadaLink.Commons.Tests/Types/TrackedOperationIdTests.cs`. Commit: `feat(commons): TrackedOperationId strong type (#23 M3)`. + +### Task A2: OperationTrackingStore (site-local SQLite) +File: `src/ScadaLink.Commons/Interfaces/IOperationTrackingStore.cs` — `RecordEnqueueAsync`, `RecordAttemptAsync`, `RecordTerminalAsync`, `GetStatusAsync(TrackedOperationId)`, `PurgeTerminalAsync(olderThanUtc)`. +File: `src/ScadaLink.SiteRuntime/Tracking/OperationTrackingStore.cs` — SQLite-backed, mirror SqliteAuditWriter pattern: Channel + background writer Task + write-lock. Schema: +```sql +CREATE TABLE IF NOT EXISTS OperationTracking ( + TrackedOperationId TEXT NOT NULL PRIMARY KEY, + Kind TEXT NOT NULL, + TargetSummary TEXT NULL, + Status TEXT NOT NULL, + RetryCount INTEGER NOT NULL DEFAULT 0, + LastError TEXT NULL, + HttpStatus INTEGER NULL, + CreatedAtUtc TEXT NOT NULL, + UpdatedAtUtc TEXT NOT NULL, + TerminalAtUtc TEXT NULL, + SourceInstanceId TEXT NULL, + SourceScript TEXT NULL); +CREATE INDEX IF NOT EXISTS IX_OperationTracking_Status_Updated ON OperationTracking(Status, UpdatedAtUtc); +``` +Tests: schema, insert+update sequence, terminal purge (only terminal rows older than threshold). Commit: `feat(siteruntime): OperationTrackingStore site-local SQLite (#23 M3)`. + +### Task A3: Tracking.Status script API +File: `src/ScadaLink.SiteRuntime/Scripts/ScriptRuntimeContext.cs` — add a `Tracking` accessor exposing `Status(TrackedOperationId)` reading via `IOperationTrackingStore.GetStatusAsync`. Returns a `TrackingStatusSnapshot` record (Commons/Types) with `Status`, `RetryCount`, `LastError`, `CreatedAtUtc`, `UpdatedAtUtc`, `TerminalAtUtc`. Returns null for unknown IDs. +Tests: known, unknown, terminal IDs. Commit: `feat(siteruntime): Tracking.Status script API (#23 M3)`. + +### Task A4: CachedCallTelemetry Commons message +File: `src/ScadaLink.Commons/Messages/Integration/CachedCallTelemetry.cs` — `public sealed record CachedCallTelemetry(TrackedOperationId TrackedOperationId, AuditEvent Audit, SiteCallOperational Operational)` plus `SiteCallOperational` record (TrackedOperationId, Channel, Target, SourceSite, Status, RetryCount, LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc?). +Tests: round-trip; lifecycle-specific construction (Submit/Attempted/Resolve). Commit: `feat(commons): CachedCallTelemetry combined operational+audit packet (#23 M3)`. + +--- + +## Bundle B — SiteCalls EF + migration + repo + +### Task B1: SiteCall entity + EF mapping +File: `src/ScadaLink.Commons/Entities/Audit/SiteCall.cs` — `public sealed record SiteCall` with fields per `SiteCallOperational` plus `IngestedAtUtc`. +File: `src/ScadaLink.ConfigurationDatabase/Configurations/SiteCallEntityTypeConfiguration.cs` — table `SiteCalls`, PK on `TrackedOperationId`, indexes `IX_SiteCalls_Source_Created` on (SourceSite, CreatedAtUtc), `IX_SiteCalls_Status_Updated` on (Status, UpdatedAtUtc). +Modify: `ScadaLinkDbContext.cs` — `public DbSet SiteCalls => Set();`. +Tests as M1 pattern. Commit: `feat(configdb): map SiteCall to SiteCalls table (#23 M3)`. + +### Task B2: AddSiteCallsTable migration +Generate via `dotnet ef migrations add AddSiteCallsTable --project src/ScadaLink.ConfigurationDatabase --startup-project src/ScadaLink.Host`. No partitioning (operational state, not audit). Use MsSqlMigrationFixture for integration test. Commit: `feat(configdb): add SiteCalls migration (#23 M3)`. + +### Task B3: ISiteCallAuditRepository + EF impl +File: `src/ScadaLink.Commons/Interfaces/Repositories/ISiteCallAuditRepository.cs` — `UpsertAsync(SiteCall)` with **monotonic status progression** (later status wins; earlier status is no-op), `GetAsync(TrackedOperationId)`, `QueryAsync(filter, paging)`, `PurgeTerminalAsync(olderThanUtc)`. +File: `src/ScadaLink.ConfigurationDatabase/Repositories/SiteCallAuditRepository.cs` — implement via `MERGE` or `INSERT ... WHERE NOT EXISTS` + `UPDATE WHERE TerminalAtUtc IS NULL AND `. Tests use MsSqlMigrationFixture. Commit: `feat(configdb): ISiteCallAuditRepository + EF impl (#23 M3)`. + +--- + +## Bundle C — SiteCallAudit project + actor + +### Task C1: ScadaLink.SiteCallAudit project + actor +Create: `src/ScadaLink.SiteCallAudit/ScadaLink.SiteCallAudit.csproj` (mirrors ScadaLink.AuditLog csproj style — net10.0, references Commons + ConfigurationDatabase). +Create: `src/ScadaLink.SiteCallAudit/SiteCallAuditActor.cs` — central singleton actor handling `UpsertSiteCallCommand(SiteCall siteCall)` by calling `ISiteCallAuditRepository.UpsertAsync` (scope-per-message via IServiceProvider, mirror AuditLogIngestActor). Idempotent via repo's monotonic upsert. +Create: `src/ScadaLink.SiteCallAudit/ServiceCollectionExtensions.cs` — `AddSiteCallAudit()` registering actor props factory. +Create: `tests/ScadaLink.SiteCallAudit.Tests/` project. +Modify: `ScadaLink.slnx` — add src + tests entries. +Commit: `feat(scaudit): SiteCallAuditActor minimum surface (#22, #23 M3)`. + +--- + +## Bundle D — Proto + central dual-write transaction + +### Task D1: Extend sitestream.proto with IngestCachedTelemetry RPC +Follow the documented protobuf regen procedure from M2 Bundle C (temporarily uncomment ItemGroup, build, copy back, recomment). Add: +```proto +message SiteCallOperationalDto { + string tracked_operation_id = 1; + string channel = 2; + string target = 3; + string source_site = 4; + string status = 5; + int32 retry_count = 6; + string last_error = 7; + google.protobuf.Int32Value http_status = 8; + google.protobuf.Timestamp created_at_utc = 9; + google.protobuf.Timestamp updated_at_utc = 10; + google.protobuf.Timestamp terminal_at_utc = 11; // null when active +} +message CachedTelemetryPacket { + AuditEventDto audit_event = 1; + SiteCallOperationalDto operational = 2; +} +message CachedTelemetryBatch { repeated CachedTelemetryPacket packets = 1; } + +service SiteStreamService { + rpc IngestCachedTelemetry(CachedTelemetryBatch) returns (IngestAck); +} +``` +Test round-trips. Commit: `feat(comms): IngestCachedTelemetry RPC + CachedTelemetryPacket proto (#23 M3)`. + +### Task D2: Dual-write transaction in AuditLogIngestActor +File: `src/ScadaLink.AuditLog/Central/AuditLogIngestActor.cs` (extend) — add `IngestCachedTelemetryCommand` handler. Inside one `DbContext.Database.BeginTransactionAsync()`: +1. Call `IAuditLogRepository.InsertIfNotExistsAsync(auditEvent)` (idempotent already from M2 Bundle A). +2. Call `ISiteCallAuditRepository.UpsertAsync(siteCallOperational)` (monotonic). +3. Commit on both-success; rollback on either-throw (the central singleton SUPERVISES — actor doesn't crash). +4. Reply `IngestAck(acceptedIds)`. + +Modify: `src/ScadaLink.Communication/SiteStreamGrpc/SiteStreamGrpcServer.cs` — implement `IngestCachedTelemetry` gRPC handler routing to actor. Same inline FromDto pattern as M2 (move to mapper if time permits per M2 reviewer recommendation). + +Add: `src/ScadaLink.Commons/Messages/Audit/IngestCachedTelemetryCommand.cs` and `IngestCachedTelemetryReply.cs` (Akka messages). + +Tests: +- Single packet → 1 AuditLog + 1 SiteCalls row. +- Duplicate `EventId` + same status → AuditLog no-op, SiteCalls no-op (monotonic), no error. +- Duplicate `EventId` + ADVANCED status → AuditLog no-op, SiteCalls updates. +- SiteCalls upsert throws → AuditLog rolled back (no orphan). +- AuditLog throws (non-duplicate) → SiteCalls rolled back. + +Commit: `feat(auditlog): combined telemetry dual-write transaction (#23 M3)`. + +--- + +## Bundle E — ESG / DB / S&F lifecycle emissions + +### Task E1: ScriptRuntimeContext.ExternalSystem.CachedCall wrapper +Mirror M2 Bundle F's `Call` wrapper. Differences: +- Emit on enqueue: AuditEvent(Kind=CachedSubmit, Status=Submitted) + SiteCallOperational(Status=Submitted, RetryCount=0). +- Calls `_externalSystemClient.CachedCallAsync` (resolves what S&F existing API surface looks like — discover by reading ExternalSystemClient). +- Returns a `TrackedOperationId` immediately (a TrackedOperationId tracking handle). +- Hands the operation to the existing StoreAndForward retry loop. + +For the per-attempt + terminal emissions, hook into the S&F dispatch loop (Bundle E2/E3). + +### Task E2: S&F retry-loop emission +Find the S&F retry-attempt callback site in `src/ScadaLink.StoreAndForward/`. On each attempt (success/transient/permanent): +- Build AuditEvent(Kind=ApiCallCached or DbWriteCached, Status=Attempted). +- Build SiteCallOperational(Status=Attempted, RetryCount=N, LastError, HttpStatus). +- Hand to `CachedCallTelemetryForwarder` which writes both to SQLite (AuditLog + OperationTracking tables, in one SQLite transaction) and lets SiteAuditTelemetryActor's drain loop push them. + +### Task E3: S&F terminal-state emission +On final state transition (Delivered / Failed / Parked / Discarded): +- Build AuditEvent(Kind=CachedResolve, Status={final state}). +- Build SiteCallOperational(Status={final state}, TerminalAtUtc=DateTime.UtcNow). +- Forward. + +### Task E4: Database.CachedWrite mirror +Same three-event pattern but Channel=DbOutbound, Kind=DbWriteCached for attempts, Kind=CachedSubmit for enqueue, Kind=CachedResolve for terminal. + +Tests in ExternalSystemGateway.Tests + StoreAndForward.Tests. + +Commit (bundle-level): one commit per task, descriptive messages following M2 style. + +--- + +## Bundle F — Host registration + +### Task F1: Register SiteCallAuditActor central singleton +File: `src/ScadaLink.Host/Actors/AkkaHostedService.cs` — register `SiteCallAuditActor` central singleton + proxy alongside `AuditLogIngestActor`. Hand the proxy to `SiteStreamGrpcServer.SetSiteCallAuditActor(proxy)` (mirroring `SetAuditIngestActor`). +File: `src/ScadaLink.Host/Program.cs` — call `.AddSiteCallAudit()` on the central role's services. +Tests in `tests/ScadaLink.Host.Tests/AkkaHostedServiceAuditWiringTests.cs` (extend). +Commit: `feat(host): register SiteCallAuditActor central singleton (#22, #23 M3)`. + +--- + +## Bundle G — Integration tests + +### Task G1: Extract DirectActorSiteStreamAuditClient to shared infrastructure +Move from `tests/ScadaLink.AuditLog.Tests/Integration/SyncCallEmissionEndToEndTests.cs` private inner class into `tests/ScadaLink.AuditLog.Tests/Integration/Infrastructure/DirectActorSiteStreamAuditClient.cs`. Extend to also implement the new `IngestCachedTelemetryAsync` method (mirror pattern). + +### Task G2: Cached call E2E test +File: `tests/ScadaLink.AuditLog.Tests/Integration/CachedCallCombinedTelemetryTests.cs` (use AuditLog.Tests, not IntegrationTests, because the existing IntegrationTests harness disables Akka per M2 reality). Test: cached call that fails twice then succeeds produces 5 AuditLog rows (1 Submit + 1 Forwarded + 2 Attempted + 1 Resolve) + 1 SiteCalls row (Status=Delivered) + Tracking.Status reports Delivered. + +### Task G3: Cached DB write E2E test +File: `tests/ScadaLink.AuditLog.Tests/Integration/CachedWriteCombinedTelemetryTests.cs`. Mirror G2 for DB. + +### Task G4: Idempotency test +File: `tests/ScadaLink.AuditLog.Tests/Integration/CombinedTelemetryIdempotencyTests.cs`. Send the same packet twice; assert exactly 1 AuditLog row + 1 SiteCalls row. + +--- + +## Final cross-bundle review + merge + +Same template as M1/M2. diff --git a/src/ScadaLink.AuditLog/Central/AuditLogIngestActor.cs b/src/ScadaLink.AuditLog/Central/AuditLogIngestActor.cs index 01496bb..2b2c580 100644 --- a/src/ScadaLink.AuditLog/Central/AuditLogIngestActor.cs +++ b/src/ScadaLink.AuditLog/Central/AuditLogIngestActor.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using ScadaLink.Commons.Entities.Audit; using ScadaLink.Commons.Interfaces.Repositories; using ScadaLink.Commons.Messages.Audit; +using ScadaLink.ConfigurationDatabase; namespace ScadaLink.AuditLog.Central; @@ -61,6 +62,11 @@ public class AuditLogIngestActor : ReceiveActor _logger = logger; ReceiveAsync(OnIngestAsync); + // The single-repository test ctor cannot service the M3 dual-write — + // it has no SiteCalls repo and no DbContext. The handler still + // registers (so callers don't dead-letter) but replies empty so the + // test surface stays explicit about what this ctor supports. + ReceiveAsync(OnCachedTelemetryWithoutDualWriteAsync); } /// @@ -81,6 +87,7 @@ public class AuditLogIngestActor : ReceiveActor _logger = logger; ReceiveAsync(OnIngestAsync); + ReceiveAsync(OnCachedTelemetryAsync); } /// @@ -150,4 +157,98 @@ public class AuditLogIngestActor : ReceiveActor replyTo.Tell(new IngestAuditEventsReply(accepted)); } + + /// + /// M3 dual-write handler. For every the + /// actor opens a fresh MS SQL transaction, inserts the AuditLog row + /// idempotently AND upserts the SiteCalls row monotonically. Both succeed + /// or both roll back, so the audit and operational mirrors never drift + /// mid-row. The IngestedAtUtc stamp is unified between the two rows so a + /// downstream join lines up cleanly. + /// + /// + /// Per-entry isolation — one entry's failed transaction does NOT abort + /// other entries in the batch (each gets its own + /// + /// scope and a try/catch around it). Audit-write failure NEVER aborts the + /// user-facing action — the site keeps the row Pending and retries on the + /// next drain. + /// + private async Task OnCachedTelemetryAsync(IngestCachedTelemetryCommand cmd) + { + var replyTo = Sender; + var accepted = new List(cmd.Entries.Count); + + try + { + await using var scope = _serviceProvider!.CreateAsyncScope(); + var auditRepo = scope.ServiceProvider.GetRequiredService(); + var siteCallRepo = scope.ServiceProvider.GetRequiredService(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + foreach (var entry in cmd.Entries) + { + try + { + await using var tx = await dbContext.Database + .BeginTransactionAsync() + .ConfigureAwait(false); + + // Stamp IngestedAtUtc on both rows from a single + // central-side instant so a join on the two tables sees + // matching timestamps (debugging convenience, not a + // correctness invariant). + var ingestedAt = DateTime.UtcNow; + var auditStamped = entry.Audit with { IngestedAtUtc = ingestedAt }; + var siteCallStamped = entry.SiteCall with { IngestedAtUtc = ingestedAt }; + + await auditRepo.InsertIfNotExistsAsync(auditStamped) + .ConfigureAwait(false); + await siteCallRepo.UpsertAsync(siteCallStamped) + .ConfigureAwait(false); + + await tx.CommitAsync().ConfigureAwait(false); + accepted.Add(entry.Audit.EventId); + } + catch (Exception ex) + { + // Both rows rolled back via the disposing transaction. The + // EventId is NOT added to `accepted` so the site keeps its + // row Pending and retries on the next drain. Other entries + // in the batch continue with their own transactions. + _logger.LogError( + ex, + "Combined telemetry dual-write failed for AuditEvent {EventId} / TrackedOperationId {TrackedOpId}; rolled back.", + entry.Audit.EventId, + entry.SiteCall.TrackedOperationId); + } + } + } + catch (Exception ex) + { + // Resolving the scope itself threw (e.g. DI mis-wiring). Log and + // reply with whatever we managed to accept (likely empty) — the + // central singleton MUST stay alive. + _logger.LogError( + ex, + "Combined telemetry batch ingest failed before per-entry processing."); + } + + replyTo.Tell(new IngestCachedTelemetryReply(accepted)); + } + + /// + /// Fallback handler installed on the single-repository test ctor — that + /// ctor has no DbContext and no , so + /// it cannot service the dual-write. Logs a warning and replies with an + /// empty ack so callers fall through to their retry path. + /// + private Task OnCachedTelemetryWithoutDualWriteAsync(IngestCachedTelemetryCommand cmd) + { + _logger.LogWarning( + "AuditLogIngestActor received IngestCachedTelemetryCommand on the single-repository ctor; dual-write requires the IServiceProvider ctor. Replying with empty ack ({Count} entries).", + cmd.Entries.Count); + Sender.Tell(new IngestCachedTelemetryReply(Array.Empty())); + return Task.CompletedTask; + } } diff --git a/src/ScadaLink.AuditLog/ServiceCollectionExtensions.cs b/src/ScadaLink.AuditLog/ServiceCollectionExtensions.cs index 34d3a23..e420f4f 100644 --- a/src/ScadaLink.AuditLog/ServiceCollectionExtensions.cs +++ b/src/ScadaLink.AuditLog/ServiceCollectionExtensions.cs @@ -102,6 +102,33 @@ public static class ServiceCollectionExtensions // SiteAuditTelemetryActor's Props.Create call. services.AddSingleton(); + // M3 Bundle F: site-side dual emitter for cached-call lifecycle + // telemetry. ScriptRuntimeContext.ExternalSystem.CachedCall / + // Database.CachedWrite resolves this through DI and pushes one combined + // packet per lifecycle event; the forwarder writes the audit half + // through IAuditWriter and the operational half through the + // IOperationTrackingStore. The audit writer is always wired (the M2 + // chain above); the operational tracking store is SITE-ONLY (registered + // by ScadaLink.SiteRuntime). On a Central composition root the tracking + // store has no registration, so the factory resolves it with GetService + // (returning null) — the forwarder degrades to "audit-only" emission, + // mirroring the lazy IAuditWriter chain established in M2. + services.AddSingleton(sp => + new CachedCallTelemetryForwarder( + sp.GetRequiredService(), + sp.GetService(), + sp.GetRequiredService>())); + + // M3 Bundle F: bridge the store-and-forward retry-loop observer hook + // to the cached-call forwarder so per-attempt + terminal telemetry + // emitted from the S&F retry sweep lands on the same SQLite hot-path + // as the script-thread CachedSubmit row. Registered as a singleton + // and also bound to ICachedCallLifecycleObserver so AddStoreAndForward + // can resolve it through DI (Bundle F StoreAndForward wiring change). + services.AddSingleton(); + services.AddSingleton( + sp => sp.GetRequiredService()); + return services; } diff --git a/src/ScadaLink.AuditLog/Site/Telemetry/CachedCallLifecycleBridge.cs b/src/ScadaLink.AuditLog/Site/Telemetry/CachedCallLifecycleBridge.cs new file mode 100644 index 0000000..cc6a975 --- /dev/null +++ b/src/ScadaLink.AuditLog/Site/Telemetry/CachedCallLifecycleBridge.cs @@ -0,0 +1,202 @@ +using Microsoft.Extensions.Logging; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Site.Telemetry; + +/// +/// Audit Log #23 (M3 Bundle E — Tasks E4/E5): translates per-attempt +/// notifications from the store-and-forward retry loop into one (or two) +/// packets and pushes them through +/// . +/// +/// +/// +/// The S&F loop's reports a +/// single coarse outcome per attempt; the audit pipeline however models the +/// lifecycle as TWO rows on terminal outcomes — an Attempted +/// ( / ) +/// row capturing the per-attempt mechanics, plus a +/// row marking the terminal state for downstream consumers. The bridge fans +/// out per outcome: +/// +/// +/// TransientFailure -> one Attempted(Failed) row. +/// Delivered -> Attempted(Delivered) + CachedResolve(Delivered). +/// PermanentFailure -> Attempted(Failed) + CachedResolve(Parked). +/// ParkedMaxRetries -> Attempted(Failed) + CachedResolve(Parked). +/// +/// +/// Best-effort emission (alog.md §7): the bridge itself never throws; +/// the underlying forwarder swallows + logs its own failures. +/// +/// +public sealed class CachedCallLifecycleBridge : ICachedCallLifecycleObserver +{ + private readonly ICachedCallTelemetryForwarder _forwarder; + private readonly ILogger _logger; + + public CachedCallLifecycleBridge( + ICachedCallTelemetryForwarder forwarder, + ILogger logger) + { + _forwarder = forwarder ?? throw new ArgumentNullException(nameof(forwarder)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public async Task OnAttemptCompletedAsync( + CachedCallAttemptContext context, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(context); + + try + { + await EmitAttemptedAsync(context, ct).ConfigureAwait(false); + + if (IsTerminal(context.Outcome)) + { + await EmitResolveAsync(context, ct).ConfigureAwait(false); + } + } + catch (Exception ex) + { + // Defensive — both EmitX paths call the forwarder which is itself + // best-effort. A throw here is unexpected, but the alog.md §7 + // contract requires we never propagate. + _logger.LogWarning(ex, + "CachedCallLifecycleBridge: unexpected throw for {TrackedOperationId} (Outcome {Outcome})", + context.TrackedOperationId, context.Outcome); + } + } + + private async Task EmitAttemptedAsync(CachedCallAttemptContext context, CancellationToken ct) + { + // Per-attempt row: kind discriminates channel; status is always + // Attempted regardless of outcome (success vs. failure is captured + // by the companion HttpStatus / ErrorMessage fields, NOT by flipping + // the status — CachedResolve carries the terminal Status). Per the + // M3 brief and alog.md §4. + var kind = ChannelToAttemptKind(context.Channel); + var status = AuditStatus.Attempted; + + var packet = BuildPacket( + context, + kind: kind, + status: status, + // Operational status mirror — for the per-attempt row the + // operational state is the running status; the bridge always + // writes "Attempted" so reconciliation can't roll back. + operationalStatus: "Attempted", + terminalAtUtc: null, + lastError: context.LastError, + httpStatus: context.HttpStatus); + + await _forwarder.ForwardAsync(packet, ct).ConfigureAwait(false); + } + + private async Task EmitResolveAsync(CachedCallAttemptContext context, CancellationToken ct) + { + var (auditStatus, operationalStatus) = TerminalOutcomeToStatuses(context.Outcome); + + var packet = BuildPacket( + context, + kind: AuditKind.CachedResolve, + status: auditStatus, + operationalStatus: operationalStatus, + terminalAtUtc: context.OccurredAtUtc, + lastError: context.LastError, + httpStatus: context.HttpStatus); + + await _forwarder.ForwardAsync(packet, ct).ConfigureAwait(false); + } + + private static CachedCallTelemetry BuildPacket( + CachedCallAttemptContext context, + AuditKind kind, + AuditStatus status, + string operationalStatus, + DateTime? terminalAtUtc, + string? lastError, + int? httpStatus) + { + var channel = ChannelStringToEnum(context.Channel); + + return new CachedCallTelemetry( + Audit: new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.SpecifyKind(context.OccurredAtUtc, DateTimeKind.Utc), + Channel = channel, + Kind = kind, + CorrelationId = context.TrackedOperationId.Value, + SourceSiteId = string.IsNullOrEmpty(context.SourceSite) ? null : context.SourceSite, + SourceInstanceId = context.SourceInstanceId, + SourceScript = null, // Not threaded through S&F; left null on retry-loop rows. + Target = context.Target, + Status = status, + HttpStatus = httpStatus, + DurationMs = context.DurationMs, + ErrorMessage = lastError, + ForwardState = AuditForwardState.Pending, + }, + Operational: new SiteCallOperational( + TrackedOperationId: context.TrackedOperationId, + Channel: context.Channel, + Target: context.Target, + SourceSite: context.SourceSite, + Status: operationalStatus, + RetryCount: context.RetryCount, + LastError: lastError, + HttpStatus: httpStatus, + CreatedAtUtc: DateTime.SpecifyKind(context.CreatedAtUtc, DateTimeKind.Utc), + UpdatedAtUtc: DateTime.SpecifyKind(context.OccurredAtUtc, DateTimeKind.Utc), + TerminalAtUtc: terminalAtUtc is null + ? null + : DateTime.SpecifyKind(terminalAtUtc.Value, DateTimeKind.Utc))); + } + + private static AuditKind ChannelToAttemptKind(string channel) => channel switch + { + "ApiOutbound" => AuditKind.ApiCallCached, + "DbOutbound" => AuditKind.DbWriteCached, + // Defensive default — the S&F observer is filtered to cached-call + // categories so this branch shouldn't fire in practice. + _ => AuditKind.ApiCallCached, + }; + + private static AuditChannel ChannelStringToEnum(string channel) => channel switch + { + "ApiOutbound" => AuditChannel.ApiOutbound, + "DbOutbound" => AuditChannel.DbOutbound, + _ => AuditChannel.ApiOutbound, + }; + + private static (AuditStatus auditStatus, string operationalStatus) TerminalOutcomeToStatuses( + CachedCallAttemptOutcome outcome) => outcome switch + { + CachedCallAttemptOutcome.Delivered => + (AuditStatus.Delivered, "Delivered"), + CachedCallAttemptOutcome.PermanentFailure => + (AuditStatus.Parked, "Parked"), + CachedCallAttemptOutcome.ParkedMaxRetries => + (AuditStatus.Parked, "Parked"), + // TransientFailure isn't terminal — see IsTerminal — but the switch + // is exhaustive so we route it through Failed for safety. + CachedCallAttemptOutcome.TransientFailure => + (AuditStatus.Failed, "Failed"), + _ => (AuditStatus.Failed, "Failed"), + }; + + private static bool IsTerminal(CachedCallAttemptOutcome outcome) => outcome switch + { + CachedCallAttemptOutcome.Delivered => true, + CachedCallAttemptOutcome.PermanentFailure => true, + CachedCallAttemptOutcome.ParkedMaxRetries => true, + CachedCallAttemptOutcome.TransientFailure => false, + _ => false, + }; +} diff --git a/src/ScadaLink.AuditLog/Site/Telemetry/CachedCallTelemetryForwarder.cs b/src/ScadaLink.AuditLog/Site/Telemetry/CachedCallTelemetryForwarder.cs new file mode 100644 index 0000000..7f45453 --- /dev/null +++ b/src/ScadaLink.AuditLog/Site/Telemetry/CachedCallTelemetryForwarder.cs @@ -0,0 +1,179 @@ +using Microsoft.Extensions.Logging; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Site.Telemetry; + +/// +/// Site-side dual emitter for cached-call lifecycle telemetry (Audit Log #23 / +/// M3). Sister to : where the M2 actor +/// drains audit-only events, this forwarder takes a combined +/// packet and fans it out to the two +/// site-local stores in a single call: +/// +/// The row is written via +/// (the site FallbackAuditWriter + +/// SqliteAuditWriter chain established in M2). +/// The operational half +/// updates the site-local OperationTracking SQLite store via +/// , with the per-lifecycle method +/// (Enqueue / Attempt / Terminal) selected from the +/// audit row's . +/// +/// +/// +/// +/// Best-effort contract (alog.md §7): a thrown writer OR a thrown +/// tracking store must never propagate to the calling script. Both emission +/// halves are wrapped in independent try/catch blocks so a SQLite outage on +/// one side cannot starve the other — the failure is logged and the call +/// returns normally. +/// +/// +/// Wire push deferred to M6. M3 keeps this forwarder synchronous +/// against the local stores: there is no site→central gRPC channel yet, so +/// the RPC +/// is registered on the interface (Bundle E1) but the production binding +/// remains NoOpSiteStreamAuditClient. Once M6 wires a real client the +/// drain pattern from SiteAuditTelemetryActor can be reused — the +/// AuditEvent rows already live in SQLite tagged +/// , so a single drain loop sweeps +/// both M2 and M3 emissions. +/// +/// +public sealed class CachedCallTelemetryForwarder : ICachedCallTelemetryForwarder +{ + private readonly IAuditWriter _auditWriter; + private readonly IOperationTrackingStore? _trackingStore; + private readonly ILogger _logger; + + /// + /// Construct the forwarder. is optional — + /// when null only the audit half of the packet is emitted, which matches + /// the M3 Bundle F composition-root contract on Central nodes: the + /// AuditLog DI surface registers the forwarder unconditionally (mirroring + /// the IAuditWriter chain) but the site-only tracking store has no central + /// registration. Production site nodes wire both — the central lazy + /// resolution is a no-op path kept symmetric with the M2 writer chain. + /// + public CachedCallTelemetryForwarder( + IAuditWriter auditWriter, + IOperationTrackingStore? trackingStore, + ILogger logger) + { + _auditWriter = auditWriter ?? throw new ArgumentNullException(nameof(auditWriter)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _trackingStore = trackingStore; + } + + /// + /// Fan out one combined-telemetry packet to the audit writer and the + /// tracking store. Returns once both halves have been attempted (success + /// OR logged failure). NEVER throws — exceptions are caught per-half and + /// logged at warning level so the calling script's outbound action is not + /// disturbed. + /// + public async Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(telemetry); + + // Independent try/catch — a thrown audit writer must not prevent the + // tracking-store update from running (and vice-versa). Both halves + // are best-effort. + await TryEmitAuditAsync(telemetry, ct).ConfigureAwait(false); + await TryEmitTrackingAsync(telemetry, ct).ConfigureAwait(false); + } + + private async Task TryEmitAuditAsync(CachedCallTelemetry telemetry, CancellationToken ct) + { + try + { + await _auditWriter.WriteAsync(telemetry.Audit, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + // alog.md §7 best-effort contract — log and swallow. The audit + // pipeline's own retry/recovery (RingBufferFallback in the + // FallbackAuditWriter) handles transient writer failures upstream; + // a throw bubbling up here means the writer's own swallow contract + // failed, which is itself best-effort-handled. + _logger.LogWarning(ex, + "CachedCallTelemetryForwarder: audit emission threw for EventId {EventId} (Kind {Kind}, Status {Status})", + telemetry.Audit.EventId, telemetry.Audit.Kind, telemetry.Audit.Status); + } + } + + private async Task TryEmitTrackingAsync(CachedCallTelemetry telemetry, CancellationToken ct) + { + if (_trackingStore is null) + { + // No site-local tracking store wired — Central composition root or + // an integration-test host that skipped AddSiteRuntime. Emitting + // through the audit half is still meaningful; the tracking half + // is a no-op rather than an error. + return; + } + + try + { + switch (telemetry.Audit.Kind) + { + case AuditKind.CachedSubmit: + // Enqueue — insert-if-not-exists with the operational + // channel as the kind discriminator. RetryCount is fixed + // at 0 by the tracking store's INSERT contract. + await _trackingStore.RecordEnqueueAsync( + telemetry.Operational.TrackedOperationId, + telemetry.Operational.Channel, + telemetry.Operational.Target, + telemetry.Audit.SourceInstanceId, + telemetry.Audit.SourceScript, + ct).ConfigureAwait(false); + break; + + case AuditKind.ApiCallCached: + case AuditKind.DbWriteCached: + // Attempt — advance retry counter + last-error/HTTP-status. + // Terminal rows are guarded by the store's WHERE clause. + await _trackingStore.RecordAttemptAsync( + telemetry.Operational.TrackedOperationId, + telemetry.Operational.Status, + telemetry.Operational.RetryCount, + telemetry.Operational.LastError, + telemetry.Operational.HttpStatus, + ct).ConfigureAwait(false); + break; + + case AuditKind.CachedResolve: + // Terminal — first-write-wins on the resolve flip. + await _trackingStore.RecordTerminalAsync( + telemetry.Operational.TrackedOperationId, + telemetry.Operational.Status, + telemetry.Operational.LastError, + telemetry.Operational.HttpStatus, + ct).ConfigureAwait(false); + break; + + default: + // Defensive — only the four cached-lifecycle kinds are + // expected on this path. Anything else is logged so a + // mis-routed packet is visible but never crashes the + // forwarder. + _logger.LogWarning( + "CachedCallTelemetryForwarder: unexpected audit kind {Kind} on tracking emission for EventId {EventId}", + telemetry.Audit.Kind, telemetry.Audit.EventId); + break; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "CachedCallTelemetryForwarder: tracking-store emission threw for TrackedOperationId {Id} (Status {Status})", + telemetry.Operational.TrackedOperationId, telemetry.Operational.Status); + } + } +} diff --git a/src/ScadaLink.AuditLog/Site/Telemetry/ISiteStreamAuditClient.cs b/src/ScadaLink.AuditLog/Site/Telemetry/ISiteStreamAuditClient.cs index c25b05a..6314bba 100644 --- a/src/ScadaLink.AuditLog/Site/Telemetry/ISiteStreamAuditClient.cs +++ b/src/ScadaLink.AuditLog/Site/Telemetry/ISiteStreamAuditClient.cs @@ -20,4 +20,23 @@ public interface ISiteStreamAuditClient /// in the site SQLite queue. /// Task IngestAuditEventsAsync(AuditEventBatch batch, CancellationToken ct); + + /// + /// Pushes the combined (Audit Log #23 / M3) + /// to the central IngestCachedTelemetry RPC. Each packet carries both + /// the audit row and the operational SiteCalls upsert; central writes + /// both in a single MS SQL transaction. Returns the same + /// shape as so + /// the M3 site-side forwarder can flip the underlying audit rows to + /// + /// once central has acknowledged them. + /// + /// + /// The production gRPC-backed implementation lands in M6 (no site→central + /// gRPC channel exists today); until then the default + /// binding returns an empty ack and + /// integration tests substitute a direct-actor client that routes the batch + /// straight into the in-process AuditLogIngestActor. + /// + Task IngestCachedTelemetryAsync(CachedTelemetryBatch batch, CancellationToken ct); } diff --git a/src/ScadaLink.AuditLog/Site/Telemetry/NoOpSiteStreamAuditClient.cs b/src/ScadaLink.AuditLog/Site/Telemetry/NoOpSiteStreamAuditClient.cs index b1a0190..b83a215 100644 --- a/src/ScadaLink.AuditLog/Site/Telemetry/NoOpSiteStreamAuditClient.cs +++ b/src/ScadaLink.AuditLog/Site/Telemetry/NoOpSiteStreamAuditClient.cs @@ -38,4 +38,16 @@ public sealed class NoOpSiteStreamAuditClient : ISiteStreamAuditClient // Pending until M6's real client (or a Bundle H test stub) takes over. return Task.FromResult(EmptyAck); } + + /// + public Task IngestCachedTelemetryAsync(CachedTelemetryBatch batch, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(batch); + // Empty ack — same rationale as IngestAuditEventsAsync. The M3 + // CachedCallTelemetryForwarder still writes the audit + tracking rows to + // the site SQLite stores authoritatively; central-side state only + // materialises once M6's real gRPC client (or a Bundle G test stub) is + // wired in. + return Task.FromResult(EmptyAck); + } } diff --git a/src/ScadaLink.Commons/Entities/Audit/SiteCall.cs b/src/ScadaLink.Commons/Entities/Audit/SiteCall.cs new file mode 100644 index 0000000..f83f3ab --- /dev/null +++ b/src/ScadaLink.Commons/Entities/Audit/SiteCall.cs @@ -0,0 +1,60 @@ +using ScadaLink.Commons.Types; + +namespace ScadaLink.Commons.Entities.Audit; + +/// +/// Central operational state row for a cached call (Site Call Audit #22, Audit Log #23 M3). +/// One row per in the SiteCalls table — append-once +/// then monotonic status update. Status transitions are forward-only +/// (Submitted → Forwarded → Attempted → Delivered|Failed|Parked|Discarded); an +/// out-of-order or duplicate upsert is a silent no-op so duplicate gRPC packets and +/// reconciliation pulls can both feed the same writer without rolling state back. +/// +/// +/// Sites remain the source of truth — this row is the eventually-consistent mirror the +/// Central UI's Site Calls page reads. Unlike the partitioned AuditLog table this +/// entity backs operational (mutable) state on a standard non-partitioned table on +/// [PRIMARY]; no DB-role restriction applies. +/// +public sealed record SiteCall +{ + /// Strong-typed idempotency key shared with every audit row for the operation. + public required TrackedOperationId TrackedOperationId { get; init; } + + /// Trust-boundary channel — "ApiOutbound" or "DbOutbound". + public required string Channel { get; init; } + + /// Human-readable target (e.g. "ERP.GetOrder"). + public required string Target { get; init; } + + /// Site id that submitted the cached call. + public required string SourceSite { get; init; } + + /// + /// Lifecycle status — string form of + /// . Monotonic: later rank + /// wins, earlier rank is a no-op. + /// + public required string Status { get; init; } + + /// Number of dispatch attempts so far; 0 prior to first attempt. + public required int RetryCount { get; init; } + + /// Most recent error message; null when no failures have occurred. + public string? LastError { get; init; } + + /// Most recent HTTP status code (API calls only); null otherwise. + public int? HttpStatus { get; init; } + + /// UTC timestamp the cached call was first submitted at the site. + public required DateTime CreatedAtUtc { get; init; } + + /// UTC timestamp of the latest status mutation at the site. + public required DateTime UpdatedAtUtc { get; init; } + + /// UTC timestamp the row reached a terminal status; null while still active. + public DateTime? TerminalAtUtc { get; init; } + + /// UTC timestamp central ingested (or last refreshed) this row. + public required DateTime IngestedAtUtc { get; init; } +} diff --git a/src/ScadaLink.Commons/Interfaces/IOperationTrackingStore.cs b/src/ScadaLink.Commons/Interfaces/IOperationTrackingStore.cs new file mode 100644 index 0000000..add0a8c --- /dev/null +++ b/src/ScadaLink.Commons/Interfaces/IOperationTrackingStore.cs @@ -0,0 +1,87 @@ +using ScadaLink.Commons.Types; + +namespace ScadaLink.Commons.Interfaces; + +/// +/// Site-local source of truth for cached-operation tracking +/// (ExternalSystem.CachedCall / Database.CachedWrite) — alongside the +/// Store-and-Forward buffer, this is the row that Tracking.Status(id) +/// reads (Audit Log #23 / M3). One row per ; +/// terminal rows are purged after a configurable retention window +/// (default 7 days). +/// +/// +/// +/// The store is intentionally a thin write-API on top of SQLite — not a +/// dispatcher. Status transitions follow +/// Submitted → Retrying → Delivered / Parked / Failed / Discarded; rows +/// in a terminal state never roll back. Implementations must: +/// +/// is insert-if-not-exists +/// (caller-supplied id is the idempotency key — duplicate enqueues are no-ops). +/// only updates non-terminal rows. +/// only flips a non-terminal row to terminal. +/// deletes terminal rows whose +/// TerminalAtUtc is strictly older than the supplied threshold. +/// +/// +/// +public interface IOperationTrackingStore +{ + /// + /// Insert a new tracking row in Submitted state with RetryCount = 0. + /// Idempotent — a duplicate id is silently ignored (the existing row is left + /// untouched), matching the at-least-once semantics of the calling site + /// store-and-forward path. + /// + Task RecordEnqueueAsync( + TrackedOperationId id, + string kind, + string? targetSummary, + string? sourceInstanceId, + string? sourceScript, + CancellationToken ct = default); + + /// + /// Advance an in-flight tracking row's status, retry counter, and most- + /// recent error/HTTP-status. Terminal rows ( + /// already applied) are NOT mutated — the operation has reached its final + /// outcome and any late-arriving attempt telemetry is dropped on the floor. + /// + Task RecordAttemptAsync( + TrackedOperationId id, + string status, + int retryCount, + string? lastError, + int? httpStatus, + CancellationToken ct = default); + + /// + /// Flip a non-terminal tracking row to terminal — sets + /// TerminalAtUtc = now and writes the final status / error. A row + /// already in terminal state is left untouched (first-write-wins). + /// + Task RecordTerminalAsync( + TrackedOperationId id, + string status, + string? lastError, + int? httpStatus, + CancellationToken ct = default); + + /// + /// Return the latest snapshot for the supplied id, or null when no + /// tracking row exists (purged or never recorded). + /// + Task GetStatusAsync( + TrackedOperationId id, + CancellationToken ct = default); + + /// + /// Delete terminal rows whose TerminalAtUtc is strictly older than + /// . Non-terminal rows are kept regardless + /// of age (the operation is still in flight). + /// + Task PurgeTerminalAsync( + DateTime olderThanUtc, + CancellationToken ct = default); +} diff --git a/src/ScadaLink.Commons/Interfaces/Repositories/ISiteCallAuditRepository.cs b/src/ScadaLink.Commons/Interfaces/Repositories/ISiteCallAuditRepository.cs new file mode 100644 index 0000000..7bb3790 --- /dev/null +++ b/src/ScadaLink.Commons/Interfaces/Repositories/ISiteCallAuditRepository.cs @@ -0,0 +1,66 @@ +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Audit; + +namespace ScadaLink.Commons.Interfaces.Repositories; + +/// +/// Operational-state data access for the central SiteCalls table +/// (Site Call Audit #22, Audit Log #23 M3 Bundle B). One row per +/// ; sites remain the source of truth and this +/// table is an eventually-consistent mirror fed by best-effort gRPC telemetry +/// plus periodic reconciliation pulls. +/// +/// +/// +/// Unlike the partitioned append-only AuditLog (M1), this table holds +/// mutable operational state. is insert-if-not-exists +/// then monotonic update — a status update with rank less than or equal to the +/// stored status is a silent no-op so out-of-order telemetry, duplicate gRPC +/// packets, and reconciliation pulls can all feed the same writer without +/// rolling state backward. +/// +/// +/// Status rank for monotonic comparison (lower wins): Submitted=0, +/// Forwarded=1, Attempted=2, Skipped=2, Delivered=3, Failed=3, Parked=3, +/// Discarded=3. Terminal statuses share rank 3 and are mutually exclusive +/// — an attempt to upsert e.g. Delivered over an existing Parked +/// row is a no-op. +/// +/// +public interface ISiteCallAuditRepository +{ + /// + /// Inserts if no row with the same + /// exists; otherwise updates the + /// existing row IF AND ONLY IF the incoming status' rank strictly exceeds + /// the stored status' rank. Out-of-order / duplicate updates are silently + /// dropped (monotonic forward-only progression). + /// + Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default); + + /// + /// Returns the row for the given id, or null if none exists. + /// + Task GetAsync(TrackedOperationId id, CancellationToken ct = default); + + /// + /// Returns up to rows matching + /// , ordered by (CreatedAtUtc DESC, + /// TrackedOperationId DESC). Use keyset paging via + /// + + /// to fetch subsequent pages. + /// + Task> QueryAsync( + SiteCallQueryFilter filter, + SiteCallPaging paging, + CancellationToken ct = default); + + /// + /// Deletes terminal rows whose is + /// strictly older than . Non-terminal rows + /// (TerminalAtUtc IS NULL) are NEVER purged. Returns the number of rows + /// deleted. + /// + Task PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default); +} diff --git a/src/ScadaLink.Commons/Interfaces/Services/ICachedCallLifecycleObserver.cs b/src/ScadaLink.Commons/Interfaces/Services/ICachedCallLifecycleObserver.cs new file mode 100644 index 0000000..8e34cb4 --- /dev/null +++ b/src/ScadaLink.Commons/Interfaces/Services/ICachedCallLifecycleObserver.cs @@ -0,0 +1,93 @@ +using ScadaLink.Commons.Types; + +namespace ScadaLink.Commons.Interfaces.Services; + +/// +/// Audit Log #23 (M3 Bundle E — Tasks E4/E5): site-side hook the +/// store-and-forward retry loop invokes after every cached-call attempt and +/// at terminal-state transitions, so the audit pipeline can emit +/// ApiCallCached/DbWriteCached per-attempt rows and the +/// CachedResolve terminal row under the original +/// . +/// +/// +/// +/// The interface deliberately uses +/// rather than so the +/// S&F project does not need to depend on the audit vocabulary — the +/// bridge living in ScadaLink.AuditLog maps the outcome to the right +/// audit kind + status when materialising the CachedCallTelemetry +/// packet. +/// +/// +/// Best-effort contract (alog.md §7): implementations MUST swallow +/// internal failures rather than propagating to the S&F service — a +/// thrown observer must not be misclassified as a transient delivery +/// failure and must not corrupt the retry-count bookkeeping. +/// +/// +public interface ICachedCallLifecycleObserver +{ + /// + /// Called by the store-and-forward retry loop after every cached-call + /// delivery attempt. Receives the message's TrackedOperationId-bearing id, + /// the per-category channel discriminator, retry-count + last-error + /// context, and whether the outcome reached a terminal state. + /// + Task OnAttemptCompletedAsync(CachedCallAttemptContext context, CancellationToken ct = default); +} + +/// +/// Per-attempt context handed to . +/// +/// +/// Tracking id parsed from the underlying StoreAndForwardMessage.Id. +/// +/// +/// Trust-boundary channel string — "ApiOutbound" for ExternalSystem +/// cached calls, "DbOutbound" for cached DB writes. +/// +/// Human-readable target (system name / DB connection). +/// Site id that submitted the cached call. +/// Per-attempt outcome. +/// Number of retries performed so far (S&F bookkeeping). +/// Most recent error message (null on success). +/// Most recent HTTP status (null when not applicable). +/// When the underlying S&F message was first enqueued. +/// When this attempt completed. +/// Duration of the attempt in milliseconds (null when not measured). +/// Originating instance, when known. +public sealed record CachedCallAttemptContext( + TrackedOperationId TrackedOperationId, + string Channel, + string Target, + string SourceSite, + CachedCallAttemptOutcome Outcome, + int RetryCount, + string? LastError, + int? HttpStatus, + DateTime CreatedAtUtc, + DateTime OccurredAtUtc, + int? DurationMs, + string? SourceInstanceId); + +/// +/// Coarse outcome of one cached-call delivery attempt, observed from inside +/// the store-and-forward retry loop. The audit bridge maps this to the +/// ApiCallCached/DbWriteCached Attempted row and, when terminal, +/// the corresponding CachedResolve row. +/// +public enum CachedCallAttemptOutcome +{ + /// Attempt delivered successfully — terminal Delivered state. + Delivered, + + /// Attempt failed transiently; another retry will follow. + TransientFailure, + + /// Attempt returned permanent failure — terminal Parked state (S&F semantics). + PermanentFailure, + + /// Retry budget exhausted — terminal Parked state. + ParkedMaxRetries, +} diff --git a/src/ScadaLink.Commons/Interfaces/Services/ICachedCallTelemetryForwarder.cs b/src/ScadaLink.Commons/Interfaces/Services/ICachedCallTelemetryForwarder.cs new file mode 100644 index 0000000..60ff6bb --- /dev/null +++ b/src/ScadaLink.Commons/Interfaces/Services/ICachedCallTelemetryForwarder.cs @@ -0,0 +1,34 @@ +using ScadaLink.Commons.Messages.Integration; + +namespace ScadaLink.Commons.Interfaces.Services; + +/// +/// Site-side fan-out abstraction for cached-call lifecycle telemetry +/// (Audit Log #23 / M3). One packet carries +/// both an audit row and an operational SiteCalls upsert; the +/// implementation routes the audit half through +/// and the operational half through the site-local tracking SQLite store. +/// +/// +/// +/// Defined in Commons so the script runtime (and the StoreAndForward retry +/// loop, Bundle E4) can take a dependency on the abstraction rather than on +/// the concrete forwarder living inside ScadaLink.AuditLog — the +/// existing dependency arrow runs from SiteRuntime to Commons, not to +/// AuditLog. +/// +/// +/// Best-effort contract (alog.md §7): implementations MUST swallow +/// internal failures rather than propagating to the calling script. +/// +/// +public interface ICachedCallTelemetryForwarder +{ + /// + /// Fan one combined-telemetry packet out to the audit writer and the + /// tracking store. Best-effort — failures on either half are logged and + /// swallowed; the returned Task completes when both halves have been + /// attempted. + /// + Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default); +} diff --git a/src/ScadaLink.Commons/Interfaces/Services/IDatabaseGateway.cs b/src/ScadaLink.Commons/Interfaces/Services/IDatabaseGateway.cs index f21c8f6..60defe9 100644 --- a/src/ScadaLink.Commons/Interfaces/Services/IDatabaseGateway.cs +++ b/src/ScadaLink.Commons/Interfaces/Services/IDatabaseGateway.cs @@ -1,4 +1,5 @@ using System.Data.Common; +using ScadaLink.Commons.Types; namespace ScadaLink.Commons.Interfaces.Services; @@ -20,10 +21,19 @@ public interface IDatabaseGateway /// /// Submits a SQL write to the store-and-forward engine for reliable delivery. /// + /// + /// Audit Log #23 (M3): caller-supplied tracking id used as the + /// store-and-forward message id so the S&F retry loop can read it + /// back via StoreAndForwardMessage.Id and emit per-attempt / + /// terminal cached-write telemetry under the same id. Defaults to + /// null — when omitted the S&F engine mints a fresh GUID and no + /// M3 telemetry is correlated (pre-M3 caller behaviour). + /// Task CachedWriteAsync( string connectionName, string sql, IReadOnlyDictionary? parameters = null, string? originInstanceName = null, - CancellationToken cancellationToken = default); + CancellationToken cancellationToken = default, + TrackedOperationId? trackedOperationId = null); } diff --git a/src/ScadaLink.Commons/Interfaces/Services/IExternalSystemClient.cs b/src/ScadaLink.Commons/Interfaces/Services/IExternalSystemClient.cs index c875ebf..627a81d 100644 --- a/src/ScadaLink.Commons/Interfaces/Services/IExternalSystemClient.cs +++ b/src/ScadaLink.Commons/Interfaces/Services/IExternalSystemClient.cs @@ -21,12 +21,22 @@ public interface IExternalSystemClient /// Attempt immediate delivery; on transient failure, hand to S&F engine. /// Permanent failures returned to caller. /// + /// + /// Audit Log #23 (M3): caller-supplied tracking id used as the + /// store-and-forward message id so the S&F retry loop can read it + /// back via StoreAndForwardMessage.Id and emit per-attempt / + /// terminal cached-call telemetry under the same id. Defaults to + /// null — when omitted the S&F engine mints a fresh GUID and no + /// M3 telemetry is correlated (the legacy behaviour pre-M3 callers rely + /// on). + /// Task CachedCallAsync( string systemName, string methodName, IReadOnlyDictionary? parameters = null, string? originInstanceName = null, - CancellationToken cancellationToken = default); + CancellationToken cancellationToken = default, + TrackedOperationId? trackedOperationId = null); } /// diff --git a/src/ScadaLink.Commons/Messages/Audit/IngestCachedTelemetryCommand.cs b/src/ScadaLink.Commons/Messages/Audit/IngestCachedTelemetryCommand.cs new file mode 100644 index 0000000..e7c63a1 --- /dev/null +++ b/src/ScadaLink.Commons/Messages/Audit/IngestCachedTelemetryCommand.cs @@ -0,0 +1,30 @@ +using ScadaLink.Commons.Entities.Audit; + +namespace ScadaLink.Commons.Messages.Audit; + +/// +/// Akka message sent to the central AuditLogIngestActor (Audit Log #23 M3 +/// Bundle D dual-write transaction) carrying a batch of combined audit + +/// site-call telemetry packets decoded by the SiteStreamGrpcServer from a +/// site's IngestCachedTelemetry gRPC RPC. For each entry the actor writes +/// the row AND the upsert inside +/// a single MS SQL transaction — both succeed or both roll back, so the audit +/// and operational mirrors never drift mid-row. +/// +/// +/// Lives in ScadaLink.Commons for the same reason as +/// IngestAuditEventsCommand: the gRPC server in +/// ScadaLink.Communication constructs it and ScadaLink.AuditLog +/// already references Communication. Putting the message in Commons avoids a +/// project-reference cycle. +/// +public sealed record IngestCachedTelemetryCommand(IReadOnlyList Entries); + +/// +/// One lifecycle event of a cached call: the to insert +/// (idempotent on ) plus the +/// to upsert (monotonic on +/// ). The two rows are paired so the +/// central dual-write transaction can commit them atomically. +/// +public sealed record CachedTelemetryEntry(AuditEvent Audit, SiteCall SiteCall); diff --git a/src/ScadaLink.Commons/Messages/Audit/IngestCachedTelemetryReply.cs b/src/ScadaLink.Commons/Messages/Audit/IngestCachedTelemetryReply.cs new file mode 100644 index 0000000..1de259d --- /dev/null +++ b/src/ScadaLink.Commons/Messages/Audit/IngestCachedTelemetryReply.cs @@ -0,0 +1,10 @@ +namespace ScadaLink.Commons.Messages.Audit; + +/// +/// Reply from the central AuditLogIngestActor for an +/// . +/// lists every entry whose dual-write transaction (AuditLog INSERT + SiteCalls +/// UPSERT) committed; entries whose transaction rolled back are absent so the +/// site can leave the row Pending and retry on the next drain. +/// +public sealed record IngestCachedTelemetryReply(IReadOnlyList AcceptedEventIds); diff --git a/src/ScadaLink.Commons/Messages/Audit/UpsertSiteCallCommand.cs b/src/ScadaLink.Commons/Messages/Audit/UpsertSiteCallCommand.cs new file mode 100644 index 0000000..ec0da24 --- /dev/null +++ b/src/ScadaLink.Commons/Messages/Audit/UpsertSiteCallCommand.cs @@ -0,0 +1,19 @@ +using ScadaLink.Commons.Entities.Audit; + +namespace ScadaLink.Commons.Messages.Audit; + +/// +/// Akka message sent to the central SiteCallAuditActor (Site Call Audit +/// #22, Audit Log #23 M3 Bundle C) carrying one row to +/// be persisted via ISiteCallAuditRepository.UpsertAsync. The repository +/// performs an insert-if-not-exists then monotonic update — duplicate gRPC +/// packets and reconciliation pulls can both feed the actor without rolling +/// state back. +/// +/// +/// Lives in ScadaLink.Commons rather than ScadaLink.SiteCallAudit +/// so the gRPC server in ScadaLink.Communication can construct it +/// without taking a project reference on the actor's host project (Bundle D +/// adds the IngestCachedTelemetry RPC that will Tell this command). +/// +public sealed record UpsertSiteCallCommand(SiteCall SiteCall); diff --git a/src/ScadaLink.Commons/Messages/Audit/UpsertSiteCallReply.cs b/src/ScadaLink.Commons/Messages/Audit/UpsertSiteCallReply.cs new file mode 100644 index 0000000..de31cec --- /dev/null +++ b/src/ScadaLink.Commons/Messages/Audit/UpsertSiteCallReply.cs @@ -0,0 +1,14 @@ +using ScadaLink.Commons.Types; + +namespace ScadaLink.Commons.Messages.Audit; + +/// +/// Reply from the central SiteCallAuditActor for an +/// . is true +/// when the upsert reached the repository without throwing (including the +/// monotonic-no-op case where the stored status' rank wins) and false +/// when persistence raised an exception. The actor itself stays alive in +/// either case — audit-write failures must NEVER abort the user-facing action +/// (Audit Log #23 §13). +/// +public sealed record UpsertSiteCallReply(TrackedOperationId TrackedOperationId, bool Accepted); diff --git a/src/ScadaLink.Commons/Messages/Integration/CachedCallTelemetry.cs b/src/ScadaLink.Commons/Messages/Integration/CachedCallTelemetry.cs new file mode 100644 index 0000000..37532aa --- /dev/null +++ b/src/ScadaLink.Commons/Messages/Integration/CachedCallTelemetry.cs @@ -0,0 +1,34 @@ +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types; + +namespace ScadaLink.Commons.Messages.Integration; + +/// +/// Combined audit + operational telemetry packet for cached outbound calls +/// (Audit Log #23 / M3). The site emits one packet per lifecycle event +/// — Submit (Audit kind CachedSubmit), per-attempt +/// ApiCallCached/DbWriteCached, terminal CachedResolve — +/// and central writes the row plus the +/// upsert in one MS SQL transaction. Two +/// payload shapes ride on a single wire message so the same on-the-wire batch +/// can carry mixed lifecycle events without the central dual-write needing a +/// second RPC for the operational state. +/// +/// +/// +/// Both inner records carry the same TrackedOperationId — the +/// idempotency key end-to-end. The +/// pattern (used by Audit Log #23 to thread cached-call rows together) is +/// honoured by the site emitter; the packet itself is shape-only and makes no +/// independent correlation guarantees. +/// +/// +/// Additive-only per Commons REQ-COM-5a (M2 reviewer note) — this is a new +/// message, not a rename of any existing M2 envelope. +/// +/// +/// The Audit Log #23 row to insert at central. +/// The SiteCalls upsert mirroring this lifecycle event. +public sealed record CachedCallTelemetry( + AuditEvent Audit, + SiteCallOperational Operational); diff --git a/src/ScadaLink.Commons/Types/Audit/SiteCallPaging.cs b/src/ScadaLink.Commons/Types/Audit/SiteCallPaging.cs new file mode 100644 index 0000000..4a8f790 --- /dev/null +++ b/src/ScadaLink.Commons/Types/Audit/SiteCallPaging.cs @@ -0,0 +1,15 @@ +namespace ScadaLink.Commons.Types.Audit; + +/// +/// Keyset paging cursor for +/// . +/// The repository orders by (CreatedAtUtc DESC, TrackedOperationId DESC) — newest +/// calls first, with the strong-typed id breaking ties when two calls share an exact +/// CreatedAtUtc. Callers pass the last row of the previous page back as +/// + to fetch the next page. +/// Both must be non-null together, or both null (first page). +/// +public sealed record SiteCallPaging( + int PageSize, + DateTime? AfterCreatedAtUtc = null, + TrackedOperationId? AfterId = null); diff --git a/src/ScadaLink.Commons/Types/Audit/SiteCallQueryFilter.cs b/src/ScadaLink.Commons/Types/Audit/SiteCallQueryFilter.cs new file mode 100644 index 0000000..cf7e7d4 --- /dev/null +++ b/src/ScadaLink.Commons/Types/Audit/SiteCallQueryFilter.cs @@ -0,0 +1,21 @@ +namespace ScadaLink.Commons.Types.Audit; + +/// +/// Filter predicate for . +/// Any field left null means "do not constrain on that column". Time bounds +/// are half-open in the spec sense — is inclusive and +/// is inclusive of the upper bound; the repository SQL uses +/// >= / <= respectively. All filter fields are AND-combined. +/// +/// +/// Channel / Status / SourceSite / Target are matched as exact strings — the +/// underlying columns are bounded ASCII (varchar) and the Central UI Site Calls +/// page exposes them as drop-down filters, not free-text search. +/// +public sealed record SiteCallQueryFilter( + string? Channel = null, + string? SourceSite = null, + string? Status = null, + string? Target = null, + DateTime? FromUtc = null, + DateTime? ToUtc = null); diff --git a/src/ScadaLink.Commons/Types/SiteCallOperational.cs b/src/ScadaLink.Commons/Types/SiteCallOperational.cs new file mode 100644 index 0000000..25d9414 --- /dev/null +++ b/src/ScadaLink.Commons/Types/SiteCallOperational.cs @@ -0,0 +1,46 @@ +namespace ScadaLink.Commons.Types; + +/// +/// Operational state of one cached call as seen by the site, carried on the +/// combined CachedCallTelemetry packet (Audit Log #23 / M3) and persisted +/// at central as the SiteCalls row mirroring the call's status. +/// +/// +/// +/// One row per at central; ingest is +/// insert-if-not-exists then upsert-on-newer-status (monotonic — never rolls +/// back). The site remains the source of truth — this record is the +/// "eventually-consistent mirror" the central UI's Site Calls page reads. +/// +/// +/// Idempotency key shared with the audit row. +/// +/// Trust-boundary channel — "ApiOutbound" or "DbOutbound". String form +/// (not the enum) so the +/// record serialises identically across SQL / gRPC / JSON boundaries. +/// +/// Human-readable target (e.g. "ERP.GetOrder"). +/// Site id that submitted the cached call. +/// +/// Lifecycle status — string form of : +/// Submitted, Retrying, Attempted, Delivered, +/// Failed, Parked, Discarded. +/// +/// Number of dispatch attempts so far; 0 prior to first attempt. +/// Most recent error message; null when no failures have occurred. +/// Most recent HTTP status code (API calls only); null otherwise. +/// UTC timestamp the cached call was first submitted. +/// UTC timestamp of the latest status mutation. +/// UTC timestamp the row reached a terminal status; null while still active. +public sealed record SiteCallOperational( + TrackedOperationId TrackedOperationId, + string Channel, + string Target, + string SourceSite, + string Status, + int RetryCount, + string? LastError, + int? HttpStatus, + DateTime CreatedAtUtc, + DateTime UpdatedAtUtc, + DateTime? TerminalAtUtc); diff --git a/src/ScadaLink.Commons/Types/TrackedOperationId.cs b/src/ScadaLink.Commons/Types/TrackedOperationId.cs new file mode 100644 index 0000000..3a6bb22 --- /dev/null +++ b/src/ScadaLink.Commons/Types/TrackedOperationId.cs @@ -0,0 +1,53 @@ +namespace ScadaLink.Commons.Types; + +/// +/// Strongly-typed identifier for a cached outbound operation +/// (ExternalSystem.CachedCall / Database.CachedWrite) — the unified +/// tracking handle introduced by Audit Log #23 (M3). The same id is the +/// idempotency key end-to-end: it is stamped on every AuditLog row +/// produced for the operation's lifecycle (CachedSubmit → ApiCallCached / +/// DbWriteCached × N attempts → CachedResolve) and is the PK on the central +/// SiteCalls row that mirrors the operation's operational state. +/// +/// +/// +/// The struct wraps a so it serialises identically to a +/// 36-character "D"-format string anywhere the existing GUID conventions are +/// used (gRPC strings, JSON, SQL TEXT columns). returns +/// the lower-case 8-4-4-4-12 form unconditionally; never the brace- / parens- +/// wrapped variants — central ingest parses with , which +/// is format-tolerant but the wire shape is fixed for log readability. +/// +/// +public readonly record struct TrackedOperationId(Guid Value) +{ + /// Mint a fresh id at the call site (script-thread safe). + public static TrackedOperationId New() => new(Guid.NewGuid()); + + /// + /// Parse a serialised id back into the strong type. Throws when the input + /// is not a valid GUID — callers crossing untrusted boundaries should use + /// instead. + /// + public static TrackedOperationId Parse(string s) => new(Guid.Parse(s)); + + /// + /// Attempt to parse a serialised id. Returns false for null, empty + /// or non-GUID input; is default on + /// failure. + /// + public static bool TryParse(string? s, out TrackedOperationId result) + { + if (Guid.TryParse(s, out var g)) + { + result = new TrackedOperationId(g); + return true; + } + + result = default; + return false; + } + + /// + public override string ToString() => Value.ToString("D"); +} diff --git a/src/ScadaLink.Commons/Types/TrackingStatusSnapshot.cs b/src/ScadaLink.Commons/Types/TrackingStatusSnapshot.cs new file mode 100644 index 0000000..22136ff --- /dev/null +++ b/src/ScadaLink.Commons/Types/TrackingStatusSnapshot.cs @@ -0,0 +1,40 @@ +namespace ScadaLink.Commons.Types; + +/// +/// Site-local snapshot of a cached operation's tracking state, returned by the +/// Tracking.Status(TrackedOperationId) script API (Audit Log #23 / M3). +/// +/// Tracking handle returned by CachedCall/CachedWrite. +/// +/// Operation category — "ApiCallCached" or "DbWriteCached" — mirroring +/// the per-attempt vocabulary. +/// +/// +/// Human-readable target (e.g. "ERP.GetOrder" or "WarehouseDb"); may be +/// null for early-lifecycle rows recorded before the target was resolved. +/// +/// +/// Lifecycle status — one of Submitted, Forwarded, Retrying, +/// Attempted, Delivered, Failed, Parked, Discarded. +/// +/// Number of attempts made; 0 prior to first dispatch. +/// Most recent error message; null while non-terminal-and-no-failures. +/// Most recent HTTP status code where applicable; null otherwise. +/// UTC timestamp the tracking row was first recorded. +/// UTC timestamp of the latest status mutation. +/// UTC timestamp the row reached a terminal status; null while still active. +/// Instance id that issued the cached call, when known. +/// Script that issued the cached call, when known. +public sealed record TrackingStatusSnapshot( + TrackedOperationId Id, + string Kind, + string? TargetSummary, + string Status, + int RetryCount, + string? LastError, + int? HttpStatus, + DateTime CreatedAtUtc, + DateTime UpdatedAtUtc, + DateTime? TerminalAtUtc, + string? SourceInstanceId, + string? SourceScript); diff --git a/src/ScadaLink.Communication/Grpc/SiteStreamGrpcServer.cs b/src/ScadaLink.Communication/Grpc/SiteStreamGrpcServer.cs index 71560c3..1da14ec 100644 --- a/src/ScadaLink.Communication/Grpc/SiteStreamGrpcServer.cs +++ b/src/ScadaLink.Communication/Grpc/SiteStreamGrpcServer.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ScadaLink.Commons.Entities.Audit; using ScadaLink.Commons.Messages.Audit; +using ScadaLink.Commons.Types; using ScadaLink.Commons.Types.Enums; using GrpcStatus = Grpc.Core.Status; @@ -298,9 +299,132 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase return ack; } + /// + /// Audit Log (#23) M3 site→central combined-telemetry push RPC. Decodes a + /// batch of entries into matched + /// (AuditEvent, SiteCall) pairs, Asks the central AuditLogIngestActor + /// proxy to persist them in dual-write transactions, and echoes the + /// AuditEvent EventIds that committed back so the site can flip its local + /// rows to Forwarded. + /// + /// + /// Same wiring-incomplete fallback as : when + /// the actor proxy has not been set the RPC replies with an empty ack so + /// sites treat the outcome as transient and retry, never a hard fault. + /// + public override async Task IngestCachedTelemetry( + CachedTelemetryBatch request, + ServerCallContext context) + { + if (request.Packets.Count == 0) + { + return new IngestAck(); + } + + var actor = _auditIngestActor; + if (actor is null) + { + _logger.LogWarning( + "IngestCachedTelemetry received {Count} packets before SetAuditIngestActor was called; returning empty ack.", + request.Packets.Count); + return new IngestAck(); + } + + var entries = new List(request.Packets.Count); + foreach (var packet in request.Packets) + { + var auditEvent = MapAuditEventFromDto(packet.AuditEvent); + var siteCall = MapSiteCallFromDto(packet.Operational); + entries.Add(new CachedTelemetryEntry(auditEvent, siteCall)); + } + + var cmd = new IngestCachedTelemetryCommand(entries); + IngestCachedTelemetryReply reply; + try + { + reply = await actor.Ask( + cmd, AuditIngestAskTimeout, context.CancellationToken); + } + catch (Exception ex) + { + _logger.LogError(ex, + "AuditLogIngestActor Ask failed for combined telemetry batch of {Count} packets; returning empty ack.", + request.Packets.Count); + return new IngestAck(); + } + + var ack = new IngestAck(); + foreach (var id in reply.AcceptedEventIds) + { + ack.AcceptedEventIds.Add(id.ToString()); + } + return ack; + } + private static string? NullIfEmpty(string? value) => string.IsNullOrEmpty(value) ? null : value; + /// + /// Inlined audit-event DTO→entity translation, kept in sync with the + /// handler above. Extracted to a private + /// helper so the M3 dual-write RPC can reuse it without duplicating yet + /// another copy. The shape still mirrors + /// AuditEventMapper.FromDto in ScadaLink.AuditLog.Telemetry; + /// the two must evolve together (the project-reference cycle that + /// prevents calling the AuditLog mapper directly is documented on + /// ). + /// + private static AuditEvent MapAuditEventFromDto(AuditEventDto dto) => + new() + { + EventId = Guid.Parse(dto.EventId), + OccurredAtUtc = DateTime.SpecifyKind(dto.OccurredAtUtc.ToDateTime(), DateTimeKind.Utc), + IngestedAtUtc = null, + Channel = Enum.Parse(dto.Channel), + Kind = Enum.Parse(dto.Kind), + CorrelationId = NullIfEmpty(dto.CorrelationId) is { } cid ? Guid.Parse(cid) : null, + SourceSiteId = NullIfEmpty(dto.SourceSiteId), + SourceInstanceId = NullIfEmpty(dto.SourceInstanceId), + SourceScript = NullIfEmpty(dto.SourceScript), + Actor = NullIfEmpty(dto.Actor), + Target = NullIfEmpty(dto.Target), + Status = Enum.Parse(dto.Status), + HttpStatus = dto.HttpStatus, + DurationMs = dto.DurationMs, + ErrorMessage = NullIfEmpty(dto.ErrorMessage), + ErrorDetail = NullIfEmpty(dto.ErrorDetail), + RequestSummary = NullIfEmpty(dto.RequestSummary), + ResponseSummary = NullIfEmpty(dto.ResponseSummary), + PayloadTruncated = dto.PayloadTruncated, + Extra = NullIfEmpty(dto.Extra), + ForwardState = null, + }; + + /// + /// Translates a into the persistence + /// entity. is stamped here as a + /// placeholder; the central ingest actor overwrites it inside the + /// dual-write transaction so the AuditLog and SiteCalls rows share one + /// instant. + /// + private static SiteCall MapSiteCallFromDto(SiteCallOperationalDto dto) => new() + { + TrackedOperationId = TrackedOperationId.Parse(dto.TrackedOperationId), + Channel = dto.Channel, + Target = dto.Target, + SourceSite = dto.SourceSite, + Status = dto.Status, + RetryCount = dto.RetryCount, + LastError = string.IsNullOrEmpty(dto.LastError) ? null : dto.LastError, + HttpStatus = dto.HttpStatus, + CreatedAtUtc = DateTime.SpecifyKind(dto.CreatedAtUtc.ToDateTime(), DateTimeKind.Utc), + UpdatedAtUtc = DateTime.SpecifyKind(dto.UpdatedAtUtc.ToDateTime(), DateTimeKind.Utc), + TerminalAtUtc = dto.TerminalAtUtc is null + ? null + : DateTime.SpecifyKind(dto.TerminalAtUtc.ToDateTime(), DateTimeKind.Utc), + IngestedAtUtc = DateTime.UtcNow, // overwritten by AuditLogIngestActor + }; + /// /// Tracks a single active stream so cleanup only removes its own entry. /// diff --git a/src/ScadaLink.Communication/Protos/sitestream.proto b/src/ScadaLink.Communication/Protos/sitestream.proto index d01852f..43ffbe3 100644 --- a/src/ScadaLink.Communication/Protos/sitestream.proto +++ b/src/ScadaLink.Communication/Protos/sitestream.proto @@ -8,6 +8,7 @@ import "google/protobuf/wrappers.proto"; // Int32Value service SiteStreamService { rpc SubscribeInstance(InstanceStreamRequest) returns (stream SiteStreamEvent); rpc IngestAuditEvents(AuditEventBatch) returns (IngestAck); + rpc IngestCachedTelemetry(CachedTelemetryBatch) returns (IngestAck); } message InstanceStreamRequest { @@ -93,3 +94,28 @@ message AuditEventDto { message AuditEventBatch { repeated AuditEventDto events = 1; } message IngestAck { repeated string accepted_event_ids = 1; } + +// Audit Log (#23) M3 cached-call combined telemetry: a single packet carries +// both the AuditEvent row to insert and the SiteCalls operational-state upsert +// for one lifecycle event of a cached outbound call. Central writes both rows +// in one MS SQL transaction so the audit and operational mirrors never drift. +message SiteCallOperationalDto { + string tracked_operation_id = 1; // GUID string ("D" format) + string channel = 2; // "ApiOutbound" | "DbOutbound" + string target = 3; + string source_site = 4; + string status = 5; // AuditStatus name + int32 retry_count = 6; + string last_error = 7; // empty when null + google.protobuf.Int32Value http_status = 8; + google.protobuf.Timestamp created_at_utc = 9; + google.protobuf.Timestamp updated_at_utc = 10; + google.protobuf.Timestamp terminal_at_utc = 11; // absent when not terminal +} + +message CachedTelemetryPacket { + AuditEventDto audit_event = 1; + SiteCallOperationalDto operational = 2; +} + +message CachedTelemetryBatch { repeated CachedTelemetryPacket packets = 1; } diff --git a/src/ScadaLink.Communication/SiteStreamGrpc/Sitestream.cs b/src/ScadaLink.Communication/SiteStreamGrpc/Sitestream.cs index 3a843eb..9639242 100644 --- a/src/ScadaLink.Communication/SiteStreamGrpc/Sitestream.cs +++ b/src/ScadaLink.Communication/SiteStreamGrpc/Sitestream.cs @@ -55,19 +55,34 @@ namespace ScadaLink.Communication.Grpc { "cGF5bG9hZF90cnVuY2F0ZWQYEiABKAgSDQoFZXh0cmEYEyABKAkiPAoPQXVk", "aXRFdmVudEJhdGNoEikKBmV2ZW50cxgBIAMoCzIZLnNpdGVzdHJlYW0uQXVk", "aXRFdmVudER0byInCglJbmdlc3RBY2sSGgoSYWNjZXB0ZWRfZXZlbnRfaWRz", - "GAEgAygJKlwKB1F1YWxpdHkSFwoTUVVBTElUWV9VTlNQRUNJRklFRBAAEhAK", - "DFFVQUxJVFlfR09PRBABEhUKEVFVQUxJVFlfVU5DRVJUQUlOEAISDwoLUVVB", - "TElUWV9CQUQQAypdCg5BbGFybVN0YXRlRW51bRIbChdBTEFSTV9TVEFURV9V", - "TlNQRUNJRklFRBAAEhYKEkFMQVJNX1NUQVRFX05PUk1BTBABEhYKEkFMQVJN", - "X1NUQVRFX0FDVElWRRACKoUBCg5BbGFybUxldmVsRW51bRIUChBBTEFSTV9M", - "RVZFTF9OT05FEAASEwoPQUxBUk1fTEVWRUxfTE9XEAESFwoTQUxBUk1fTEVW", - "RUxfTE9XX0xPVxACEhQKEEFMQVJNX0xFVkVMX0hJR0gQAxIZChVBTEFSTV9M", - "RVZFTF9ISUdIX0hJR0gQBDKzAQoRU2l0ZVN0cmVhbVNlcnZpY2USVQoRU3Vi", - "c2NyaWJlSW5zdGFuY2USIS5zaXRlc3RyZWFtLkluc3RhbmNlU3RyZWFtUmVx", - "dWVzdBobLnNpdGVzdHJlYW0uU2l0ZVN0cmVhbUV2ZW50MAESRwoRSW5nZXN0", - "QXVkaXRFdmVudHMSGy5zaXRlc3RyZWFtLkF1ZGl0RXZlbnRCYXRjaBoVLnNp", - "dGVzdHJlYW0uSW5nZXN0QWNrQh+qAhxTY2FkYUxpbmsuQ29tbXVuaWNhdGlv", - "bi5HcnBjYgZwcm90bzM=")); + "GAEgAygJIvQCChZTaXRlQ2FsbE9wZXJhdGlvbmFsRHRvEhwKFHRyYWNrZWRf", + "b3BlcmF0aW9uX2lkGAEgASgJEg8KB2NoYW5uZWwYAiABKAkSDgoGdGFyZ2V0", + "GAMgASgJEhMKC3NvdXJjZV9zaXRlGAQgASgJEg4KBnN0YXR1cxgFIAEoCRIT", + "CgtyZXRyeV9jb3VudBgGIAEoBRISCgpsYXN0X2Vycm9yGAcgASgJEjAKC2h0", + "dHBfc3RhdHVzGAggASgLMhsuZ29vZ2xlLnByb3RvYnVmLkludDMyVmFsdWUS", + "MgoOY3JlYXRlZF9hdF91dGMYCSABKAsyGi5nb29nbGUucHJvdG9idWYuVGlt", + "ZXN0YW1wEjIKDnVwZGF0ZWRfYXRfdXRjGAogASgLMhouZ29vZ2xlLnByb3Rv", + "YnVmLlRpbWVzdGFtcBIzCg90ZXJtaW5hbF9hdF91dGMYCyABKAsyGi5nb29n", + "bGUucHJvdG9idWYuVGltZXN0YW1wIoABChVDYWNoZWRUZWxlbWV0cnlQYWNr", + "ZXQSLgoLYXVkaXRfZXZlbnQYASABKAsyGS5zaXRlc3RyZWFtLkF1ZGl0RXZl", + "bnREdG8SNwoLb3BlcmF0aW9uYWwYAiABKAsyIi5zaXRlc3RyZWFtLlNpdGVD", + "YWxsT3BlcmF0aW9uYWxEdG8iSgoUQ2FjaGVkVGVsZW1ldHJ5QmF0Y2gSMgoH", + "cGFja2V0cxgBIAMoCzIhLnNpdGVzdHJlYW0uQ2FjaGVkVGVsZW1ldHJ5UGFj", + "a2V0KlwKB1F1YWxpdHkSFwoTUVVBTElUWV9VTlNQRUNJRklFRBAAEhAKDFFV", + "QUxJVFlfR09PRBABEhUKEVFVQUxJVFlfVU5DRVJUQUlOEAISDwoLUVVBTElU", + "WV9CQUQQAypdCg5BbGFybVN0YXRlRW51bRIbChdBTEFSTV9TVEFURV9VTlNQ", + "RUNJRklFRBAAEhYKEkFMQVJNX1NUQVRFX05PUk1BTBABEhYKEkFMQVJNX1NU", + "QVRFX0FDVElWRRACKoUBCg5BbGFybUxldmVsRW51bRIUChBBTEFSTV9MRVZF", + "TF9OT05FEAASEwoPQUxBUk1fTEVWRUxfTE9XEAESFwoTQUxBUk1fTEVWRUxf", + "TE9XX0xPVxACEhQKEEFMQVJNX0xFVkVMX0hJR0gQAxIZChVBTEFSTV9MRVZF", + "TF9ISUdIX0hJR0gQBDKFAgoRU2l0ZVN0cmVhbVNlcnZpY2USVQoRU3Vic2Ny", + "aWJlSW5zdGFuY2USIS5zaXRlc3RyZWFtLkluc3RhbmNlU3RyZWFtUmVxdWVz", + "dBobLnNpdGVzdHJlYW0uU2l0ZVN0cmVhbUV2ZW50MAESRwoRSW5nZXN0QXVk", + "aXRFdmVudHMSGy5zaXRlc3RyZWFtLkF1ZGl0RXZlbnRCYXRjaBoVLnNpdGVz", + "dHJlYW0uSW5nZXN0QWNrElAKFUluZ2VzdENhY2hlZFRlbGVtZXRyeRIgLnNp", + "dGVzdHJlYW0uQ2FjaGVkVGVsZW1ldHJ5QmF0Y2gaFS5zaXRlc3RyZWFtLklu", + "Z2VzdEFja0IfqgIcU2NhZGFMaW5rLkNvbW11bmljYXRpb24uR3JwY2IGcHJv", + "dG8z")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.TimestampReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.WrappersReflection.Descriptor, }, new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ScadaLink.Communication.Grpc.Quality), typeof(global::ScadaLink.Communication.Grpc.AlarmStateEnum), typeof(global::ScadaLink.Communication.Grpc.AlarmLevelEnum), }, null, new pbr::GeneratedClrTypeInfo[] { @@ -77,7 +92,10 @@ namespace ScadaLink.Communication.Grpc { new pbr::GeneratedClrTypeInfo(typeof(global::ScadaLink.Communication.Grpc.AlarmStateUpdate), global::ScadaLink.Communication.Grpc.AlarmStateUpdate.Parser, new[]{ "InstanceUniqueName", "AlarmName", "State", "Priority", "Timestamp", "Level", "Message" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ScadaLink.Communication.Grpc.AuditEventDto), global::ScadaLink.Communication.Grpc.AuditEventDto.Parser, new[]{ "EventId", "OccurredAtUtc", "Channel", "Kind", "CorrelationId", "SourceSiteId", "SourceInstanceId", "SourceScript", "Actor", "Target", "Status", "HttpStatus", "DurationMs", "ErrorMessage", "ErrorDetail", "RequestSummary", "ResponseSummary", "PayloadTruncated", "Extra" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ScadaLink.Communication.Grpc.AuditEventBatch), global::ScadaLink.Communication.Grpc.AuditEventBatch.Parser, new[]{ "Events" }, null, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::ScadaLink.Communication.Grpc.IngestAck), global::ScadaLink.Communication.Grpc.IngestAck.Parser, new[]{ "AcceptedEventIds" }, null, null, null, null) + new pbr::GeneratedClrTypeInfo(typeof(global::ScadaLink.Communication.Grpc.IngestAck), global::ScadaLink.Communication.Grpc.IngestAck.Parser, new[]{ "AcceptedEventIds" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::ScadaLink.Communication.Grpc.SiteCallOperationalDto), global::ScadaLink.Communication.Grpc.SiteCallOperationalDto.Parser, new[]{ "TrackedOperationId", "Channel", "Target", "SourceSite", "Status", "RetryCount", "LastError", "HttpStatus", "CreatedAtUtc", "UpdatedAtUtc", "TerminalAtUtc" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::ScadaLink.Communication.Grpc.CachedTelemetryPacket), global::ScadaLink.Communication.Grpc.CachedTelemetryPacket.Parser, new[]{ "AuditEvent", "Operational" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::ScadaLink.Communication.Grpc.CachedTelemetryBatch), global::ScadaLink.Communication.Grpc.CachedTelemetryBatch.Parser, new[]{ "Packets" }, null, null, null, null) })); } #endregion @@ -2780,6 +2798,1070 @@ namespace ScadaLink.Communication.Grpc { } + /// + /// Audit Log (#23) M3 cached-call combined telemetry: a single packet carries + /// both the AuditEvent row to insert and the SiteCalls operational-state upsert + /// for one lifecycle event of a cached outbound call. Central writes both rows + /// in one MS SQL transaction so the audit and operational mirrors never drift. + /// + [global::System.Diagnostics.DebuggerDisplayAttribute("{ToString(),nq}")] + public sealed partial class SiteCallOperationalDto : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new SiteCallOperationalDto()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::ScadaLink.Communication.Grpc.SitestreamReflection.Descriptor.MessageTypes[7]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public SiteCallOperationalDto() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public SiteCallOperationalDto(SiteCallOperationalDto other) : this() { + trackedOperationId_ = other.trackedOperationId_; + channel_ = other.channel_; + target_ = other.target_; + sourceSite_ = other.sourceSite_; + status_ = other.status_; + retryCount_ = other.retryCount_; + lastError_ = other.lastError_; + HttpStatus = other.HttpStatus; + createdAtUtc_ = other.createdAtUtc_ != null ? other.createdAtUtc_.Clone() : null; + updatedAtUtc_ = other.updatedAtUtc_ != null ? other.updatedAtUtc_.Clone() : null; + terminalAtUtc_ = other.terminalAtUtc_ != null ? other.terminalAtUtc_.Clone() : null; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public SiteCallOperationalDto Clone() { + return new SiteCallOperationalDto(this); + } + + /// Field number for the "tracked_operation_id" field. + public const int TrackedOperationIdFieldNumber = 1; + private string trackedOperationId_ = ""; + /// + /// GUID string ("D" format) + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string TrackedOperationId { + get { return trackedOperationId_; } + set { + trackedOperationId_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + /// Field number for the "channel" field. + public const int ChannelFieldNumber = 2; + private string channel_ = ""; + /// + /// "ApiOutbound" | "DbOutbound" + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Channel { + get { return channel_; } + set { + channel_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + /// Field number for the "target" field. + public const int TargetFieldNumber = 3; + private string target_ = ""; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Target { + get { return target_; } + set { + target_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + /// Field number for the "source_site" field. + public const int SourceSiteFieldNumber = 4; + private string sourceSite_ = ""; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string SourceSite { + get { return sourceSite_; } + set { + sourceSite_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + /// Field number for the "status" field. + public const int StatusFieldNumber = 5; + private string status_ = ""; + /// + /// AuditStatus name + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string Status { + get { return status_; } + set { + status_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + /// Field number for the "retry_count" field. + public const int RetryCountFieldNumber = 6; + private int retryCount_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int RetryCount { + get { return retryCount_; } + set { + retryCount_ = value; + } + } + + /// Field number for the "last_error" field. + public const int LastErrorFieldNumber = 7; + private string lastError_ = ""; + /// + /// empty when null + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string LastError { + get { return lastError_; } + set { + lastError_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + + /// Field number for the "http_status" field. + public const int HttpStatusFieldNumber = 8; + private static readonly pb::FieldCodec _single_httpStatus_codec = pb::FieldCodec.ForStructWrapper(66); + private int? httpStatus_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int? HttpStatus { + get { return httpStatus_; } + set { + httpStatus_ = value; + } + } + + + /// Field number for the "created_at_utc" field. + public const int CreatedAtUtcFieldNumber = 9; + private global::Google.Protobuf.WellKnownTypes.Timestamp createdAtUtc_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::Google.Protobuf.WellKnownTypes.Timestamp CreatedAtUtc { + get { return createdAtUtc_; } + set { + createdAtUtc_ = value; + } + } + + /// Field number for the "updated_at_utc" field. + public const int UpdatedAtUtcFieldNumber = 10; + private global::Google.Protobuf.WellKnownTypes.Timestamp updatedAtUtc_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::Google.Protobuf.WellKnownTypes.Timestamp UpdatedAtUtc { + get { return updatedAtUtc_; } + set { + updatedAtUtc_ = value; + } + } + + /// Field number for the "terminal_at_utc" field. + public const int TerminalAtUtcFieldNumber = 11; + private global::Google.Protobuf.WellKnownTypes.Timestamp terminalAtUtc_; + /// + /// absent when not terminal + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::Google.Protobuf.WellKnownTypes.Timestamp TerminalAtUtc { + get { return terminalAtUtc_; } + set { + terminalAtUtc_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as SiteCallOperationalDto); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(SiteCallOperationalDto other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (TrackedOperationId != other.TrackedOperationId) return false; + if (Channel != other.Channel) return false; + if (Target != other.Target) return false; + if (SourceSite != other.SourceSite) return false; + if (Status != other.Status) return false; + if (RetryCount != other.RetryCount) return false; + if (LastError != other.LastError) return false; + if (HttpStatus != other.HttpStatus) return false; + if (!object.Equals(CreatedAtUtc, other.CreatedAtUtc)) return false; + if (!object.Equals(UpdatedAtUtc, other.UpdatedAtUtc)) return false; + if (!object.Equals(TerminalAtUtc, other.TerminalAtUtc)) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (TrackedOperationId.Length != 0) hash ^= TrackedOperationId.GetHashCode(); + if (Channel.Length != 0) hash ^= Channel.GetHashCode(); + if (Target.Length != 0) hash ^= Target.GetHashCode(); + if (SourceSite.Length != 0) hash ^= SourceSite.GetHashCode(); + if (Status.Length != 0) hash ^= Status.GetHashCode(); + if (RetryCount != 0) hash ^= RetryCount.GetHashCode(); + if (LastError.Length != 0) hash ^= LastError.GetHashCode(); + if (httpStatus_ != null) hash ^= HttpStatus.GetHashCode(); + if (createdAtUtc_ != null) hash ^= CreatedAtUtc.GetHashCode(); + if (updatedAtUtc_ != null) hash ^= UpdatedAtUtc.GetHashCode(); + if (terminalAtUtc_ != null) hash ^= TerminalAtUtc.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (TrackedOperationId.Length != 0) { + output.WriteRawTag(10); + output.WriteString(TrackedOperationId); + } + if (Channel.Length != 0) { + output.WriteRawTag(18); + output.WriteString(Channel); + } + if (Target.Length != 0) { + output.WriteRawTag(26); + output.WriteString(Target); + } + if (SourceSite.Length != 0) { + output.WriteRawTag(34); + output.WriteString(SourceSite); + } + if (Status.Length != 0) { + output.WriteRawTag(42); + output.WriteString(Status); + } + if (RetryCount != 0) { + output.WriteRawTag(48); + output.WriteInt32(RetryCount); + } + if (LastError.Length != 0) { + output.WriteRawTag(58); + output.WriteString(LastError); + } + if (httpStatus_ != null) { + _single_httpStatus_codec.WriteTagAndValue(output, HttpStatus); + } + if (createdAtUtc_ != null) { + output.WriteRawTag(74); + output.WriteMessage(CreatedAtUtc); + } + if (updatedAtUtc_ != null) { + output.WriteRawTag(82); + output.WriteMessage(UpdatedAtUtc); + } + if (terminalAtUtc_ != null) { + output.WriteRawTag(90); + output.WriteMessage(TerminalAtUtc); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (TrackedOperationId.Length != 0) { + output.WriteRawTag(10); + output.WriteString(TrackedOperationId); + } + if (Channel.Length != 0) { + output.WriteRawTag(18); + output.WriteString(Channel); + } + if (Target.Length != 0) { + output.WriteRawTag(26); + output.WriteString(Target); + } + if (SourceSite.Length != 0) { + output.WriteRawTag(34); + output.WriteString(SourceSite); + } + if (Status.Length != 0) { + output.WriteRawTag(42); + output.WriteString(Status); + } + if (RetryCount != 0) { + output.WriteRawTag(48); + output.WriteInt32(RetryCount); + } + if (LastError.Length != 0) { + output.WriteRawTag(58); + output.WriteString(LastError); + } + if (httpStatus_ != null) { + _single_httpStatus_codec.WriteTagAndValue(ref output, HttpStatus); + } + if (createdAtUtc_ != null) { + output.WriteRawTag(74); + output.WriteMessage(CreatedAtUtc); + } + if (updatedAtUtc_ != null) { + output.WriteRawTag(82); + output.WriteMessage(UpdatedAtUtc); + } + if (terminalAtUtc_ != null) { + output.WriteRawTag(90); + output.WriteMessage(TerminalAtUtc); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (TrackedOperationId.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(TrackedOperationId); + } + if (Channel.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Channel); + } + if (Target.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Target); + } + if (SourceSite.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(SourceSite); + } + if (Status.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(Status); + } + if (RetryCount != 0) { + size += 1 + pb::CodedOutputStream.ComputeInt32Size(RetryCount); + } + if (LastError.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(LastError); + } + if (httpStatus_ != null) { + size += _single_httpStatus_codec.CalculateSizeWithTag(HttpStatus); + } + if (createdAtUtc_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(CreatedAtUtc); + } + if (updatedAtUtc_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(UpdatedAtUtc); + } + if (terminalAtUtc_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(TerminalAtUtc); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(SiteCallOperationalDto other) { + if (other == null) { + return; + } + if (other.TrackedOperationId.Length != 0) { + TrackedOperationId = other.TrackedOperationId; + } + if (other.Channel.Length != 0) { + Channel = other.Channel; + } + if (other.Target.Length != 0) { + Target = other.Target; + } + if (other.SourceSite.Length != 0) { + SourceSite = other.SourceSite; + } + if (other.Status.Length != 0) { + Status = other.Status; + } + if (other.RetryCount != 0) { + RetryCount = other.RetryCount; + } + if (other.LastError.Length != 0) { + LastError = other.LastError; + } + if (other.httpStatus_ != null) { + if (httpStatus_ == null || other.HttpStatus != 0) { + HttpStatus = other.HttpStatus; + } + } + if (other.createdAtUtc_ != null) { + if (createdAtUtc_ == null) { + CreatedAtUtc = new global::Google.Protobuf.WellKnownTypes.Timestamp(); + } + CreatedAtUtc.MergeFrom(other.CreatedAtUtc); + } + if (other.updatedAtUtc_ != null) { + if (updatedAtUtc_ == null) { + UpdatedAtUtc = new global::Google.Protobuf.WellKnownTypes.Timestamp(); + } + UpdatedAtUtc.MergeFrom(other.UpdatedAtUtc); + } + if (other.terminalAtUtc_ != null) { + if (terminalAtUtc_ == null) { + TerminalAtUtc = new global::Google.Protobuf.WellKnownTypes.Timestamp(); + } + TerminalAtUtc.MergeFrom(other.TerminalAtUtc); + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + TrackedOperationId = input.ReadString(); + break; + } + case 18: { + Channel = input.ReadString(); + break; + } + case 26: { + Target = input.ReadString(); + break; + } + case 34: { + SourceSite = input.ReadString(); + break; + } + case 42: { + Status = input.ReadString(); + break; + } + case 48: { + RetryCount = input.ReadInt32(); + break; + } + case 58: { + LastError = input.ReadString(); + break; + } + case 66: { + int? value = _single_httpStatus_codec.Read(input); + if (httpStatus_ == null || value != 0) { + HttpStatus = value; + } + break; + } + case 74: { + if (createdAtUtc_ == null) { + CreatedAtUtc = new global::Google.Protobuf.WellKnownTypes.Timestamp(); + } + input.ReadMessage(CreatedAtUtc); + break; + } + case 82: { + if (updatedAtUtc_ == null) { + UpdatedAtUtc = new global::Google.Protobuf.WellKnownTypes.Timestamp(); + } + input.ReadMessage(UpdatedAtUtc); + break; + } + case 90: { + if (terminalAtUtc_ == null) { + TerminalAtUtc = new global::Google.Protobuf.WellKnownTypes.Timestamp(); + } + input.ReadMessage(TerminalAtUtc); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + TrackedOperationId = input.ReadString(); + break; + } + case 18: { + Channel = input.ReadString(); + break; + } + case 26: { + Target = input.ReadString(); + break; + } + case 34: { + SourceSite = input.ReadString(); + break; + } + case 42: { + Status = input.ReadString(); + break; + } + case 48: { + RetryCount = input.ReadInt32(); + break; + } + case 58: { + LastError = input.ReadString(); + break; + } + case 66: { + int? value = _single_httpStatus_codec.Read(ref input); + if (httpStatus_ == null || value != 0) { + HttpStatus = value; + } + break; + } + case 74: { + if (createdAtUtc_ == null) { + CreatedAtUtc = new global::Google.Protobuf.WellKnownTypes.Timestamp(); + } + input.ReadMessage(CreatedAtUtc); + break; + } + case 82: { + if (updatedAtUtc_ == null) { + UpdatedAtUtc = new global::Google.Protobuf.WellKnownTypes.Timestamp(); + } + input.ReadMessage(UpdatedAtUtc); + break; + } + case 90: { + if (terminalAtUtc_ == null) { + TerminalAtUtc = new global::Google.Protobuf.WellKnownTypes.Timestamp(); + } + input.ReadMessage(TerminalAtUtc); + break; + } + } + } + } + #endif + + } + + [global::System.Diagnostics.DebuggerDisplayAttribute("{ToString(),nq}")] + public sealed partial class CachedTelemetryPacket : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new CachedTelemetryPacket()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::ScadaLink.Communication.Grpc.SitestreamReflection.Descriptor.MessageTypes[8]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CachedTelemetryPacket() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CachedTelemetryPacket(CachedTelemetryPacket other) : this() { + auditEvent_ = other.auditEvent_ != null ? other.auditEvent_.Clone() : null; + operational_ = other.operational_ != null ? other.operational_.Clone() : null; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CachedTelemetryPacket Clone() { + return new CachedTelemetryPacket(this); + } + + /// Field number for the "audit_event" field. + public const int AuditEventFieldNumber = 1; + private global::ScadaLink.Communication.Grpc.AuditEventDto auditEvent_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::ScadaLink.Communication.Grpc.AuditEventDto AuditEvent { + get { return auditEvent_; } + set { + auditEvent_ = value; + } + } + + /// Field number for the "operational" field. + public const int OperationalFieldNumber = 2; + private global::ScadaLink.Communication.Grpc.SiteCallOperationalDto operational_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::ScadaLink.Communication.Grpc.SiteCallOperationalDto Operational { + get { return operational_; } + set { + operational_ = value; + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as CachedTelemetryPacket); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(CachedTelemetryPacket other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (!object.Equals(AuditEvent, other.AuditEvent)) return false; + if (!object.Equals(Operational, other.Operational)) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (auditEvent_ != null) hash ^= AuditEvent.GetHashCode(); + if (operational_ != null) hash ^= Operational.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (auditEvent_ != null) { + output.WriteRawTag(10); + output.WriteMessage(AuditEvent); + } + if (operational_ != null) { + output.WriteRawTag(18); + output.WriteMessage(Operational); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (auditEvent_ != null) { + output.WriteRawTag(10); + output.WriteMessage(AuditEvent); + } + if (operational_ != null) { + output.WriteRawTag(18); + output.WriteMessage(Operational); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (auditEvent_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(AuditEvent); + } + if (operational_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(Operational); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(CachedTelemetryPacket other) { + if (other == null) { + return; + } + if (other.auditEvent_ != null) { + if (auditEvent_ == null) { + AuditEvent = new global::ScadaLink.Communication.Grpc.AuditEventDto(); + } + AuditEvent.MergeFrom(other.AuditEvent); + } + if (other.operational_ != null) { + if (operational_ == null) { + Operational = new global::ScadaLink.Communication.Grpc.SiteCallOperationalDto(); + } + Operational.MergeFrom(other.Operational); + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + if (auditEvent_ == null) { + AuditEvent = new global::ScadaLink.Communication.Grpc.AuditEventDto(); + } + input.ReadMessage(AuditEvent); + break; + } + case 18: { + if (operational_ == null) { + Operational = new global::ScadaLink.Communication.Grpc.SiteCallOperationalDto(); + } + input.ReadMessage(Operational); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + if (auditEvent_ == null) { + AuditEvent = new global::ScadaLink.Communication.Grpc.AuditEventDto(); + } + input.ReadMessage(AuditEvent); + break; + } + case 18: { + if (operational_ == null) { + Operational = new global::ScadaLink.Communication.Grpc.SiteCallOperationalDto(); + } + input.ReadMessage(Operational); + break; + } + } + } + } + #endif + + } + + [global::System.Diagnostics.DebuggerDisplayAttribute("{ToString(),nq}")] + public sealed partial class CachedTelemetryBatch : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new CachedTelemetryBatch()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::ScadaLink.Communication.Grpc.SitestreamReflection.Descriptor.MessageTypes[9]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CachedTelemetryBatch() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CachedTelemetryBatch(CachedTelemetryBatch other) : this() { + packets_ = other.packets_.Clone(); + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public CachedTelemetryBatch Clone() { + return new CachedTelemetryBatch(this); + } + + /// Field number for the "packets" field. + public const int PacketsFieldNumber = 1; + private static readonly pb::FieldCodec _repeated_packets_codec + = pb::FieldCodec.ForMessage(10, global::ScadaLink.Communication.Grpc.CachedTelemetryPacket.Parser); + private readonly pbc::RepeatedField packets_ = new pbc::RepeatedField(); + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pbc::RepeatedField Packets { + get { return packets_; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as CachedTelemetryBatch); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(CachedTelemetryBatch other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if(!packets_.Equals(other.packets_)) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + hash ^= packets_.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + packets_.WriteTo(output, _repeated_packets_codec); + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + packets_.WriteTo(ref output, _repeated_packets_codec); + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + size += packets_.CalculateSize(_repeated_packets_codec); + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(CachedTelemetryBatch other) { + if (other == null) { + return; + } + packets_.Add(other.packets_); + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + packets_.AddEntriesFrom(input, _repeated_packets_codec); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + packets_.AddEntriesFrom(ref input, _repeated_packets_codec); + break; + } + } + } + } + #endif + + } + #endregion } diff --git a/src/ScadaLink.Communication/SiteStreamGrpc/SitestreamGrpc.cs b/src/ScadaLink.Communication/SiteStreamGrpc/SitestreamGrpc.cs index 0a900cb..e7b9b33 100644 --- a/src/ScadaLink.Communication/SiteStreamGrpc/SitestreamGrpc.cs +++ b/src/ScadaLink.Communication/SiteStreamGrpc/SitestreamGrpc.cs @@ -53,6 +53,8 @@ namespace ScadaLink.Communication.Grpc { static readonly grpc::Marshaller __Marshaller_sitestream_AuditEventBatch = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::ScadaLink.Communication.Grpc.AuditEventBatch.Parser)); [global::System.CodeDom.Compiler.GeneratedCode("grpc_csharp_plugin", null)] static readonly grpc::Marshaller __Marshaller_sitestream_IngestAck = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::ScadaLink.Communication.Grpc.IngestAck.Parser)); + [global::System.CodeDom.Compiler.GeneratedCode("grpc_csharp_plugin", null)] + static readonly grpc::Marshaller __Marshaller_sitestream_CachedTelemetryBatch = grpc::Marshallers.Create(__Helper_SerializeMessage, context => __Helper_DeserializeMessage(context, global::ScadaLink.Communication.Grpc.CachedTelemetryBatch.Parser)); [global::System.CodeDom.Compiler.GeneratedCode("grpc_csharp_plugin", null)] static readonly grpc::Method __Method_SubscribeInstance = new grpc::Method( @@ -70,6 +72,14 @@ namespace ScadaLink.Communication.Grpc { __Marshaller_sitestream_AuditEventBatch, __Marshaller_sitestream_IngestAck); + [global::System.CodeDom.Compiler.GeneratedCode("grpc_csharp_plugin", null)] + static readonly grpc::Method __Method_IngestCachedTelemetry = new grpc::Method( + grpc::MethodType.Unary, + __ServiceName, + "IngestCachedTelemetry", + __Marshaller_sitestream_CachedTelemetryBatch, + __Marshaller_sitestream_IngestAck); + /// Service descriptor public static global::Google.Protobuf.Reflection.ServiceDescriptor Descriptor { @@ -92,6 +102,12 @@ namespace ScadaLink.Communication.Grpc { throw new grpc::RpcException(new grpc::Status(grpc::StatusCode.Unimplemented, "")); } + [global::System.CodeDom.Compiler.GeneratedCode("grpc_csharp_plugin", null)] + public virtual global::System.Threading.Tasks.Task IngestCachedTelemetry(global::ScadaLink.Communication.Grpc.CachedTelemetryBatch request, grpc::ServerCallContext context) + { + throw new grpc::RpcException(new grpc::Status(grpc::StatusCode.Unimplemented, "")); + } + } /// Client for SiteStreamService @@ -151,6 +167,26 @@ namespace ScadaLink.Communication.Grpc { { return CallInvoker.AsyncUnaryCall(__Method_IngestAuditEvents, null, options, request); } + [global::System.CodeDom.Compiler.GeneratedCode("grpc_csharp_plugin", null)] + public virtual global::ScadaLink.Communication.Grpc.IngestAck IngestCachedTelemetry(global::ScadaLink.Communication.Grpc.CachedTelemetryBatch request, grpc::Metadata headers = null, global::System.DateTime? deadline = null, global::System.Threading.CancellationToken cancellationToken = default(global::System.Threading.CancellationToken)) + { + return IngestCachedTelemetry(request, new grpc::CallOptions(headers, deadline, cancellationToken)); + } + [global::System.CodeDom.Compiler.GeneratedCode("grpc_csharp_plugin", null)] + public virtual global::ScadaLink.Communication.Grpc.IngestAck IngestCachedTelemetry(global::ScadaLink.Communication.Grpc.CachedTelemetryBatch request, grpc::CallOptions options) + { + return CallInvoker.BlockingUnaryCall(__Method_IngestCachedTelemetry, null, options, request); + } + [global::System.CodeDom.Compiler.GeneratedCode("grpc_csharp_plugin", null)] + public virtual grpc::AsyncUnaryCall IngestCachedTelemetryAsync(global::ScadaLink.Communication.Grpc.CachedTelemetryBatch request, grpc::Metadata headers = null, global::System.DateTime? deadline = null, global::System.Threading.CancellationToken cancellationToken = default(global::System.Threading.CancellationToken)) + { + return IngestCachedTelemetryAsync(request, new grpc::CallOptions(headers, deadline, cancellationToken)); + } + [global::System.CodeDom.Compiler.GeneratedCode("grpc_csharp_plugin", null)] + public virtual grpc::AsyncUnaryCall IngestCachedTelemetryAsync(global::ScadaLink.Communication.Grpc.CachedTelemetryBatch request, grpc::CallOptions options) + { + return CallInvoker.AsyncUnaryCall(__Method_IngestCachedTelemetry, null, options, request); + } /// Creates a new instance of client from given ClientBaseConfiguration. [global::System.CodeDom.Compiler.GeneratedCode("grpc_csharp_plugin", null)] protected override SiteStreamServiceClient NewInstance(ClientBaseConfiguration configuration) @@ -166,7 +202,8 @@ namespace ScadaLink.Communication.Grpc { { return grpc::ServerServiceDefinition.CreateBuilder() .AddMethod(__Method_SubscribeInstance, serviceImpl.SubscribeInstance) - .AddMethod(__Method_IngestAuditEvents, serviceImpl.IngestAuditEvents).Build(); + .AddMethod(__Method_IngestAuditEvents, serviceImpl.IngestAuditEvents) + .AddMethod(__Method_IngestCachedTelemetry, serviceImpl.IngestCachedTelemetry).Build(); } /// Register service method with a service binder with or without implementation. Useful when customizing the service binding logic. @@ -178,6 +215,7 @@ namespace ScadaLink.Communication.Grpc { { serviceBinder.AddMethod(__Method_SubscribeInstance, serviceImpl == null ? null : new grpc::ServerStreamingServerMethod(serviceImpl.SubscribeInstance)); serviceBinder.AddMethod(__Method_IngestAuditEvents, serviceImpl == null ? null : new grpc::UnaryServerMethod(serviceImpl.IngestAuditEvents)); + serviceBinder.AddMethod(__Method_IngestCachedTelemetry, serviceImpl == null ? null : new grpc::UnaryServerMethod(serviceImpl.IngestCachedTelemetry)); } } diff --git a/src/ScadaLink.ConfigurationDatabase/Configurations/SiteCallEntityTypeConfiguration.cs b/src/ScadaLink.ConfigurationDatabase/Configurations/SiteCallEntityTypeConfiguration.cs new file mode 100644 index 0000000..78b7ccc --- /dev/null +++ b/src/ScadaLink.ConfigurationDatabase/Configurations/SiteCallEntityTypeConfiguration.cs @@ -0,0 +1,75 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types; + +namespace ScadaLink.ConfigurationDatabase.Configurations; + +/// +/// Maps the record to the central SiteCalls table +/// (Site Call Audit #22, Audit Log #23 M3 Bundle B). Operational state — NOT audit — +/// so the table is non-partitioned, standard [PRIMARY] filegroup, no DB-role +/// restriction. Two named indexes back the Central UI's "from this site" and +/// "in this status" queries. +/// +public class SiteCallEntityTypeConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("SiteCalls"); + + // PK is the strong-typed TrackedOperationId. Stored as varchar(36) by converting + // through the canonical "D"-format GUID string. Going through the string surface + // (rather than uniqueidentifier) keeps the column shape identical to how the id + // is serialised on the wire (gRPC strings, SQLite TEXT on the site) — one + // consistent format everywhere makes operational debugging far easier than + // mixing a uniqueidentifier central column with TEXT site columns. + builder.HasKey(s => s.TrackedOperationId); + + builder.Property(s => s.TrackedOperationId) + .HasConversion( + id => id.Value.ToString("D"), + s => new TrackedOperationId(Guid.Parse(s))) + .HasMaxLength(36) + .IsUnicode(false) + .IsRequired(); + + // Enum-as-string columns: bounded varchar, ASCII. + builder.Property(s => s.Channel) + .HasMaxLength(32) + .IsUnicode(false) + .IsRequired(); + + builder.Property(s => s.Status) + .HasMaxLength(32) + .IsUnicode(false) + .IsRequired(); + + builder.Property(s => s.SourceSite) + .HasMaxLength(64) + .IsUnicode(false) + .IsRequired(); + + builder.Property(s => s.Target) + .HasMaxLength(256) + .IsUnicode(false) + .IsRequired(); + + // Bounded unicode message column. + builder.Property(s => s.LastError) + .HasMaxLength(1024); + + // Indexes — names locked for reconciliation/migration discoverability. + // Source_Created backs "calls from this site" (Central UI Site Calls page, + // filter by SourceSite, newest first). + builder.HasIndex(s => new { s.SourceSite, s.CreatedAtUtc }) + .IsDescending(false, true) + .HasDatabaseName("IX_SiteCalls_Source_Created"); + + // Status_Updated backs "calls in this status" (e.g. parked rows awaiting + // operator action, newest UpdatedAtUtc first). + builder.HasIndex(s => new { s.Status, s.UpdatedAtUtc }) + .IsDescending(false, true) + .HasDatabaseName("IX_SiteCalls_Status_Updated"); + } +} diff --git a/src/ScadaLink.ConfigurationDatabase/Migrations/20260520180431_AddSiteCallsTable.Designer.cs b/src/ScadaLink.ConfigurationDatabase/Migrations/20260520180431_AddSiteCallsTable.Designer.cs new file mode 100644 index 0000000..a1bfd93 --- /dev/null +++ b/src/ScadaLink.ConfigurationDatabase/Migrations/20260520180431_AddSiteCallsTable.Designer.cs @@ -0,0 +1,1619 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ScadaLink.ConfigurationDatabase; + +#nullable disable + +namespace ScadaLink.ConfigurationDatabase.Migrations +{ + [DbContext(typeof(ScadaLinkDbContext))] + [Migration("20260520180431_AddSiteCallsTable")] + partial class AddSiteCallsTable + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("Microsoft.AspNetCore.DataProtection.EntityFrameworkCore.DataProtectionKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("FriendlyName") + .HasColumnType("nvarchar(max)"); + + b.Property("Xml") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("DataProtectionKeys"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Audit.AuditEvent", b => + { + b.Property("EventId") + .HasColumnType("uniqueidentifier"); + + b.Property("OccurredAtUtc") + .HasColumnType("datetime2"); + + b.Property("Actor") + .HasMaxLength(128) + .IsUnicode(false) + .HasColumnType("varchar(128)"); + + b.Property("Channel") + .IsRequired() + .HasMaxLength(32) + .IsUnicode(false) + .HasColumnType("varchar(32)"); + + b.Property("CorrelationId") + .HasColumnType("uniqueidentifier"); + + b.Property("DurationMs") + .HasColumnType("int"); + + b.Property("ErrorDetail") + .HasColumnType("nvarchar(max)"); + + b.Property("ErrorMessage") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("Extra") + .HasColumnType("nvarchar(max)"); + + b.Property("ForwardState") + .HasMaxLength(32) + .IsUnicode(false) + .HasColumnType("varchar(32)"); + + b.Property("HttpStatus") + .HasColumnType("int"); + + b.Property("IngestedAtUtc") + .HasColumnType("datetime2"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(32) + .IsUnicode(false) + .HasColumnType("varchar(32)"); + + b.Property("PayloadTruncated") + .HasColumnType("bit"); + + b.Property("RequestSummary") + .HasColumnType("nvarchar(max)"); + + b.Property("ResponseSummary") + .HasColumnType("nvarchar(max)"); + + b.Property("SourceInstanceId") + .HasMaxLength(128) + .IsUnicode(false) + .HasColumnType("varchar(128)"); + + b.Property("SourceScript") + .HasMaxLength(128) + .IsUnicode(false) + .HasColumnType("varchar(128)"); + + b.Property("SourceSiteId") + .HasMaxLength(64) + .IsUnicode(false) + .HasColumnType("varchar(64)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .IsUnicode(false) + .HasColumnType("varchar(32)"); + + b.Property("Target") + .HasMaxLength(256) + .IsUnicode(false) + .HasColumnType("varchar(256)"); + + b.HasKey("EventId", "OccurredAtUtc"); + + b.HasIndex("CorrelationId") + .HasDatabaseName("IX_AuditLog_CorrelationId") + .HasFilter("[CorrelationId] IS NOT NULL"); + + b.HasIndex("EventId") + .IsUnique() + .HasDatabaseName("UX_AuditLog_EventId"); + + b.HasIndex("OccurredAtUtc") + .IsDescending() + .HasDatabaseName("IX_AuditLog_OccurredAtUtc"); + + b.HasIndex("SourceSiteId", "OccurredAtUtc") + .IsDescending(false, true) + .HasDatabaseName("IX_AuditLog_Site_Occurred"); + + b.HasIndex("Target", "OccurredAtUtc") + .IsDescending(false, true) + .HasDatabaseName("IX_AuditLog_Target_Occurred") + .HasFilter("[Target] IS NOT NULL"); + + b.HasIndex("Channel", "Status", "OccurredAtUtc") + .IsDescending(false, false, true) + .HasDatabaseName("IX_AuditLog_Channel_Status_Occurred"); + + b.ToTable("AuditLog", (string)null); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Audit.AuditLogEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Action") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("AfterStateJson") + .HasColumnType("nvarchar(max)"); + + b.Property("EntityId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("EntityName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("EntityType") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Timestamp") + .HasColumnType("datetimeoffset"); + + b.Property("User") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("Action"); + + b.HasIndex("EntityId"); + + b.HasIndex("EntityType"); + + b.HasIndex("Timestamp"); + + b.HasIndex("User"); + + b.ToTable("AuditLogEntries"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Audit.SiteCall", b => + { + b.Property("TrackedOperationId") + .HasMaxLength(36) + .IsUnicode(false) + .HasColumnType("varchar(36)"); + + b.Property("Channel") + .IsRequired() + .HasMaxLength(32) + .IsUnicode(false) + .HasColumnType("varchar(32)"); + + b.Property("CreatedAtUtc") + .HasColumnType("datetime2"); + + b.Property("HttpStatus") + .HasColumnType("int"); + + b.Property("IngestedAtUtc") + .HasColumnType("datetime2"); + + b.Property("LastError") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("RetryCount") + .HasColumnType("int"); + + b.Property("SourceSite") + .IsRequired() + .HasMaxLength(64) + .IsUnicode(false) + .HasColumnType("varchar(64)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .IsUnicode(false) + .HasColumnType("varchar(32)"); + + b.Property("Target") + .IsRequired() + .HasMaxLength(256) + .IsUnicode(false) + .HasColumnType("varchar(256)"); + + b.Property("TerminalAtUtc") + .HasColumnType("datetime2"); + + b.Property("UpdatedAtUtc") + .HasColumnType("datetime2"); + + b.HasKey("TrackedOperationId"); + + b.HasIndex("SourceSite", "CreatedAtUtc") + .IsDescending(false, true) + .HasDatabaseName("IX_SiteCalls_Source_Created"); + + b.HasIndex("Status", "UpdatedAtUtc") + .IsDescending(false, true) + .HasDatabaseName("IX_SiteCalls_Status_Updated"); + + b.ToTable("SiteCalls", (string)null); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Deployment.DeployedConfigSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ConfigurationJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DeployedAt") + .HasColumnType("datetimeoffset"); + + b.Property("DeploymentId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("InstanceId") + .HasColumnType("int"); + + b.Property("RevisionHash") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.HasKey("Id"); + + b.HasIndex("DeploymentId"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DeployedConfigSnapshots"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Deployment.DeploymentRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CompletedAt") + .HasColumnType("datetimeoffset"); + + b.Property("DeployedAt") + .HasColumnType("datetimeoffset"); + + b.Property("DeployedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("DeploymentId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("ErrorMessage") + .HasColumnType("nvarchar(max)"); + + b.Property("InstanceId") + .HasColumnType("int"); + + b.Property("RevisionHash") + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("DeployedAt"); + + b.HasIndex("DeploymentId") + .IsUnique(); + + b.HasIndex("InstanceId"); + + b.ToTable("DeploymentRecords"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Deployment.SystemArtifactDeploymentRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ArtifactType") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("DeployedAt") + .HasColumnType("datetimeoffset"); + + b.Property("DeployedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("PerSiteStatus") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.HasKey("Id"); + + b.HasIndex("DeployedAt"); + + b.ToTable("SystemArtifactDeploymentRecords"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.ExternalSystems.DatabaseConnectionDefinition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ConnectionString") + .IsRequired() + .HasMaxLength(8000) + .HasColumnType("nvarchar(max)"); + + b.Property("MaxRetries") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("RetryDelay") + .HasColumnType("time"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("DatabaseConnectionDefinitions"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.ExternalSystems.ExternalSystemDefinition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AuthConfiguration") + .HasMaxLength(8000) + .HasColumnType("nvarchar(max)"); + + b.Property("AuthType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("EndpointUrl") + .IsRequired() + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)"); + + b.Property("MaxRetries") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("RetryDelay") + .HasColumnType("time"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("ExternalSystemDefinitions"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.ExternalSystems.ExternalSystemMethod", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ExternalSystemDefinitionId") + .HasColumnType("int"); + + b.Property("HttpMethod") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("nvarchar(10)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ParameterDefinitions") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)"); + + b.Property("ReturnDefinition") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.HasKey("Id"); + + b.HasIndex("ExternalSystemDefinitionId", "Name") + .IsUnique(); + + b.ToTable("ExternalSystemMethods"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.InboundApi.ApiKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("IsEnabled") + .HasColumnType("bit"); + + b.Property("KeyHash") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("KeyHash") + .IsUnique(); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("ApiKeys"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.InboundApi.ApiMethod", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ApprovedApiKeyIds") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ParameterDefinitions") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("ReturnDefinition") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("Script") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TimeoutSeconds") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("ApiMethods"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.Area", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ParentAreaId") + .HasColumnType("int"); + + b.Property("SiteId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ParentAreaId"); + + b.HasIndex("SiteId", "ParentAreaId", "Name") + .IsUnique() + .HasFilter("[ParentAreaId] IS NOT NULL"); + + b.ToTable("Areas"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.Instance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AreaId") + .HasColumnType("int"); + + b.Property("SiteId") + .HasColumnType("int"); + + b.Property("State") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("TemplateId") + .HasColumnType("int"); + + b.Property("UniqueName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("AreaId"); + + b.HasIndex("TemplateId"); + + b.HasIndex("SiteId", "UniqueName") + .IsUnique(); + + b.ToTable("Instances"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.InstanceAlarmOverride", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AlarmCanonicalName") + .IsRequired() + .HasMaxLength(400) + .HasColumnType("nvarchar(400)"); + + b.Property("InstanceId") + .HasColumnType("int"); + + b.Property("PriorityLevelOverride") + .HasColumnType("int"); + + b.Property("TriggerConfigurationOverride") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId", "AlarmCanonicalName") + .IsUnique(); + + b.ToTable("InstanceAlarmOverrides"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.InstanceAttributeOverride", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AttributeName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("InstanceId") + .HasColumnType("int"); + + b.Property("OverrideValue") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.HasKey("Id"); + + b.HasIndex("InstanceId", "AttributeName") + .IsUnique(); + + b.ToTable("InstanceAttributeOverrides"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.InstanceConnectionBinding", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AttributeName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("DataConnectionId") + .HasColumnType("int"); + + b.Property("InstanceId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("DataConnectionId"); + + b.HasIndex("InstanceId", "AttributeName") + .IsUnique(); + + b.ToTable("InstanceConnectionBindings"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Notifications.Notification", b => + { + b.Property("NotificationId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Body") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("CreatedAt") + .HasColumnType("datetimeoffset"); + + b.Property("DeliveredAt") + .HasColumnType("datetimeoffset"); + + b.Property("LastAttemptAt") + .HasColumnType("datetimeoffset"); + + b.Property("LastError") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("ListName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("NextAttemptAt") + .HasColumnType("datetimeoffset"); + + b.Property("ResolvedTargets") + .HasColumnType("nvarchar(max)"); + + b.Property("RetryCount") + .HasColumnType("int"); + + b.Property("SiteEnqueuedAt") + .HasColumnType("datetimeoffset"); + + b.Property("SourceInstanceId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("SourceScript") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("SourceSiteId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Subject") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("TypeData") + .HasColumnType("nvarchar(max)"); + + b.HasKey("NotificationId"); + + b.HasIndex("SourceSiteId", "CreatedAt"); + + b.HasIndex("Status", "NextAttemptAt"); + + b.ToTable("Notifications"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Notifications.NotificationList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("NotificationLists"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Notifications.NotificationRecipient", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("EmailAddress") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("NotificationListId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("NotificationListId"); + + b.ToTable("NotificationRecipients"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Notifications.SmtpConfiguration", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("AuthType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("ConnectionTimeoutSeconds") + .HasColumnType("int"); + + b.Property("Credentials") + .HasMaxLength(8000) + .HasColumnType("nvarchar(max)"); + + b.Property("FromAddress") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("MaxConcurrentConnections") + .HasColumnType("int"); + + b.Property("MaxRetries") + .HasColumnType("int"); + + b.Property("Port") + .HasColumnType("int"); + + b.Property("RetryDelay") + .HasColumnType("time"); + + b.Property("TlsMode") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.ToTable("SmtpConfigurations"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Scripts.SharedScript", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Code") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ParameterDefinitions") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("ReturnDefinition") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("SharedScripts"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Security.LdapGroupMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("LdapGroupName") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("Role") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.HasKey("Id"); + + b.HasIndex("LdapGroupName") + .IsUnique(); + + b.ToTable("LdapGroupMappings"); + + b.HasData( + new + { + Id = 1, + LdapGroupName = "SCADA-Admins", + Role = "Admin" + }, + new + { + Id = 2, + LdapGroupName = "SCADA-Designers", + Role = "Design" + }, + new + { + Id = 3, + LdapGroupName = "SCADA-Deploy-All", + Role = "Deployment" + }, + new + { + Id = 4, + LdapGroupName = "SCADA-Deploy-SiteA", + Role = "Deployment" + }); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Security.SiteScopeRule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("LdapGroupMappingId") + .HasColumnType("int"); + + b.Property("SiteId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SiteId"); + + b.HasIndex("LdapGroupMappingId", "SiteId") + .IsUnique(); + + b.ToTable("SiteScopeRules"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Sites.DataConnection", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("BackupConfiguration") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("FailoverRetryCount") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValue(3); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("PrimaryConfiguration") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("Protocol") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("SiteId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("SiteId", "Name") + .IsUnique(); + + b.ToTable("DataConnections"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Sites.Site", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Description") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)"); + + b.Property("GrpcNodeAAddress") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("GrpcNodeBAddress") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("NodeAAddress") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("NodeBAddress") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("SiteIdentifier") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique(); + + b.HasIndex("SiteIdentifier") + .IsUnique(); + + b.ToTable("Sites"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.Template", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Description") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)"); + + b.Property("FolderId") + .HasColumnType("int"); + + b.Property("IsDerived") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("OwnerCompositionId") + .HasColumnType("int"); + + b.Property("ParentTemplateId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("FolderId"); + + b.HasIndex("Name") + .IsUnique() + .HasFilter("[IsDerived] = 0"); + + b.HasIndex("ParentTemplateId"); + + b.ToTable("Templates"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.TemplateAlarm", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Description") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)"); + + b.Property("IsInherited") + .HasColumnType("bit"); + + b.Property("IsLocked") + .HasColumnType("bit"); + + b.Property("LockedInDerived") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("OnTriggerScriptId") + .HasColumnType("int"); + + b.Property("PriorityLevel") + .HasColumnType("int"); + + b.Property("TemplateId") + .HasColumnType("int"); + + b.Property("TriggerConfiguration") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("TriggerType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("TemplateId", "Name") + .IsUnique(); + + b.ToTable("TemplateAlarms"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.TemplateAttribute", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("DataSourceReference") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("DataType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Description") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)"); + + b.Property("IsInherited") + .HasColumnType("bit"); + + b.Property("IsLocked") + .HasColumnType("bit"); + + b.Property("LockedInDerived") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("TemplateId") + .HasColumnType("int"); + + b.Property("Value") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.HasKey("Id"); + + b.HasIndex("TemplateId", "Name") + .IsUnique(); + + b.ToTable("TemplateAttributes"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.TemplateComposition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ComposedTemplateId") + .HasColumnType("int"); + + b.Property("InstanceName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("TemplateId") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ComposedTemplateId"); + + b.HasIndex("TemplateId", "InstanceName") + .IsUnique(); + + b.ToTable("TemplateCompositions"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.TemplateFolder", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ParentFolderId") + .HasColumnType("int"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ParentFolderId", "Name") + .IsUnique() + .HasFilter("[ParentFolderId] IS NOT NULL"); + + b.ToTable("TemplateFolders"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.TemplateScript", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Code") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsInherited") + .HasColumnType("bit"); + + b.Property("IsLocked") + .HasColumnType("bit"); + + b.Property("LockedInDerived") + .HasColumnType("bit"); + + b.Property("MinTimeBetweenRuns") + .HasColumnType("time"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ParameterDefinitions") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("ReturnDefinition") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("TemplateId") + .HasColumnType("int"); + + b.Property("TriggerConfiguration") + .HasMaxLength(4000) + .HasColumnType("nvarchar(4000)"); + + b.Property("TriggerType") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("TemplateId", "Name") + .IsUnique(); + + b.ToTable("TemplateScripts"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Deployment.DeployedConfigSnapshot", b => + { + b.HasOne("ScadaLink.Commons.Entities.Instances.Instance", null) + .WithMany() + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Deployment.DeploymentRecord", b => + { + b.HasOne("ScadaLink.Commons.Entities.Instances.Instance", null) + .WithMany() + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.ExternalSystems.ExternalSystemMethod", b => + { + b.HasOne("ScadaLink.Commons.Entities.ExternalSystems.ExternalSystemDefinition", null) + .WithMany() + .HasForeignKey("ExternalSystemDefinitionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.Area", b => + { + b.HasOne("ScadaLink.Commons.Entities.Instances.Area", null) + .WithMany("Children") + .HasForeignKey("ParentAreaId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("ScadaLink.Commons.Entities.Sites.Site", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.Instance", b => + { + b.HasOne("ScadaLink.Commons.Entities.Instances.Area", null) + .WithMany() + .HasForeignKey("AreaId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("ScadaLink.Commons.Entities.Sites.Site", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("ScadaLink.Commons.Entities.Templates.Template", null) + .WithMany() + .HasForeignKey("TemplateId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.InstanceAlarmOverride", b => + { + b.HasOne("ScadaLink.Commons.Entities.Instances.Instance", null) + .WithMany("AlarmOverrides") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.InstanceAttributeOverride", b => + { + b.HasOne("ScadaLink.Commons.Entities.Instances.Instance", null) + .WithMany("AttributeOverrides") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.InstanceConnectionBinding", b => + { + b.HasOne("ScadaLink.Commons.Entities.Sites.DataConnection", null) + .WithMany() + .HasForeignKey("DataConnectionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("ScadaLink.Commons.Entities.Instances.Instance", null) + .WithMany("ConnectionBindings") + .HasForeignKey("InstanceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Notifications.NotificationRecipient", b => + { + b.HasOne("ScadaLink.Commons.Entities.Notifications.NotificationList", null) + .WithMany("Recipients") + .HasForeignKey("NotificationListId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Security.SiteScopeRule", b => + { + b.HasOne("ScadaLink.Commons.Entities.Security.LdapGroupMapping", null) + .WithMany() + .HasForeignKey("LdapGroupMappingId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("ScadaLink.Commons.Entities.Sites.Site", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Sites.DataConnection", b => + { + b.HasOne("ScadaLink.Commons.Entities.Sites.Site", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.Template", b => + { + b.HasOne("ScadaLink.Commons.Entities.Templates.TemplateFolder", null) + .WithMany() + .HasForeignKey("FolderId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("ScadaLink.Commons.Entities.Templates.Template", null) + .WithMany() + .HasForeignKey("ParentTemplateId") + .OnDelete(DeleteBehavior.Restrict); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.TemplateAlarm", b => + { + b.HasOne("ScadaLink.Commons.Entities.Templates.Template", null) + .WithMany("Alarms") + .HasForeignKey("TemplateId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.TemplateAttribute", b => + { + b.HasOne("ScadaLink.Commons.Entities.Templates.Template", null) + .WithMany("Attributes") + .HasForeignKey("TemplateId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.TemplateComposition", b => + { + b.HasOne("ScadaLink.Commons.Entities.Templates.Template", null) + .WithMany() + .HasForeignKey("ComposedTemplateId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("ScadaLink.Commons.Entities.Templates.Template", null) + .WithMany("Compositions") + .HasForeignKey("TemplateId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.TemplateFolder", b => + { + b.HasOne("ScadaLink.Commons.Entities.Templates.TemplateFolder", null) + .WithMany() + .HasForeignKey("ParentFolderId") + .OnDelete(DeleteBehavior.Restrict); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.TemplateScript", b => + { + b.HasOne("ScadaLink.Commons.Entities.Templates.Template", null) + .WithMany("Scripts") + .HasForeignKey("TemplateId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.Area", b => + { + b.Navigation("Children"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Instances.Instance", b => + { + b.Navigation("AlarmOverrides"); + + b.Navigation("AttributeOverrides"); + + b.Navigation("ConnectionBindings"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Notifications.NotificationList", b => + { + b.Navigation("Recipients"); + }); + + modelBuilder.Entity("ScadaLink.Commons.Entities.Templates.Template", b => + { + b.Navigation("Alarms"); + + b.Navigation("Attributes"); + + b.Navigation("Compositions"); + + b.Navigation("Scripts"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ScadaLink.ConfigurationDatabase/Migrations/20260520180431_AddSiteCallsTable.cs b/src/ScadaLink.ConfigurationDatabase/Migrations/20260520180431_AddSiteCallsTable.cs new file mode 100644 index 0000000..45bbda1 --- /dev/null +++ b/src/ScadaLink.ConfigurationDatabase/Migrations/20260520180431_AddSiteCallsTable.cs @@ -0,0 +1,56 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ScadaLink.ConfigurationDatabase.Migrations +{ + /// + public partial class AddSiteCallsTable : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "SiteCalls", + columns: table => new + { + TrackedOperationId = table.Column(type: "varchar(36)", unicode: false, maxLength: 36, nullable: false), + Channel = table.Column(type: "varchar(32)", unicode: false, maxLength: 32, nullable: false), + Target = table.Column(type: "varchar(256)", unicode: false, maxLength: 256, nullable: false), + SourceSite = table.Column(type: "varchar(64)", unicode: false, maxLength: 64, nullable: false), + Status = table.Column(type: "varchar(32)", unicode: false, maxLength: 32, nullable: false), + RetryCount = table.Column(type: "int", nullable: false), + LastError = table.Column(type: "nvarchar(1024)", maxLength: 1024, nullable: true), + HttpStatus = table.Column(type: "int", nullable: true), + CreatedAtUtc = table.Column(type: "datetime2", nullable: false), + UpdatedAtUtc = table.Column(type: "datetime2", nullable: false), + TerminalAtUtc = table.Column(type: "datetime2", nullable: true), + IngestedAtUtc = table.Column(type: "datetime2", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_SiteCalls", x => x.TrackedOperationId); + }); + + migrationBuilder.CreateIndex( + name: "IX_SiteCalls_Source_Created", + table: "SiteCalls", + columns: new[] { "SourceSite", "CreatedAtUtc" }, + descending: new[] { false, true }); + + migrationBuilder.CreateIndex( + name: "IX_SiteCalls_Status_Updated", + table: "SiteCalls", + columns: new[] { "Status", "UpdatedAtUtc" }, + descending: new[] { false, true }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "SiteCalls"); + } + } +} diff --git a/src/ScadaLink.ConfigurationDatabase/Migrations/ScadaLinkDbContextModelSnapshot.cs b/src/ScadaLink.ConfigurationDatabase/Migrations/ScadaLinkDbContextModelSnapshot.cs index 02b7745..328a8a5 100644 --- a/src/ScadaLink.ConfigurationDatabase/Migrations/ScadaLinkDbContextModelSnapshot.cs +++ b/src/ScadaLink.ConfigurationDatabase/Migrations/ScadaLinkDbContextModelSnapshot.cs @@ -212,6 +212,72 @@ namespace ScadaLink.ConfigurationDatabase.Migrations b.ToTable("AuditLogEntries"); }); + modelBuilder.Entity("ScadaLink.Commons.Entities.Audit.SiteCall", b => + { + b.Property("TrackedOperationId") + .HasMaxLength(36) + .IsUnicode(false) + .HasColumnType("varchar(36)"); + + b.Property("Channel") + .IsRequired() + .HasMaxLength(32) + .IsUnicode(false) + .HasColumnType("varchar(32)"); + + b.Property("CreatedAtUtc") + .HasColumnType("datetime2"); + + b.Property("HttpStatus") + .HasColumnType("int"); + + b.Property("IngestedAtUtc") + .HasColumnType("datetime2"); + + b.Property("LastError") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("RetryCount") + .HasColumnType("int"); + + b.Property("SourceSite") + .IsRequired() + .HasMaxLength(64) + .IsUnicode(false) + .HasColumnType("varchar(64)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .IsUnicode(false) + .HasColumnType("varchar(32)"); + + b.Property("Target") + .IsRequired() + .HasMaxLength(256) + .IsUnicode(false) + .HasColumnType("varchar(256)"); + + b.Property("TerminalAtUtc") + .HasColumnType("datetime2"); + + b.Property("UpdatedAtUtc") + .HasColumnType("datetime2"); + + b.HasKey("TrackedOperationId"); + + b.HasIndex("SourceSite", "CreatedAtUtc") + .IsDescending(false, true) + .HasDatabaseName("IX_SiteCalls_Source_Created"); + + b.HasIndex("Status", "UpdatedAtUtc") + .IsDescending(false, true) + .HasDatabaseName("IX_SiteCalls_Status_Updated"); + + b.ToTable("SiteCalls", (string)null); + }); + modelBuilder.Entity("ScadaLink.Commons.Entities.Deployment.DeployedConfigSnapshot", b => { b.Property("Id") diff --git a/src/ScadaLink.ConfigurationDatabase/Repositories/SiteCallAuditRepository.cs b/src/ScadaLink.ConfigurationDatabase/Repositories/SiteCallAuditRepository.cs new file mode 100644 index 0000000..3fdff7e --- /dev/null +++ b/src/ScadaLink.ConfigurationDatabase/Repositories/SiteCallAuditRepository.cs @@ -0,0 +1,214 @@ +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces.Repositories; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Audit; + +namespace ScadaLink.ConfigurationDatabase.Repositories; + +/// +/// EF Core implementation of . See the +/// interface for the monotonic-upsert contract; this class adds notes on the +/// data-access strategy used by each method. +/// +public class SiteCallAuditRepository : ISiteCallAuditRepository +{ + // SQL Server duplicate-key error numbers, identical to the AuditLogRepository + // race-fix: 2601 = unique-index violation, 2627 = PK/unique-constraint + // violation. The IF NOT EXISTS … INSERT pattern has a check-then-act window + // and the loser surfaces as one of these; monotonic-upsert semantics demand + // we swallow them. + private const int SqlErrorUniqueIndexViolation = 2601; + private const int SqlErrorPrimaryKeyViolation = 2627; + + // Monotonic status ordering. Lower rank wins on tie (same-rank upserts are + // no-ops, including terminal-over-terminal). Spec from Bundle B3 plan: + // Submitted < Forwarded < Attempted == Skipped < Delivered == Failed == Parked == Discarded. + private static readonly Dictionary StatusRank = new(StringComparer.Ordinal) + { + ["Submitted"] = 0, + ["Forwarded"] = 1, + ["Attempted"] = 2, + ["Skipped"] = 2, + ["Delivered"] = 3, + ["Failed"] = 3, + ["Parked"] = 3, + ["Discarded"] = 3, + }; + + private readonly ScadaLinkDbContext _context; + private readonly ILogger _logger; + + public SiteCallAuditRepository(ScadaLinkDbContext context, ILogger? logger = null) + { + _context = context ?? throw new ArgumentNullException(nameof(context)); + _logger = logger ?? NullLogger.Instance; + } + + /// + /// Two-step: IF NOT EXISTS INSERT then conditional UPDATE with + /// an inline CASE rank comparison. Both go through + /// + /// so the change tracker is bypassed and the value-converted PK column is + /// written as the canonical "D"-format GUID string. Duplicate-key violations + /// from the insert race are swallowed. + /// + public async Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) + { + if (siteCall is null) + { + throw new ArgumentNullException(nameof(siteCall)); + } + + var idText = siteCall.TrackedOperationId.Value.ToString("D"); + var incomingRank = GetRankOrThrow(siteCall.Status); + + // Step 1: insert-if-not-exists. Like AuditLogRepository.InsertIfNotExistsAsync + // this is check-then-act so a duplicate-key violation may surface under + // concurrent inserts on the same id — caught + logged at Debug. + try + { + await _context.Database.ExecuteSqlInterpolatedAsync( + $@"IF NOT EXISTS (SELECT 1 FROM dbo.SiteCalls WHERE TrackedOperationId = {idText}) +INSERT INTO dbo.SiteCalls + (TrackedOperationId, Channel, Target, SourceSite, Status, RetryCount, + LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc) +VALUES + ({idText}, {siteCall.Channel}, {siteCall.Target}, {siteCall.SourceSite}, {siteCall.Status}, {siteCall.RetryCount}, + {siteCall.LastError}, {siteCall.HttpStatus}, {siteCall.CreatedAtUtc}, {siteCall.UpdatedAtUtc}, {siteCall.TerminalAtUtc}, {siteCall.IngestedAtUtc});", + ct); + } + catch (SqlException ex) when ( + ex.Number == SqlErrorUniqueIndexViolation + || ex.Number == SqlErrorPrimaryKeyViolation) + { + _logger.LogDebug( + ex, + "SiteCallAuditRepository.UpsertAsync swallowed duplicate-key violation (error {SqlErrorNumber}) for TrackedOperationId {TrackedOperationId}; falling through to monotonic update.", + ex.Number, + idText); + } + + // Step 2: monotonic update. The CASE expression maps the stored Status + // string to the same rank table the caller uses; we only mutate if the + // incoming rank is strictly greater. Same-rank (including + // terminal-over-terminal) is a no-op — first-write-wins at each rank. + await _context.Database.ExecuteSqlInterpolatedAsync( + $@"UPDATE dbo.SiteCalls +SET Status = {siteCall.Status}, + RetryCount = {siteCall.RetryCount}, + LastError = {siteCall.LastError}, + HttpStatus = {siteCall.HttpStatus}, + UpdatedAtUtc = {siteCall.UpdatedAtUtc}, + TerminalAtUtc = {siteCall.TerminalAtUtc}, + IngestedAtUtc = {siteCall.IngestedAtUtc} +WHERE TrackedOperationId = {idText} + AND {incomingRank} > (CASE Status + WHEN 'Submitted' THEN 0 + WHEN 'Forwarded' THEN 1 + WHEN 'Attempted' THEN 2 + WHEN 'Skipped' THEN 2 + WHEN 'Delivered' THEN 3 + WHEN 'Failed' THEN 3 + WHEN 'Parked' THEN 3 + WHEN 'Discarded' THEN 3 + ELSE -1 + END);", + ct); + } + + /// + /// Single FindAsync against the PK. Returns null for unknown ids. + /// + public async Task GetAsync(TrackedOperationId id, CancellationToken ct = default) + { + return await _context.Set().FindAsync(new object?[] { id }, ct); + } + + /// + /// Builds a parameterised SQL query against dbo.SiteCalls ordered by + /// (CreatedAtUtc DESC, TrackedOperationId DESC), with keyset paging. + /// Raw SQL is used here (rather than LINQ) because EF Core 10 cannot + /// translate the lexicographic string comparison against the value-converted + /// column inside an expression tree — the + /// converter is applied to equality but not to inequality comparisons + /// against the underlying Guid. The keyset tiebreaker is varchar lex order, + /// which is deterministic and gives "no overlap, every row exactly once" + /// paging without depending on Guid byte ordering. + /// + public async Task> QueryAsync( + SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) + { + if (filter is null) + { + throw new ArgumentNullException(nameof(filter)); + } + + if (paging is null) + { + throw new ArgumentNullException(nameof(paging)); + } + + // FormattableString interpolation parameterises every value (no concatenation) + // so this is injection-safe. EF Core resolves the parameter values, the + // composed sql is shaped to SQL Server's grammar and projected into the + // SiteCall entity via FromSqlInterpolated. The CASE expressions wrap each + // optional predicate so a null filter field degrades to a no-op (matches + // every row) instead of branching at C# level into N variants. + var afterCreated = paging.AfterCreatedAtUtc; + var afterIdString = paging.AfterId?.Value.ToString("D"); + var hasCursor = afterCreated is not null && afterIdString is not null; + + var fromUtc = filter.FromUtc; + var toUtc = filter.ToUtc; + + FormattableString sql = $@" +SELECT TOP ({paging.PageSize}) + TrackedOperationId, Channel, Target, SourceSite, Status, RetryCount, + LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc +FROM dbo.SiteCalls +WHERE ({filter.Channel} IS NULL OR Channel = {filter.Channel}) + AND ({filter.SourceSite} IS NULL OR SourceSite = {filter.SourceSite}) + AND ({filter.Status} IS NULL OR Status = {filter.Status}) + AND ({filter.Target} IS NULL OR Target = {filter.Target}) + AND ({fromUtc} IS NULL OR CreatedAtUtc >= {fromUtc}) + AND ({toUtc} IS NULL OR CreatedAtUtc <= {toUtc}) + AND ({(hasCursor ? 1 : 0)} = 0 + OR CreatedAtUtc < {afterCreated} + OR (CreatedAtUtc = {afterCreated} AND TrackedOperationId < {afterIdString})) +ORDER BY CreatedAtUtc DESC, TrackedOperationId DESC;"; + + var rows = await _context.Set() + .FromSqlInterpolated(sql) + .AsNoTracking() + .ToListAsync(ct); + + return rows; + } + + /// + /// Deletes rows whose is non-null AND + /// strictly less than . Non-terminal rows are + /// never touched. Returns the number of rows removed. + /// + public async Task PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) + { + return await _context.Database.ExecuteSqlInterpolatedAsync( + $"DELETE FROM dbo.SiteCalls WHERE TerminalAtUtc IS NOT NULL AND TerminalAtUtc < {olderThanUtc};", + ct); + } + + private static int GetRankOrThrow(string status) + { + if (!StatusRank.TryGetValue(status, out var rank)) + { + throw new ArgumentException( + $"Unknown SiteCall status '{status}'. Expected one of: {string.Join(", ", StatusRank.Keys)}.", + nameof(status)); + } + return rank; + } +} diff --git a/src/ScadaLink.ConfigurationDatabase/ScadaLinkDbContext.cs b/src/ScadaLink.ConfigurationDatabase/ScadaLinkDbContext.cs index f25118f..4e35590 100644 --- a/src/ScadaLink.ConfigurationDatabase/ScadaLinkDbContext.cs +++ b/src/ScadaLink.ConfigurationDatabase/ScadaLinkDbContext.cs @@ -85,6 +85,7 @@ public class ScadaLinkDbContext : DbContext, IDataProtectionKeyContext // Audit public DbSet AuditLogEntries => Set(); public DbSet AuditLogs => Set(); + public DbSet SiteCalls => Set(); // Data Protection Keys (for shared ASP.NET Data Protection across nodes) public DbSet DataProtectionKeys => Set(); diff --git a/src/ScadaLink.ConfigurationDatabase/ServiceCollectionExtensions.cs b/src/ScadaLink.ConfigurationDatabase/ServiceCollectionExtensions.cs index b7ba6b4..bf79b29 100644 --- a/src/ScadaLink.ConfigurationDatabase/ServiceCollectionExtensions.cs +++ b/src/ScadaLink.ConfigurationDatabase/ServiceCollectionExtensions.cs @@ -47,6 +47,7 @@ public static class ServiceCollectionExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/ScadaLink.ExternalSystemGateway/DatabaseGateway.cs b/src/ScadaLink.ExternalSystemGateway/DatabaseGateway.cs index 94d3495..7e48288 100644 --- a/src/ScadaLink.ExternalSystemGateway/DatabaseGateway.cs +++ b/src/ScadaLink.ExternalSystemGateway/DatabaseGateway.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using ScadaLink.Commons.Entities.ExternalSystems; using ScadaLink.Commons.Interfaces.Repositories; using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Types; using ScadaLink.Commons.Types.Enums; using ScadaLink.StoreAndForward; @@ -71,12 +72,19 @@ public class DatabaseGateway : IDatabaseGateway /// /// Submits a SQL write to the store-and-forward engine for reliable delivery. /// + /// + /// Audit Log #23 (M3): used as the S&F message id so the retry loop can + /// recover it via StoreAndForwardMessage.Id and emit per-attempt / + /// terminal cached-write telemetry (Tasks E4/E5). Null preserves the + /// pre-M3 behaviour (S&F mints a random GUID). + /// public async Task CachedWriteAsync( string connectionName, string sql, IReadOnlyDictionary? parameters = null, string? originInstanceName = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + TrackedOperationId? trackedOperationId = null) { var definition = await ResolveConnectionAsync(connectionName, cancellationToken); if (definition == null) @@ -110,7 +118,13 @@ public class DatabaseGateway : IDatabaseGateway payload, originInstanceName, definition.MaxRetries > 0 ? definition.MaxRetries : null, - definition.RetryDelay > TimeSpan.Zero ? definition.RetryDelay : null); + definition.RetryDelay > TimeSpan.Zero ? definition.RetryDelay : null, + // Audit Log #23 (M3): pin the S&F message id to the + // TrackedOperationId so the retry loop (Bundle E Tasks E4/E5) can + // read it back via StoreAndForwardMessage.Id and emit per-attempt + + // terminal cached-write telemetry. Null -> S&F mints its own GUID + // (legacy pre-M3 behaviour). + messageId: trackedOperationId?.ToString()); } /// diff --git a/src/ScadaLink.ExternalSystemGateway/ExternalSystemClient.cs b/src/ScadaLink.ExternalSystemGateway/ExternalSystemClient.cs index c5f80b3..0b88de6 100644 --- a/src/ScadaLink.ExternalSystemGateway/ExternalSystemClient.cs +++ b/src/ScadaLink.ExternalSystemGateway/ExternalSystemClient.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Options; using ScadaLink.Commons.Entities.ExternalSystems; using ScadaLink.Commons.Interfaces.Repositories; using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Types; using ScadaLink.Commons.Types.Enums; using ScadaLink.StoreAndForward; @@ -72,12 +73,20 @@ public class ExternalSystemClient : IExternalSystemClient /// /// WP-7: CachedCall — attempt immediate, transient failure goes to S&F, permanent returned to script. /// + /// + /// Audit Log #23 (M3): used as the S&F message id so the retry loop can + /// recover it from StoreAndForwardMessage.Id and emit per-attempt / + /// terminal cached-call telemetry (Tasks E4/E5). When null the S&F engine + /// mints its own GUID — preserving the pre-M3 behaviour for callers that + /// don't participate in the M3 audit pipeline. + /// public async Task CachedCallAsync( string systemName, string methodName, IReadOnlyDictionary? parameters = null, string? originInstanceName = null, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + TrackedOperationId? trackedOperationId = null) { var (system, method) = await ResolveSystemAndMethodAsync(systemName, methodName, cancellationToken); if (system == null || method == null) @@ -129,7 +138,13 @@ public class ExternalSystemClient : IExternalSystemClient originInstanceName, system.MaxRetries > 0 ? system.MaxRetries : null, system.RetryDelay > TimeSpan.Zero ? system.RetryDelay : null, - attemptImmediateDelivery: false); + attemptImmediateDelivery: false, + // Audit Log #23 (M3): pin the S&F message id to the + // TrackedOperationId so the retry loop can read it back via + // StoreAndForwardMessage.Id and emit per-attempt + terminal + // cached-call telemetry (Bundle E Tasks E4/E5). Null -> S&F + // mints its own GUID (legacy pre-M3 behaviour). + messageId: trackedOperationId?.ToString()); return new ExternalCallResult(true, null, null, WasBuffered: true); } diff --git a/src/ScadaLink.Host/Actors/AkkaHostedService.cs b/src/ScadaLink.Host/Actors/AkkaHostedService.cs index 5bc5c7e..fe4fe15 100644 --- a/src/ScadaLink.Host/Actors/AkkaHostedService.cs +++ b/src/ScadaLink.Host/Actors/AkkaHostedService.cs @@ -342,6 +342,35 @@ akka {{ "AuditLogIngestActor singleton created (gRPC server bound: {GrpcBound})", grpcServer is not null); + // Site Call Audit (#22) — central singleton mirrors the AuditLogIngest + // and NotificationOutbox patterns. M3's dual-write transaction routes + // SiteCalls upserts through AuditLogIngestActor's own scope-per-message + // ISiteCallAuditRepository resolution, so this singleton is not on the + // M3 happy-path hot path; it exists so future direct-write callers + // (reconciliation puller, central→site Retry/Discard relay, KPI + // projector) Ask through a stable cluster proxy without further wiring. + // Like AuditLogIngestActor, the actor takes the root IServiceProvider + // and creates a fresh scope per message because ISiteCallAuditRepository + // is a scoped EF Core service. + var siteCallAuditLogger = _serviceProvider.GetRequiredService() + .CreateLogger(); + + var siteCallAuditSingletonProps = ClusterSingletonManager.Props( + singletonProps: Props.Create(() => new ScadaLink.SiteCallAudit.SiteCallAuditActor( + _serviceProvider, + siteCallAuditLogger)), + terminationMessage: PoisonPill.Instance, + settings: ClusterSingletonManagerSettings.Create(_actorSystem!) + .WithSingletonName("site-call-audit")); + _actorSystem!.ActorOf(siteCallAuditSingletonProps, "site-call-audit-singleton"); + + var siteCallAuditProxyProps = ClusterSingletonProxy.Props( + singletonManagerPath: "/user/site-call-audit-singleton", + settings: ClusterSingletonProxySettings.Create(_actorSystem) + .WithSingletonName("site-call-audit")); + _actorSystem.ActorOf(siteCallAuditProxyProps, "site-call-audit-proxy"); + _logger.LogInformation("SiteCallAuditActor singleton created"); + _logger.LogInformation("Central actors registered. CentralCommunicationActor created."); } diff --git a/src/ScadaLink.Host/Program.cs b/src/ScadaLink.Host/Program.cs index a42f93d..ed15327 100644 --- a/src/ScadaLink.Host/Program.cs +++ b/src/ScadaLink.Host/Program.cs @@ -16,6 +16,7 @@ using ScadaLink.ManagementService; using ScadaLink.NotificationOutbox; using ScadaLink.NotificationService; using ScadaLink.Security; +using ScadaLink.SiteCallAudit; using ScadaLink.TemplateEngine; using Serilog; @@ -82,6 +83,12 @@ try // IAuditLogRepository. The site writer chain is still registered (lazy // singletons) but is never resolved on a central node. builder.Services.AddAuditLog(builder.Configuration); + // Site Call Audit (#22) — central node owns the SiteCallAuditActor + // singleton (M3 Bundle F). The extension itself currently registers + // nothing — actor Props are constructed inline in AkkaHostedService — + // but the call is here for symmetry with the other audit composition + // roots so future per-actor DI lands without touching Program.cs. + builder.Services.AddSiteCallAudit(); builder.Services.AddTemplateEngine(); builder.Services.AddDeploymentManager(); builder.Services.AddSecurity(); diff --git a/src/ScadaLink.Host/ScadaLink.Host.csproj b/src/ScadaLink.Host/ScadaLink.Host.csproj index 4b45288..0f027e6 100644 --- a/src/ScadaLink.Host/ScadaLink.Host.csproj +++ b/src/ScadaLink.Host/ScadaLink.Host.csproj @@ -39,6 +39,7 @@ + diff --git a/src/ScadaLink.Host/SiteServiceRegistration.cs b/src/ScadaLink.Host/SiteServiceRegistration.cs index dd13484..0e59f9b 100644 --- a/src/ScadaLink.Host/SiteServiceRegistration.cs +++ b/src/ScadaLink.Host/SiteServiceRegistration.cs @@ -42,6 +42,14 @@ public static class SiteServiceRegistration var siteDbPath = config["ScadaLink:Database:SiteDbPath"] ?? "site.db"; services.AddSiteRuntime($"Data Source={siteDbPath}"); services.AddDataConnectionLayer(); + // Audit Log #23 (M3 Bundle F): adapter that surfaces the site id to + // StoreAndForwardService through DI WITHOUT introducing a + // StoreAndForward → HealthMonitoring project-reference cycle. Must be + // registered BEFORE AddStoreAndForward so the S&F factory resolves a + // non-empty SiteId at construction time (otherwise the S&F service is + // a singleton and the empty-string value would be cached for the + // lifetime of the process). + services.AddSingleton(); services.AddStoreAndForward(); services.AddSiteEventLogging(); diff --git a/src/ScadaLink.Host/StoreAndForwardSiteContext.cs b/src/ScadaLink.Host/StoreAndForwardSiteContext.cs new file mode 100644 index 0000000..365959e --- /dev/null +++ b/src/ScadaLink.Host/StoreAndForwardSiteContext.cs @@ -0,0 +1,32 @@ +using Microsoft.Extensions.Options; +using ScadaLink.StoreAndForward; + +namespace ScadaLink.Host; + +/// +/// Audit Log #23 (M3 Bundle F): Host-side adapter implementing the +/// optional the Store-and-Forward +/// service consults to stamp cached-call audit telemetry with the site id. +/// +/// +/// Forwards verbatim — the same value +/// exposes to HealthMonitoring. Defined as +/// a separate adapter (rather than reusing ) +/// to avoid pulling HealthMonitoring into the StoreAndForward project's +/// dependency graph, which would create a project-reference cycle. +/// +public class StoreAndForwardSiteContext : IStoreAndForwardSiteContext +{ + public string SiteId { get; } + + public StoreAndForwardSiteContext(IOptions nodeOptions) + { + // NodeOptions.SiteId is nullable; SiteServiceRegistration ONLY adds + // this binding on the site role, so a non-null site id is expected + // here. Mirror SiteIdentityProvider's hard fail so a missing site id + // surfaces at composition time rather than at the first cached call. + SiteId = nodeOptions.Value.SiteId + ?? throw new InvalidOperationException( + "ScadaLink:Node:SiteId is required for the site role's StoreAndForward wiring."); + } +} diff --git a/src/ScadaLink.SiteCallAudit/ScadaLink.SiteCallAudit.csproj b/src/ScadaLink.SiteCallAudit/ScadaLink.SiteCallAudit.csproj new file mode 100644 index 0000000..d8b0e7a --- /dev/null +++ b/src/ScadaLink.SiteCallAudit/ScadaLink.SiteCallAudit.csproj @@ -0,0 +1,31 @@ + + + + net10.0 + enable + enable + true + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ScadaLink.SiteCallAudit/ServiceCollectionExtensions.cs b/src/ScadaLink.SiteCallAudit/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..6eb0d80 --- /dev/null +++ b/src/ScadaLink.SiteCallAudit/ServiceCollectionExtensions.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace ScadaLink.SiteCallAudit; + +/// +/// Composition root for the Site Call Audit (#22) component. +/// +/// +/// +/// M3 Bundle C ships the ingest-only minimum surface (the actor itself); the +/// full DI surface — reconciliation puller, KPI projector, central→site +/// Retry/Discard relay, options + validators — is deferred to a follow-up. +/// +/// +/// The repository (ISiteCallAuditRepository) is registered by +/// ScadaLink.ConfigurationDatabase.ServiceCollectionExtensions.AddConfigurationDatabase, +/// so callers (the Host on the central node) must also call that. The actor's +/// Props are wired up in Host registration (Bundle F); this extension +/// is currently a no-op placeholder kept for symmetry with the AuditLog and +/// NotificationOutbox composition roots — adding it now means consumers can +/// reference the method without re-touching the Host project later. +/// +/// +public static class ServiceCollectionExtensions +{ + /// + /// Registers Site Call Audit (#22) services. Currently a no-op + /// placeholder — Bundle F will populate this with the actor's Props + /// factory + options bindings. The method is exposed now so the Host + /// wiring call already exists at the API boundary. + /// + public static IServiceCollection AddSiteCallAudit(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + // Actor props are constructed in Host wiring (Bundle F). This + // extension is a placeholder for future config + DI. + return services; + } +} diff --git a/src/ScadaLink.SiteCallAudit/SiteCallAuditActor.cs b/src/ScadaLink.SiteCallAudit/SiteCallAuditActor.cs new file mode 100644 index 0000000..7506681 --- /dev/null +++ b/src/ScadaLink.SiteCallAudit/SiteCallAuditActor.cs @@ -0,0 +1,140 @@ +using Akka.Actor; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ScadaLink.Commons.Interfaces.Repositories; +using ScadaLink.Commons.Messages.Audit; + +namespace ScadaLink.SiteCallAudit; + +/// +/// Central singleton for Site Call Audit (#22). Receives +/// messages and persists each +/// row via +/// — idempotent monotonic +/// upsert. Out-of-order or duplicate updates are silent no-ops at the +/// repository layer; the actor always replies +/// with Accepted=true in that case because storage state is consistent +/// and the site is free to consider its packet acked. +/// +/// +/// +/// M3 ships the minimum surface: ingest only. Reconciliation, KPIs, and +/// central→site Retry/Discard relay are deferred (per CLAUDE.md scope +/// discipline — Site Call Audit's KPIs and the Retry/Discard relay land in a +/// follow-up). +/// +/// +/// Per CLAUDE.md "audit-write failure NEVER aborts the user-facing action" — +/// the actor catches every exception from the repository call and replies +/// Accepted=false without rethrowing, so the central singleton stays +/// alive. The uses Resume so an +/// unexpected throw before the catch (defence in depth) does not restart the +/// actor and reset in-flight state. +/// +/// +/// Two constructors exist for the same reason as +/// AuditLogIngestActor: production wiring (Bundle F) resolves the +/// scoped EF repository from a fresh DI scope per message because the actor +/// is a long-lived cluster singleton, while tests inject a concrete +/// against a per-test MSSQL fixture +/// so the actor exercises the real monotonic upsert SQL end to end. +/// +/// +public class SiteCallAuditActor : ReceiveActor +{ + private readonly IServiceProvider? _serviceProvider; + private readonly ISiteCallAuditRepository? _injectedRepository; + private readonly ILogger _logger; + + /// + /// Test-mode constructor — injects a concrete repository instance whose + /// lifetime exceeds the test, so the actor reuses the same instance + /// across every message. Used by Bundle C's MSSQL-backed TestKit fixture. + /// + public SiteCallAuditActor( + ISiteCallAuditRepository repository, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(repository); + ArgumentNullException.ThrowIfNull(logger); + + _injectedRepository = repository; + _logger = logger; + + ReceiveAsync(OnUpsertAsync); + } + + /// + /// Production constructor — resolves + /// from a fresh DI scope per message because the repository is a scoped EF + /// Core service registered by AddConfigurationDatabase. The actor + /// itself is a long-lived cluster singleton, so it cannot hold a scope + /// across messages. + /// + public SiteCallAuditActor( + IServiceProvider serviceProvider, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(serviceProvider); + ArgumentNullException.ThrowIfNull(logger); + + _serviceProvider = serviceProvider; + _logger = logger; + + ReceiveAsync(OnUpsertAsync); + } + + /// + /// Audit-write failures are best-effort by design (CLAUDE.md §Audit): a + /// thrown exception in the upsert pipeline must not crash the actor. + /// Resume keeps the actor's state intact so the next packet is processed + /// against the same repository instance. + /// + protected override SupervisorStrategy SupervisorStrategy() + { + return new OneForOneStrategy(maxNrOfRetries: 0, withinTimeRange: TimeSpan.Zero, decider: + Akka.Actor.SupervisorStrategy.DefaultDecider); + } + + private async Task OnUpsertAsync(UpsertSiteCallCommand cmd) + { + // Sender is captured before the first await — Akka resets Sender + // between message dispatches, so a post-await Tell would go to + // DeadLetters. + var replyTo = Sender; + var id = cmd.SiteCall.TrackedOperationId; + + // Scope-per-message mirrors AuditLogIngestActor — production EF + // repository is scoped; the injected-repository mode (tests) skips + // the scope entirely. + IServiceScope? scope = null; + ISiteCallAuditRepository repository; + if (_injectedRepository is not null) + { + repository = _injectedRepository; + } + else + { + scope = _serviceProvider!.CreateScope(); + repository = scope.ServiceProvider.GetRequiredService(); + } + + try + { + await repository.UpsertAsync(cmd.SiteCall).ConfigureAwait(false); + replyTo.Tell(new UpsertSiteCallReply(id, Accepted: true)); + } + catch (Exception ex) + { + // Per CLAUDE.md: audit-write failure NEVER aborts the user-facing + // action — log and reply Accepted=false; do NOT rethrow (the + // central singleton MUST stay alive). + _logger.LogError(ex, "SiteCallAudit upsert failed for {TrackedOperationId}", id); + replyTo.Tell(new UpsertSiteCallReply(id, Accepted: false)); + } + finally + { + scope?.Dispose(); + } + } +} diff --git a/src/ScadaLink.SiteRuntime/Actors/ScriptExecutionActor.cs b/src/ScadaLink.SiteRuntime/Actors/ScriptExecutionActor.cs index a812158..e5b84ae 100644 --- a/src/ScadaLink.SiteRuntime/Actors/ScriptExecutionActor.cs +++ b/src/ScadaLink.SiteRuntime/Actors/ScriptExecutionActor.cs @@ -2,6 +2,7 @@ using Akka.Actor; using Microsoft.CodeAnalysis.Scripting; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using ScadaLink.Commons.Interfaces; using ScadaLink.Commons.Interfaces.Services; using ScadaLink.Commons.Messages.ScriptExecution; using ScadaLink.Commons.Types; @@ -105,6 +106,18 @@ public class ScriptExecutionActor : ReceiveActor // composes the SQLite hot-path + drop-oldest ring); null in tests / hosts // that haven't called AddAuditLog, which the helper handles as a no-op. IAuditWriter? auditWriter = null; + // Audit Log #23 (M3 Bundle A — Task A3): site-local tracking store + // backing Tracking.Status(id). Singleton; null in tests / hosts + // that haven't wired the store, which the helper handles by + // throwing on access. + IOperationTrackingStore? operationTrackingStore = null; + // Audit Log #23 (M3 Bundle F — Task F1): site-side cached-call + // telemetry forwarder. Singleton bound to the AuditLog + // composition root; null in tests / hosts that haven't called + // AddAuditLog, in which case the cached-call helpers degrade + // to the no-emission path (the underlying S&F handoff still + // happens and a TrackedOperationId is still returned). + ICachedCallTelemetryForwarder? cachedForwarder = null; if (serviceProvider != null) { @@ -115,6 +128,8 @@ public class ScriptExecutionActor : ReceiveActor siteId = serviceScope.ServiceProvider.GetService()?.SiteId ?? string.Empty; auditWriter = serviceScope.ServiceProvider.GetService(); + operationTrackingStore = serviceScope.ServiceProvider.GetService(); + cachedForwarder = serviceScope.ServiceProvider.GetService(); } var context = new ScriptRuntimeContext( @@ -138,7 +153,18 @@ public class ScriptExecutionActor : ReceiveActor // ExternalSystem.Call. Writer is best-effort; failures are logged // and swallowed inside the helper so the script's call path is // never aborted by an audit failure. - auditWriter: auditWriter); + auditWriter: auditWriter, + // Audit Log #23 (M3 Bundle A — Task A3): site-local tracking store + // backing Tracking.Status(id). Authoritative source of truth for + // cached-call status — read directly by the script API. + operationTrackingStore: operationTrackingStore, + // Audit Log #23 (M3 Bundle F — Task F1): cached-call telemetry + // forwarder for ExternalSystem.CachedCall / Database.CachedWrite + // CachedSubmit emission + the immediate-success terminal-row + // emission. Best-effort: null degrades the helpers to a + // no-emission path; the S&F handoff and TrackedOperationId + // return are unaffected. + cachedForwarder: cachedForwarder); var globals = new ScriptGlobals { diff --git a/src/ScadaLink.SiteRuntime/Scripts/ScriptRuntimeContext.cs b/src/ScadaLink.SiteRuntime/Scripts/ScriptRuntimeContext.cs index 2ac8a38..b5eceaa 100644 --- a/src/ScadaLink.SiteRuntime/Scripts/ScriptRuntimeContext.cs +++ b/src/ScadaLink.SiteRuntime/Scripts/ScriptRuntimeContext.cs @@ -4,8 +4,10 @@ using System.Text.RegularExpressions; using Akka.Actor; using Microsoft.Extensions.Logging; using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces; using ScadaLink.Commons.Interfaces.Services; using ScadaLink.Commons.Messages.Instance; +using ScadaLink.Commons.Messages.Integration; using ScadaLink.Commons.Messages.Notification; using ScadaLink.Commons.Messages.ScriptExecution; using ScadaLink.Commons.Types; @@ -85,6 +87,24 @@ public class ScriptRuntimeContext /// private readonly IAuditWriter? _auditWriter; + /// + /// Audit Log #23 (M3): site-local tracking store consulted by + /// Tracking.Status(TrackedOperationId). Optional — when null the + /// helper throws on access, mirroring the existing + /// "service-not-wired" behaviour of the other integration helpers. + /// + private readonly IOperationTrackingStore? _operationTrackingStore; + + /// + /// Audit Log #23 (M3 Bundle E — Task E3): site-side dual emitter for + /// cached-call lifecycle telemetry. Optional — when null + /// ExternalSystem.CachedCall / Database.CachedWrite still + /// return a and invoke the underlying + /// store-and-forward path, but no audit / SiteCalls telemetry is emitted + /// (tests / minimal hosts that don't wire the audit pipeline). + /// + private readonly ICachedCallTelemetryForwarder? _cachedForwarder; + public ScriptRuntimeContext( IActorRef instanceActor, IActorRef self, @@ -100,7 +120,9 @@ public class ScriptRuntimeContext ICanTell? siteCommunicationActor = null, string siteId = "", string? sourceScript = null, - IAuditWriter? auditWriter = null) + IAuditWriter? auditWriter = null, + IOperationTrackingStore? operationTrackingStore = null, + ICachedCallTelemetryForwarder? cachedForwarder = null) { _instanceActor = instanceActor; _self = self; @@ -117,6 +139,8 @@ public class ScriptRuntimeContext _siteId = siteId; _sourceScript = sourceScript; _auditWriter = auditWriter; + _operationTrackingStore = operationTrackingStore; + _cachedForwarder = cachedForwarder; } /// @@ -217,14 +241,21 @@ public class ScriptRuntimeContext /// ExternalSystem.CachedCall("systemName", "methodName", params) /// public ExternalSystemHelper ExternalSystem => new( - _externalSystemClient, _instanceName, _logger, _auditWriter, _siteId, _sourceScript); + _externalSystemClient, _instanceName, _logger, _auditWriter, _siteId, _sourceScript, + // Audit Log #23 (M3 Bundle E — Task E3): emit CachedSubmit telemetry + // on every ExternalSystem.CachedCall enqueue. + _cachedForwarder); /// /// WP-13: Provides access to database operations. /// Database.Connection("name") /// Database.CachedWrite("name", "sql", params) /// - public DatabaseHelper Database => new(_databaseGateway, _instanceName, _logger); + public DatabaseHelper Database => new( + _databaseGateway, _instanceName, _logger, _siteId, _sourceScript, + // Audit Log #23 (M3 Bundle E — Task E6): emit CachedSubmit telemetry on + // every Database.CachedWrite enqueue. + _cachedForwarder); /// /// Provides access to the Notification Outbox API. @@ -235,6 +266,15 @@ public class ScriptRuntimeContext public NotifyHelper Notify => new( _storeAndForward, _siteCommunicationActor, _siteId, _instanceName, _sourceScript, _askTimeout, _logger); + /// + /// Audit Log #23 (M3): site-local tracking-status API for cached operations. + /// Tracking.Status(trackedOperationId) reads the site SQLite tracking row + /// directly (authoritative source of truth — no central round-trip) and + /// returns a , or null when the + /// id is unknown / has already been purged. + /// + public TrackingHelper Tracking => new(_operationTrackingStore, _logger); + /// /// Helper class for Scripts.CallShared() syntax. /// @@ -308,14 +348,19 @@ public class ScriptRuntimeContext private readonly IAuditWriter? _auditWriter; private readonly string _siteId; private readonly string? _sourceScript; + private readonly ICachedCallTelemetryForwarder? _cachedForwarder; + // Internal constructor for tests living in ScadaLink.SiteRuntime.Tests + // (via InternalsVisibleTo). Production sites resolve the helper through + // ScriptRuntimeContext.ExternalSystem. internal ExternalSystemHelper( IExternalSystemClient? client, string instanceName, ILogger logger, IAuditWriter? auditWriter = null, string siteId = "", - string? sourceScript = null) + string? sourceScript = null, + ICachedCallTelemetryForwarder? cachedForwarder = null) { _client = client; _instanceName = instanceName; @@ -323,6 +368,7 @@ public class ScriptRuntimeContext _auditWriter = auditWriter; _siteId = siteId; _sourceScript = sourceScript; + _cachedForwarder = cachedForwarder; } public async Task Call( @@ -361,7 +407,22 @@ public class ScriptRuntimeContext } } - public async Task CachedCall( + /// + /// Submit a cached outbound API call (Audit Log #23 / M3). Mints a + /// fresh , emits the lifecycle's first + /// CachedSubmit telemetry packet, hands the call to the + /// store-and-forward retry loop (which emits per-attempt and terminal + /// telemetry under the same id — Bundle E Tasks E4/E5), and returns + /// the id immediately so the script can later query + /// Tracking.Status(id). + /// + /// + /// Best-effort emission (alog.md §7): if the forwarder throws, + /// the failure is logged and swallowed; the underlying cached-call + /// path still runs and the id is still returned. The script must never + /// be aborted by an audit-pipeline failure. + /// + public async Task CachedCall( string systemName, string methodName, IReadOnlyDictionary? parameters = null, @@ -370,7 +431,307 @@ public class ScriptRuntimeContext if (_client == null) throw new InvalidOperationException("External system client not available"); - return await _client.CachedCallAsync(systemName, methodName, parameters, _instanceName, cancellationToken); + var trackedId = TrackedOperationId.New(); + var occurredAtUtc = DateTime.UtcNow; + var target = $"{systemName}.{methodName}"; + + // Emit CachedSubmit telemetry BEFORE handing off to the S&F + // engine — that way the SiteCalls row is materialised before the + // first delivery attempt and Tracking.Status(id) can observe a + // Submitted row even if the immediate-delivery attempt happens to + // resolve before this method returns. + await EmitCachedSubmitTelemetryAsync( + systemName, methodName, target, trackedId, occurredAtUtc, cancellationToken) + .ConfigureAwait(false); + + // Hand off to the existing cached-call path. The TrackedOperationId + // becomes the S&F message id so the retry loop (Bundle E Tasks + // E4/E5) can read it back via StoreAndForwardMessage.Id. + // + // M3 Bundle F (F2): the result is now retained because the + // immediate-success path (WasBuffered=false) bypasses S&F entirely + // — no retry loop, no ICachedCallLifecycleObserver fire. The + // helper must emit the Attempted + CachedResolve terminal rows + // itself, otherwise Tracking.Status(id) would stay in Submitted + // forever and the audit log would be missing the M3 lifecycle. + // The WasBuffered=true path is unaffected — the S&F retry loop + // owns the Attempted + Resolve emissions in that case. + ExternalCallResult? result; + try + { + result = await _client.CachedCallAsync( + systemName, + methodName, + parameters, + _instanceName, + cancellationToken, + trackedId).ConfigureAwait(false); + } + catch (Exception ex) + { + // The cached-call surface returns ExternalCallResult on permanent + // failure rather than throwing; a throw here is exceptional + // (e.g. cancellation, resolver outage). Log it and rethrow — the + // script does need to learn about catastrophic failures. The + // tracked id was still returned via the telemetry submit above. + _logger.LogWarning(ex, + "ExternalSystem.CachedCall threw for {System}.{Method} (TrackedOperationId {Id})", + systemName, methodName, trackedId); + throw; + } + + // M3 Bundle F (F2): immediate-completion lifecycle — emit the + // missing Attempted + CachedResolve rows when the underlying call + // resolved without engaging the store-and-forward retry loop. + if (result is { WasBuffered: false }) + { + await EmitImmediateTerminalTelemetryAsync( + systemName, methodName, target, trackedId, result, cancellationToken) + .ConfigureAwait(false); + } + + return trackedId; + } + + /// + /// Best-effort emission of the CachedSubmit lifecycle event. Any + /// exception thrown by the forwarder is logged and swallowed so the + /// calling script's enqueue is not disturbed. + /// + private async Task EmitCachedSubmitTelemetryAsync( + string systemName, + string methodName, + string target, + TrackedOperationId trackedId, + DateTime occurredAtUtc, + CancellationToken cancellationToken) + { + if (_cachedForwarder == null) + { + return; + } + + CachedCallTelemetry telemetry; + try + { + telemetry = new CachedCallTelemetry( + Audit: new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc), + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.CachedSubmit, + CorrelationId = trackedId.Value, + SourceSiteId = string.IsNullOrEmpty(_siteId) ? null : _siteId, + SourceInstanceId = _instanceName, + SourceScript = _sourceScript, + Target = target, + Status = AuditStatus.Submitted, + ForwardState = AuditForwardState.Pending, + }, + Operational: new SiteCallOperational( + TrackedOperationId: trackedId, + Channel: "ApiOutbound", + Target: target, + SourceSite: _siteId, + Status: "Submitted", + RetryCount: 0, + LastError: null, + HttpStatus: null, + CreatedAtUtc: occurredAtUtc, + UpdatedAtUtc: occurredAtUtc, + TerminalAtUtc: null)); + } + catch (Exception buildEx) + { + _logger.LogWarning(buildEx, + "Failed to build CachedSubmit telemetry for {System}.{Method} (TrackedOperationId {Id}) — skipping emission", + systemName, methodName, trackedId); + return; + } + + try + { + await _cachedForwarder.ForwardAsync(telemetry, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "CachedSubmit telemetry forward failed for {System}.{Method} (TrackedOperationId {Id})", + systemName, methodName, trackedId); + } + } + + /// + /// M3 Bundle F (F2): emit the Attempted + CachedResolve lifecycle + /// rows for an immediate-completion CachedCall (WasBuffered=false). + /// The S&F retry loop never engaged, so the + /// ICachedCallLifecycleObserver never fires — the helper must + /// produce both rows itself to keep the M3 audit contract whole + /// (Submit → Attempted → Resolve under one TrackedOperationId). + /// + /// + /// Best-effort emission: a throwing forwarder is logged and swallowed + /// per alog.md §7. The two rows are emitted INDEPENDENTLY so a single + /// forwarder fault doesn't drop both halves of the terminal pair. + /// + private async Task EmitImmediateTerminalTelemetryAsync( + string systemName, + string methodName, + string target, + TrackedOperationId trackedId, + ExternalCallResult result, + CancellationToken cancellationToken) + { + if (_cachedForwarder == null) + { + return; + } + + var occurredAtUtc = DateTime.UtcNow; + // Extract an HTTP status from the error message when present + // (mirrors EmitCallAudit's existing HttpStatusRegex behaviour so + // the immediate-failure row carries the same HttpStatus value the + // synchronous Call() audit row would have stamped). + int? httpStatus = null; + if (!result.Success && !string.IsNullOrEmpty(result.ErrorMessage)) + { + var match = HttpStatusRegex.Match(result.ErrorMessage); + if (match.Success && int.TryParse(match.Groups["code"].Value, out var code)) + { + httpStatus = code; + } + } + + // Status mapping for immediate completion: + // Success=true -> Delivered (audit) / "Delivered" (operational) + // Success=false -> Failed (audit) / "Failed" (operational) + // Permanent vs transient is not relevant here: a permanent failure + // returns Success=false WasBuffered=false (parked-equivalent); a + // transient failure with NO S&F engine wired likewise lands here + // with Success=false. Either way the terminal state is "the + // immediate attempt failed and the operation is done". + var auditTerminalStatus = result.Success + ? AuditStatus.Delivered + : AuditStatus.Failed; + var operationalTerminalStatus = result.Success ? "Delivered" : "Failed"; + + // --- Attempted row ------------------------------------------------- + CachedCallTelemetry attempted; + try + { + attempted = new CachedCallTelemetry( + Audit: new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc), + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCallCached, + CorrelationId = trackedId.Value, + SourceSiteId = string.IsNullOrEmpty(_siteId) ? null : _siteId, + SourceInstanceId = _instanceName, + SourceScript = _sourceScript, + Target = target, + Status = AuditStatus.Attempted, + HttpStatus = httpStatus, + ErrorMessage = result.Success ? null : result.ErrorMessage, + ForwardState = AuditForwardState.Pending, + }, + Operational: new SiteCallOperational( + TrackedOperationId: trackedId, + Channel: "ApiOutbound", + Target: target, + SourceSite: _siteId, + Status: "Attempted", + // RetryCount stays 0 — the operation never reached the + // S&F retry sweep, so no retries were performed. + RetryCount: 0, + LastError: result.Success ? null : result.ErrorMessage, + HttpStatus: httpStatus, + CreatedAtUtc: occurredAtUtc, + UpdatedAtUtc: occurredAtUtc, + TerminalAtUtc: null)); + } + catch (Exception buildEx) + { + _logger.LogWarning(buildEx, + "Failed to build immediate-Attempted telemetry for {System}.{Method} (TrackedOperationId {Id}) — skipping emission", + systemName, methodName, trackedId); + attempted = null!; + } + + if (attempted is not null) + { + try + { + await _cachedForwarder.ForwardAsync(attempted, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Immediate-Attempted telemetry forward failed for {System}.{Method} (TrackedOperationId {Id})", + systemName, methodName, trackedId); + } + } + + // --- CachedResolve row -------------------------------------------- + CachedCallTelemetry resolve; + try + { + resolve = new CachedCallTelemetry( + Audit: new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc), + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.CachedResolve, + CorrelationId = trackedId.Value, + SourceSiteId = string.IsNullOrEmpty(_siteId) ? null : _siteId, + SourceInstanceId = _instanceName, + SourceScript = _sourceScript, + Target = target, + Status = auditTerminalStatus, + HttpStatus = httpStatus, + ErrorMessage = result.Success ? null : result.ErrorMessage, + ForwardState = AuditForwardState.Pending, + }, + Operational: new SiteCallOperational( + TrackedOperationId: trackedId, + Channel: "ApiOutbound", + Target: target, + SourceSite: _siteId, + Status: operationalTerminalStatus, + RetryCount: 0, + LastError: result.Success ? null : result.ErrorMessage, + HttpStatus: httpStatus, + CreatedAtUtc: occurredAtUtc, + UpdatedAtUtc: occurredAtUtc, + // Immediate-completion terminal — mark TerminalAtUtc so + // SiteCallAudit can move the row to its purge eligible + // set. + TerminalAtUtc: occurredAtUtc)); + } + catch (Exception buildEx) + { + _logger.LogWarning(buildEx, + "Failed to build immediate-CachedResolve telemetry for {System}.{Method} (TrackedOperationId {Id}) — skipping emission", + systemName, methodName, trackedId); + return; + } + + try + { + await _cachedForwarder.ForwardAsync(resolve, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Immediate-CachedResolve telemetry forward failed for {System}.{Method} (TrackedOperationId {Id})", + systemName, methodName, trackedId); + } } /// @@ -516,17 +877,37 @@ public class ScriptRuntimeContext /// /// WP-13: Helper for Database.Connection/CachedWrite syntax. /// + /// + /// Audit Log #23 (M3 Bundle E — Task E6): mirrors + /// — mints a + /// , emits the lifecycle's first + /// CachedSubmit packet (Channel DbOutbound), hands off to the S&F + /// retry loop, and returns the id. Per-attempt + terminal telemetry is + /// emitted by the retry loop (Tasks E4/E5). + /// public class DatabaseHelper { private readonly IDatabaseGateway? _gateway; private readonly string _instanceName; private readonly ILogger _logger; + private readonly string _siteId; + private readonly string? _sourceScript; + private readonly ICachedCallTelemetryForwarder? _cachedForwarder; - internal DatabaseHelper(IDatabaseGateway? gateway, string instanceName, ILogger logger) + internal DatabaseHelper( + IDatabaseGateway? gateway, + string instanceName, + ILogger logger, + string siteId = "", + string? sourceScript = null, + ICachedCallTelemetryForwarder? cachedForwarder = null) { _gateway = gateway; _instanceName = instanceName; _logger = logger; + _siteId = siteId; + _sourceScript = sourceScript; + _cachedForwarder = cachedForwarder; } public async Task Connection( @@ -539,7 +920,13 @@ public class ScriptRuntimeContext return await _gateway.GetConnectionAsync(name, cancellationToken); } - public async Task CachedWrite( + /// + /// Submit a cached outbound database write. Mints a fresh + /// , emits CachedSubmit telemetry on + /// DbOutbound, hands off to the cached-write S&F path, and + /// returns the id. Best-effort emission per alog.md §7. + /// + public async Task CachedWrite( string name, string sql, IReadOnlyDictionary? parameters = null, @@ -548,7 +935,95 @@ public class ScriptRuntimeContext if (_gateway == null) throw new InvalidOperationException("Database gateway not available"); - await _gateway.CachedWriteAsync(name, sql, parameters, _instanceName, cancellationToken); + var trackedId = TrackedOperationId.New(); + var occurredAtUtc = DateTime.UtcNow; + // The DB cached-write target uses the connection name (the only + // human-readable handle the gateway carries on the buffered row). + var target = name; + + await EmitCachedDbSubmitTelemetryAsync( + name, trackedId, target, occurredAtUtc, cancellationToken) + .ConfigureAwait(false); + + try + { + await _gateway.CachedWriteAsync( + name, sql, parameters, _instanceName, cancellationToken, trackedId) + .ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Database.CachedWrite threw for {Connection} (TrackedOperationId {Id})", + name, trackedId); + throw; + } + + return trackedId; + } + + private async Task EmitCachedDbSubmitTelemetryAsync( + string connectionName, + TrackedOperationId trackedId, + string target, + DateTime occurredAtUtc, + CancellationToken cancellationToken) + { + if (_cachedForwarder == null) + { + return; + } + + CachedCallTelemetry telemetry; + try + { + telemetry = new CachedCallTelemetry( + Audit: new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc), + Channel = AuditChannel.DbOutbound, + Kind = AuditKind.CachedSubmit, + CorrelationId = trackedId.Value, + SourceSiteId = string.IsNullOrEmpty(_siteId) ? null : _siteId, + SourceInstanceId = _instanceName, + SourceScript = _sourceScript, + Target = target, + Status = AuditStatus.Submitted, + ForwardState = AuditForwardState.Pending, + }, + Operational: new SiteCallOperational( + TrackedOperationId: trackedId, + Channel: "DbOutbound", + Target: target, + SourceSite: _siteId, + Status: "Submitted", + RetryCount: 0, + LastError: null, + HttpStatus: null, + CreatedAtUtc: occurredAtUtc, + UpdatedAtUtc: occurredAtUtc, + TerminalAtUtc: null)); + } + catch (Exception buildEx) + { + _logger.LogWarning(buildEx, + "Failed to build CachedSubmit telemetry for Database.CachedWrite {Connection} (TrackedOperationId {Id}) — skipping emission", + connectionName, trackedId); + return; + } + + try + { + await _cachedForwarder.ForwardAsync(telemetry, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "CachedSubmit telemetry forward failed for Database.CachedWrite {Connection} (TrackedOperationId {Id})", + connectionName, trackedId); + } } } @@ -746,4 +1221,46 @@ public class ScriptRuntimeContext return notificationId; } } + + /// + /// Audit Log #23 (M3): script-side accessor for cached-operation tracking. + /// Tracking.Status(trackedOperationId) reads the site-local SQLite + /// row directly via — + /// the site is the single source of truth for cached-call status, so no + /// central round-trip is needed and the call is answered authoritatively. + /// + public class TrackingHelper + { + private readonly IOperationTrackingStore? _store; + private readonly ILogger _logger; + + internal TrackingHelper(IOperationTrackingStore? store, ILogger logger) + { + _store = store; + _logger = logger; + } + + /// + /// Returns the latest tracking snapshot for the supplied id, or + /// null when the id is unknown (never recorded, or purged after + /// the retention window). + /// + /// + /// Thrown when the script runtime was constructed without an + /// — mirrors the + /// "service-not-wired" failure mode of the other integration helpers. + /// + public Task Status( + TrackedOperationId trackedOperationId, + CancellationToken cancellationToken = default) + { + if (_store == null) + { + throw new InvalidOperationException( + "Operation tracking store not available"); + } + + return _store.GetStatusAsync(trackedOperationId, cancellationToken); + } + } } diff --git a/src/ScadaLink.SiteRuntime/Tracking/OperationTrackingOptions.cs b/src/ScadaLink.SiteRuntime/Tracking/OperationTrackingOptions.cs new file mode 100644 index 0000000..b2fcf7d --- /dev/null +++ b/src/ScadaLink.SiteRuntime/Tracking/OperationTrackingOptions.cs @@ -0,0 +1,21 @@ +namespace ScadaLink.SiteRuntime.Tracking; + +/// +/// Options for — site-local cached-call +/// tracking SQLite store (Audit Log #23 / M3). +/// +public class OperationTrackingOptions +{ + /// + /// Full ADO.NET connection string for the SQLite database (e.g. + /// "Data Source=site-tracking.db"). Tests use the + /// Mode=Memory;Cache=Shared form to keep the database in-memory. + /// + public string ConnectionString { get; set; } = "Data Source=site-tracking.db"; + + /// + /// Retention window for terminal tracking rows. The default purge cadence + /// (driven by the host) deletes terminal rows older than this many days. + /// + public int RetentionDays { get; set; } = 7; +} diff --git a/src/ScadaLink.SiteRuntime/Tracking/OperationTrackingStore.cs b/src/ScadaLink.SiteRuntime/Tracking/OperationTrackingStore.cs new file mode 100644 index 0000000..8ef0d2f --- /dev/null +++ b/src/ScadaLink.SiteRuntime/Tracking/OperationTrackingStore.cs @@ -0,0 +1,333 @@ +using System.Globalization; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ScadaLink.Commons.Interfaces; +using ScadaLink.Commons.Types; + +namespace ScadaLink.SiteRuntime.Tracking; + +/// +/// Site-local SQLite source-of-truth for cached-operation tracking — the row +/// that Tracking.Status(TrackedOperationId) reads (Audit Log #23 / M3). +/// +/// +/// +/// One row per ; lifecycle is +/// Submitted → Retrying → Delivered / Parked / Failed / Discarded; terminal +/// rows are purged after the configured retention window +/// (). Volume is bounded — +/// only cached calls produce rows, and only a handful of lifecycle events per +/// call — so we keep the implementation deliberately simple: a single owned +/// serialised behind a +/// (one async writer at a time). This is the pattern the M3 brief calls out as +/// "cleaner than the M2 Channel<T> pipeline given the volume"; the M2 +/// audit-writer's batched-channel design is reserved for the high-volume audit +/// hot-path. +/// +/// +/// All mutations are idempotent / monotonic: is +/// INSERT OR IGNORE, filters out terminal +/// rows in the WHERE clause, and only +/// fires on rows that haven't terminated yet (first-write-wins). This makes the +/// store safe under the at-least-once semantics of the site→central telemetry +/// path. +/// +/// +public class OperationTrackingStore : IOperationTrackingStore, IAsyncDisposable, IDisposable +{ + private readonly SqliteConnection _connection; + private readonly SemaphoreSlim _gate = new(1, 1); + private readonly ILogger _logger; + private bool _disposed; + + public OperationTrackingStore( + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + + _logger = logger; + _connection = new SqliteConnection(options.Value.ConnectionString); + _connection.Open(); + InitializeSchema(); + } + + private void InitializeSchema() + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + CREATE TABLE IF NOT EXISTS OperationTracking ( + TrackedOperationId TEXT NOT NULL PRIMARY KEY, + Kind TEXT NOT NULL, + TargetSummary TEXT NULL, + Status TEXT NOT NULL, + RetryCount INTEGER NOT NULL DEFAULT 0, + LastError TEXT NULL, + HttpStatus INTEGER NULL, + CreatedAtUtc TEXT NOT NULL, + UpdatedAtUtc TEXT NOT NULL, + TerminalAtUtc TEXT NULL, + SourceInstanceId TEXT NULL, + SourceScript TEXT NULL + ); + CREATE INDEX IF NOT EXISTS IX_OperationTracking_Status_Updated + ON OperationTracking (Status, UpdatedAtUtc); + """; + cmd.ExecuteNonQuery(); + } + + /// + public async Task RecordEnqueueAsync( + TrackedOperationId id, + string kind, + string? targetSummary, + string? sourceInstanceId, + string? sourceScript, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(kind); + + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + ObjectDisposedException.ThrowIf(_disposed, this); + + var now = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture); + + using var cmd = _connection.CreateCommand(); + // INSERT OR IGNORE: duplicate ids are no-ops (first-write-wins) — + // matches the at-least-once semantics the site emits under. + cmd.CommandText = """ + INSERT OR IGNORE INTO OperationTracking ( + TrackedOperationId, Kind, TargetSummary, Status, + RetryCount, LastError, HttpStatus, + CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, + SourceInstanceId, SourceScript + ) VALUES ( + $id, $kind, $targetSummary, $status, + 0, NULL, NULL, + $now, $now, NULL, + $sourceInstanceId, $sourceScript + ); + """; + cmd.Parameters.AddWithValue("$id", id.ToString()); + cmd.Parameters.AddWithValue("$kind", kind); + cmd.Parameters.AddWithValue("$targetSummary", (object?)targetSummary ?? DBNull.Value); + cmd.Parameters.AddWithValue("$status", "Submitted"); + cmd.Parameters.AddWithValue("$now", now); + cmd.Parameters.AddWithValue("$sourceInstanceId", (object?)sourceInstanceId ?? DBNull.Value); + cmd.Parameters.AddWithValue("$sourceScript", (object?)sourceScript ?? DBNull.Value); + + cmd.ExecuteNonQuery(); + } + finally + { + _gate.Release(); + } + } + + /// + public async Task RecordAttemptAsync( + TrackedOperationId id, + string status, + int retryCount, + string? lastError, + int? httpStatus, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(status); + + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + ObjectDisposedException.ThrowIf(_disposed, this); + + var now = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture); + + using var cmd = _connection.CreateCommand(); + // Terminal rows are immutable — the WHERE clause filters them out so + // late-arriving attempt telemetry never overwrites a resolved row. + cmd.CommandText = """ + UPDATE OperationTracking + SET Status = $status, + RetryCount = $retryCount, + LastError = $lastError, + HttpStatus = $httpStatus, + UpdatedAtUtc = $now + WHERE TrackedOperationId = $id + AND TerminalAtUtc IS NULL; + """; + cmd.Parameters.AddWithValue("$id", id.ToString()); + cmd.Parameters.AddWithValue("$status", status); + cmd.Parameters.AddWithValue("$retryCount", retryCount); + cmd.Parameters.AddWithValue("$lastError", (object?)lastError ?? DBNull.Value); + cmd.Parameters.AddWithValue("$httpStatus", (object?)httpStatus ?? DBNull.Value); + cmd.Parameters.AddWithValue("$now", now); + + cmd.ExecuteNonQuery(); + } + finally + { + _gate.Release(); + } + } + + /// + public async Task RecordTerminalAsync( + TrackedOperationId id, + string status, + string? lastError, + int? httpStatus, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(status); + + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + ObjectDisposedException.ThrowIf(_disposed, this); + + var now = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture); + + using var cmd = _connection.CreateCommand(); + // First-write-wins on the terminal flip: only update rows that + // haven't already terminated. + cmd.CommandText = """ + UPDATE OperationTracking + SET Status = $status, + LastError = $lastError, + HttpStatus = $httpStatus, + UpdatedAtUtc = $now, + TerminalAtUtc = $now + WHERE TrackedOperationId = $id + AND TerminalAtUtc IS NULL; + """; + cmd.Parameters.AddWithValue("$id", id.ToString()); + cmd.Parameters.AddWithValue("$status", status); + cmd.Parameters.AddWithValue("$lastError", (object?)lastError ?? DBNull.Value); + cmd.Parameters.AddWithValue("$httpStatus", (object?)httpStatus ?? DBNull.Value); + cmd.Parameters.AddWithValue("$now", now); + + cmd.ExecuteNonQuery(); + } + finally + { + _gate.Release(); + } + } + + /// + public async Task GetStatusAsync( + TrackedOperationId id, + CancellationToken ct = default) + { + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + ObjectDisposedException.ThrowIf(_disposed, this); + + using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + SELECT TrackedOperationId, Kind, TargetSummary, Status, + RetryCount, LastError, HttpStatus, + CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, + SourceInstanceId, SourceScript + FROM OperationTracking + WHERE TrackedOperationId = $id; + """; + cmd.Parameters.AddWithValue("$id", id.ToString()); + + using var reader = cmd.ExecuteReader(); + if (!reader.Read()) + { + return null; + } + + return new TrackingStatusSnapshot( + Id: TrackedOperationId.Parse(reader.GetString(0)), + Kind: reader.GetString(1), + TargetSummary: reader.IsDBNull(2) ? null : reader.GetString(2), + Status: reader.GetString(3), + RetryCount: reader.GetInt32(4), + LastError: reader.IsDBNull(5) ? null : reader.GetString(5), + HttpStatus: reader.IsDBNull(6) ? null : reader.GetInt32(6), + CreatedAtUtc: ParseUtc(reader.GetString(7)), + UpdatedAtUtc: ParseUtc(reader.GetString(8)), + TerminalAtUtc: reader.IsDBNull(9) ? null : ParseUtc(reader.GetString(9)), + SourceInstanceId: reader.IsDBNull(10) ? null : reader.GetString(10), + SourceScript: reader.IsDBNull(11) ? null : reader.GetString(11)); + } + finally + { + _gate.Release(); + } + } + + /// + public async Task PurgeTerminalAsync( + DateTime olderThanUtc, + CancellationToken ct = default) + { + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + ObjectDisposedException.ThrowIf(_disposed, this); + + using var cmd = _connection.CreateCommand(); + // Non-terminal rows (TerminalAtUtc IS NULL) are kept regardless of + // age — the operation is still in flight. + cmd.CommandText = """ + DELETE FROM OperationTracking + WHERE TerminalAtUtc IS NOT NULL + AND TerminalAtUtc < $threshold; + """; + cmd.Parameters.AddWithValue( + "$threshold", + olderThanUtc.ToString("o", CultureInfo.InvariantCulture)); + + cmd.ExecuteNonQuery(); + } + finally + { + _gate.Release(); + } + } + + private static DateTime ParseUtc(string raw) + { + return DateTime.Parse( + raw, + CultureInfo.InvariantCulture, + DateTimeStyles.RoundtripKind); + } + + public void Dispose() + { + DisposeAsyncCore().AsTask().GetAwaiter().GetResult(); + GC.SuppressFinalize(this); + } + + public async ValueTask DisposeAsync() + { + await DisposeAsyncCore().ConfigureAwait(false); + GC.SuppressFinalize(this); + } + + private async ValueTask DisposeAsyncCore() + { + await _gate.WaitAsync().ConfigureAwait(false); + try + { + if (_disposed) return; + _disposed = true; + _connection.Dispose(); + } + finally + { + _gate.Release(); + _gate.Dispose(); + } + } +} diff --git a/src/ScadaLink.StoreAndForward/IStoreAndForwardSiteContext.cs b/src/ScadaLink.StoreAndForward/IStoreAndForwardSiteContext.cs new file mode 100644 index 0000000..e15290f --- /dev/null +++ b/src/ScadaLink.StoreAndForward/IStoreAndForwardSiteContext.cs @@ -0,0 +1,27 @@ +namespace ScadaLink.StoreAndForward; + +/// +/// Optional ambient site context the Store-and-Forward service consults at +/// construction time. Carries the site identifier the S&F retry loop +/// stamps onto cached-call audit telemetry (Audit Log #23 / M3 Bundle F). +/// +/// +/// +/// Defined here (not in HealthMonitoring alongside the existing +/// ISiteIdentityProvider) so the dependency arrow does not flip: +/// HealthMonitoring already references StoreAndForward, and +/// having S&F take a dependency on HealthMonitoring would create a +/// project-reference cycle. +/// +/// +/// The Host registers a trivial adapter that forwards to the same +/// NodeOptions.SiteId the existing ISiteIdentityProvider reads. +/// Resolution is optional: when no binding is registered the S&F service +/// stamps an empty site id, preserving the legacy pre-M3 behaviour exactly. +/// +/// +public interface IStoreAndForwardSiteContext +{ + /// The site id stamped onto cached-call audit telemetry. + string SiteId { get; } +} diff --git a/src/ScadaLink.StoreAndForward/ServiceCollectionExtensions.cs b/src/ScadaLink.StoreAndForward/ServiceCollectionExtensions.cs index d1e50ac..4e9998f 100644 --- a/src/ScadaLink.StoreAndForward/ServiceCollectionExtensions.cs +++ b/src/ScadaLink.StoreAndForward/ServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ScadaLink.Commons.Interfaces.Services; namespace ScadaLink.StoreAndForward; @@ -23,7 +24,28 @@ public static class ServiceCollectionExtensions var options = sp.GetRequiredService>().Value; var logger = sp.GetRequiredService>(); var replication = sp.GetRequiredService(); - return new StoreAndForwardService(storage, options, logger, replication); + // Audit Log #23 (M3 Bundle F): Wire the cached-call lifecycle + // observer + site identity through DI so the S&F retry loop emits + // per-attempt + terminal telemetry under the same TrackedOperationId + // the script-thread CachedSubmit row used. Both bindings are + // optional — when null the legacy pre-M3 retry behaviour is + // preserved exactly (tests, central nodes without sites, hosts + // that haven't called AddAuditLog). + // + // Site identity is resolved through the optional + // IStoreAndForwardSiteContext binding (registered by the Host) to + // avoid a project-reference cycle with HealthMonitoring's + // ISiteIdentityProvider — HealthMonitoring already references S&F. + var cachedCallObserver = sp.GetService(); + var siteContext = sp.GetService(); + var siteId = siteContext?.SiteId ?? string.Empty; + return new StoreAndForwardService( + storage, + options, + logger, + replication, + cachedCallObserver, + siteId); }); services.AddSingleton(sp => diff --git a/src/ScadaLink.StoreAndForward/StoreAndForwardService.cs b/src/ScadaLink.StoreAndForward/StoreAndForwardService.cs index 4a38216..70f6c23 100644 --- a/src/ScadaLink.StoreAndForward/StoreAndForwardService.cs +++ b/src/ScadaLink.StoreAndForward/StoreAndForwardService.cs @@ -1,4 +1,6 @@ using Microsoft.Extensions.Logging; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Types; using ScadaLink.Commons.Types.Enums; namespace ScadaLink.StoreAndForward; @@ -33,6 +35,19 @@ public class StoreAndForwardService private readonly StoreAndForwardOptions _options; private readonly ReplicationService? _replication; private readonly ILogger _logger; + /// + /// Audit Log #23 (M3 Bundle E — Task E4): site-side observer notified + /// after every cached-call delivery attempt. Optional — when null no + /// telemetry is emitted; the legacy pre-M3 retry loop behaviour is + /// preserved exactly. + /// + private readonly ICachedCallLifecycleObserver? _cachedCallObserver; + /// + /// Audit Log #23 (M3 Bundle E — Task E4): site id stamped onto the + /// cached-call attempt context so the audit bridge can build the + /// half of the telemetry packet. + /// + private readonly string _siteId; private Timer? _retryTimer; private int _retryInProgress; @@ -63,12 +78,16 @@ public class StoreAndForwardService StoreAndForwardStorage storage, StoreAndForwardOptions options, ILogger logger, - ReplicationService? replication = null) + ReplicationService? replication = null, + ICachedCallLifecycleObserver? cachedCallObserver = null, + string siteId = "") { _storage = storage; _options = options; _logger = logger; _replication = replication; + _cachedCallObserver = cachedCallObserver; + _siteId = siteId; } /// @@ -280,15 +299,33 @@ public class StoreAndForwardService return; } + // Audit Log #23 (M3 Bundle E — Tasks E4/E5): measure per-attempt + // duration so the audit row carries a meaningful DurationMs. Captured + // around the handler invocation only — storage / replication overhead + // is excluded. + var attemptStartUtc = DateTime.UtcNow; + var attemptStopwatch = System.Diagnostics.Stopwatch.StartNew(); + try { var success = await handler(message); + attemptStopwatch.Stop(); if (success) { await _storage.RemoveMessageAsync(message.Id); _replication?.ReplicateRemove(message.Id); RaiseActivity("Delivered", message.Category, $"Delivered to {message.Target} after {message.RetryCount} retries"); + + // M3: terminal Delivered observer notification — the audit + // bridge maps this to Attempted + CachedResolve(Delivered). + await NotifyCachedCallObserverAsync( + message, + CachedCallAttemptOutcome.Delivered, + lastError: null, + httpStatus: null, + occurredAtUtc: attemptStartUtc, + durationMs: (int)attemptStopwatch.ElapsedMilliseconds); return; } @@ -311,9 +348,20 @@ public class StoreAndForwardService _replication?.ReplicatePark(message); RaiseActivity("Parked", message.Category, $"Permanent failure for {message.Target}: handler returned false"); + + // M3: terminal PermanentFailure observer notification — the + // audit bridge maps this to Attempted(Failed) + CachedResolve(Parked). + await NotifyCachedCallObserverAsync( + message, + CachedCallAttemptOutcome.PermanentFailure, + lastError: message.LastError, + httpStatus: null, + occurredAtUtc: attemptStartUtc, + durationMs: (int)attemptStopwatch.ElapsedMilliseconds); } catch (Exception ex) { + attemptStopwatch.Stop(); // Transient failure — increment retry, check max message.RetryCount++; message.LastAttemptAt = DateTimeOffset.UtcNow; @@ -339,6 +387,16 @@ public class StoreAndForwardService _logger.LogWarning( "Message {MessageId} parked after {MaxRetries} retries to {Target}", message.Id, message.MaxRetries, message.Target); + + // M3: terminal ParkedMaxRetries observer notification — the + // audit bridge maps this to Attempted(Failed) + CachedResolve(Parked). + await NotifyCachedCallObserverAsync( + message, + CachedCallAttemptOutcome.ParkedMaxRetries, + lastError: ex.Message, + httpStatus: null, + occurredAtUtc: attemptStartUtc, + durationMs: (int)attemptStopwatch.ElapsedMilliseconds); } else { @@ -355,10 +413,113 @@ public class StoreAndForwardService } RaiseActivity("Retried", message.Category, $"Retry {message.RetryCount}/{message.MaxRetries} for {message.Target}: {ex.Message}"); + + // M3: per-attempt TransientFailure observer notification — + // the audit bridge maps this to Attempted(Failed). + await NotifyCachedCallObserverAsync( + message, + CachedCallAttemptOutcome.TransientFailure, + lastError: ex.Message, + httpStatus: null, + occurredAtUtc: attemptStartUtc, + durationMs: (int)attemptStopwatch.ElapsedMilliseconds); } } } + /// + /// Audit Log #23 (M3 Bundle E — Tasks E4/E5): notify the registered + /// of the just-completed + /// attempt. Only fires for cached-call categories + /// ( and + /// ); the + /// category has its + /// own central-side audit pipeline (Notification Outbox / #21) and must + /// not surface on this hook. + /// + /// + /// Best-effort: an observer that throws is logged and swallowed so a + /// failing audit pipeline cannot corrupt S&F retry bookkeeping + /// (alog.md §7 contract). Messages whose ids are not valid GUIDs (pre-M3 + /// callers that didn't thread a TrackedOperationId in) are silently + /// skipped — the observer requires a parseable id by contract. + /// + private async Task NotifyCachedCallObserverAsync( + StoreAndForwardMessage message, + CachedCallAttemptOutcome outcome, + string? lastError, + int? httpStatus, + DateTime occurredAtUtc, + int? durationMs) + { + if (_cachedCallObserver == null) + { + return; + } + + // Only cached-call categories generate audit telemetry on this hook — + // notifications have their own outbox-side audit pipeline. + var channel = message.Category switch + { + StoreAndForwardCategory.ExternalSystem => "ApiOutbound", + StoreAndForwardCategory.CachedDbWrite => "DbOutbound", + _ => null, + }; + if (channel is null) + { + return; + } + + if (!TrackedOperationId.TryParse(message.Id, out var trackedId)) + { + // Pre-M3 message (random GUID-N id from S&F itself, no + // TrackedOperationId threaded in). Skip — no audit row to bind to. + return; + } + + CachedCallAttemptContext context; + try + { + context = new CachedCallAttemptContext( + TrackedOperationId: trackedId, + Channel: channel, + Target: message.Target, + SourceSite: _siteId, + Outcome: outcome, + RetryCount: message.RetryCount, + LastError: lastError, + HttpStatus: httpStatus, + CreatedAtUtc: message.CreatedAt.UtcDateTime, + OccurredAtUtc: DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc), + DurationMs: durationMs, + SourceInstanceId: message.OriginInstanceName); + } + catch (Exception buildEx) + { + // Defensive — record construction shouldn't throw, but the alog.md + // §7 contract requires this path be exception-safe regardless. + _logger.LogWarning(buildEx, + "Failed to build cached-call attempt context for {MessageId}; observer skipped", + message.Id); + return; + } + + try + { + await _cachedCallObserver.OnAttemptCompletedAsync(context, CancellationToken.None) + .ConfigureAwait(false); + } + catch (Exception ex) + { + // alog.md §7 best-effort: an audit observer outage must NEVER be + // misclassified as a transient delivery failure or corrupt the + // S&F retry bookkeeping. + _logger.LogWarning(ex, + "ICachedCallLifecycleObserver threw for {MessageId} (Outcome {Outcome}); ignored", + message.Id, outcome); + } + } + /// /// WP-12: Gets parked messages for central query (Pattern 8). /// diff --git a/tests/ScadaLink.AuditLog.Tests/Central/AuditLogIngestActorCombinedTelemetryTests.cs b/tests/ScadaLink.AuditLog.Tests/Central/AuditLogIngestActorCombinedTelemetryTests.cs new file mode 100644 index 0000000..f61d1cd --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Central/AuditLogIngestActorCombinedTelemetryTests.cs @@ -0,0 +1,391 @@ +using Akka.Actor; +using Akka.TestKit.Xunit2; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using ScadaLink.AuditLog.Central; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces.Repositories; +using ScadaLink.Commons.Messages.Audit; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Audit; +using ScadaLink.Commons.Types.Enums; +using ScadaLink.ConfigurationDatabase; +using ScadaLink.ConfigurationDatabase.Repositories; +using ScadaLink.ConfigurationDatabase.Tests.Migrations; + +namespace ScadaLink.AuditLog.Tests.Central; + +/// +/// Bundle D D2 tests for 's M3 combined- +/// telemetry dual-write transaction. Uses the same +/// as the M1 + M2 repository tests so the actor exercises real +/// + +/// against a per-test MSSQL +/// database. The transaction commits or rolls back inside one +/// . +/// +public class AuditLogIngestActorCombinedTelemetryTests : TestKit, IClassFixture +{ + private readonly MsSqlMigrationFixture _fixture; + + public AuditLogIngestActorCombinedTelemetryTests(MsSqlMigrationFixture fixture) + { + _fixture = fixture; + } + + private ScadaLinkDbContext CreateReadContext() + { + var options = new DbContextOptionsBuilder() + .UseSqlServer(_fixture.ConnectionString) + .Options; + return new ScadaLinkDbContext(options); + } + + private static string NewSiteId() => + "test-bundle-d2-cached-" + Guid.NewGuid().ToString("N").Substring(0, 8); + + private static (AuditEvent audit, SiteCall siteCall) NewEntry( + string siteId, + TrackedOperationId? trackedOperationId = null, + Guid? eventId = null, + string status = "Submitted", + AuditStatus auditStatus = AuditStatus.Submitted) + { + var trackedId = trackedOperationId ?? TrackedOperationId.New(); + var now = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc); + + var audit = new AuditEvent + { + EventId = eventId ?? Guid.NewGuid(), + OccurredAtUtc = now, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.CachedSubmit, + Status = auditStatus, + SourceSiteId = siteId, + CorrelationId = trackedId.Value, + }; + + var siteCall = new SiteCall + { + TrackedOperationId = trackedId, + Channel = "ApiOutbound", + Target = "ERP.GetOrder", + SourceSite = siteId, + Status = status, + RetryCount = 0, + CreatedAtUtc = now, + UpdatedAtUtc = now, + IngestedAtUtc = now, // overwritten by the actor + }; + + return (audit, siteCall); + } + + /// + /// Builds a minimal DI container around the per-test MSSQL fixture's + /// connection string — DbContext + the two repositories the dual-write + /// handler resolves. Mirrors AddConfigurationDatabase without the + /// DataProtection wiring (we never write secret columns in these tests). + /// + private IServiceProvider BuildServiceProvider( + Func? siteCallRepoFactory = null) + { + var services = new ServiceCollection(); + services.AddDbContext(opts => + opts.UseSqlServer(_fixture.ConnectionString) + .ConfigureWarnings(w => w.Ignore( + Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))); + services.AddScoped(sp => + new AuditLogRepository(sp.GetRequiredService())); + if (siteCallRepoFactory is null) + { + services.AddScoped(sp => + new SiteCallAuditRepository(sp.GetRequiredService())); + } + else + { + services.AddScoped(sp => + siteCallRepoFactory(sp.GetRequiredService())); + } + return services.BuildServiceProvider(); + } + + private IActorRef CreateActor(IServiceProvider serviceProvider) => + Sys.ActorOf(Props.Create(() => new AuditLogIngestActor( + serviceProvider, + NullLogger.Instance))); + + [SkippableFact] + public async Task Receive_OneCachedPacket_WritesAuditRow_AND_SiteCallRow_AcksId() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var (audit, siteCall) = NewEntry(siteId); + + var sp = BuildServiceProvider(); + var actor = CreateActor(sp); + + actor.Tell( + new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(audit, siteCall) }), + TestActor); + + var reply = ExpectMsg(TimeSpan.FromSeconds(15)); + Assert.Single(reply.AcceptedEventIds); + Assert.Equal(audit.EventId, reply.AcceptedEventIds[0]); + + // Verify rows landed in both tables. + await using var read = CreateReadContext(); + var auditRow = await read.Set().SingleOrDefaultAsync(e => e.EventId == audit.EventId); + Assert.NotNull(auditRow); + Assert.NotNull(auditRow!.IngestedAtUtc); + + var siteCallRow = await read.Set() + .SingleOrDefaultAsync(s => s.TrackedOperationId == siteCall.TrackedOperationId); + Assert.NotNull(siteCallRow); + Assert.Equal(siteCall.Status, siteCallRow!.Status); + } + + [SkippableFact] + public async Task Receive_DuplicateEventId_SameStatus_NoOp_RowCountUnchanged_AcksId() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var trackedId = TrackedOperationId.New(); + var eventId = Guid.NewGuid(); + var (audit, siteCall) = NewEntry(siteId, trackedId, eventId); + + var sp = BuildServiceProvider(); + var actor = CreateActor(sp); + + // First write + actor.Tell( + new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(audit, siteCall) }), + TestActor); + ExpectMsg(TimeSpan.FromSeconds(15)); + + // Second write — same EventId and TrackedOperationId, same status. Both + // the audit insert (idempotent) and the SiteCalls upsert (monotonic + // same-rank → no-op) should silently do nothing while still acking. + actor.Tell( + new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(audit, siteCall) }), + TestActor); + var reply = ExpectMsg(TimeSpan.FromSeconds(15)); + + Assert.Single(reply.AcceptedEventIds); + Assert.Equal(eventId, reply.AcceptedEventIds[0]); + + await using var read = CreateReadContext(); + var auditCount = await read.Set().CountAsync(e => e.EventId == eventId); + Assert.Equal(1, auditCount); + + var siteCallCount = await read.Set() + .CountAsync(s => s.TrackedOperationId == trackedId); + Assert.Equal(1, siteCallCount); + } + + [SkippableFact] + public async Task Receive_DuplicateEventId_AdvancedSiteCallStatus_UpdatesSiteCall() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var trackedId = TrackedOperationId.New(); + + var sp = BuildServiceProvider(); + var actor = CreateActor(sp); + + // 1st packet — Submitted (audit EventId #1, SiteCalls Status=Submitted). + var (auditSubmit, siteCallSubmit) = NewEntry( + siteId, trackedId, status: "Submitted", auditStatus: AuditStatus.Submitted); + actor.Tell( + new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(auditSubmit, siteCallSubmit) }), + TestActor); + ExpectMsg(TimeSpan.FromSeconds(15)); + + // 2nd packet — Attempted with retry count 1 (audit EventId #2, + // SiteCalls Status=Attempted — monotonic upsert wins). Same + // TrackedOperationId throughout. + var (auditAttempt, siteCallAttempt) = NewEntry( + siteId, trackedId, status: "Attempted", auditStatus: AuditStatus.Attempted); + var advanced = siteCallAttempt with { RetryCount = 1, UpdatedAtUtc = siteCallAttempt.UpdatedAtUtc.AddMinutes(1) }; + actor.Tell( + new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(auditAttempt, advanced) }), + TestActor); + var reply = ExpectMsg(TimeSpan.FromSeconds(15)); + + Assert.Single(reply.AcceptedEventIds); + Assert.Equal(auditAttempt.EventId, reply.AcceptedEventIds[0]); + + // Both audit rows exist. + await using var read = CreateReadContext(); + var auditRows = await read.Set() + .Where(e => e.SourceSiteId == siteId) + .ToListAsync(); + Assert.Equal(2, auditRows.Count); + + // SiteCalls row advanced to Attempted with retry count 1. + var siteCallRow = await read.Set() + .SingleAsync(s => s.TrackedOperationId == trackedId); + Assert.Equal("Attempted", siteCallRow.Status); + Assert.Equal(1, siteCallRow.RetryCount); + } + + [SkippableFact] + public async Task Receive_AuditInsertSucceeds_SiteCallThrows_BothRolledBack_NoOrphanRow() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var (audit, siteCall) = NewEntry(siteId); + + // Wrap the SiteCalls repo so UpsertAsync always throws — the dual-write + // transaction must roll back the AuditLog INSERT done in the same + // transaction, leaving no orphan row. + var sp = BuildServiceProvider( + ctx => new ThrowingSiteCallRepo(new SiteCallAuditRepository(ctx))); + var actor = CreateActor(sp); + + actor.Tell( + new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(audit, siteCall) }), + TestActor); + + var reply = ExpectMsg(TimeSpan.FromSeconds(15)); + Assert.Empty(reply.AcceptedEventIds); + + await using var read = CreateReadContext(); + var auditRow = await read.Set().SingleOrDefaultAsync(e => e.EventId == audit.EventId); + Assert.Null(auditRow); + + var siteCallRow = await read.Set() + .SingleOrDefaultAsync(s => s.TrackedOperationId == siteCall.TrackedOperationId); + Assert.Null(siteCallRow); + } + + [SkippableFact] + public async Task Receive_FiveCachedPackets_AllPersistedSeparately_AllAcked() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var entries = Enumerable.Range(0, 5).Select(_ => + { + var (audit, siteCall) = NewEntry(siteId); + return new CachedTelemetryEntry(audit, siteCall); + }).ToList(); + + var sp = BuildServiceProvider(); + var actor = CreateActor(sp); + + actor.Tell(new IngestCachedTelemetryCommand(entries), TestActor); + + var reply = ExpectMsg(TimeSpan.FromSeconds(15)); + Assert.Equal(5, reply.AcceptedEventIds.Count); + Assert.True(entries.Select(e => e.Audit.EventId).ToHashSet() + .SetEquals(reply.AcceptedEventIds.ToHashSet())); + + await using var read = CreateReadContext(); + var auditCount = await read.Set().CountAsync(e => e.SourceSiteId == siteId); + Assert.Equal(5, auditCount); + + var siteCallCount = await read.Set().CountAsync(s => s.SourceSite == siteId); + Assert.Equal(5, siteCallCount); + } + + [SkippableFact] + public async Task Receive_OnePacketSucceeds_NextPacketThrows_FirstStillCommitted_BatchContinues() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var (audit1, siteCall1) = NewEntry(siteId); + var (audit2, siteCall2) = NewEntry(siteId); + var (audit3, siteCall3) = NewEntry(siteId); + var poisonTrackedId = siteCall2.TrackedOperationId; + + // Throw only for the middle entry's TrackedOperationId — entries on + // either side must commit their own transactions independently. + var sp = BuildServiceProvider( + ctx => new PoisonOnIdSiteCallRepo(new SiteCallAuditRepository(ctx), poisonTrackedId)); + var actor = CreateActor(sp); + + actor.Tell( + new IngestCachedTelemetryCommand(new[] + { + new CachedTelemetryEntry(audit1, siteCall1), + new CachedTelemetryEntry(audit2, siteCall2), + new CachedTelemetryEntry(audit3, siteCall3), + }), + TestActor); + + var reply = ExpectMsg(TimeSpan.FromSeconds(15)); + + // Two entries committed; poison entry rolled back. + Assert.Equal(2, reply.AcceptedEventIds.Count); + Assert.Contains(audit1.EventId, reply.AcceptedEventIds); + Assert.Contains(audit3.EventId, reply.AcceptedEventIds); + Assert.DoesNotContain(audit2.EventId, reply.AcceptedEventIds); + + await using var read = CreateReadContext(); + var auditRows = await read.Set().Where(e => e.SourceSiteId == siteId).ToListAsync(); + Assert.Equal(2, auditRows.Count); + Assert.DoesNotContain(auditRows, r => r.EventId == audit2.EventId); + + var siteCallRows = await read.Set().Where(s => s.SourceSite == siteId).ToListAsync(); + Assert.Equal(2, siteCallRows.Count); + Assert.DoesNotContain(siteCallRows, r => r.TrackedOperationId == poisonTrackedId); + } + + /// + /// Test double — throws unconditionally from so + /// the dual-write transaction is forced to roll back. Lets the AuditLog + /// row insert succeed in-transaction; the rollback must remove it. + /// + private sealed class ThrowingSiteCallRepo : ISiteCallAuditRepository + { + private readonly ISiteCallAuditRepository _inner; + public ThrowingSiteCallRepo(ISiteCallAuditRepository inner) { _inner = inner; } + public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) => + throw new InvalidOperationException("simulated SiteCalls upsert failure"); + public Task GetAsync(TrackedOperationId id, CancellationToken ct = default) => + _inner.GetAsync(id, ct); + public Task> QueryAsync( + SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) => + _inner.QueryAsync(filter, paging, ct); + public Task PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) => + _inner.PurgeTerminalAsync(olderThanUtc, ct); + } + + /// + /// Test double — throws only when the supplied poison TrackedOperationId + /// is the one being upserted. Demonstrates per-entry transaction isolation: + /// one entry's failed transaction must not abort the batch's other entries. + /// + private sealed class PoisonOnIdSiteCallRepo : ISiteCallAuditRepository + { + private readonly ISiteCallAuditRepository _inner; + private readonly TrackedOperationId _poisonId; + public PoisonOnIdSiteCallRepo(ISiteCallAuditRepository inner, TrackedOperationId poisonId) + { + _inner = inner; + _poisonId = poisonId; + } + public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) + { + if (siteCall.TrackedOperationId == _poisonId) + { + throw new InvalidOperationException("simulated SiteCalls upsert failure for poison id"); + } + return _inner.UpsertAsync(siteCall, ct); + } + public Task GetAsync(TrackedOperationId id, CancellationToken ct = default) => + _inner.GetAsync(id, ct); + public Task> QueryAsync( + SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) => + _inner.QueryAsync(filter, paging, ct); + public Task PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) => + _inner.PurgeTerminalAsync(olderThanUtc, ct); + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Integration/CachedCallCombinedTelemetryTests.cs b/tests/ScadaLink.AuditLog.Tests/Integration/CachedCallCombinedTelemetryTests.cs new file mode 100644 index 0000000..342b028 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Integration/CachedCallCombinedTelemetryTests.cs @@ -0,0 +1,270 @@ +using Akka.TestKit.Xunit2; +using Microsoft.EntityFrameworkCore; +using ScadaLink.AuditLog.Tests.Integration.Infrastructure; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; +using ScadaLink.ConfigurationDatabase.Tests.Migrations; + +namespace ScadaLink.AuditLog.Tests.Integration; + +/// +/// Bundle G G2 end-to-end suite for cached ExternalSystem.CachedCall +/// lifecycle telemetry (Audit Log #23 / M3). Wires the full M3 pipeline: +/// site-local SQLite audit writer + operation tracking store + the production +/// + the test-side +/// that ALSO pushes each combined +/// packet through the stub gRPC client into the central +/// AuditLogIngestActor's dual-write transaction against a per-test +/// MSSQL database. Asserts the audit rows + the SiteCalls row + the +/// site-local tracking row converge to the expected shape for each lifecycle. +/// +/// +/// +/// The bridge is driven directly via +/// — these tests do NOT spin up the actual S&F retry loop; that would +/// require a full SiteRuntime host and is out of scope for M3 (the S&F +/// observer hooks are exercised in ScadaLink.StoreAndForward.Tests at +/// unit level). The submit row is emitted via +/// because the +/// production submit emission happens at the script-call site, not inside the +/// S&F loop. +/// +/// +/// Each test uses a unique SourceSite id (Guid suffix) so concurrent +/// tests sharing the per-fixture MSSQL database don't interfere with each +/// other. +/// +/// +public class CachedCallCombinedTelemetryTests : TestKit, IClassFixture +{ + private readonly MsSqlMigrationFixture _fixture; + + public CachedCallCombinedTelemetryTests(MsSqlMigrationFixture fixture) + { + _fixture = fixture; + } + + private static string NewSiteId() => + "test-g2-cached-" + Guid.NewGuid().ToString("N").Substring(0, 8); + + private static CachedCallTelemetry SubmitPacket( + TrackedOperationId id, string siteId, DateTime nowUtc, string target = "ERP.GetOrder") => + new( + Audit: new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = nowUtc, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.CachedSubmit, + CorrelationId = id.Value, + SourceSiteId = siteId, + SourceInstanceId = "Plant.Pump42", + SourceScript = "ScriptActor:doStuff", + Target = target, + Status = AuditStatus.Submitted, + ForwardState = AuditForwardState.Pending, + }, + Operational: new SiteCallOperational( + TrackedOperationId: id, + Channel: "ApiOutbound", + Target: target, + SourceSite: siteId, + Status: "Submitted", + RetryCount: 0, + LastError: null, + HttpStatus: null, + CreatedAtUtc: nowUtc, + UpdatedAtUtc: nowUtc, + TerminalAtUtc: null)); + + private static CachedCallAttemptContext AttemptContext( + TrackedOperationId id, + string siteId, + CachedCallAttemptOutcome outcome, + int retryCount, + string? lastError, + int? httpStatus, + DateTime createdUtc, + DateTime occurredUtc, + string target = "ERP.GetOrder", + string channel = "ApiOutbound") => + new( + TrackedOperationId: id, + Channel: channel, + Target: target, + SourceSite: siteId, + Outcome: outcome, + RetryCount: retryCount, + LastError: lastError, + HttpStatus: httpStatus, + CreatedAtUtc: createdUtc, + OccurredAtUtc: occurredUtc, + DurationMs: 42, + SourceInstanceId: "Plant.Pump42"); + + [SkippableFact] + public async Task CachedCall_FailFailSuccess_Emits_5_AuditRows_AND_1_SiteCall_Delivered() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var trackedId = TrackedOperationId.New(); + var t0 = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc); + + await using var harness = new CombinedTelemetryHarness(_fixture, this); + + // Submit + await harness.EmitSubmitAsync(SubmitPacket(trackedId, siteId, t0)); + + // Attempt 1: transient HTTP 500 + await harness.EmitAttemptAsync(AttemptContext( + trackedId, siteId, + CachedCallAttemptOutcome.TransientFailure, + retryCount: 1, lastError: "HTTP 500", httpStatus: 500, + createdUtc: t0, occurredUtc: t0.AddSeconds(5))); + + // Attempt 2: transient HTTP 500 + await harness.EmitAttemptAsync(AttemptContext( + trackedId, siteId, + CachedCallAttemptOutcome.TransientFailure, + retryCount: 2, lastError: "HTTP 500", httpStatus: 500, + createdUtc: t0, occurredUtc: t0.AddSeconds(15))); + + // Attempt 3: delivered (terminal — emits Attempted + CachedResolve) + await harness.EmitAttemptAsync(AttemptContext( + trackedId, siteId, + CachedCallAttemptOutcome.Delivered, + retryCount: 3, lastError: null, httpStatus: 200, + createdUtc: t0, occurredUtc: t0.AddSeconds(25))); + + // Central side: each forward through the dispatcher round-trips + // through the stub client + ingest actor, so by the time the awaits + // complete the rows are visible in MSSQL. + await using var read = harness.CreateReadContext(); + + // 1 Submit + 2 transient Attempted + 1 terminal Attempted + 1 + // CachedResolve = 5 audit rows. The plan allows 4-5; this is the + // happy path emitting exactly 5. + var auditRows = await read.Set() + .Where(e => e.SourceSiteId == siteId) + .ToListAsync(); + Assert.InRange(auditRows.Count, 4, 5); + + // All audit rows must share the same CorrelationId (= TrackedOperationId). + Assert.All(auditRows, r => Assert.Equal(trackedId.Value, r.CorrelationId)); + + // Exactly one CachedSubmit row. + Assert.Single(auditRows, r => r.Kind == AuditKind.CachedSubmit); + // Exactly one terminal CachedResolve row, status Delivered. + var resolve = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedResolve); + Assert.Equal(AuditStatus.Delivered, resolve.Status); + + // SiteCalls row: Delivered, RetryCount=3, TerminalAtUtc set. + var siteCall = await read.Set() + .SingleAsync(s => s.TrackedOperationId == trackedId); + Assert.Equal("Delivered", siteCall.Status); + Assert.Equal(3, siteCall.RetryCount); + Assert.NotNull(siteCall.TerminalAtUtc); + + // Site-local Tracking.Status mirrors the same outcome. + var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId); + Assert.NotNull(snapshot); + Assert.Equal("Delivered", snapshot!.Status); + Assert.NotNull(snapshot.TerminalAtUtc); + } + + [SkippableFact] + public async Task CachedCall_AllAttemptsFailedAndParked_Emits_Terminal_Parked() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var trackedId = TrackedOperationId.New(); + var t0 = new DateTime(2026, 5, 20, 11, 0, 0, DateTimeKind.Utc); + + await using var harness = new CombinedTelemetryHarness(_fixture, this); + + await harness.EmitSubmitAsync(SubmitPacket(trackedId, siteId, t0)); + + // Three transient failures... + for (int i = 1; i <= 3; i++) + { + await harness.EmitAttemptAsync(AttemptContext( + trackedId, siteId, + CachedCallAttemptOutcome.TransientFailure, + retryCount: i, lastError: "HTTP 500", httpStatus: 500, + createdUtc: t0, occurredUtc: t0.AddSeconds(i * 5))); + } + + // ...then S&F gives up — ParkedMaxRetries. + await harness.EmitAttemptAsync(AttemptContext( + trackedId, siteId, + CachedCallAttemptOutcome.ParkedMaxRetries, + retryCount: 4, lastError: "HTTP 500", httpStatus: 500, + createdUtc: t0, occurredUtc: t0.AddSeconds(30))); + + await using var read = harness.CreateReadContext(); + + var siteCall = await read.Set() + .SingleAsync(s => s.TrackedOperationId == trackedId); + Assert.Equal("Parked", siteCall.Status); + Assert.NotNull(siteCall.TerminalAtUtc); + + // Terminal audit row should also be Parked. + var resolve = await read.Set() + .Where(e => e.SourceSiteId == siteId && e.Kind == AuditKind.CachedResolve) + .SingleAsync(); + Assert.Equal(AuditStatus.Parked, resolve.Status); + + // Site-local tracking matches. + var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId); + Assert.NotNull(snapshot); + Assert.Equal("Parked", snapshot!.Status); + Assert.NotNull(snapshot.TerminalAtUtc); + } + + [SkippableFact] + public async Task CachedCall_ImmediateSuccess_NoSF_Emits_Attempted_And_Resolve_Delivered() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var trackedId = TrackedOperationId.New(); + var t0 = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc); + + await using var harness = new CombinedTelemetryHarness(_fixture, this); + + // Submit + immediate delivered attempt (RetryCount = 0). + await harness.EmitSubmitAsync(SubmitPacket(trackedId, siteId, t0)); + await harness.EmitAttemptAsync(AttemptContext( + trackedId, siteId, + CachedCallAttemptOutcome.Delivered, + retryCount: 0, lastError: null, httpStatus: 200, + createdUtc: t0, occurredUtc: t0.AddMilliseconds(50))); + + await using var read = harness.CreateReadContext(); + + var siteCall = await read.Set() + .SingleAsync(s => s.TrackedOperationId == trackedId); + Assert.Equal("Delivered", siteCall.Status); + Assert.Equal(0, siteCall.RetryCount); + Assert.NotNull(siteCall.TerminalAtUtc); + + // 1 Submit + 1 Attempted + 1 CachedResolve = 3 audit rows. + var auditRows = await read.Set() + .Where(e => e.SourceSiteId == siteId) + .ToListAsync(); + Assert.Equal(3, auditRows.Count); + Assert.Single(auditRows, r => r.Kind == AuditKind.CachedSubmit); + Assert.Single(auditRows, r => r.Kind == AuditKind.ApiCallCached); + var resolve = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedResolve); + Assert.Equal(AuditStatus.Delivered, resolve.Status); + + var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId); + Assert.NotNull(snapshot); + Assert.Equal("Delivered", snapshot!.Status); + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Integration/CachedWriteCombinedTelemetryTests.cs b/tests/ScadaLink.AuditLog.Tests/Integration/CachedWriteCombinedTelemetryTests.cs new file mode 100644 index 0000000..b67488e --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Integration/CachedWriteCombinedTelemetryTests.cs @@ -0,0 +1,196 @@ +using Akka.TestKit.Xunit2; +using Microsoft.EntityFrameworkCore; +using ScadaLink.AuditLog.Tests.Integration.Infrastructure; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; +using ScadaLink.ConfigurationDatabase.Tests.Migrations; + +namespace ScadaLink.AuditLog.Tests.Integration; + +/// +/// Bundle G G3 mirror of for +/// Database.CachedWrite. Same pipeline composition, same dual-write +/// transaction, but the lifecycle bridge maps channel "DbOutbound" to +/// on the per-attempt audit row (vs. +/// for API calls). The +/// on the audit row, the SiteCalls.Channel +/// column, and the per-attempt all need to come +/// through as the DB variants for this path to be considered exercised. +/// +/// +/// As with G2, the bridge is driven directly via the harness — we do not +/// stand up a real Database.CachedWrite caller. The site-side +/// unit-level emission for the DB path is exercised in +/// ScadaLink.SiteRuntime.Tests; this suite verifies the end-to-end +/// combined-telemetry path produces the right central rows. +/// +public class CachedWriteCombinedTelemetryTests : TestKit, IClassFixture +{ + private readonly MsSqlMigrationFixture _fixture; + + public CachedWriteCombinedTelemetryTests(MsSqlMigrationFixture fixture) + { + _fixture = fixture; + } + + private static string NewSiteId() => + "test-g3-cachedwrite-" + Guid.NewGuid().ToString("N").Substring(0, 8); + + private static CachedCallTelemetry DbSubmitPacket( + TrackedOperationId id, string siteId, DateTime nowUtc, string target = "OperationsDb.UpdateOrder") => + new( + Audit: new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = nowUtc, + Channel = AuditChannel.DbOutbound, + Kind = AuditKind.CachedSubmit, + CorrelationId = id.Value, + SourceSiteId = siteId, + SourceInstanceId = "Plant.Pump42", + SourceScript = "ScriptActor:doStuff", + Target = target, + Status = AuditStatus.Submitted, + ForwardState = AuditForwardState.Pending, + }, + Operational: new SiteCallOperational( + TrackedOperationId: id, + Channel: "DbOutbound", + Target: target, + SourceSite: siteId, + Status: "Submitted", + RetryCount: 0, + LastError: null, + HttpStatus: null, + CreatedAtUtc: nowUtc, + UpdatedAtUtc: nowUtc, + TerminalAtUtc: null)); + + private static CachedCallAttemptContext DbAttemptContext( + TrackedOperationId id, + string siteId, + CachedCallAttemptOutcome outcome, + int retryCount, + string? lastError, + DateTime createdUtc, + DateTime occurredUtc, + string target = "OperationsDb.UpdateOrder") => + new( + TrackedOperationId: id, + Channel: "DbOutbound", + Target: target, + SourceSite: siteId, + Outcome: outcome, + RetryCount: retryCount, + LastError: lastError, + HttpStatus: null, + CreatedAtUtc: createdUtc, + OccurredAtUtc: occurredUtc, + DurationMs: 12, + SourceInstanceId: "Plant.Pump42"); + + [SkippableFact] + public async Task CachedWrite_Success_Emits_Delivered_Lifecycle() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var trackedId = TrackedOperationId.New(); + var t0 = new DateTime(2026, 5, 20, 13, 0, 0, DateTimeKind.Utc); + + await using var harness = new CombinedTelemetryHarness(_fixture, this); + + // Submit + immediate delivered attempt. + await harness.EmitSubmitAsync(DbSubmitPacket(trackedId, siteId, t0)); + await harness.EmitAttemptAsync(DbAttemptContext( + trackedId, siteId, + CachedCallAttemptOutcome.Delivered, + retryCount: 0, lastError: null, + createdUtc: t0, occurredUtc: t0.AddMilliseconds(50))); + + await using var read = harness.CreateReadContext(); + + // Central SiteCalls row — DbOutbound channel, Delivered. + var siteCall = await read.Set() + .SingleAsync(s => s.TrackedOperationId == trackedId); + Assert.Equal("DbOutbound", siteCall.Channel); + Assert.Equal("Delivered", siteCall.Status); + Assert.Equal(0, siteCall.RetryCount); + Assert.NotNull(siteCall.TerminalAtUtc); + + var auditRows = await read.Set() + .Where(e => e.SourceSiteId == siteId) + .ToListAsync(); + Assert.Equal(3, auditRows.Count); + // Submit row: CachedSubmit + DbOutbound channel. + var submit = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedSubmit); + Assert.Equal(AuditChannel.DbOutbound, submit.Channel); + // Per-attempt row: DbWriteCached (NOT ApiCallCached). + var attempt = Assert.Single(auditRows, r => r.Kind == AuditKind.DbWriteCached); + Assert.Equal(AuditStatus.Attempted, attempt.Status); + Assert.Equal(AuditChannel.DbOutbound, attempt.Channel); + // Terminal: CachedResolve Delivered. + var resolve = Assert.Single(auditRows, r => r.Kind == AuditKind.CachedResolve); + Assert.Equal(AuditStatus.Delivered, resolve.Status); + Assert.Equal(AuditChannel.DbOutbound, resolve.Channel); + + // Site-local tracking row mirrors the same outcome. + var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId); + Assert.NotNull(snapshot); + Assert.Equal("Delivered", snapshot!.Status); + } + + [SkippableFact] + public async Task CachedWrite_Parked_Emits_Terminal_Parked() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var trackedId = TrackedOperationId.New(); + var t0 = new DateTime(2026, 5, 20, 14, 0, 0, DateTimeKind.Utc); + + await using var harness = new CombinedTelemetryHarness(_fixture, this); + + await harness.EmitSubmitAsync(DbSubmitPacket(trackedId, siteId, t0)); + + // Two transient SQL-error attempts... + for (int i = 1; i <= 2; i++) + { + await harness.EmitAttemptAsync(DbAttemptContext( + trackedId, siteId, + CachedCallAttemptOutcome.TransientFailure, + retryCount: i, lastError: "Deadlock victim", + createdUtc: t0, occurredUtc: t0.AddSeconds(i * 5))); + } + + // ...then permanent failure → Parked terminal. + await harness.EmitAttemptAsync(DbAttemptContext( + trackedId, siteId, + CachedCallAttemptOutcome.PermanentFailure, + retryCount: 3, lastError: "ConstraintViolation: FK_Orders_Customer", + createdUtc: t0, occurredUtc: t0.AddSeconds(20))); + + await using var read = harness.CreateReadContext(); + + var siteCall = await read.Set() + .SingleAsync(s => s.TrackedOperationId == trackedId); + Assert.Equal("DbOutbound", siteCall.Channel); + Assert.Equal("Parked", siteCall.Status); + Assert.NotNull(siteCall.TerminalAtUtc); + + var resolve = await read.Set() + .Where(e => e.SourceSiteId == siteId && e.Kind == AuditKind.CachedResolve) + .SingleAsync(); + Assert.Equal(AuditStatus.Parked, resolve.Status); + Assert.Equal(AuditChannel.DbOutbound, resolve.Channel); + + // Tracking store mirrors Parked. + var snapshot = await harness.TrackingStore.GetStatusAsync(trackedId); + Assert.NotNull(snapshot); + Assert.Equal("Parked", snapshot!.Status); + Assert.NotNull(snapshot.TerminalAtUtc); + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Integration/CombinedTelemetryIdempotencyTests.cs b/tests/ScadaLink.AuditLog.Tests/Integration/CombinedTelemetryIdempotencyTests.cs new file mode 100644 index 0000000..1d67ed7 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Integration/CombinedTelemetryIdempotencyTests.cs @@ -0,0 +1,202 @@ +using Akka.TestKit.Xunit2; +using Microsoft.EntityFrameworkCore; +using ScadaLink.AuditLog.Tests.Integration.Infrastructure; +using ScadaLink.AuditLog.Telemetry; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; +using ScadaLink.ConfigurationDatabase.Tests.Migrations; +using ScadaLink.Communication.Grpc; +using Timestamp = Google.Protobuf.WellKnownTypes.Timestamp; + +namespace ScadaLink.AuditLog.Tests.Integration; + +/// +/// Bundle G G4 idempotency suite. Telemetry packets MUST round-trip safely +/// under retried delivery (at-least-once site→central) AND under out-of-order +/// arrival (a stale Submit packet arriving after the central row has already +/// advanced to Attempted must not regress the SiteCalls status, but must +/// still insert its own audit row because audit rows are append-only and the +/// lifecycle history is the source of truth for forensics). +/// +/// +/// Pushes packets directly through the +/// stub client (bypassing the local SQLite writer + tracking store) — the +/// scenario being modeled is a wire-level retry, not a fresh site call, so +/// the local stores' insert/no-op behaviour is already covered by the G2/G3 +/// happy-path tests. This suite focuses on the central ingest actor's +/// dual-write transaction's idempotency contract. +/// +public class CombinedTelemetryIdempotencyTests : TestKit, IClassFixture +{ + private readonly MsSqlMigrationFixture _fixture; + + public CombinedTelemetryIdempotencyTests(MsSqlMigrationFixture fixture) + { + _fixture = fixture; + } + + private static string NewSiteId() => + "test-g4-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8); + + private static CachedTelemetryPacket BuildPacket( + Guid eventId, + TrackedOperationId trackedId, + string siteId, + AuditKind kind, + AuditStatus auditStatus, + string operationalStatus, + int retryCount, + DateTime nowUtc, + DateTime? terminalUtc = null, + string? lastError = null, + int? httpStatus = null) + { + var dto = new CachedTelemetryPacket + { + AuditEvent = AuditEventMapper.ToDto(new AuditEvent + { + EventId = eventId, + OccurredAtUtc = nowUtc, + Channel = AuditChannel.ApiOutbound, + Kind = kind, + CorrelationId = trackedId.Value, + SourceSiteId = siteId, + Target = "ERP.GetOrder", + Status = auditStatus, + HttpStatus = httpStatus, + ErrorMessage = lastError, + ForwardState = AuditForwardState.Pending, + }), + Operational = new SiteCallOperationalDto + { + TrackedOperationId = trackedId.Value.ToString("D"), + Channel = "ApiOutbound", + Target = "ERP.GetOrder", + SourceSite = siteId, + Status = operationalStatus, + RetryCount = retryCount, + LastError = lastError ?? string.Empty, + CreatedAtUtc = Timestamp.FromDateTime(DateTime.SpecifyKind(nowUtc, DateTimeKind.Utc)), + UpdatedAtUtc = Timestamp.FromDateTime(DateTime.SpecifyKind(nowUtc, DateTimeKind.Utc)), + }, + }; + if (httpStatus.HasValue) + { + dto.Operational.HttpStatus = httpStatus.Value; + } + if (terminalUtc.HasValue) + { + dto.Operational.TerminalAtUtc = + Timestamp.FromDateTime(DateTime.SpecifyKind(terminalUtc.Value, DateTimeKind.Utc)); + } + return dto; + } + + private static CachedTelemetryBatch BatchOf(params CachedTelemetryPacket[] packets) + { + var batch = new CachedTelemetryBatch(); + batch.Packets.AddRange(packets); + return batch; + } + + [SkippableFact] + public async Task DuplicatePacket_AuditLogStaysAtOneRow_SiteCallsUpserted_Monotonically() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var trackedId = TrackedOperationId.New(); + var eventId = Guid.NewGuid(); + var t0 = new DateTime(2026, 5, 20, 15, 0, 0, DateTimeKind.Utc); + + await using var harness = new CombinedTelemetryHarness(_fixture, this); + + var packet = BuildPacket( + eventId, trackedId, siteId, + AuditKind.CachedSubmit, AuditStatus.Submitted, "Submitted", + retryCount: 0, nowUtc: t0); + + // First delivery + var ack1 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(packet), CancellationToken.None); + Assert.Single(ack1.AcceptedEventIds); + + // Second delivery — the exact same packet (simulates a retried gRPC). + var ack2 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(packet), CancellationToken.None); + // Central acks both deliveries because storage state is consistent — + // the site is free to treat its local row as Forwarded either way. + Assert.Single(ack2.AcceptedEventIds); + Assert.Equal(eventId.ToString(), ack2.AcceptedEventIds[0]); + + await using var read = harness.CreateReadContext(); + + // AuditLog: exactly ONE row for the EventId (insert-if-not-exists). + var auditCount = await read.Set() + .CountAsync(e => e.EventId == eventId); + Assert.Equal(1, auditCount); + + // SiteCalls: exactly ONE row for the TrackedOperationId. + var siteCalls = await read.Set() + .Where(s => s.TrackedOperationId == trackedId) + .ToListAsync(); + Assert.Single(siteCalls); + Assert.Equal("Submitted", siteCalls[0].Status); + Assert.Equal(0, siteCalls[0].RetryCount); + } + + [SkippableFact] + public async Task OutOfOrderPackets_OlderStatus_ArrivesAfterNewer_IsNoOp() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var trackedId = TrackedOperationId.New(); + var t0 = new DateTime(2026, 5, 20, 16, 0, 0, DateTimeKind.Utc); + + await using var harness = new CombinedTelemetryHarness(_fixture, this); + + // First: the Attempted (RetryCount=2) row arrives at central — perhaps + // the Submit packet got delayed in flight. SiteCalls advances straight + // to Attempted with retry count 2. + var attemptedEventId = Guid.NewGuid(); + var attemptedPacket = BuildPacket( + attemptedEventId, trackedId, siteId, + AuditKind.ApiCallCached, AuditStatus.Attempted, "Attempted", + retryCount: 2, nowUtc: t0.AddSeconds(10), + lastError: "HTTP 500", httpStatus: 500); + var ack1 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(attemptedPacket), CancellationToken.None); + Assert.Single(ack1.AcceptedEventIds); + + // Now the stale Submit packet shows up. The audit row should still be + // inserted (audit is append-only — preserve the lifecycle history), + // but SiteCalls must NOT regress to Submitted/RetryCount=0. + var submitEventId = Guid.NewGuid(); + var submitPacket = BuildPacket( + submitEventId, trackedId, siteId, + AuditKind.CachedSubmit, AuditStatus.Submitted, "Submitted", + retryCount: 0, nowUtc: t0); + var ack2 = await harness.StubClient.IngestCachedTelemetryAsync(BatchOf(submitPacket), CancellationToken.None); + Assert.Single(ack2.AcceptedEventIds); + + await using var read = harness.CreateReadContext(); + + // AuditLog: TWO rows now exist for this lifecycle — the Submit and + // the Attempted. Their order is by OccurredAtUtc; the test doesn't + // assert ordering, only count + correlation. + var auditRows = await read.Set() + .Where(e => e.SourceSiteId == siteId) + .ToListAsync(); + Assert.Equal(2, auditRows.Count); + Assert.All(auditRows, r => Assert.Equal(trackedId.Value, r.CorrelationId)); + + // SiteCalls: stuck at Attempted (monotonic — Submitted is rank 0, + // Attempted is rank 2, the upsert for the older row is a no-op). + var siteCall = await read.Set() + .SingleAsync(s => s.TrackedOperationId == trackedId); + Assert.Equal("Attempted", siteCall.Status); + Assert.Equal(2, siteCall.RetryCount); + Assert.Equal("HTTP 500", siteCall.LastError); + Assert.Equal(500, siteCall.HttpStatus); + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Integration/Infrastructure/CombinedTelemetryDispatcher.cs b/tests/ScadaLink.AuditLog.Tests/Integration/Infrastructure/CombinedTelemetryDispatcher.cs new file mode 100644 index 0000000..83c8303 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Integration/Infrastructure/CombinedTelemetryDispatcher.cs @@ -0,0 +1,125 @@ +using ScadaLink.AuditLog.Site.Telemetry; +using ScadaLink.AuditLog.Telemetry; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.Commons.Types; +using ScadaLink.Communication.Grpc; +using Google.Protobuf.WellKnownTypes; +using Timestamp = Google.Protobuf.WellKnownTypes.Timestamp; + +namespace ScadaLink.AuditLog.Tests.Integration.Infrastructure; + +/// +/// Test-side combined-telemetry dispatcher: wraps a production +/// so the local audit + tracking +/// stores still get written, then projects the same packet onto the wire as a +/// and pushes it through the supplied +/// . The bridge can be composed into the +/// existing chain as the +/// implementation so a single +/// observer callback fans out to both halves. +/// +/// +/// +/// Production wiring keeps the wire push deferred to M6 — the site SQLite hot +/// path is the source of truth and a future M6 drain will push the rows +/// through the gRPC client. For end-to-end testing today we need a way to +/// exercise the central dual-write transaction immediately, so this +/// dispatcher synthesises the wire packet inline and round-trips it through +/// the stub client. The shape mirrors what the M6 drain will eventually emit. +/// +/// +/// Best-effort: both the inner forwarder call and the wire push are +/// wrapped in independent try/catch blocks. A thrown wire client doesn't +/// abort the local writes (the audit row is already in SQLite); a thrown +/// local forwarder doesn't abort the wire push (central still gets the +/// dual-write attempt). +/// +/// +public sealed class CombinedTelemetryDispatcher : ICachedCallTelemetryForwarder +{ + private readonly ICachedCallTelemetryForwarder _inner; + private readonly ISiteStreamAuditClient _wireClient; + + public CombinedTelemetryDispatcher( + ICachedCallTelemetryForwarder inner, + ISiteStreamAuditClient wireClient) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _wireClient = wireClient ?? throw new ArgumentNullException(nameof(wireClient)); + } + + /// + public async Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(telemetry); + + // Inner forwarder writes the audit row to SQLite + updates the + // tracking store. Best-effort — exceptions are already swallowed + // inside the production forwarder, but wrap defensively here too in + // case a test substitutes a stricter inner. + try + { + await _inner.ForwardAsync(telemetry, ct).ConfigureAwait(false); + } + catch + { + // Swallow — alog.md §7 best-effort contract. + } + + // Project the same packet onto the wire and push it through the stub + // client. This is the bit a future M6 drain will subsume — until + // then the test wraps the two halves into one observer-driven step. + try + { + var batch = new CachedTelemetryBatch(); + batch.Packets.Add(BuildPacket(telemetry)); + await _wireClient.IngestCachedTelemetryAsync(batch, ct).ConfigureAwait(false); + } + catch + { + // Swallow — the audit row is still in SQLite for a future drain; + // the central row will materialise the next time the wire path + // is exercised (or via the M6 reconciliation pull). + } + } + + private static CachedTelemetryPacket BuildPacket(CachedCallTelemetry telemetry) + { + return new CachedTelemetryPacket + { + AuditEvent = AuditEventMapper.ToDto(telemetry.Audit), + Operational = ToOperationalDto(telemetry.Operational), + }; + } + + private static SiteCallOperationalDto ToOperationalDto(SiteCallOperational op) + { + var dto = new SiteCallOperationalDto + { + TrackedOperationId = op.TrackedOperationId.Value.ToString("D"), + Channel = op.Channel, + Target = op.Target, + SourceSite = op.SourceSite, + Status = op.Status, + RetryCount = op.RetryCount, + LastError = op.LastError ?? string.Empty, + CreatedAtUtc = Timestamp.FromDateTime(EnsureUtc(op.CreatedAtUtc)), + UpdatedAtUtc = Timestamp.FromDateTime(EnsureUtc(op.UpdatedAtUtc)), + }; + if (op.HttpStatus.HasValue) + { + dto.HttpStatus = op.HttpStatus.Value; + } + if (op.TerminalAtUtc.HasValue) + { + dto.TerminalAtUtc = Timestamp.FromDateTime(EnsureUtc(op.TerminalAtUtc.Value)); + } + return dto; + } + + private static DateTime EnsureUtc(DateTime value) => + value.Kind == DateTimeKind.Utc + ? value + : DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc); +} diff --git a/tests/ScadaLink.AuditLog.Tests/Integration/Infrastructure/CombinedTelemetryHarness.cs b/tests/ScadaLink.AuditLog.Tests/Integration/Infrastructure/CombinedTelemetryHarness.cs new file mode 100644 index 0000000..538b269 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Integration/Infrastructure/CombinedTelemetryHarness.cs @@ -0,0 +1,175 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Central; +using ScadaLink.AuditLog.Site; +using ScadaLink.AuditLog.Site.Telemetry; +using ScadaLink.Commons.Interfaces; +using ScadaLink.Commons.Interfaces.Repositories; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.ConfigurationDatabase; +using ScadaLink.ConfigurationDatabase.Repositories; +using ScadaLink.ConfigurationDatabase.Tests.Migrations; +using ScadaLink.SiteRuntime.Tracking; + +namespace ScadaLink.AuditLog.Tests.Integration.Infrastructure; + +/// +/// Shared end-to-end harness for the M3 cached-call combined telemetry tests +/// (G2/G3/G4). Composes the full pipeline: +/// +/// Site-local SQLite (in-memory) + +/// + . +/// Site-local SQLite (in-memory). +/// Production wrapped by a +/// test-side that also pushes each +/// packet through the stub gRPC client. +/// wired to the +/// dispatcher so a single observer call fans out audit + tracking + wire. +/// connected +/// to an backed by the real +/// + +/// against the per-test database. +/// +/// +/// +/// +/// Disposal cleans up the in-memory SQLite stores. The Akka actor system is +/// owned by the calling ; the harness +/// only owns the ingest actor IActorRef and the underlying repositories' +/// DbContext lifecycle. +/// +/// +public sealed class CombinedTelemetryHarness : IAsyncDisposable +{ + public SqliteAuditWriter SqliteWriter { get; } + public RingBufferFallback Ring { get; } + public FallbackAuditWriter FallbackWriter { get; } + public OperationTrackingStore TrackingStore { get; } + public CachedCallTelemetryForwarder InnerForwarder { get; } + public CombinedTelemetryDispatcher Dispatcher { get; } + public CachedCallLifecycleBridge Bridge { get; } + public DirectActorSiteStreamAuditClient StubClient { get; } + public IActorRef IngestActor { get; } + public IServiceProvider ServiceProvider { get; } + + private readonly MsSqlMigrationFixture _fixture; + private bool _disposed; + + public CombinedTelemetryHarness( + MsSqlMigrationFixture fixture, + Akka.TestKit.Xunit2.TestKit testKit, + Func? siteCallRepoOverride = null) + { + _fixture = fixture ?? throw new ArgumentNullException(nameof(fixture)); + ArgumentNullException.ThrowIfNull(testKit); + + // Site SQLite — unique in-memory database per harness so tests don't share + // an audit queue. Mode=Memory + Cache=Shared keeps the file alive for the + // lifetime of the writer connection. + SqliteWriter = new SqliteAuditWriter( + Options.Create(new SqliteAuditWriterOptions + { + DatabasePath = "ignored", + BatchSize = 64, + ChannelCapacity = 1024, + }), + NullLogger.Instance, + connectionStringOverride: + $"Data Source=file:cachedcall-g-{Guid.NewGuid():N}?mode=memory&cache=shared"); + + Ring = new RingBufferFallback(); + FallbackWriter = new FallbackAuditWriter( + SqliteWriter, Ring, new NoOpAuditWriteFailureCounter(), + NullLogger.Instance); + + TrackingStore = new OperationTrackingStore( + Options.Create(new OperationTrackingOptions + { + // Same shared-in-memory pattern as the audit writer. + ConnectionString = + $"Data Source=file:tracking-g-{Guid.NewGuid():N}?mode=memory&cache=shared", + }), + NullLogger.Instance); + + // Central wiring: real repositories backed by the MSSQL fixture's DB. + ServiceProvider = BuildCentralServiceProvider(siteCallRepoOverride); + IngestActor = testKit.Sys.ActorOf(Props.Create(() => new AuditLogIngestActor( + ServiceProvider, + NullLogger.Instance))); + + StubClient = new DirectActorSiteStreamAuditClient(IngestActor); + + // Production forwarder writes the local stores; the dispatcher wraps + // it to ALSO push the same packet to central via the stub client. + InnerForwarder = new CachedCallTelemetryForwarder( + FallbackWriter, TrackingStore, NullLogger.Instance); + Dispatcher = new CombinedTelemetryDispatcher(InnerForwarder, StubClient); + + Bridge = new CachedCallLifecycleBridge(Dispatcher, NullLogger.Instance); + } + + /// + /// Convenience: emit the initial submit packet directly through the + /// dispatcher (the bridge's hooks fire only for S&F retry-loop + /// attempts; submit-row emission happens at the script call site). + /// + public Task EmitSubmitAsync(CachedCallTelemetry submit, CancellationToken ct = default) => + Dispatcher.ForwardAsync(submit, ct); + + /// + /// Convenience: route a per-attempt or terminal outcome through the bridge. + /// + public Task EmitAttemptAsync(CachedCallAttemptContext context, CancellationToken ct = default) => + Bridge.OnAttemptCompletedAsync(context, ct); + + public ScadaLinkDbContext CreateReadContext() + { + var options = new DbContextOptionsBuilder() + .UseSqlServer(_fixture.ConnectionString) + .Options; + return new ScadaLinkDbContext(options); + } + + private IServiceProvider BuildCentralServiceProvider( + Func? siteCallRepoOverride) + { + var services = new ServiceCollection(); + services.AddDbContext(opts => + opts.UseSqlServer(_fixture.ConnectionString) + .ConfigureWarnings(w => w.Ignore( + Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))); + services.AddScoped(sp => + new AuditLogRepository(sp.GetRequiredService())); + if (siteCallRepoOverride is null) + { + services.AddScoped(sp => + new SiteCallAuditRepository(sp.GetRequiredService())); + } + else + { + services.AddScoped(sp => + siteCallRepoOverride(sp.GetRequiredService())); + } + return services.BuildServiceProvider(); + } + + public async ValueTask DisposeAsync() + { + if (_disposed) return; + _disposed = true; + await SqliteWriter.DisposeAsync().ConfigureAwait(false); + await TrackingStore.DisposeAsync().ConfigureAwait(false); + if (ServiceProvider is IAsyncDisposable asyncSp) + { + await asyncSp.DisposeAsync().ConfigureAwait(false); + } + else if (ServiceProvider is IDisposable sp) + { + sp.Dispose(); + } + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Integration/Infrastructure/DirectActorSiteStreamAuditClient.cs b/tests/ScadaLink.AuditLog.Tests/Integration/Infrastructure/DirectActorSiteStreamAuditClient.cs new file mode 100644 index 0000000..903f917 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Integration/Infrastructure/DirectActorSiteStreamAuditClient.cs @@ -0,0 +1,177 @@ +using Akka.Actor; +using ScadaLink.AuditLog.Site.Telemetry; +using ScadaLink.AuditLog.Telemetry; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Messages.Audit; +using ScadaLink.Commons.Types; +using ScadaLink.Communication.Grpc; + +namespace ScadaLink.AuditLog.Tests.Integration.Infrastructure; + +/// +/// Shared component-level test double that +/// short-circuits the gRPC wire and forwards each batch directly to a central +/// via Akka . +/// Lives under Integration/Infrastructure/ so both the M2 sync-call and +/// M3 cached-call end-to-end suites can reuse it. +/// +/// +/// +/// The class deliberately mirrors the production SiteStreamGrpcServer +/// flow: decode each DTO into the in-process entity, Ask the central ingest +/// actor with the matching Akka command, and convert the Akka reply's accepted +/// id list into the proto the telemetry actor / forwarder +/// expects. The actor wiring (single-repository vs. +/// ctor) lives in the central ingest actor itself — this stub just routes the +/// command. +/// +/// +/// arms a deterministic number of failures +/// before the stub recovers; it applies to BOTH RPCs because the M2 sync-call +/// retry behaviour and the M3 cached-telemetry retry behaviour share a single +/// SiteAuditTelemetryActor drain. Tests that need to differentiate per-RPC +/// failures should reach for a per-test wrapper rather than extending this +/// shared infrastructure. +/// +/// +public sealed class DirectActorSiteStreamAuditClient : ISiteStreamAuditClient +{ + private readonly IActorRef _ingestActor; + private int _failsRemaining; + private int _callCount; + private int _cachedTelemetryCallCount; + + public DirectActorSiteStreamAuditClient(IActorRef ingestActor) + { + _ingestActor = ingestActor ?? throw new ArgumentNullException(nameof(ingestActor)); + } + + /// + /// When > 0, the next FailNextCallCount invocations of either + /// RPC throw to simulate a gRPC error; after that count is exhausted, calls + /// succeed normally. + /// + public int FailNextCallCount + { + get => _failsRemaining; + set => _failsRemaining = value; + } + + /// + /// Total successful + failed invocations of . + /// + public int CallCount => Volatile.Read(ref _callCount); + + /// + /// Total successful + failed invocations of . + /// Separate counter so cached-call tests can assert dispatch independently of + /// any sync-call traffic going through the same stub. + /// + public int CachedTelemetryCallCount => Volatile.Read(ref _cachedTelemetryCallCount); + + public async Task IngestAuditEventsAsync(AuditEventBatch batch, CancellationToken ct) + { + Interlocked.Increment(ref _callCount); + + // Atomically consume one of the queued failures, if any. This lets the + // test arm a deterministic number of failures before the stub recovers. + if (Interlocked.Decrement(ref _failsRemaining) >= 0) + { + throw new InvalidOperationException("simulated gRPC failure for test"); + } + + // Clamp at -1 to keep the field bounded under many calls. + Interlocked.Exchange(ref _failsRemaining, -1); + + // Decode the proto batch back into AuditEvent records — mirrors what + // SiteStreamGrpcServer does before dispatching to the ingest actor. + var events = new List(batch.Events.Count); + foreach (var dto in batch.Events) + { + events.Add(AuditEventMapper.FromDto(dto)); + } + + // Ask the central actor; the reply carries the accepted EventIds. + var reply = await _ingestActor + .Ask( + new IngestAuditEventsCommand(events), + TimeSpan.FromSeconds(10)) + .ConfigureAwait(false); + + var ack = new IngestAck(); + foreach (var id in reply.AcceptedEventIds) + { + ack.AcceptedEventIds.Add(id.ToString()); + } + return ack; + } + + /// + /// M3 dual-write path: decode each into + /// the paired (, ) entry and + /// Ask the central ingest actor with an . + /// The accepted EventIds returned by the actor's dual-write transaction map + /// back into the proto ack. + /// + /// + /// Uses the shared for the audit half; + /// the SiteCall DTO is decoded inline because the AuditLog mapper does not + /// (and should not) know about — the + /// production gRPC server (Bundle D) uses the same inline shape. + /// + public async Task IngestCachedTelemetryAsync(CachedTelemetryBatch batch, CancellationToken ct) + { + Interlocked.Increment(ref _cachedTelemetryCallCount); + + if (Interlocked.Decrement(ref _failsRemaining) >= 0) + { + throw new InvalidOperationException("simulated gRPC failure for test"); + } + Interlocked.Exchange(ref _failsRemaining, -1); + + var entries = new List(batch.Packets.Count); + foreach (var packet in batch.Packets) + { + var audit = AuditEventMapper.FromDto(packet.AuditEvent); + var siteCall = MapSiteCallFromDto(packet.Operational); + entries.Add(new CachedTelemetryEntry(audit, siteCall)); + } + + var reply = await _ingestActor + .Ask( + new IngestCachedTelemetryCommand(entries), + TimeSpan.FromSeconds(10)) + .ConfigureAwait(false); + + var ack = new IngestAck(); + foreach (var id in reply.AcceptedEventIds) + { + ack.AcceptedEventIds.Add(id.ToString()); + } + return ack; + } + + /// + /// Mirrors SiteStreamGrpcServer.MapSiteCallFromDto — keep the two in + /// sync. The placeholder stamped here + /// is overwritten by the central ingest actor inside the dual-write + /// transaction, so the value sent on the wire is informational only. + /// + private static SiteCall MapSiteCallFromDto(SiteCallOperationalDto dto) => new() + { + TrackedOperationId = TrackedOperationId.Parse(dto.TrackedOperationId), + Channel = dto.Channel, + Target = dto.Target, + SourceSite = dto.SourceSite, + Status = dto.Status, + RetryCount = dto.RetryCount, + LastError = string.IsNullOrEmpty(dto.LastError) ? null : dto.LastError, + HttpStatus = dto.HttpStatus, + CreatedAtUtc = DateTime.SpecifyKind(dto.CreatedAtUtc.ToDateTime(), DateTimeKind.Utc), + UpdatedAtUtc = DateTime.SpecifyKind(dto.UpdatedAtUtc.ToDateTime(), DateTimeKind.Utc), + TerminalAtUtc = dto.TerminalAtUtc is null + ? null + : DateTime.SpecifyKind(dto.TerminalAtUtc.ToDateTime(), DateTimeKind.Utc), + IngestedAtUtc = DateTime.UtcNow, + }; +} diff --git a/tests/ScadaLink.AuditLog.Tests/Integration/SyncCallEmissionEndToEndTests.cs b/tests/ScadaLink.AuditLog.Tests/Integration/SyncCallEmissionEndToEndTests.cs index 5f16ce9..a0c5c85 100644 --- a/tests/ScadaLink.AuditLog.Tests/Integration/SyncCallEmissionEndToEndTests.cs +++ b/tests/ScadaLink.AuditLog.Tests/Integration/SyncCallEmissionEndToEndTests.cs @@ -6,15 +6,14 @@ using Microsoft.Extensions.Options; using ScadaLink.AuditLog.Central; using ScadaLink.AuditLog.Site; using ScadaLink.AuditLog.Site.Telemetry; +using ScadaLink.AuditLog.Tests.Integration.Infrastructure; using ScadaLink.Commons.Entities.Audit; using ScadaLink.Commons.Interfaces.Repositories; -using ScadaLink.Commons.Messages.Audit; using ScadaLink.Commons.Types.Audit; using ScadaLink.Commons.Types.Enums; using ScadaLink.ConfigurationDatabase; using ScadaLink.ConfigurationDatabase.Repositories; using ScadaLink.ConfigurationDatabase.Tests.Migrations; -using ScadaLink.Communication.Grpc; namespace ScadaLink.AuditLog.Tests.Integration; @@ -267,75 +266,4 @@ public class SyncCallEmissionEndToEndTests : TestKit, IClassFixture - /// Test double for that short-circuits - /// the gRPC wire and forwards the batch directly to a central - /// via Akka . The - /// Akka is converted to the proto - /// that the telemetry actor expects. - /// - private sealed class DirectActorSiteStreamAuditClient : ISiteStreamAuditClient - { - private readonly IActorRef _ingestActor; - private int _failsRemaining; - private int _callCount; - - public DirectActorSiteStreamAuditClient(IActorRef ingestActor) - { - _ingestActor = ingestActor ?? throw new ArgumentNullException(nameof(ingestActor)); - } - - /// - /// When > 0, the next FailNextCallCount invocations of - /// throw to simulate a gRPC error; - /// after that count is exhausted, calls succeed normally. - /// - public int FailNextCallCount - { - get => _failsRemaining; - set => _failsRemaining = value; - } - - public int CallCount => Volatile.Read(ref _callCount); - - public async Task IngestAuditEventsAsync(AuditEventBatch batch, CancellationToken ct) - { - Interlocked.Increment(ref _callCount); - - // Atomically consume one of the queued failures, if any. This - // lets the test arm a deterministic number of failures before the - // stub recovers. - if (Interlocked.Decrement(ref _failsRemaining) >= 0) - { - throw new InvalidOperationException("simulated gRPC failure for test"); - } - - // Decrement under-ran into negative territory; clamp at -1 to keep - // the field bounded even under many calls. - Interlocked.Exchange(ref _failsRemaining, -1); - - // Decode the proto batch back into AuditEvent records — this - // mirrors what the production SiteStreamGrpcServer does before - // dispatching to the ingest actor (see Bundle D's gRPC handler). - var events = new List(batch.Events.Count); - foreach (var dto in batch.Events) - { - events.Add(ScadaLink.AuditLog.Telemetry.AuditEventMapper.FromDto(dto)); - } - - // Ask the central actor; the reply carries the accepted EventIds. - var reply = await _ingestActor - .Ask( - new IngestAuditEventsCommand(events), - TimeSpan.FromSeconds(10)) - .ConfigureAwait(false); - - var ack = new IngestAck(); - foreach (var id in reply.AcceptedEventIds) - { - ack.AcceptedEventIds.Add(id.ToString()); - } - return ack; - } - } } diff --git a/tests/ScadaLink.AuditLog.Tests/ScadaLink.AuditLog.Tests.csproj b/tests/ScadaLink.AuditLog.Tests/ScadaLink.AuditLog.Tests.csproj index 625f9c6..bd99f49 100644 --- a/tests/ScadaLink.AuditLog.Tests/ScadaLink.AuditLog.Tests.csproj +++ b/tests/ScadaLink.AuditLog.Tests/ScadaLink.AuditLog.Tests.csproj @@ -48,6 +48,13 @@ the fixture + EF migrations come along without duplicating them. --> + + diff --git a/tests/ScadaLink.AuditLog.Tests/Site/Telemetry/CachedCallLifecycleBridgeTests.cs b/tests/ScadaLink.AuditLog.Tests/Site/Telemetry/CachedCallLifecycleBridgeTests.cs new file mode 100644 index 0000000..97cbb84 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Site/Telemetry/CachedCallLifecycleBridgeTests.cs @@ -0,0 +1,187 @@ +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using ScadaLink.AuditLog.Site.Telemetry; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Tests.Site.Telemetry; + +/// +/// Bundle E Tasks E4/E5 bridge tests. The bridge ingests +/// notifications from the S&F +/// retry loop and routes them through +/// as one or two packets: +/// +/// Per-attempt: one ApiCallCached/DbWriteCached Attempted row. +/// Terminal (Delivered/PermanentFailure/ParkedMaxRetries): adds a CachedResolve row carrying the terminal Status. +/// +/// +public class CachedCallLifecycleBridgeTests +{ + private readonly ICachedCallTelemetryForwarder _forwarder = Substitute.For(); + private readonly TrackedOperationId _id = TrackedOperationId.New(); + + private CachedCallLifecycleBridge CreateSut() => new( + _forwarder, NullLogger.Instance); + + private CachedCallAttemptContext Ctx( + CachedCallAttemptOutcome outcome, + string channel = "ApiOutbound", + int retryCount = 1, + string? lastError = null, + int? httpStatus = null) => + new( + TrackedOperationId: _id, + Channel: channel, + Target: "ERP.GetOrder", + SourceSite: "site-77", + Outcome: outcome, + RetryCount: retryCount, + LastError: lastError, + HttpStatus: httpStatus, + CreatedAtUtc: new DateTime(2026, 5, 20, 9, 0, 0, DateTimeKind.Utc), + OccurredAtUtc: new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc), + DurationMs: 42, + SourceInstanceId: "Plant.Pump42"); + + [Fact] + public async Task TransientFailure_EmitsOneAttemptedRow_NoResolve() + { + var captured = new List(); + _forwarder.ForwardAsync(Arg.Do(t => captured.Add(t)), Arg.Any()) + .Returns(Task.CompletedTask); + + var sut = CreateSut(); + await sut.OnAttemptCompletedAsync(Ctx( + CachedCallAttemptOutcome.TransientFailure, + retryCount: 2, + lastError: "HTTP 503", + httpStatus: 503)); + + var packet = Assert.Single(captured); + Assert.Equal(AuditKind.ApiCallCached, packet.Audit.Kind); + Assert.Equal(AuditStatus.Attempted, packet.Audit.Status); + Assert.Equal(503, packet.Audit.HttpStatus); + Assert.Equal("HTTP 503", packet.Audit.ErrorMessage); + Assert.Equal(_id.Value, packet.Audit.CorrelationId); + Assert.Equal("Attempted", packet.Operational.Status); + Assert.Equal(2, packet.Operational.RetryCount); + Assert.Null(packet.Operational.TerminalAtUtc); + } + + [Fact] + public async Task Delivered_EmitsAttemptedRow_AndCachedResolveDelivered() + { + var captured = new List(); + _forwarder.ForwardAsync(Arg.Do(t => captured.Add(t)), Arg.Any()) + .Returns(Task.CompletedTask); + + var sut = CreateSut(); + await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.Delivered)); + + Assert.Equal(2, captured.Count); + + var attempted = captured[0]; + Assert.Equal(AuditKind.ApiCallCached, attempted.Audit.Kind); + Assert.Equal(AuditStatus.Attempted, attempted.Audit.Status); + Assert.Equal("Attempted", attempted.Operational.Status); + Assert.Null(attempted.Operational.TerminalAtUtc); + + var resolve = captured[1]; + Assert.Equal(AuditKind.CachedResolve, resolve.Audit.Kind); + Assert.Equal(AuditStatus.Delivered, resolve.Audit.Status); + Assert.Equal("Delivered", resolve.Operational.Status); + Assert.NotNull(resolve.Operational.TerminalAtUtc); + Assert.Equal(_id.Value, resolve.Audit.CorrelationId); + } + + [Fact] + public async Task PermanentFailure_EmitsAttempted_AndCachedResolveParked() + { + var captured = new List(); + _forwarder.ForwardAsync(Arg.Do(t => captured.Add(t)), Arg.Any()) + .Returns(Task.CompletedTask); + + var sut = CreateSut(); + await sut.OnAttemptCompletedAsync(Ctx( + CachedCallAttemptOutcome.PermanentFailure, + lastError: "Permanent failure (handler returned false)")); + + Assert.Equal(2, captured.Count); + Assert.Equal(AuditKind.ApiCallCached, captured[0].Audit.Kind); + Assert.Equal(AuditKind.CachedResolve, captured[1].Audit.Kind); + Assert.Equal(AuditStatus.Parked, captured[1].Audit.Status); + Assert.Equal("Parked", captured[1].Operational.Status); + } + + [Fact] + public async Task ParkedMaxRetries_EmitsAttempted_AndCachedResolveParked() + { + var captured = new List(); + _forwarder.ForwardAsync(Arg.Do(t => captured.Add(t)), Arg.Any()) + .Returns(Task.CompletedTask); + + var sut = CreateSut(); + await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.ParkedMaxRetries)); + + Assert.Equal(2, captured.Count); + Assert.Equal(AuditKind.CachedResolve, captured[1].Audit.Kind); + Assert.Equal(AuditStatus.Parked, captured[1].Audit.Status); + } + + [Fact] + public async Task DbChannel_MapsToDbWriteCachedKind_AndDbOutboundChannel() + { + var captured = new List(); + _forwarder.ForwardAsync(Arg.Do(t => captured.Add(t)), Arg.Any()) + .Returns(Task.CompletedTask); + + var sut = CreateSut(); + await sut.OnAttemptCompletedAsync(Ctx( + CachedCallAttemptOutcome.Delivered, channel: "DbOutbound")); + + Assert.Equal(2, captured.Count); + Assert.Equal(AuditKind.DbWriteCached, captured[0].Audit.Kind); + Assert.Equal(AuditChannel.DbOutbound, captured[0].Audit.Channel); + Assert.Equal("DbOutbound", captured[0].Operational.Channel); + Assert.Equal(AuditKind.CachedResolve, captured[1].Audit.Kind); + Assert.Equal(AuditChannel.DbOutbound, captured[1].Audit.Channel); + } + + [Fact] + public async Task BridgeDoesNotThrow_WhenForwarderThrows() + { + _forwarder + .ForwardAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromException(new InvalidOperationException("forwarder down"))); + + var sut = CreateSut(); + + // Must not throw — best-effort emission. + await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.Delivered)); + } + + [Fact] + public async Task BridgePopulatesProvenance_FromAttemptContext() + { + CachedCallTelemetry? captured = null; + _forwarder.ForwardAsync(Arg.Do(t => captured = t), Arg.Any()) + .Returns(Task.CompletedTask); + + var sut = CreateSut(); + await sut.OnAttemptCompletedAsync(Ctx( + CachedCallAttemptOutcome.TransientFailure, + retryCount: 3, + lastError: "transient", + httpStatus: 500)); + + Assert.NotNull(captured); + Assert.Equal("site-77", captured!.Audit.SourceSiteId); + Assert.Equal("Plant.Pump42", captured.Audit.SourceInstanceId); + Assert.Equal("ERP.GetOrder", captured.Audit.Target); + Assert.Equal(42, captured.Audit.DurationMs); + Assert.Equal(_id.Value, captured.Audit.CorrelationId); + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Site/Telemetry/CachedCallTelemetryForwarderTests.cs b/tests/ScadaLink.AuditLog.Tests/Site/Telemetry/CachedCallTelemetryForwarderTests.cs new file mode 100644 index 0000000..61cc353 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Site/Telemetry/CachedCallTelemetryForwarderTests.cs @@ -0,0 +1,245 @@ +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using NSubstitute.ExceptionExtensions; +using ScadaLink.AuditLog.Site.Telemetry; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Tests.Site.Telemetry; + +/// +/// Bundle E E2 tests for . The +/// forwarder is the site-side dual emitter: every cached-call lifecycle event +/// writes one to and one +/// operational tracking-row mutation to . +/// Audit-emission contract: best-effort — a thrown writer or tracking store +/// must be logged and swallowed; the forwarder must never propagate the +/// exception to the calling script. +/// +public class CachedCallTelemetryForwarderTests +{ + private readonly IAuditWriter _writer = Substitute.For(); + private readonly IOperationTrackingStore _tracking = Substitute.For(); + private readonly TrackedOperationId _id = TrackedOperationId.New(); + private readonly DateTime _now = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc); + + private CachedCallTelemetryForwarder CreateSut() => new( + _writer, _tracking, NullLogger.Instance); + + private CachedCallTelemetry SubmitPacket() => + new( + Audit: new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = _now, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.CachedSubmit, + CorrelationId = _id.Value, + SourceSiteId = "site-1", + SourceInstanceId = "inst-1", + SourceScript = "ScriptActor:doStuff", + Target = "ERP.GetOrder", + Status = AuditStatus.Submitted, + ForwardState = AuditForwardState.Pending, + }, + Operational: new SiteCallOperational( + TrackedOperationId: _id, + Channel: "ApiOutbound", + Target: "ERP.GetOrder", + SourceSite: "site-1", + Status: "Submitted", + RetryCount: 0, + LastError: null, + HttpStatus: null, + CreatedAtUtc: _now, + UpdatedAtUtc: _now, + TerminalAtUtc: null)); + + private CachedCallTelemetry AttemptedPacket(int retryCount = 1, string? lastError = "HTTP 500", int? httpStatus = 500) => + new( + Audit: new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = _now, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCallCached, + CorrelationId = _id.Value, + SourceSiteId = "site-1", + Target = "ERP.GetOrder", + Status = AuditStatus.Attempted, + HttpStatus = httpStatus, + ErrorMessage = lastError, + ForwardState = AuditForwardState.Pending, + }, + Operational: new SiteCallOperational( + TrackedOperationId: _id, + Channel: "ApiOutbound", + Target: "ERP.GetOrder", + SourceSite: "site-1", + Status: "Attempted", + RetryCount: retryCount, + LastError: lastError, + HttpStatus: httpStatus, + CreatedAtUtc: _now, + UpdatedAtUtc: _now, + TerminalAtUtc: null)); + + private CachedCallTelemetry ResolvePacket(string status = "Delivered") => + new( + Audit: new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = _now, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.CachedResolve, + CorrelationId = _id.Value, + SourceSiteId = "site-1", + Target = "ERP.GetOrder", + Status = Enum.Parse(status), + ForwardState = AuditForwardState.Pending, + }, + Operational: new SiteCallOperational( + TrackedOperationId: _id, + Channel: "ApiOutbound", + Target: "ERP.GetOrder", + SourceSite: "site-1", + Status: status, + RetryCount: 2, + LastError: null, + HttpStatus: null, + CreatedAtUtc: _now, + UpdatedAtUtc: _now, + TerminalAtUtc: _now)); + + [Fact] + public async Task ForwardAsync_Submit_WritesAuditEvent_AndRecordsEnqueue() + { + var sut = CreateSut(); + var packet = SubmitPacket(); + + await sut.ForwardAsync(packet, CancellationToken.None); + + // Audit row: one WriteAsync of the submit event. + await _writer.Received(1).WriteAsync( + Arg.Is(e => + e.EventId == packet.Audit.EventId + && e.Kind == AuditKind.CachedSubmit + && e.Status == AuditStatus.Submitted), + Arg.Any()); + + // Tracking row: insert-if-not-exists with kind discriminator. + await _tracking.Received(1).RecordEnqueueAsync( + _id, + "ApiOutbound", + "ERP.GetOrder", + "inst-1", + "ScriptActor:doStuff", + Arg.Any()); + await _tracking.DidNotReceiveWithAnyArgs().RecordAttemptAsync( + default, default!, default, default, default, default); + await _tracking.DidNotReceiveWithAnyArgs().RecordTerminalAsync( + default, default!, default, default, default); + } + + [Fact] + public async Task ForwardAsync_Attempted_WritesAuditEvent_AndRecordsAttempt() + { + var sut = CreateSut(); + var packet = AttemptedPacket(retryCount: 2, lastError: "HTTP 503", httpStatus: 503); + + await sut.ForwardAsync(packet, CancellationToken.None); + + await _writer.Received(1).WriteAsync( + Arg.Is(e => + e.EventId == packet.Audit.EventId + && e.Kind == AuditKind.ApiCallCached + && e.Status == AuditStatus.Attempted), + Arg.Any()); + + await _tracking.Received(1).RecordAttemptAsync( + _id, "Attempted", 2, "HTTP 503", 503, Arg.Any()); + await _tracking.DidNotReceiveWithAnyArgs().RecordEnqueueAsync( + default, default!, default, default, default, default); + await _tracking.DidNotReceiveWithAnyArgs().RecordTerminalAsync( + default, default!, default, default, default); + } + + [Fact] + public async Task ForwardAsync_Resolve_WritesAuditEvent_AndRecordsTerminal() + { + var sut = CreateSut(); + var packet = ResolvePacket("Delivered"); + + await sut.ForwardAsync(packet, CancellationToken.None); + + await _writer.Received(1).WriteAsync( + Arg.Is(e => + e.EventId == packet.Audit.EventId + && e.Kind == AuditKind.CachedResolve + && e.Status == AuditStatus.Delivered), + Arg.Any()); + + await _tracking.Received(1).RecordTerminalAsync( + _id, "Delivered", null, null, Arg.Any()); + await _tracking.DidNotReceiveWithAnyArgs().RecordEnqueueAsync( + default, default!, default, default, default, default); + await _tracking.DidNotReceiveWithAnyArgs().RecordAttemptAsync( + default, default!, default, default, default, default); + } + + [Fact] + public async Task ForwardAsync_WriterThrows_Logs_DoesNotPropagate() + { + _writer.WriteAsync(Arg.Any(), Arg.Any()) + .Throws(new InvalidOperationException("primary down")); + + var sut = CreateSut(); + + // Must not throw. + await sut.ForwardAsync(SubmitPacket(), CancellationToken.None); + + // Tracking still attempted — emission of the two halves is independent + // so a writer outage cannot starve the operational row (and vice-versa). + await _tracking.Received(1).RecordEnqueueAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()); + } + + [Fact] + public async Task ForwardAsync_TrackingStoreThrows_Logs_DoesNotPropagate() + { + _tracking.RecordEnqueueAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Throws(new InvalidOperationException("sqlite locked")); + + var sut = CreateSut(); + + await sut.ForwardAsync(SubmitPacket(), CancellationToken.None); + + // Writer still attempted — emission halves are independent. + await _writer.Received(1).WriteAsync( + Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task ForwardAsync_NullPacket_Throws() + { + var sut = CreateSut(); + + await Assert.ThrowsAsync( + () => sut.ForwardAsync(null!, CancellationToken.None)); + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Site/Telemetry/NoOpSiteStreamAuditClientTests.cs b/tests/ScadaLink.AuditLog.Tests/Site/Telemetry/NoOpSiteStreamAuditClientTests.cs new file mode 100644 index 0000000..b673bbb --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Site/Telemetry/NoOpSiteStreamAuditClientTests.cs @@ -0,0 +1,58 @@ +using ScadaLink.AuditLog.Site.Telemetry; +using ScadaLink.Communication.Grpc; + +namespace ScadaLink.AuditLog.Tests.Site.Telemetry; + +/// +/// Bundle E E1 tests for . The NoOp +/// client is the default binding until M6 +/// delivers the gRPC-backed implementation; both IngestAuditEventsAsync +/// (M2) and IngestCachedTelemetryAsync (M3) must return an empty ack +/// (no rows flipped to Forwarded) without throwing or partially handling the +/// batch. +/// +public class NoOpSiteStreamAuditClientTests +{ + [Fact] + public async Task IngestCachedTelemetryAsync_EmptyBatch_ReturnsEmptyAck() + { + var sut = new NoOpSiteStreamAuditClient(); + var batch = new CachedTelemetryBatch(); + + var ack = await sut.IngestCachedTelemetryAsync(batch, CancellationToken.None); + + Assert.NotNull(ack); + Assert.Empty(ack.AcceptedEventIds); + } + + [Fact] + public async Task IngestCachedTelemetryAsync_PopulatedBatch_ReturnsEmptyAck() + { + var sut = new NoOpSiteStreamAuditClient(); + var batch = new CachedTelemetryBatch(); + batch.Packets.Add(new CachedTelemetryPacket + { + AuditEvent = new AuditEventDto + { + EventId = Guid.NewGuid().ToString(), + Channel = "ApiOutbound", + Kind = "CachedSubmit", + Status = "Submitted", + }, + }); + + var ack = await sut.IngestCachedTelemetryAsync(batch, CancellationToken.None); + + // No EventIds flipped — NoOp does not forward to anyone. + Assert.Empty(ack.AcceptedEventIds); + } + + [Fact] + public async Task IngestCachedTelemetryAsync_NullBatch_Throws() + { + var sut = new NoOpSiteStreamAuditClient(); + + await Assert.ThrowsAsync( + () => sut.IngestCachedTelemetryAsync(null!, CancellationToken.None)); + } +} diff --git a/tests/ScadaLink.Commons.Tests/Messages/Integration/CachedCallTelemetryTests.cs b/tests/ScadaLink.Commons.Tests/Messages/Integration/CachedCallTelemetryTests.cs new file mode 100644 index 0000000..03980d8 --- /dev/null +++ b/tests/ScadaLink.Commons.Tests/Messages/Integration/CachedCallTelemetryTests.cs @@ -0,0 +1,203 @@ +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.Commons.Tests.Messages.Integration; + +/// +/// Audit Log #23 (M3 Bundle A — Task A4) — tests for the combined +/// audit + operational telemetry packet emitted per cached-call lifecycle event +/// (Submit → per-attempt ApiCallCached / DbWriteCached → +/// terminal Resolve). The site emits one packet per event; central writes +/// AuditLog + SiteCalls in one MS SQL transaction. +/// +public class CachedCallTelemetryTests +{ + private static readonly DateTime FixedNowUtc = new(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc); + + private const string SiteId = "site-77"; + private const string InstanceName = "Plant.Pump42"; + private const string SourceScript = "ScriptActor:OnTick"; + + private static AuditEvent BuildAuditEvent( + TrackedOperationId trackedId, + AuditKind kind, + AuditStatus status, + Guid? correlationId = null, + string? errorMessage = null, + int? httpStatus = null) + { + return new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = FixedNowUtc, + Channel = AuditChannel.ApiOutbound, + Kind = kind, + CorrelationId = correlationId ?? trackedId.Value, + SourceSiteId = SiteId, + SourceInstanceId = InstanceName, + SourceScript = SourceScript, + Target = "ERP.GetOrder", + Status = status, + HttpStatus = httpStatus, + ErrorMessage = errorMessage, + PayloadTruncated = false, + ForwardState = AuditForwardState.Pending, + }; + } + + private static SiteCallOperational BuildOperational( + TrackedOperationId trackedId, + AuditStatus status, + int retryCount, + string? lastError = null, + int? httpStatus = null, + DateTime? terminalAtUtc = null) + { + return new SiteCallOperational( + TrackedOperationId: trackedId, + Channel: nameof(AuditChannel.ApiOutbound), + Target: "ERP.GetOrder", + SourceSite: SiteId, + Status: status.ToString(), + RetryCount: retryCount, + LastError: lastError, + HttpStatus: httpStatus, + CreatedAtUtc: FixedNowUtc, + UpdatedAtUtc: terminalAtUtc ?? FixedNowUtc, + TerminalAtUtc: terminalAtUtc); + } + + [Fact] + public void SubmitPacket_AuditCarriesCachedSubmit_AndOperationalRetryCountZero() + { + var trackedId = TrackedOperationId.New(); + var audit = BuildAuditEvent(trackedId, AuditKind.CachedSubmit, AuditStatus.Submitted); + var operational = BuildOperational(trackedId, AuditStatus.Submitted, retryCount: 0); + + var packet = new CachedCallTelemetry(audit, operational); + + Assert.Equal(AuditKind.CachedSubmit, packet.Audit.Kind); + Assert.Equal(AuditStatus.Submitted, packet.Audit.Status); + Assert.Equal(nameof(AuditStatus.Submitted), packet.Operational.Status); + Assert.Equal(0, packet.Operational.RetryCount); + Assert.Null(packet.Operational.TerminalAtUtc); + Assert.Equal(trackedId, packet.Operational.TrackedOperationId); + } + + [Fact] + public void AttemptedPacket_AuditCarriesApiCallCached_RetryCountAlignsBetweenAuditAndOperational() + { + var trackedId = TrackedOperationId.New(); + var audit = BuildAuditEvent( + trackedId, + AuditKind.ApiCallCached, + AuditStatus.Attempted, + errorMessage: "HTTP 503 from ERP", + httpStatus: 503); + var operational = BuildOperational( + trackedId, + AuditStatus.Attempted, + retryCount: 2, + lastError: "HTTP 503 from ERP", + httpStatus: 503); + + var packet = new CachedCallTelemetry(audit, operational); + + Assert.Equal(AuditKind.ApiCallCached, packet.Audit.Kind); + Assert.Equal(AuditStatus.Attempted, packet.Audit.Status); + Assert.Equal(nameof(AuditStatus.Attempted), packet.Operational.Status); + // Retry-count alignment: the operational row carries the canonical N; + // the audit row's error/http surface the same attempt's outcome. + Assert.Equal(packet.Audit.ErrorMessage, packet.Operational.LastError); + Assert.Equal(packet.Audit.HttpStatus, packet.Operational.HttpStatus); + Assert.Equal(2, packet.Operational.RetryCount); + Assert.Null(packet.Operational.TerminalAtUtc); + } + + [Fact] + public void AttemptedPacket_DbWriteCached_CarriesDbWriteCachedKind() + { + var trackedId = TrackedOperationId.New(); + var audit = BuildAuditEvent( + trackedId, + AuditKind.DbWriteCached, + AuditStatus.Attempted, + errorMessage: "Timeout", + httpStatus: null); + var operational = BuildOperational( + trackedId, + AuditStatus.Attempted, + retryCount: 1, + lastError: "Timeout"); + + var packet = new CachedCallTelemetry(audit, operational); + + Assert.Equal(AuditKind.DbWriteCached, packet.Audit.Kind); + Assert.Equal(AuditStatus.Attempted, packet.Audit.Status); + Assert.Equal(1, packet.Operational.RetryCount); + } + + [Theory] + [InlineData(AuditStatus.Delivered)] + [InlineData(AuditStatus.Failed)] + [InlineData(AuditStatus.Parked)] + [InlineData(AuditStatus.Discarded)] + public void ResolvePacket_AuditCarriesCachedResolve_OperationalTerminalAtUtcSet(AuditStatus terminalStatus) + { + var trackedId = TrackedOperationId.New(); + var terminalAt = FixedNowUtc.AddMinutes(5); + var audit = BuildAuditEvent(trackedId, AuditKind.CachedResolve, terminalStatus); + var operational = BuildOperational( + trackedId, + terminalStatus, + retryCount: 3, + terminalAtUtc: terminalAt); + + var packet = new CachedCallTelemetry(audit, operational); + + Assert.Equal(AuditKind.CachedResolve, packet.Audit.Kind); + Assert.Equal(terminalStatus, packet.Audit.Status); + Assert.Equal(terminalStatus.ToString(), packet.Operational.Status); + Assert.NotNull(packet.Operational.TerminalAtUtc); + Assert.Equal(terminalAt, packet.Operational.TerminalAtUtc); + } + + [Fact] + public void CachedCallTelemetry_RoundTripEquality() + { + var trackedId = TrackedOperationId.New(); + var audit = BuildAuditEvent(trackedId, AuditKind.CachedSubmit, AuditStatus.Submitted); + var operational = BuildOperational(trackedId, AuditStatus.Submitted, retryCount: 0); + + var a = new CachedCallTelemetry(audit, operational); + var b = new CachedCallTelemetry(audit, operational); + + Assert.Equal(a, b); + Assert.Equal(a.GetHashCode(), b.GetHashCode()); + + var differentOperational = operational with { RetryCount = 1 }; + var c = a with { Operational = differentOperational }; + + Assert.NotEqual(a, c); + Assert.Equal(0, a.Operational.RetryCount); + Assert.Equal(1, c.Operational.RetryCount); + } + + [Fact] + public void SiteCallOperational_RoundTripEquality_AndWithExpression() + { + var trackedId = TrackedOperationId.New(); + var a = BuildOperational(trackedId, AuditStatus.Submitted, retryCount: 0); + var b = BuildOperational(trackedId, AuditStatus.Submitted, retryCount: 0); + + Assert.Equal(a, b); + Assert.Equal(a.GetHashCode(), b.GetHashCode()); + + var withDifferentRetry = a with { RetryCount = 5 }; + Assert.NotEqual(a, withDifferentRetry); + Assert.Equal(0, a.RetryCount); + Assert.Equal(5, withDifferentRetry.RetryCount); + } +} diff --git a/tests/ScadaLink.Commons.Tests/Types/TrackedOperationIdTests.cs b/tests/ScadaLink.Commons.Tests/Types/TrackedOperationIdTests.cs new file mode 100644 index 0000000..3e8822d --- /dev/null +++ b/tests/ScadaLink.Commons.Tests/Types/TrackedOperationIdTests.cs @@ -0,0 +1,80 @@ +using ScadaLink.Commons.Types; + +namespace ScadaLink.Commons.Tests.Types; + +/// +/// Audit Log #23 (M3): tests for the strongly-typed cached-operation identifier +/// produced by ExternalSystem.CachedCall / Database.CachedWrite and +/// surfaced to scripts via Tracking.Status(id). +/// +public class TrackedOperationIdTests +{ + [Fact] + public void New_ProducesUniqueIds() + { + var a = TrackedOperationId.New(); + var b = TrackedOperationId.New(); + + Assert.NotEqual(a, b); + Assert.NotEqual(Guid.Empty, a.Value); + Assert.NotEqual(Guid.Empty, b.Value); + } + + [Fact] + public void Parse_RoundTrip_PreservesValue() + { + var original = TrackedOperationId.New(); + var serialized = original.ToString(); + + var parsed = TrackedOperationId.Parse(serialized); + + Assert.Equal(original, parsed); + Assert.Equal(original.Value, parsed.Value); + } + + [Fact] + public void TryParse_InvalidInput_ReturnsFalse() + { + Assert.False(TrackedOperationId.TryParse("not-a-guid", out var result)); + Assert.Equal(default, result); + + Assert.False(TrackedOperationId.TryParse(null, out var nullResult)); + Assert.Equal(default, nullResult); + + Assert.False(TrackedOperationId.TryParse(string.Empty, out var emptyResult)); + Assert.Equal(default, emptyResult); + } + + [Fact] + public void TryParse_ValidInput_ReturnsTrueAndId() + { + var original = TrackedOperationId.New(); + var serialized = original.ToString(); + + Assert.True(TrackedOperationId.TryParse(serialized, out var parsed)); + Assert.Equal(original, parsed); + } + + [Fact] + public void Equality_BasedOnValue() + { + var guid = Guid.NewGuid(); + var a = new TrackedOperationId(guid); + var b = new TrackedOperationId(guid); + + Assert.Equal(a, b); + Assert.True(a == b); + Assert.False(a != b); + Assert.Equal(a.GetHashCode(), b.GetHashCode()); + } + + [Fact] + public void ToString_StandardGuidFormat() + { + var guid = Guid.Parse("12345678-1234-1234-1234-1234567890ab"); + var id = new TrackedOperationId(guid); + + // "D" format: 32 hex digits separated by hyphens (8-4-4-4-12). + Assert.Equal("12345678-1234-1234-1234-1234567890ab", id.ToString()); + } +} diff --git a/tests/ScadaLink.Communication.Tests/Protos/CachedTelemetryProtoTests.cs b/tests/ScadaLink.Communication.Tests/Protos/CachedTelemetryProtoTests.cs new file mode 100644 index 0000000..4405768 --- /dev/null +++ b/tests/ScadaLink.Communication.Tests/Protos/CachedTelemetryProtoTests.cs @@ -0,0 +1,173 @@ +using Google.Protobuf; +using Google.Protobuf.WellKnownTypes; +using ScadaLink.Communication.Grpc; + +namespace ScadaLink.Communication.Tests.Protos; + +/// +/// Wire-format round-trip tests for the Audit Log (#23) M3 cached-telemetry +/// proto messages (, +/// , ). +/// Locks the additive contract the central dual-write transaction depends on. +/// +public class CachedTelemetryProtoTests +{ + private static AuditEventDto NewAuditDto(Guid? id = null) => new() + { + EventId = (id ?? Guid.NewGuid()).ToString(), + OccurredAtUtc = Timestamp.FromDateTimeOffset( + new DateTimeOffset(2026, 5, 20, 10, 15, 30, 123, TimeSpan.Zero)), + Channel = "ApiOutbound", + Kind = "CachedSubmit", + Status = "Submitted", + SourceSiteId = "site-1", + }; + + [Fact] + public void SiteCallOperationalDto_RoundTrip_PreservesAllFields() + { + var createdAt = Timestamp.FromDateTimeOffset( + new DateTimeOffset(2026, 5, 20, 10, 0, 0, TimeSpan.Zero)); + var updatedAt = Timestamp.FromDateTimeOffset( + new DateTimeOffset(2026, 5, 20, 10, 5, 0, TimeSpan.Zero)); + var terminalAt = Timestamp.FromDateTimeOffset( + new DateTimeOffset(2026, 5, 20, 10, 10, 0, TimeSpan.Zero)); + + var original = new SiteCallOperationalDto + { + TrackedOperationId = Guid.NewGuid().ToString(), + Channel = "ApiOutbound", + Target = "ERP.GetOrder", + SourceSite = "site-melbourne", + Status = "Delivered", + RetryCount = 3, + LastError = "transient 503", + HttpStatus = 200, + CreatedAtUtc = createdAt, + UpdatedAtUtc = updatedAt, + TerminalAtUtc = terminalAt, + }; + + var bytes = original.ToByteArray(); + var deserialized = SiteCallOperationalDto.Parser.ParseFrom(bytes); + + Assert.Equal(original.TrackedOperationId, deserialized.TrackedOperationId); + Assert.Equal(original.Channel, deserialized.Channel); + Assert.Equal(original.Target, deserialized.Target); + Assert.Equal(original.SourceSite, deserialized.SourceSite); + Assert.Equal(original.Status, deserialized.Status); + Assert.Equal(original.RetryCount, deserialized.RetryCount); + Assert.Equal(original.LastError, deserialized.LastError); + Assert.Equal(original.HttpStatus, deserialized.HttpStatus); + Assert.Equal(original.CreatedAtUtc, deserialized.CreatedAtUtc); + Assert.Equal(original.UpdatedAtUtc, deserialized.UpdatedAtUtc); + Assert.Equal(original.TerminalAtUtc, deserialized.TerminalAtUtc); + } + + [Fact] + public void SiteCallOperationalDto_TerminalAt_AbsentWhenNotTerminal() + { + // Lifecycle events prior to the terminal step leave TerminalAtUtc unset; + // the well-known Timestamp wrapper is absent on the wire (null in C#). + var dto = new SiteCallOperationalDto + { + TrackedOperationId = Guid.NewGuid().ToString(), + Channel = "DbOutbound", + Target = "warehouse.dbo.WriteOrder", + SourceSite = "site-brisbane", + Status = "Attempted", + RetryCount = 1, + CreatedAtUtc = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), + UpdatedAtUtc = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), + }; + + Assert.Null(dto.TerminalAtUtc); + + var bytes = dto.ToByteArray(); + var deserialized = SiteCallOperationalDto.Parser.ParseFrom(bytes); + + Assert.Null(deserialized.TerminalAtUtc); + } + + [Fact] + public void SiteCallOperationalDto_NullableHttpStatus_AbsentByDefault() + { + // Int32Value wrapper-typed http_status — unset round-trips as null, + // matching DB nullable column semantics for non-API cached writes. + var dto = new SiteCallOperationalDto + { + TrackedOperationId = Guid.NewGuid().ToString(), + Channel = "DbOutbound", + Target = "warehouse.dbo.WriteOrder", + SourceSite = "site-brisbane", + Status = "Submitted", + RetryCount = 0, + CreatedAtUtc = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), + UpdatedAtUtc = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), + }; + + Assert.Null(dto.HttpStatus); + + var bytes = dto.ToByteArray(); + var deserialized = SiteCallOperationalDto.Parser.ParseFrom(bytes); + + Assert.Null(deserialized.HttpStatus); + } + + [Fact] + public void CachedTelemetryPacket_RoundTrip_PreservesNestedEntities() + { + var trackedOpId = Guid.NewGuid().ToString(); + var auditDto = NewAuditDto(); + auditDto.Target = "ERP.GetOrder"; + auditDto.Status = "Attempted"; + + var operationalDto = new SiteCallOperationalDto + { + TrackedOperationId = trackedOpId, + Channel = "ApiOutbound", + Target = "ERP.GetOrder", + SourceSite = "site-1", + Status = "Attempted", + RetryCount = 2, + HttpStatus = 503, + LastError = "Service unavailable", + CreatedAtUtc = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), + UpdatedAtUtc = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), + }; + + var original = new CachedTelemetryPacket + { + AuditEvent = auditDto, + Operational = operationalDto, + }; + + var bytes = original.ToByteArray(); + var deserialized = CachedTelemetryPacket.Parser.ParseFrom(bytes); + + Assert.NotNull(deserialized.AuditEvent); + Assert.Equal(auditDto.EventId, deserialized.AuditEvent.EventId); + Assert.Equal(auditDto.Target, deserialized.AuditEvent.Target); + Assert.Equal(auditDto.Status, deserialized.AuditEvent.Status); + + Assert.NotNull(deserialized.Operational); + Assert.Equal(trackedOpId, deserialized.Operational.TrackedOperationId); + Assert.Equal(operationalDto.Channel, deserialized.Operational.Channel); + Assert.Equal(operationalDto.Status, deserialized.Operational.Status); + Assert.Equal(operationalDto.RetryCount, deserialized.Operational.RetryCount); + Assert.Equal(operationalDto.HttpStatus, deserialized.Operational.HttpStatus); + Assert.Equal(operationalDto.LastError, deserialized.Operational.LastError); + } + + [Fact] + public void CachedTelemetryBatch_Empty_RoundTrip_Yields_EmptyPackets() + { + var original = new CachedTelemetryBatch(); + Assert.Empty(original.Packets); + + var bytes = original.ToByteArray(); + var deserialized = CachedTelemetryBatch.Parser.ParseFrom(bytes); + + Assert.Empty(deserialized.Packets); + } +} diff --git a/tests/ScadaLink.Communication.Tests/SiteStreamIngestCachedTelemetryTests.cs b/tests/ScadaLink.Communication.Tests/SiteStreamIngestCachedTelemetryTests.cs new file mode 100644 index 0000000..b5194a0 --- /dev/null +++ b/tests/ScadaLink.Communication.Tests/SiteStreamIngestCachedTelemetryTests.cs @@ -0,0 +1,121 @@ +using Akka.Actor; +using Akka.TestKit.Xunit2; +using Google.Protobuf.WellKnownTypes; +using Grpc.Core; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using ScadaLink.Commons.Messages.Audit; +using ScadaLink.Communication.Grpc; + +namespace ScadaLink.Communication.Tests; + +/// +/// Bundle D D2 tests for . +/// Verifies the DTO→entity→actor→ack round-trip through the gRPC handler. A +/// tiny EchoCachedIngestActor stands in for the central +/// AuditLogIngestActor, replying with the EventIds it received so the +/// test asserts the wiring without depending on MSSQL. +/// +public class SiteStreamIngestCachedTelemetryTests : TestKit +{ + private readonly ISiteStreamSubscriber _subscriber = Substitute.For(); + + private SiteStreamGrpcServer CreateServer() => + new(_subscriber, NullLogger.Instance); + + private static ServerCallContext NewContext(CancellationToken ct = default) + { + var context = Substitute.For(); + context.CancellationToken.Returns(ct); + return context; + } + + private static CachedTelemetryPacket NewPacket(Guid? eventId = null, Guid? trackedId = null) + { + var now = Timestamp.FromDateTime( + DateTime.SpecifyKind(new DateTime(2026, 5, 20, 10, 0, 0), DateTimeKind.Utc)); + return new CachedTelemetryPacket + { + AuditEvent = new AuditEventDto + { + EventId = (eventId ?? Guid.NewGuid()).ToString(), + OccurredAtUtc = now, + Channel = "ApiOutbound", + Kind = "CachedSubmit", + Status = "Submitted", + SourceSiteId = "site-1", + CorrelationId = (trackedId ?? Guid.NewGuid()).ToString(), + }, + Operational = new SiteCallOperationalDto + { + TrackedOperationId = (trackedId ?? Guid.NewGuid()).ToString(), + Channel = "ApiOutbound", + Target = "ERP.GetOrder", + SourceSite = "site-1", + Status = "Submitted", + RetryCount = 0, + CreatedAtUtc = now, + UpdatedAtUtc = now, + }, + }; + } + + [Fact] + public async Task IngestCachedTelemetry_RoutesToActor_ReturnsReply() + { + // Arrange — stub actor that echoes every received EventId back. + var stubActor = Sys.ActorOf(Props.Create(() => new EchoCachedIngestActor())); + + var server = CreateServer(); + server.SetAuditIngestActor(stubActor); + + var packets = Enumerable.Range(0, 3) + .Select(_ => NewPacket()) + .ToList(); + + var batch = new CachedTelemetryBatch(); + batch.Packets.AddRange(packets); + + // Act + var ack = await server.IngestCachedTelemetry(batch, NewContext()); + + // Assert — every packet's EventId appears in the ack, demonstrating + // end-to-end routing through the actor. + Assert.Equal(3, ack.AcceptedEventIds.Count); + var expectedIds = packets.Select(p => p.AuditEvent.EventId).ToHashSet(); + Assert.True(expectedIds.SetEquals(ack.AcceptedEventIds.ToHashSet())); + } + + [Fact] + public async Task IngestCachedTelemetry_NoActorWired_ReturnsEmptyAck() + { + var server = CreateServer(); + // Intentionally do NOT call SetAuditIngestActor — simulates host + // startup race window. + + var batch = new CachedTelemetryBatch(); + batch.Packets.Add(NewPacket()); + + var ack = await server.IngestCachedTelemetry(batch, NewContext()); + + Assert.Empty(ack.AcceptedEventIds); + } + + /// + /// Tiny ReceiveActor that echoes every EventId in an incoming + /// back as an + /// . Stands in for the central + /// AuditLogIngestActor so this test never touches MSSQL. + /// + private sealed class EchoCachedIngestActor : ReceiveActor + { + public EchoCachedIngestActor() + { + Receive(cmd => + { + var ids = cmd.Entries.Select(e => e.Audit.EventId).ToList(); + Sender.Tell(new IngestCachedTelemetryReply(ids)); + }); + } + } +} diff --git a/tests/ScadaLink.ConfigurationDatabase.Tests/Configurations/SiteCallEntityTypeConfigurationTests.cs b/tests/ScadaLink.ConfigurationDatabase.Tests/Configurations/SiteCallEntityTypeConfigurationTests.cs new file mode 100644 index 0000000..ed0720b --- /dev/null +++ b/tests/ScadaLink.ConfigurationDatabase.Tests/Configurations/SiteCallEntityTypeConfigurationTests.cs @@ -0,0 +1,109 @@ +using Microsoft.EntityFrameworkCore; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.ConfigurationDatabase; +using ScadaLink.ConfigurationDatabase.Configurations; + +namespace ScadaLink.ConfigurationDatabase.Tests.Configurations; + +/// +/// Schema-level tests for (#22 / #23 M3 Bundle B). +/// Verifies the record maps to the SiteCalls table with the +/// expected primary key, value conversion on TrackedOperationId, and the two named +/// indexes that back the "calls from this site" and "calls in this status" Central UI queries. +/// Mirrors the AuditLog Bundle B test pattern — inspects EF model metadata via the existing +/// in-memory SQLite test context, no database round-trips required. +/// +public class SiteCallEntityTypeConfigurationTests : IDisposable +{ + private readonly ScadaLinkDbContext _context; + + public SiteCallEntityTypeConfigurationTests() + { + _context = SqliteTestHelper.CreateInMemoryContext(); + } + + public void Dispose() + { + _context.Database.CloseConnection(); + _context.Dispose(); + } + + [Fact] + public void Configure_MapsToSiteCallsTable() + { + var entity = _context.Model.FindEntityType(typeof(SiteCall)); + + Assert.NotNull(entity); + Assert.Equal("SiteCalls", entity!.GetTableName()); + } + + [Fact] + public void Configure_PrimaryKey_TrackedOperationId() + { + var entity = _context.Model.FindEntityType(typeof(SiteCall)); + Assert.NotNull(entity); + + var pk = entity!.FindPrimaryKey(); + Assert.NotNull(pk); + + var pkPropertyNames = pk!.Properties.Select(p => p.Name).ToArray(); + Assert.Equal(new[] { nameof(SiteCall.TrackedOperationId) }, pkPropertyNames); + } + + [Fact] + public void Configure_HasIndexes_NamedAndOrdered() + { + var entity = _context.Model.FindEntityType(typeof(SiteCall)); + Assert.NotNull(entity); + + var indexes = entity!.GetIndexes().ToList(); + + // IX_SiteCalls_Source_Created: (SourceSite ASC, CreatedAtUtc DESC). + var sourceCreated = indexes.SingleOrDefault(i => i.GetDatabaseName() == "IX_SiteCalls_Source_Created"); + Assert.NotNull(sourceCreated); + var sourceCreatedProps = sourceCreated!.Properties.Select(p => p.Name).ToArray(); + Assert.Equal(new[] { nameof(SiteCall.SourceSite), nameof(SiteCall.CreatedAtUtc) }, sourceCreatedProps); + + // IX_SiteCalls_Status_Updated: (Status ASC, UpdatedAtUtc DESC). + var statusUpdated = indexes.SingleOrDefault(i => i.GetDatabaseName() == "IX_SiteCalls_Status_Updated"); + Assert.NotNull(statusUpdated); + var statusUpdatedProps = statusUpdated!.Properties.Select(p => p.Name).ToArray(); + Assert.Equal(new[] { nameof(SiteCall.Status), nameof(SiteCall.UpdatedAtUtc) }, statusUpdatedProps); + } + + [Fact] + public void Configure_TrackedOperationId_ConvertedToString_Length36() + { + var entity = _context.Model.FindEntityType(typeof(SiteCall)); + Assert.NotNull(entity); + + var property = entity!.FindProperty(nameof(SiteCall.TrackedOperationId)); + Assert.NotNull(property); + + // Stored as varchar(36) (TrackedOperationId.ToString("D") is always 36 chars). + // The value-conversion target type is exposed via GetProviderClrType when set, or + // discovered indirectly through the configured converter; either way the on-wire + // CLR type is string. + var providerClrType = property!.GetProviderClrType() ?? property.GetValueConverter()?.ProviderClrType; + Assert.Equal(typeof(string), providerClrType); + Assert.Equal(36, property.GetMaxLength()); + Assert.False(property.IsUnicode() ?? true); + } + + [Theory] + [InlineData(nameof(SiteCall.Channel), 32)] + [InlineData(nameof(SiteCall.SourceSite), 64)] + [InlineData(nameof(SiteCall.Status), 32)] + [InlineData(nameof(SiteCall.Target), 256)] + public void Configure_AsciiBoundedColumns(string propertyName, int expectedMaxLength) + { + var entity = _context.Model.FindEntityType(typeof(SiteCall)); + Assert.NotNull(entity); + + var property = entity!.FindProperty(propertyName); + Assert.NotNull(property); + + Assert.Equal(expectedMaxLength, property!.GetMaxLength()); + Assert.False(property.IsUnicode() ?? true); + } +} diff --git a/tests/ScadaLink.ConfigurationDatabase.Tests/Migrations/AddSiteCallsTableMigrationTests.cs b/tests/ScadaLink.ConfigurationDatabase.Tests/Migrations/AddSiteCallsTableMigrationTests.cs new file mode 100644 index 0000000..ff60d18 --- /dev/null +++ b/tests/ScadaLink.ConfigurationDatabase.Tests/Migrations/AddSiteCallsTableMigrationTests.cs @@ -0,0 +1,125 @@ +using Microsoft.Data.SqlClient; +using Xunit; + +namespace ScadaLink.ConfigurationDatabase.Tests.Migrations; + +/// +/// Bundle B2 (#22, #23 M3) integration tests for the AddSiteCallsTable +/// migration: applies the EF migrations to a freshly-created MSSQL test database +/// on the running infra/mssql container and asserts that the resulting +/// SiteCalls table carries the expected columns, primary key, and the +/// two named operational indexes. +/// +/// +/// Unlike AddAuditLogTable, the SiteCalls table is operational (mutable) +/// state — no partition function, no partition scheme, no DB-role restriction. +/// Standard [PRIMARY] filegroup. Tests pair +/// with Skip.IfNot(...) so the runner reports them as Skipped (not Passed) +/// when MSSQL is unreachable. The fixture applies the migration once at +/// construction time. +/// +public class AddSiteCallsTableMigrationTests : IClassFixture +{ + private readonly MsSqlMigrationFixture _fixture; + + public AddSiteCallsTableMigrationTests(MsSqlMigrationFixture fixture) + { + _fixture = fixture; + } + + [SkippableFact] + public async Task AppliesMigration_CreatesSiteCallsTable_WithExpectedColumns() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var exists = await ScalarAsync( + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES " + + "WHERE TABLE_NAME = 'SiteCalls' AND TABLE_SCHEMA = 'dbo';"); + Assert.Equal(1, exists); + + // Every required column from SiteCall + IngestedAtUtc. We don't pin types + // here because EF's CreateTable layer already encodes them; the + // entity-config tests cover length / unicode / nullability for the + // value-converted PK column. Just confirm the schema has all twelve. + var expectedColumns = new[] + { + "TrackedOperationId", + "Channel", + "Target", + "SourceSite", + "Status", + "RetryCount", + "LastError", + "HttpStatus", + "CreatedAtUtc", + "UpdatedAtUtc", + "TerminalAtUtc", + "IngestedAtUtc", + }; + + foreach (var column in expectedColumns) + { + var present = await ScalarAsync( + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS " + + $"WHERE TABLE_NAME = 'SiteCalls' AND COLUMN_NAME = '{column}';"); + Assert.True(present == 1, $"Expected SiteCalls.{column} to exist; found {present}."); + } + } + + [SkippableFact] + public async Task AppliesMigration_CreatesPK_OnTrackedOperationId() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + // Walk sys.indexes for the table's clustered PK index and confirm its + // single key column is TrackedOperationId. SiteCalls is non-partitioned + // so the PK is a simple single-column clustered index. + var pkColumn = await ScalarAsync( + "SELECT c.name FROM sys.indexes i " + + "INNER JOIN sys.objects o ON i.object_id = o.object_id " + + "INNER JOIN sys.index_columns ic ON i.object_id = ic.object_id AND i.index_id = ic.index_id " + + "INNER JOIN sys.columns c ON ic.object_id = c.object_id AND ic.column_id = c.column_id " + + "WHERE o.name = 'SiteCalls' AND i.is_primary_key = 1;"); + + Assert.Equal("TrackedOperationId", pkColumn); + } + + [SkippableFact] + public async Task AppliesMigration_CreatesIndex_Source_Created() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var count = await ScalarAsync( + "SELECT COUNT(*) FROM sys.indexes i " + + "INNER JOIN sys.objects o ON i.object_id = o.object_id " + + "WHERE o.name = 'SiteCalls' AND i.name = 'IX_SiteCalls_Source_Created';"); + Assert.Equal(1, count); + } + + [SkippableFact] + public async Task AppliesMigration_CreatesIndex_Status_Updated() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var count = await ScalarAsync( + "SELECT COUNT(*) FROM sys.indexes i " + + "INNER JOIN sys.objects o ON i.object_id = o.object_id " + + "WHERE o.name = 'SiteCalls' AND i.name = 'IX_SiteCalls_Status_Updated';"); + Assert.Equal(1, count); + } + + // --- helpers ------------------------------------------------------------ + + private async Task ScalarAsync(string sql) + { + await using var conn = _fixture.OpenConnection(); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + var result = await cmd.ExecuteScalarAsync(); + if (result is null || result is DBNull) + { + return default!; + } + return (T)Convert.ChangeType(result, typeof(T) == typeof(string) ? typeof(string) : Nullable.GetUnderlyingType(typeof(T)) ?? typeof(T))!; + } +} diff --git a/tests/ScadaLink.ConfigurationDatabase.Tests/Repositories/SiteCallAuditRepositoryTests.cs b/tests/ScadaLink.ConfigurationDatabase.Tests/Repositories/SiteCallAuditRepositoryTests.cs new file mode 100644 index 0000000..ae5dd90 --- /dev/null +++ b/tests/ScadaLink.ConfigurationDatabase.Tests/Repositories/SiteCallAuditRepositoryTests.cs @@ -0,0 +1,388 @@ +using Microsoft.EntityFrameworkCore; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Audit; +using ScadaLink.ConfigurationDatabase; +using ScadaLink.ConfigurationDatabase.Repositories; +using ScadaLink.ConfigurationDatabase.Tests.Migrations; +using Xunit; + +namespace ScadaLink.ConfigurationDatabase.Tests.Repositories; + +/// +/// Bundle B3 (#22, #23 M3) integration tests for . +/// Uses the same as the Bundle B2 migration tests so +/// the monotonic-upsert SQL executes against the real SiteCalls schema. Each test +/// scopes its data by minting a fresh (or a per-test +/// SourceSite suffix) so tests neither collide nor require teardown. +/// +public class SiteCallAuditRepositoryTests : IClassFixture +{ + private readonly MsSqlMigrationFixture _fixture; + + public SiteCallAuditRepositoryTests(MsSqlMigrationFixture fixture) + { + _fixture = fixture; + } + + [SkippableFact] + public async Task UpsertAsync_FreshId_InsertsOneRow() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var id = TrackedOperationId.New(); + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + + var row = NewRow(id, status: "Submitted", retryCount: 0); + await repo.UpsertAsync(row); + + await using var readContext = CreateContext(); + var loaded = await readContext.Set() + .Where(s => s.TrackedOperationId == id) + .ToListAsync(); + + Assert.Single(loaded); + Assert.Equal("Submitted", loaded[0].Status); + Assert.Equal(0, loaded[0].RetryCount); + } + + [SkippableFact] + public async Task UpsertAsync_AdvancedStatus_UpdatesRow() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var id = TrackedOperationId.New(); + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + + // Submitted (rank 0) → Forwarded (rank 1) → Attempted (rank 2) — every + // step strictly advances the rank, so each upsert must mutate the row. + await repo.UpsertAsync(NewRow(id, status: "Submitted", retryCount: 0)); + await repo.UpsertAsync(NewRow(id, status: "Forwarded", retryCount: 0)); + await repo.UpsertAsync(NewRow(id, status: "Attempted", retryCount: 1, lastError: "transient 503")); + + var loaded = await repo.GetAsync(id); + Assert.NotNull(loaded); + Assert.Equal("Attempted", loaded!.Status); + Assert.Equal(1, loaded.RetryCount); + Assert.Equal("transient 503", loaded.LastError); + } + + [SkippableFact] + public async Task UpsertAsync_OlderStatus_IsNoOp_RowUnchanged() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var id = TrackedOperationId.New(); + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + + // First land Attempted (rank 2). A late-arriving Submitted (rank 0) must + // NOT roll the row back — silent no-op. + await repo.UpsertAsync(NewRow(id, status: "Attempted", retryCount: 5, lastError: "transient")); + var attemptedSnapshot = await repo.GetAsync(id); + + await repo.UpsertAsync(NewRow(id, status: "Submitted", retryCount: 0, lastError: null)); + var afterStale = await repo.GetAsync(id); + + Assert.NotNull(afterStale); + Assert.Equal("Attempted", afterStale!.Status); + Assert.Equal(5, afterStale.RetryCount); + Assert.Equal("transient", afterStale.LastError); + // UpdatedAtUtc should not have moved when the stale write was rejected. + Assert.Equal(attemptedSnapshot!.UpdatedAtUtc, afterStale.UpdatedAtUtc); + } + + [SkippableFact] + public async Task UpsertAsync_SameStatus_IsNoOp() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var id = TrackedOperationId.New(); + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + + await repo.UpsertAsync(NewRow(id, status: "Attempted", retryCount: 1, lastError: "first")); + var snapshot = await repo.GetAsync(id); + + // Same rank (2) — repository must treat this as a no-op (no fields move). + await repo.UpsertAsync(NewRow(id, status: "Attempted", retryCount: 2, lastError: "second")); + var afterDuplicate = await repo.GetAsync(id); + + Assert.NotNull(afterDuplicate); + Assert.Equal("Attempted", afterDuplicate!.Status); + Assert.Equal(1, afterDuplicate.RetryCount); + Assert.Equal("first", afterDuplicate.LastError); + Assert.Equal(snapshot!.UpdatedAtUtc, afterDuplicate.UpdatedAtUtc); + } + + [SkippableFact] + public async Task UpsertAsync_TerminalOverTerminal_IsNoOp() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + // Bundle B3 plan: terminal statuses share rank 3 and are mutually + // exclusive — Delivered cannot overwrite Parked. + var id = TrackedOperationId.New(); + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + + await repo.UpsertAsync(NewRow(id, status: "Parked", retryCount: 3, lastError: "parked-reason", terminal: true)); + var afterPark = await repo.GetAsync(id); + + await repo.UpsertAsync(NewRow(id, status: "Delivered", retryCount: 4, lastError: null, terminal: true)); + var afterDeliveredAttempt = await repo.GetAsync(id); + + Assert.NotNull(afterDeliveredAttempt); + Assert.Equal("Parked", afterDeliveredAttempt!.Status); + Assert.Equal("parked-reason", afterDeliveredAttempt.LastError); + Assert.Equal(afterPark!.UpdatedAtUtc, afterDeliveredAttempt.UpdatedAtUtc); + } + + [SkippableFact] + public async Task UpsertAsync_ConcurrentInserts_SameId_OnlyOneRow() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + // 50 parallel inserters with the same id. The IF NOT EXISTS … INSERT + // pattern has a check-then-act race; concurrent losers must surface as + // silent duplicate-key swallows, not thrown exceptions. Final row + // count must be exactly 1. + var id = TrackedOperationId.New(); + var row = NewRow(id, status: "Submitted", retryCount: 0); + + await Parallel.ForEachAsync( + Enumerable.Range(0, 50), + new ParallelOptions { MaxDegreeOfParallelism = 50 }, + async (_, ct) => + { + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + await repo.UpsertAsync(row, ct); + }); + + await using var readContext = CreateContext(); + var count = await readContext.Set() + .Where(s => s.TrackedOperationId == id) + .CountAsync(); + Assert.Equal(1, count); + } + + [SkippableFact] + public async Task GetAsync_KnownId_ReturnsRow() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var id = TrackedOperationId.New(); + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + + await repo.UpsertAsync(NewRow(id, status: "Submitted", retryCount: 0)); + + var loaded = await repo.GetAsync(id); + Assert.NotNull(loaded); + Assert.Equal(id, loaded!.TrackedOperationId); + } + + [SkippableFact] + public async Task GetAsync_UnknownId_ReturnsNull() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + + var loaded = await repo.GetAsync(TrackedOperationId.New()); + Assert.Null(loaded); + } + + [SkippableFact] + public async Task QueryAsync_FilterBySourceSite_ReturnsMatchingRows() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteA = NewSiteId(); + var siteB = NewSiteId(); + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + + var t0 = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc); + await repo.UpsertAsync(NewRow(TrackedOperationId.New(), sourceSite: siteA, createdAtUtc: t0)); + await repo.UpsertAsync(NewRow(TrackedOperationId.New(), sourceSite: siteA, createdAtUtc: t0.AddMinutes(1))); + await repo.UpsertAsync(NewRow(TrackedOperationId.New(), sourceSite: siteB, createdAtUtc: t0.AddMinutes(2))); + + var rows = await repo.QueryAsync( + new SiteCallQueryFilter(SourceSite: siteA), + new SiteCallPaging(PageSize: 10)); + + Assert.Equal(2, rows.Count); + Assert.All(rows, r => Assert.Equal(siteA, r.SourceSite)); + } + + [SkippableFact] + public async Task QueryAsync_KeysetPaging_NoOverlap() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var site = NewSiteId(); + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + + // Five rows with distinct CreatedAtUtc. Page-size 2 → page 1 returns + // minutes 4,3; cursor (minutes 3) → page 2 returns minutes 2,1; cursor + // (minutes 1) → page 3 returns minute 0. + var t0 = new DateTime(2026, 5, 20, 9, 0, 0, DateTimeKind.Utc); + for (var i = 0; i < 5; i++) + { + await repo.UpsertAsync(NewRow(TrackedOperationId.New(), sourceSite: site, createdAtUtc: t0.AddMinutes(i))); + } + + var page1 = await repo.QueryAsync( + new SiteCallQueryFilter(SourceSite: site), + new SiteCallPaging(PageSize: 2)); + Assert.Equal(2, page1.Count); + Assert.Equal(t0.AddMinutes(4), page1[0].CreatedAtUtc); + Assert.Equal(t0.AddMinutes(3), page1[1].CreatedAtUtc); + + var cursor1 = page1[^1]; + var page2 = await repo.QueryAsync( + new SiteCallQueryFilter(SourceSite: site), + new SiteCallPaging( + PageSize: 2, + AfterCreatedAtUtc: cursor1.CreatedAtUtc, + AfterId: cursor1.TrackedOperationId)); + Assert.Equal(2, page2.Count); + Assert.Equal(t0.AddMinutes(2), page2[0].CreatedAtUtc); + Assert.Equal(t0.AddMinutes(1), page2[1].CreatedAtUtc); + + var cursor2 = page2[^1]; + var page3 = await repo.QueryAsync( + new SiteCallQueryFilter(SourceSite: site), + new SiteCallPaging( + PageSize: 2, + AfterCreatedAtUtc: cursor2.CreatedAtUtc, + AfterId: cursor2.TrackedOperationId)); + Assert.Single(page3); + Assert.Equal(t0.AddMinutes(0), page3[0].CreatedAtUtc); + + // No overlap across pages. + var allIds = page1.Concat(page2).Concat(page3).Select(r => r.TrackedOperationId).ToHashSet(); + Assert.Equal(5, allIds.Count); + } + + [SkippableFact] + public async Task PurgeTerminalAsync_RemovesTerminalAndOld() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var site = NewSiteId(); + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + + // One row that's been Delivered for a long time (5 days ago) — should be purged. + var oldId = TrackedOperationId.New(); + var fiveDaysAgo = DateTime.UtcNow.AddDays(-5); + await repo.UpsertAsync(NewRow( + oldId, + sourceSite: site, + status: "Delivered", + retryCount: 1, + createdAtUtc: fiveDaysAgo.AddMinutes(-1), + updatedAtUtc: fiveDaysAgo, + terminal: true, + terminalAtUtc: fiveDaysAgo)); + + var purged = await repo.PurgeTerminalAsync(DateTime.UtcNow.AddDays(-1)); + + Assert.True(purged >= 1, $"Expected at least one purged row; got {purged}."); + Assert.Null(await repo.GetAsync(oldId)); + } + + [SkippableFact] + public async Task PurgeTerminalAsync_KeepsNonTerminalAndRecent() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var site = NewSiteId(); + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + + // Non-terminal row: never eligible. + var activeId = TrackedOperationId.New(); + await repo.UpsertAsync(NewRow( + activeId, + sourceSite: site, + status: "Attempted", + retryCount: 1, + createdAtUtc: DateTime.UtcNow.AddDays(-10), + updatedAtUtc: DateTime.UtcNow.AddDays(-10), + terminal: false)); + + // Recent terminal row: TerminalAtUtc within the keep window. + var recentTerminalId = TrackedOperationId.New(); + await repo.UpsertAsync(NewRow( + recentTerminalId, + sourceSite: site, + status: "Delivered", + retryCount: 0, + createdAtUtc: DateTime.UtcNow.AddHours(-2), + updatedAtUtc: DateTime.UtcNow.AddHours(-1), + terminal: true, + terminalAtUtc: DateTime.UtcNow.AddHours(-1))); + + // Purge older than 1 day — both rows must survive. + await repo.PurgeTerminalAsync(DateTime.UtcNow.AddDays(-1)); + + Assert.NotNull(await repo.GetAsync(activeId)); + Assert.NotNull(await repo.GetAsync(recentTerminalId)); + } + + // --- helpers ------------------------------------------------------------ + + private ScadaLinkDbContext CreateContext() + { + var options = new DbContextOptionsBuilder() + .UseSqlServer(_fixture.ConnectionString) + .Options; + return new ScadaLinkDbContext(options); + } + + private static string NewSiteId() => + "site-b3-" + Guid.NewGuid().ToString("N").Substring(0, 8); + + private static SiteCall NewRow( + TrackedOperationId id, + string? sourceSite = null, + string status = "Submitted", + int retryCount = 0, + string? lastError = null, + int? httpStatus = null, + DateTime? createdAtUtc = null, + DateTime? updatedAtUtc = null, + bool terminal = false, + DateTime? terminalAtUtc = null) + { + var created = createdAtUtc ?? DateTime.UtcNow; + var updated = updatedAtUtc ?? created; + DateTime? terminalAt = terminal + ? (terminalAtUtc ?? updated) + : null; + + return new SiteCall + { + TrackedOperationId = id, + Channel = "ApiOutbound", + Target = "ERP.GetOrder", + SourceSite = sourceSite ?? NewSiteId(), + Status = status, + RetryCount = retryCount, + LastError = lastError, + HttpStatus = httpStatus, + CreatedAtUtc = created, + UpdatedAtUtc = updated, + TerminalAtUtc = terminalAt, + IngestedAtUtc = DateTime.UtcNow, + }; + } +} diff --git a/tests/ScadaLink.Host.Tests/AkkaHostedServiceAuditWiringTests.cs b/tests/ScadaLink.Host.Tests/AkkaHostedServiceAuditWiringTests.cs index 392dc38..e9d52a4 100644 --- a/tests/ScadaLink.Host.Tests/AkkaHostedServiceAuditWiringTests.cs +++ b/tests/ScadaLink.Host.Tests/AkkaHostedServiceAuditWiringTests.cs @@ -14,6 +14,7 @@ using ScadaLink.Commons.Interfaces.Services; using ScadaLink.ConfigurationDatabase; using ScadaLink.Host; using ScadaLink.Host.Actors; +using ScadaLink.StoreAndForward; namespace ScadaLink.Host.Tests; @@ -189,6 +190,43 @@ public class CentralAuditWiringTests : IDisposable Assert.NotNull(client); Assert.IsType(client); } + + /// + /// M3 Bundle F (T15): the Central composition root calls + /// AddSiteCallAudit(). Today that extension is a no-op placeholder, + /// but invoking it must not throw and the central host's service collection + /// must build successfully — the actor's Props are constructed inline in + /// AkkaHostedService (via the root ), + /// not from a DI factory. Asserting the host built confirms the wiring + /// call is in place; this test guards against accidentally removing it + /// from Program.cs. + /// + [Fact] + public void Central_HostBuilds_With_AddSiteCallAudit_Wired() + { + // Reaching _factory.Services means WebApplicationFactory built the host + // (DI validation completed). The fact this test is in the + // CentralAuditWiringTests fixture means it ran against the Central + // composition root path through Program.cs. + Assert.NotNull(_factory.Services); + } + + /// + /// M3 Bundle F: the Central composition root registers + /// ICachedCallTelemetryForwarder as a lazy singleton (the + /// forwarder degrades to audit-only emission when the site-only + /// IOperationTrackingStore is absent, matching the M2 lazy chain + /// pattern). The binding is exercised here so a future regression that + /// removes the registration or makes IOperationTrackingStore mandatory + /// fails on the Central node, not just at first script execution. + /// + [Fact] + public void Central_Resolves_ICachedCallTelemetryForwarder_LazySingleton() + { + var forwarder = _factory.Services.GetService(); + Assert.NotNull(forwarder); + Assert.IsType(forwarder); + } } /// @@ -303,4 +341,66 @@ public class SiteAuditWiringTests : IDisposable Assert.Equal(5, opts.Value.BusyIntervalSeconds); Assert.Equal(30, opts.Value.IdleIntervalSeconds); } + + /// + /// M3 Bundle F (T15): the site composition root resolves the cached-call + /// telemetry forwarder. ScriptExecutionActor consumes this through + /// GetService<ICachedCallTelemetryForwarder>() on every script + /// execution; a missing registration would silently degrade + /// ExternalSystem.CachedCall / Database.CachedWrite to the + /// "no-emission" path and break the M3 audit pipeline. + /// + [Fact] + public void Site_Resolves_ICachedCallTelemetryForwarder() + { + var forwarder = _host.Services.GetService(); + Assert.NotNull(forwarder); + Assert.IsType(forwarder); + } + + /// + /// M3 Bundle F (T15): the site composition root resolves the lifecycle + /// bridge that translates S&F retry-loop attempt notifications into + /// cached-call telemetry packets. + /// + [Fact] + public void Site_Resolves_CachedCallLifecycleBridge_AsSingleton() + { + var a = _host.Services.GetService(); + var b = _host.Services.GetService(); + Assert.NotNull(a); + Assert.NotNull(b); + Assert.Same(a, b); + } + + /// + /// M3 Bundle F (T15): the lifecycle bridge is bound to the + /// contract that + /// StoreAndForwardService consults at construction time. Without this + /// binding the S&F service is built with a null observer and the + /// retry-loop telemetry never reaches the audit pipeline. + /// + [Fact] + public void Site_ICachedCallLifecycleObserver_IsTheLifecycleBridge() + { + var observer = _host.Services.GetService(); + var bridge = _host.Services.GetService(); + Assert.NotNull(observer); + Assert.NotNull(bridge); + Assert.Same(bridge, observer); + } + + /// + /// M3 Bundle F (T15): the Host registers an + /// adapter so the S&F service + /// can resolve the site id at composition time WITHOUT introducing a + /// StoreAndForward → HealthMonitoring project-reference cycle. + /// + [Fact] + public void Site_Resolves_IStoreAndForwardSiteContext_FromHost() + { + var ctx = _host.Services.GetService(); + Assert.NotNull(ctx); + Assert.Equal("TestSite", ctx!.SiteId); + } } diff --git a/tests/ScadaLink.SiteCallAudit.Tests/ScadaLink.SiteCallAudit.Tests.csproj b/tests/ScadaLink.SiteCallAudit.Tests/ScadaLink.SiteCallAudit.Tests.csproj new file mode 100644 index 0000000..772b7fe --- /dev/null +++ b/tests/ScadaLink.SiteCallAudit.Tests/ScadaLink.SiteCallAudit.Tests.csproj @@ -0,0 +1,51 @@ + + + + net10.0 + enable + enable + true + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/ScadaLink.SiteCallAudit.Tests/SiteCallAuditActorTests.cs b/tests/ScadaLink.SiteCallAudit.Tests/SiteCallAuditActorTests.cs new file mode 100644 index 0000000..e9ef807 --- /dev/null +++ b/tests/ScadaLink.SiteCallAudit.Tests/SiteCallAuditActorTests.cs @@ -0,0 +1,221 @@ +using Akka.Actor; +using Akka.TestKit.Xunit2; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces.Repositories; +using ScadaLink.Commons.Messages.Audit; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Audit; +using ScadaLink.ConfigurationDatabase; +using ScadaLink.ConfigurationDatabase.Repositories; +using ScadaLink.ConfigurationDatabase.Tests.Migrations; + +namespace ScadaLink.SiteCallAudit.Tests; + +/// +/// Bundle C1 (#22, #23 M3) tests for . Uses the +/// same as the Bundle B3 repository tests +/// so the actor exercises the real monotonic-upsert SQL end to end against the +/// SiteCalls schema. Each test scopes its data by minting a fresh +/// (and a per-test SourceSite suffix) +/// so tests neither collide nor require teardown. +/// +public class SiteCallAuditActorTests : TestKit, IClassFixture +{ + private readonly MsSqlMigrationFixture _fixture; + + public SiteCallAuditActorTests(MsSqlMigrationFixture fixture) + { + _fixture = fixture; + } + + private ScadaLinkDbContext CreateContext() + { + var options = new DbContextOptionsBuilder() + .UseSqlServer(_fixture.ConnectionString) + .Options; + return new ScadaLinkDbContext(options); + } + + private static string NewSiteId() => + "test-bundle-c1-" + Guid.NewGuid().ToString("N").Substring(0, 8); + + private static SiteCall NewRow( + TrackedOperationId id, + string sourceSite, + string status = "Submitted", + int retryCount = 0, + string? lastError = null, + DateTime? createdAtUtc = null, + DateTime? updatedAtUtc = null, + bool terminal = false) + { + var created = createdAtUtc ?? DateTime.UtcNow; + var updated = updatedAtUtc ?? created; + return new SiteCall + { + TrackedOperationId = id, + Channel = "ApiOutbound", + Target = "ERP.GetOrder", + SourceSite = sourceSite, + Status = status, + RetryCount = retryCount, + LastError = lastError, + HttpStatus = null, + CreatedAtUtc = created, + UpdatedAtUtc = updated, + TerminalAtUtc = terminal ? updated : null, + IngestedAtUtc = DateTime.UtcNow, + }; + } + + private IActorRef CreateActor(ISiteCallAuditRepository repository) => + Sys.ActorOf(Props.Create(() => new SiteCallAuditActor( + repository, + NullLogger.Instance))); + + [SkippableFact] + public async Task Receive_UpsertSiteCallCommand_Persists_Replies_Accepted() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var id = TrackedOperationId.New(); + var row = NewRow(id, siteId, status: "Submitted", retryCount: 0); + + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + var actor = CreateActor(repo); + + actor.Tell(new UpsertSiteCallCommand(row), TestActor); + + var reply = ExpectMsg(TimeSpan.FromSeconds(10)); + Assert.True(reply.Accepted, "Actor should reply Accepted=true on a successful upsert."); + Assert.Equal(id, reply.TrackedOperationId); + + // Verify the row landed in MSSQL via a fresh context (separate from the + // actor's repository context). + await using var readContext = CreateContext(); + var rows = await readContext.Set() + .Where(s => s.SourceSite == siteId) + .ToListAsync(); + Assert.Single(rows); + Assert.Equal("Submitted", rows[0].Status); + } + + [SkippableFact] + public async Task Receive_DuplicateUpsert_OlderStatus_NoOp_StillRepliesAccepted() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + // Idempotency contract: a stale/duplicate packet (lower rank than the + // stored status) is a silent no-op at the repository — the actor must + // still reply Accepted=true so the site is free to consider its + // packet acked. Storage state is consistent either way. + var siteId = NewSiteId(); + var id = TrackedOperationId.New(); + + await using var context = CreateContext(); + var repo = new SiteCallAuditRepository(context); + var actor = CreateActor(repo); + + // Land Attempted (rank 2) first. + actor.Tell(new UpsertSiteCallCommand(NewRow(id, siteId, status: "Attempted", retryCount: 1, lastError: "first")), TestActor); + var firstReply = ExpectMsg(TimeSpan.FromSeconds(10)); + Assert.True(firstReply.Accepted); + + // Late-arriving Submitted (rank 0) — must be no-op in storage and + // still acked by the actor. + actor.Tell(new UpsertSiteCallCommand(NewRow(id, siteId, status: "Submitted", retryCount: 0)), TestActor); + var secondReply = ExpectMsg(TimeSpan.FromSeconds(10)); + Assert.True(secondReply.Accepted, "Stale upsert must still be acked (idempotent contract)."); + + // Storage must still show the rank-2 row, not rolled back. + await using var readContext = CreateContext(); + var stored = await readContext.Set() + .Where(s => s.TrackedOperationId == id) + .ToListAsync(); + Assert.Single(stored); + Assert.Equal("Attempted", stored[0].Status); + Assert.Equal(1, stored[0].RetryCount); + Assert.Equal("first", stored[0].LastError); + } + + [SkippableFact] + public async Task Receive_RepoThrowsTransient_RepliesAccepted_False_ActorStaysAlive() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + // Per CLAUDE.md: audit-write failure NEVER aborts the user-facing + // action. The actor must catch the throw, reply Accepted=false, and + // stay alive — a follow-up message on the same actor must still be + // processed (the singleton cannot die on a transient repo error). + var siteId = NewSiteId(); + var poisonId = TrackedOperationId.New(); + var healthyId = TrackedOperationId.New(); + + await using var context = CreateContext(); + var realRepo = new SiteCallAuditRepository(context); + var wrappedRepo = new ThrowingRepository(realRepo, poisonId); + var actor = CreateActor(wrappedRepo); + + // Poison row — the wrapper throws when this id arrives. + actor.Tell(new UpsertSiteCallCommand(NewRow(poisonId, siteId, status: "Submitted")), TestActor); + var poisonReply = ExpectMsg(TimeSpan.FromSeconds(10)); + Assert.False(poisonReply.Accepted, "Actor should reply Accepted=false when the repo throws."); + Assert.Equal(poisonId, poisonReply.TrackedOperationId); + + // Healthy follow-up on the SAME actor — must still be processed + // (singleton staying alive proves the actor did not crash). + actor.Tell(new UpsertSiteCallCommand(NewRow(healthyId, siteId, status: "Submitted")), TestActor); + var healthyReply = ExpectMsg(TimeSpan.FromSeconds(10)); + Assert.True(healthyReply.Accepted, "Actor must stay alive after a transient repo failure."); + Assert.Equal(healthyId, healthyReply.TrackedOperationId); + + // Verify storage: healthy row landed, poison row did not. + await using var readContext = CreateContext(); + var rows = await readContext.Set() + .Where(s => s.SourceSite == siteId) + .ToListAsync(); + Assert.Single(rows); + Assert.Equal(healthyId, rows[0].TrackedOperationId); + } + + /// + /// Tiny test double that delegates to a real repository but throws on a + /// specified . Used to verify the actor's + /// fault-isolation behaviour: a transient repository failure must produce + /// Accepted=false without crashing the singleton. + /// + private sealed class ThrowingRepository : ISiteCallAuditRepository + { + private readonly ISiteCallAuditRepository _inner; + private readonly TrackedOperationId _poisonId; + + public ThrowingRepository(ISiteCallAuditRepository inner, TrackedOperationId poisonId) + { + _inner = inner; + _poisonId = poisonId; + } + + public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) + { + if (siteCall.TrackedOperationId == _poisonId) + { + throw new InvalidOperationException("simulated transient repo failure for poison row"); + } + return _inner.UpsertAsync(siteCall, ct); + } + + public Task GetAsync(TrackedOperationId id, CancellationToken ct = default) => + _inner.GetAsync(id, ct); + + public Task> QueryAsync( + SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) => + _inner.QueryAsync(filter, paging, ct); + + public Task PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) => + _inner.PurgeTerminalAsync(olderThanUtc, ct); + } +} diff --git a/tests/ScadaLink.SiteRuntime.Tests/Scripts/DatabaseCachedWriteEmissionTests.cs b/tests/ScadaLink.SiteRuntime.Tests/Scripts/DatabaseCachedWriteEmissionTests.cs new file mode 100644 index 0000000..991bbaf --- /dev/null +++ b/tests/ScadaLink.SiteRuntime.Tests/Scripts/DatabaseCachedWriteEmissionTests.cs @@ -0,0 +1,170 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; +using ScadaLink.SiteRuntime.Scripts; + +namespace ScadaLink.SiteRuntime.Tests.Scripts; + +/// +/// Audit Log #23 — M3 Bundle E (Task E6): every script-initiated +/// Database.CachedWrite emits exactly one CachedSubmit +/// combined-telemetry packet at enqueue time on the DbOutbound +/// channel, returns a fresh , and threads +/// the id into the database gateway so the store-and-forward retry loop can +/// emit per-attempt + terminal telemetry under the same id. +/// +public class DatabaseCachedWriteEmissionTests +{ + private sealed class CapturingForwarder : ICachedCallTelemetryForwarder + { + public List Telemetry { get; } = new(); + public Exception? ThrowOnForward { get; set; } + + public Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default) + { + if (ThrowOnForward != null) + { + return Task.FromException(ThrowOnForward); + } + Telemetry.Add(telemetry); + return Task.CompletedTask; + } + } + + private const string SiteId = "site-77"; + private const string InstanceName = "Plant.Pump42"; + private const string SourceScript = "ScriptActor:WriteAudit"; + + private static ScriptRuntimeContext.DatabaseHelper CreateHelper( + IDatabaseGateway gateway, + ICachedCallTelemetryForwarder? forwarder) + { + return new ScriptRuntimeContext.DatabaseHelper( + gateway, + InstanceName, + NullLogger.Instance, + siteId: SiteId, + sourceScript: SourceScript, + cachedForwarder: forwarder); + } + + [Fact] + public async Task CachedWrite_EmitsSubmitTelemetry_OnEnqueue_KindCachedSubmit_ChannelDbOutbound() + { + var gateway = new Mock(); + gateway + .Setup(g => g.CachedWriteAsync( + "myDb", "INSERT INTO t VALUES (1)", + It.IsAny?>(), + InstanceName, + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + var forwarder = new CapturingForwarder(); + + var helper = CreateHelper(gateway.Object, forwarder); + var trackedId = await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)"); + + Assert.NotEqual(default, trackedId); + var packet = Assert.Single(forwarder.Telemetry); + + Assert.Equal(AuditChannel.DbOutbound, packet.Audit.Channel); + Assert.Equal(AuditKind.CachedSubmit, packet.Audit.Kind); + Assert.Equal(AuditStatus.Submitted, packet.Audit.Status); + Assert.Equal("myDb", packet.Audit.Target); + Assert.Equal(trackedId.Value, packet.Audit.CorrelationId); + + Assert.Equal(trackedId, packet.Operational.TrackedOperationId); + Assert.Equal("DbOutbound", packet.Operational.Channel); + Assert.Equal("myDb", packet.Operational.Target); + Assert.Equal(SiteId, packet.Operational.SourceSite); + Assert.Equal("Submitted", packet.Operational.Status); + Assert.Equal(0, packet.Operational.RetryCount); + Assert.Null(packet.Operational.TerminalAtUtc); + } + + [Fact] + public async Task CachedWrite_ProvenancePopulated() + { + var gateway = new Mock(); + gateway + .Setup(g => g.CachedWriteAsync( + It.IsAny(), It.IsAny(), + It.IsAny?>(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + var forwarder = new CapturingForwarder(); + + var helper = CreateHelper(gateway.Object, forwarder); + await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)"); + + var packet = Assert.Single(forwarder.Telemetry); + Assert.Equal(SiteId, packet.Audit.SourceSiteId); + Assert.Equal(InstanceName, packet.Audit.SourceInstanceId); + Assert.Equal(SourceScript, packet.Audit.SourceScript); + Assert.Equal(SiteId, packet.Operational.SourceSite); + } + + [Fact] + public async Task CachedWrite_ReturnsTrackedOperationId_ThreadsIdToGateway() + { + var gateway = new Mock(); + gateway + .Setup(g => g.CachedWriteAsync( + It.IsAny(), It.IsAny(), + It.IsAny?>(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + var forwarder = new CapturingForwarder(); + + var helper = CreateHelper(gateway.Object, forwarder); + var trackedId = await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)"); + + Assert.NotEqual(default, trackedId); + gateway.Verify(g => g.CachedWriteAsync( + "myDb", "INSERT INTO t VALUES (1)", + It.IsAny?>(), + InstanceName, + It.IsAny(), + trackedId), + Times.Once); + } + + [Fact] + public async Task CachedWrite_ForwarderThrows_StillReturnsTrackedOperationId() + { + var gateway = new Mock(); + gateway + .Setup(g => g.CachedWriteAsync( + It.IsAny(), It.IsAny(), + It.IsAny?>(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + var forwarder = new CapturingForwarder + { + ThrowOnForward = new InvalidOperationException("simulated forwarder outage"), + }; + + var helper = CreateHelper(gateway.Object, forwarder); + var trackedId = await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)"); + + Assert.NotEqual(default, trackedId); + gateway.Verify(g => g.CachedWriteAsync( + "myDb", "INSERT INTO t VALUES (1)", + It.IsAny?>(), + InstanceName, + It.IsAny(), + trackedId), + Times.Once); + } +} diff --git a/tests/ScadaLink.SiteRuntime.Tests/Scripts/ExternalSystemCachedCallEmissionTests.cs b/tests/ScadaLink.SiteRuntime.Tests/Scripts/ExternalSystemCachedCallEmissionTests.cs new file mode 100644 index 0000000..3d56ffe --- /dev/null +++ b/tests/ScadaLink.SiteRuntime.Tests/Scripts/ExternalSystemCachedCallEmissionTests.cs @@ -0,0 +1,358 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Messages.Integration; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; +using ScadaLink.SiteRuntime.Scripts; + +namespace ScadaLink.SiteRuntime.Tests.Scripts; + +/// +/// Audit Log #23 — M3 Bundle E (Task E3): every script-initiated +/// ExternalSystem.CachedCall emits exactly one CachedSubmit +/// combined-telemetry packet at enqueue time, returns a fresh +/// , and threads that id down to the +/// store-and-forward layer so the retry-loop emissions (Tasks E4/E5) can join +/// them by id. The audit emission is best-effort: a thrown forwarder must +/// never abort the script's call, and the original +/// must surface to the caller unchanged. +/// +public class ExternalSystemCachedCallEmissionTests +{ + private sealed class CapturingForwarder : ICachedCallTelemetryForwarder + { + public List Telemetry { get; } = new(); + public Exception? ThrowOnForward { get; set; } + + public Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default) + { + if (ThrowOnForward != null) + { + return Task.FromException(ThrowOnForward); + } + Telemetry.Add(telemetry); + return Task.CompletedTask; + } + } + + private const string SiteId = "site-77"; + private const string InstanceName = "Plant.Pump42"; + private const string SourceScript = "ScriptActor:CheckPressure"; + + private static ScriptRuntimeContext.ExternalSystemHelper CreateHelper( + IExternalSystemClient client, + ICachedCallTelemetryForwarder? forwarder) + { + return new ScriptRuntimeContext.ExternalSystemHelper( + client, + InstanceName, + NullLogger.Instance, + auditWriter: null, + siteId: SiteId, + sourceScript: SourceScript, + cachedForwarder: forwarder); + } + + [Fact] + public async Task CachedCall_EmitsSubmitTelemetry_OnEnqueue() + { + var client = new Mock(); + client + .Setup(c => c.CachedCallAsync( + "ERP", "GetOrder", + It.IsAny?>(), + InstanceName, + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); + var forwarder = new CapturingForwarder(); + + var helper = CreateHelper(client.Object, forwarder); + var trackedId = await helper.CachedCall("ERP", "GetOrder"); + + Assert.NotEqual(default, trackedId); + Assert.Single(forwarder.Telemetry); + var packet = forwarder.Telemetry[0]; + + Assert.Equal(AuditChannel.ApiOutbound, packet.Audit.Channel); + Assert.Equal(AuditKind.CachedSubmit, packet.Audit.Kind); + Assert.Equal(AuditStatus.Submitted, packet.Audit.Status); + Assert.Equal("ERP.GetOrder", packet.Audit.Target); + Assert.Equal(trackedId.Value, packet.Audit.CorrelationId); + Assert.Equal(AuditForwardState.Pending, packet.Audit.ForwardState); + + // Operational mirror — same id, Submitted, RetryCount 0, not terminal. + Assert.Equal(trackedId, packet.Operational.TrackedOperationId); + Assert.Equal("ApiOutbound", packet.Operational.Channel); + Assert.Equal("ERP.GetOrder", packet.Operational.Target); + Assert.Equal(SiteId, packet.Operational.SourceSite); + Assert.Equal("Submitted", packet.Operational.Status); + Assert.Equal(0, packet.Operational.RetryCount); + Assert.Null(packet.Operational.LastError); + Assert.Null(packet.Operational.TerminalAtUtc); + } + + [Fact] + public async Task CachedCall_ReturnsTrackedOperationId() + { + var client = new Mock(); + client + .Setup(c => c.CachedCallAsync( + It.IsAny(), It.IsAny(), + It.IsAny?>(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); + var forwarder = new CapturingForwarder(); + + var helper = CreateHelper(client.Object, forwarder); + + var id1 = await helper.CachedCall("ERP", "GetOrder"); + var id2 = await helper.CachedCall("ERP", "GetOrder"); + + Assert.NotEqual(default, id1); + Assert.NotEqual(default, id2); + Assert.NotEqual(id1, id2); + + // Both ids were threaded into the client invocations. + client.Verify(c => c.CachedCallAsync( + "ERP", "GetOrder", + It.IsAny?>(), + InstanceName, + It.IsAny(), + id1), + Times.Once); + client.Verify(c => c.CachedCallAsync( + "ERP", "GetOrder", + It.IsAny?>(), + InstanceName, + It.IsAny(), + id2), + Times.Once); + } + + [Fact] + public async Task CachedCall_ForwarderThrows_StillReturnsTrackedOperationId_OriginalCallProceeds() + { + var client = new Mock(); + client + .Setup(c => c.CachedCallAsync( + It.IsAny(), It.IsAny(), + It.IsAny?>(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); + var forwarder = new CapturingForwarder + { + ThrowOnForward = new InvalidOperationException("simulated forwarder outage"), + }; + + var helper = CreateHelper(client.Object, forwarder); + + // Must not throw — best-effort emission contract. + var trackedId = await helper.CachedCall("ERP", "GetOrder"); + + Assert.NotEqual(default, trackedId); + // The underlying call still ran exactly once. + client.Verify(c => c.CachedCallAsync( + "ERP", "GetOrder", + It.IsAny?>(), + InstanceName, + It.IsAny(), + trackedId), + Times.Once); + } + + [Fact] + public async Task CachedCall_Provenance_Populated_FromContext() + { + var client = new Mock(); + client + .Setup(c => c.CachedCallAsync( + It.IsAny(), It.IsAny(), + It.IsAny?>(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); + var forwarder = new CapturingForwarder(); + + var helper = CreateHelper(client.Object, forwarder); + await helper.CachedCall("ERP", "GetOrder"); + + var packet = Assert.Single(forwarder.Telemetry); + Assert.Equal(SiteId, packet.Audit.SourceSiteId); + Assert.Equal(InstanceName, packet.Audit.SourceInstanceId); + Assert.Equal(SourceScript, packet.Audit.SourceScript); + Assert.Equal(SiteId, packet.Operational.SourceSite); + } + + [Fact] + public async Task CachedCall_NoForwarder_StillReturnsTrackedOperationId() + { + // Forwarder not wired (tests / minimal hosts) — must still return a + // fresh id and invoke the underlying call. + var client = new Mock(); + client + .Setup(c => c.CachedCallAsync( + It.IsAny(), It.IsAny(), + It.IsAny?>(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); + + var helper = CreateHelper(client.Object, forwarder: null); + var trackedId = await helper.CachedCall("ERP", "GetOrder"); + + Assert.NotEqual(default, trackedId); + client.Verify(c => c.CachedCallAsync( + "ERP", "GetOrder", + It.IsAny?>(), + InstanceName, + It.IsAny(), + trackedId), + Times.Once); + } + + /// + /// Audit Log #23 — M3 Bundle F (F2): when the underlying client call + /// completes immediately (no S&F buffering, WasBuffered=false), + /// the S&F retry loop never engages and the + /// ICachedCallLifecycleObserver hook never fires. The cached-call + /// helper itself must therefore emit the terminal lifecycle rows — + /// otherwise Tracking.Status(id) would return Submitted + /// forever and the audit log would be missing the Attempted / + /// CachedResolve pair the M3 contract requires. + /// + /// Expected emissions on immediate success: + /// 1. CachedSubmit / Submitted (already covered) + /// 2. ApiCallCached / Attempted + /// 3. CachedResolve / Delivered (TerminalAtUtc set) + /// + [Fact] + public async Task CachedCall_ImmediateSuccess_EmitsAttemptedAndCachedResolve() + { + var client = new Mock(); + client + .Setup(c => c.CachedCallAsync( + "ERP", "GetOrder", + It.IsAny?>(), + InstanceName, + It.IsAny(), + It.IsAny())) + // WasBuffered=false — the immediate HTTP attempt succeeded; S&F + // is bypassed entirely. + .ReturnsAsync(new ExternalCallResult(true, "{\"ok\":true}", null, WasBuffered: false)); + var forwarder = new CapturingForwarder(); + + var helper = CreateHelper(client.Object, forwarder); + var trackedId = await helper.CachedCall("ERP", "GetOrder"); + + // Three telemetry packets emitted: Submit, Attempted, Resolve. + Assert.Equal(3, forwarder.Telemetry.Count); + + var submit = forwarder.Telemetry[0]; + Assert.Equal(AuditKind.CachedSubmit, submit.Audit.Kind); + Assert.Equal(AuditStatus.Submitted, submit.Audit.Status); + Assert.Equal(trackedId, submit.Operational.TrackedOperationId); + Assert.Null(submit.Operational.TerminalAtUtc); + + var attempted = forwarder.Telemetry[1]; + Assert.Equal(AuditChannel.ApiOutbound, attempted.Audit.Channel); + Assert.Equal(AuditKind.ApiCallCached, attempted.Audit.Kind); + Assert.Equal(AuditStatus.Attempted, attempted.Audit.Status); + Assert.Equal(trackedId.Value, attempted.Audit.CorrelationId); + Assert.Equal("ERP.GetOrder", attempted.Audit.Target); + Assert.Equal(trackedId, attempted.Operational.TrackedOperationId); + Assert.Equal("Attempted", attempted.Operational.Status); + Assert.Null(attempted.Operational.TerminalAtUtc); + + var resolve = forwarder.Telemetry[2]; + Assert.Equal(AuditChannel.ApiOutbound, resolve.Audit.Channel); + Assert.Equal(AuditKind.CachedResolve, resolve.Audit.Kind); + Assert.Equal(AuditStatus.Delivered, resolve.Audit.Status); + Assert.Equal(trackedId.Value, resolve.Audit.CorrelationId); + Assert.Equal(trackedId, resolve.Operational.TrackedOperationId); + Assert.Equal("Delivered", resolve.Operational.Status); + // Terminal row carries TerminalAtUtc. + Assert.NotNull(resolve.Operational.TerminalAtUtc); + } + + /// + /// Audit Log #23 — M3 Bundle F (F2): the immediate-failure terminal + /// path. When the client returns Success=false with + /// WasBuffered=false (a permanent failure or a transient failure + /// without an S&F engine to buffer it), the cached-call helper must + /// still emit Attempted + CachedResolve with the failed status. + /// + [Fact] + public async Task CachedCall_ImmediateFailure_EmitsAttemptedAndCachedResolveFailed() + { + var client = new Mock(); + client + .Setup(c => c.CachedCallAsync( + "ERP", "GetOrder", + It.IsAny?>(), + InstanceName, + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new ExternalCallResult( + false, null, "Permanent error: HTTP 422 bad payload", WasBuffered: false)); + var forwarder = new CapturingForwarder(); + + var helper = CreateHelper(client.Object, forwarder); + var trackedId = await helper.CachedCall("ERP", "GetOrder"); + + Assert.Equal(3, forwarder.Telemetry.Count); + + var attempted = forwarder.Telemetry[1]; + Assert.Equal(AuditKind.ApiCallCached, attempted.Audit.Kind); + Assert.Equal(AuditStatus.Attempted, attempted.Audit.Status); + // The per-attempt row carries the error message. + Assert.NotNull(attempted.Audit.ErrorMessage); + + var resolve = forwarder.Telemetry[2]; + Assert.Equal(AuditKind.CachedResolve, resolve.Audit.Kind); + // Immediate permanent failure -> Failed audit status / operational Failed. + Assert.Equal(AuditStatus.Failed, resolve.Audit.Status); + Assert.Equal("Failed", resolve.Operational.Status); + Assert.NotNull(resolve.Operational.TerminalAtUtc); + Assert.NotNull(resolve.Operational.LastError); + } + + /// + /// Audit Log #23 — M3 Bundle F (F2): when the client reports + /// WasBuffered=true, the helper hands the operation to S&F and + /// the retry-loop observer owns the Attempted + Resolve emissions. The + /// helper must NOT emit those rows itself (otherwise we'd get duplicate + /// Attempted + Resolve audit rows under the same tracking id). + /// + [Fact] + public async Task CachedCall_BufferedPath_DoesNotEmitTerminalTelemetryFromHelper() + { + var client = new Mock(); + client + .Setup(c => c.CachedCallAsync( + "ERP", "GetOrder", + It.IsAny?>(), + InstanceName, + It.IsAny(), + It.IsAny())) + // S&F took ownership — Attempted + Resolve come from the + // CachedCallLifecycleBridge driven by the retry loop, not the helper. + .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); + var forwarder = new CapturingForwarder(); + + var helper = CreateHelper(client.Object, forwarder); + await helper.CachedCall("ERP", "GetOrder"); + + // Only the CachedSubmit row — no Attempted / Resolve from the helper. + var only = Assert.Single(forwarder.Telemetry); + Assert.Equal(AuditKind.CachedSubmit, only.Audit.Kind); + } +} diff --git a/tests/ScadaLink.SiteRuntime.Tests/Scripts/TrackingApiTests.cs b/tests/ScadaLink.SiteRuntime.Tests/Scripts/TrackingApiTests.cs new file mode 100644 index 0000000..a0acd1c --- /dev/null +++ b/tests/ScadaLink.SiteRuntime.Tests/Scripts/TrackingApiTests.cs @@ -0,0 +1,76 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using ScadaLink.Commons.Interfaces; +using ScadaLink.Commons.Types; +using ScadaLink.SiteRuntime.Scripts; + +namespace ScadaLink.SiteRuntime.Tests.Scripts; + +/// +/// Audit Log #23 (M3 Bundle A — Task A3) — script-side API tests for +/// Tracking.Status(TrackedOperationId). The helper reads the site-local +/// directly (no central round-trip) and +/// returns the latest , or null when +/// the id is unknown. +/// +public class TrackingApiTests +{ + private static ScriptRuntimeContext.TrackingHelper CreateHelper( + IOperationTrackingStore? store) + { + return new ScriptRuntimeContext.TrackingHelper(store, NullLogger.Instance); + } + + [Fact] + public async Task Status_UnknownId_ReturnsNull() + { + var store = new Mock(); + store + .Setup(s => s.GetStatusAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((TrackingStatusSnapshot?)null); + + var helper = CreateHelper(store.Object); + var result = await helper.Status(TrackedOperationId.New()); + + Assert.Null(result); + } + + [Fact] + public async Task Status_KnownId_ReturnsLatestSnapshot() + { + var id = TrackedOperationId.New(); + var expected = new TrackingStatusSnapshot( + Id: id, + Kind: "ApiCallCached", + TargetSummary: "ERP.GetOrder", + Status: "Delivered", + RetryCount: 2, + LastError: null, + HttpStatus: 200, + CreatedAtUtc: new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc), + UpdatedAtUtc: new DateTime(2026, 5, 20, 10, 2, 30, DateTimeKind.Utc), + TerminalAtUtc: new DateTime(2026, 5, 20, 10, 2, 30, DateTimeKind.Utc), + SourceInstanceId: "Plant.Pump42", + SourceScript: "ScriptActor:OnTick"); + + var store = new Mock(); + store + .Setup(s => s.GetStatusAsync(id, It.IsAny())) + .ReturnsAsync(expected); + + var helper = CreateHelper(store.Object); + var result = await helper.Status(id); + + Assert.NotNull(result); + Assert.Equal(expected, result); + } + + [Fact] + public async Task Status_NoStoreWired_Throws() + { + var helper = CreateHelper(store: null); + await Assert.ThrowsAsync( + () => helper.Status(TrackedOperationId.New())); + } + +} diff --git a/tests/ScadaLink.SiteRuntime.Tests/Tracking/OperationTrackingStoreTests.cs b/tests/ScadaLink.SiteRuntime.Tests/Tracking/OperationTrackingStoreTests.cs new file mode 100644 index 0000000..952d7dc --- /dev/null +++ b/tests/ScadaLink.SiteRuntime.Tests/Tracking/OperationTrackingStoreTests.cs @@ -0,0 +1,286 @@ +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; +using ScadaLink.SiteRuntime.Tracking; + +namespace ScadaLink.SiteRuntime.Tests.Tracking; + +/// +/// Audit Log #23 (M3 Bundle A — Task A2) — schema + behaviour tests for the +/// site-local . Each test uses a unique +/// shared-cache in-memory SQLite database so the store and the verifier share +/// the same store without touching disk. +/// +public class OperationTrackingStoreTests +{ + private static (OperationTrackingStore store, string dataSource) CreateStore( + string testName) + { + var dataSource = $"file:{testName}-{Guid.NewGuid():N}?mode=memory&cache=shared"; + var connectionString = $"Data Source={dataSource};Cache=Shared"; + var options = new OperationTrackingOptions + { + ConnectionString = connectionString, + }; + var store = new OperationTrackingStore( + Options.Create(options), + NullLogger.Instance); + return (store, dataSource); + } + + private static SqliteConnection OpenVerifierConnection(string dataSource) + { + var connection = new SqliteConnection($"Data Source={dataSource};Cache=Shared"); + connection.Open(); + return connection; + } + + [Fact] + public void Constructor_CreatesOperationTracking_SchemaOnFirstUse() + { + var (store, dataSource) = CreateStore(nameof(Constructor_CreatesOperationTracking_SchemaOnFirstUse)); + using (store) + { + using var connection = OpenVerifierConnection(dataSource); + using var cmd = connection.CreateCommand(); + cmd.CommandText = "PRAGMA table_info(OperationTracking);"; + using var reader = cmd.ExecuteReader(); + + var columns = new List<(string Name, int Pk, int NotNull)>(); + while (reader.Read()) + { + columns.Add((reader.GetString(1), reader.GetInt32(5), reader.GetInt32(3))); + } + + var expected = new[] + { + "TrackedOperationId", "Kind", "TargetSummary", "Status", + "RetryCount", "LastError", "HttpStatus", "CreatedAtUtc", + "UpdatedAtUtc", "TerminalAtUtc", "SourceInstanceId", "SourceScript", + }; + Assert.Equal( + expected.OrderBy(n => n), + columns.Select(c => c.Name).OrderBy(n => n)); + + var pkColumns = columns.Where(c => c.Pk > 0).Select(c => c.Name).ToList(); + Assert.Single(pkColumns); + Assert.Equal("TrackedOperationId", pkColumns[0]); + } + } + + [Fact] + public async Task RecordEnqueueAsync_InsertsSubmittedRow_WithRetryCountZero() + { + var (store, dataSource) = CreateStore(nameof(RecordEnqueueAsync_InsertsSubmittedRow_WithRetryCountZero)); + await using var _ = store; + + var id = TrackedOperationId.New(); + await store.RecordEnqueueAsync( + id, + kind: nameof(AuditKind.ApiCallCached), + targetSummary: "ERP.GetOrder", + sourceInstanceId: "Plant.Pump42", + sourceScript: "ScriptActor:OnTick"); + + var snapshot = await store.GetStatusAsync(id); + Assert.NotNull(snapshot); + Assert.Equal(id, snapshot!.Id); + Assert.Equal(nameof(AuditKind.ApiCallCached), snapshot.Kind); + Assert.Equal("ERP.GetOrder", snapshot.TargetSummary); + Assert.Equal(nameof(AuditStatus.Submitted), snapshot.Status); + Assert.Equal(0, snapshot.RetryCount); + Assert.Null(snapshot.LastError); + Assert.Null(snapshot.HttpStatus); + Assert.Null(snapshot.TerminalAtUtc); + Assert.Equal("Plant.Pump42", snapshot.SourceInstanceId); + Assert.Equal("ScriptActor:OnTick", snapshot.SourceScript); + Assert.Equal(DateTimeKind.Utc, snapshot.CreatedAtUtc.Kind); + Assert.Equal(DateTimeKind.Utc, snapshot.UpdatedAtUtc.Kind); + } + + [Fact] + public async Task RecordEnqueueAsync_Duplicate_IsNoOp_FirstWriteWins() + { + var (store, _) = CreateStore(nameof(RecordEnqueueAsync_Duplicate_IsNoOp_FirstWriteWins)); + await using var _store = store; + + var id = TrackedOperationId.New(); + await store.RecordEnqueueAsync(id, "ApiCallCached", "ERP.GetOrder", "Plant.Pump42", "ScriptActor:OnTick"); + await store.RecordEnqueueAsync(id, "ApiCallCached", "OtherTarget", "Other.Instance", "ScriptActor:Other"); + + var snapshot = await store.GetStatusAsync(id); + Assert.NotNull(snapshot); + // First-write-wins: the second enqueue is ignored — Target/Source stay first. + Assert.Equal("ERP.GetOrder", snapshot!.TargetSummary); + Assert.Equal("Plant.Pump42", snapshot.SourceInstanceId); + Assert.Equal("ScriptActor:OnTick", snapshot.SourceScript); + Assert.Equal(nameof(AuditStatus.Submitted), snapshot.Status); + Assert.Equal(0, snapshot.RetryCount); + } + + [Fact] + public async Task RecordAttemptAsync_AdvancesStatusAndRetryCount_OnNonTerminalRow() + { + var (store, _) = CreateStore(nameof(RecordAttemptAsync_AdvancesStatusAndRetryCount_OnNonTerminalRow)); + await using var _store = store; + + var id = TrackedOperationId.New(); + await store.RecordEnqueueAsync(id, "ApiCallCached", "ERP.GetOrder", null, null); + + await store.RecordAttemptAsync( + id, + status: nameof(AuditStatus.Attempted), + retryCount: 1, + lastError: "HTTP 503 from ERP", + httpStatus: 503); + + var snapshot = await store.GetStatusAsync(id); + Assert.NotNull(snapshot); + Assert.Equal(nameof(AuditStatus.Attempted), snapshot!.Status); + Assert.Equal(1, snapshot.RetryCount); + Assert.Equal("HTTP 503 from ERP", snapshot.LastError); + Assert.Equal(503, snapshot.HttpStatus); + Assert.Null(snapshot.TerminalAtUtc); + + // UpdatedAtUtc advances past CreatedAtUtc. + Assert.True(snapshot.UpdatedAtUtc >= snapshot.CreatedAtUtc); + } + + [Fact] + public async Task RecordAttemptAsync_OnTerminalRow_IsNoOp() + { + var (store, _) = CreateStore(nameof(RecordAttemptAsync_OnTerminalRow_IsNoOp)); + await using var _store = store; + + var id = TrackedOperationId.New(); + await store.RecordEnqueueAsync(id, "ApiCallCached", "ERP.GetOrder", null, null); + await store.RecordTerminalAsync( + id, + status: nameof(AuditStatus.Delivered), + lastError: null, + httpStatus: 200); + + var terminalSnapshot = await store.GetStatusAsync(id); + Assert.NotNull(terminalSnapshot); + Assert.NotNull(terminalSnapshot!.TerminalAtUtc); + + // Late attempt telemetry must NOT overwrite the terminal row. + await store.RecordAttemptAsync( + id, + status: nameof(AuditStatus.Attempted), + retryCount: 5, + lastError: "late attempt", + httpStatus: 500); + + var afterLate = await store.GetStatusAsync(id); + Assert.NotNull(afterLate); + Assert.Equal(nameof(AuditStatus.Delivered), afterLate!.Status); + Assert.Equal(0, afterLate.RetryCount); + Assert.Null(afterLate.LastError); + Assert.Equal(200, afterLate.HttpStatus); + Assert.NotNull(afterLate.TerminalAtUtc); + } + + [Fact] + public async Task RecordTerminalAsync_FlipsToTerminal_WithTerminalAtUtcSet() + { + var (store, _) = CreateStore(nameof(RecordTerminalAsync_FlipsToTerminal_WithTerminalAtUtcSet)); + await using var _store = store; + + var id = TrackedOperationId.New(); + await store.RecordEnqueueAsync(id, "ApiCallCached", "ERP.GetOrder", null, null); + + var beforeTerminal = DateTime.UtcNow; + await store.RecordTerminalAsync( + id, + status: nameof(AuditStatus.Parked), + lastError: "HTTP 503 (max retries)", + httpStatus: 503); + + var snapshot = await store.GetStatusAsync(id); + Assert.NotNull(snapshot); + Assert.Equal(nameof(AuditStatus.Parked), snapshot!.Status); + Assert.NotNull(snapshot.TerminalAtUtc); + Assert.Equal(DateTimeKind.Utc, snapshot.TerminalAtUtc!.Value.Kind); + Assert.True(snapshot.TerminalAtUtc >= beforeTerminal.AddSeconds(-1)); + Assert.Equal("HTTP 503 (max retries)", snapshot.LastError); + Assert.Equal(503, snapshot.HttpStatus); + } + + [Fact] + public async Task GetStatusAsync_Unknown_ReturnsNull() + { + var (store, _) = CreateStore(nameof(GetStatusAsync_Unknown_ReturnsNull)); + await using var _store = store; + + var unknown = TrackedOperationId.New(); + var snapshot = await store.GetStatusAsync(unknown); + + Assert.Null(snapshot); + } + + [Fact] + public async Task GetStatusAsync_ReturnsLatestSnapshot_AfterMultipleAttempts() + { + var (store, _) = CreateStore(nameof(GetStatusAsync_ReturnsLatestSnapshot_AfterMultipleAttempts)); + await using var _store = store; + + var id = TrackedOperationId.New(); + await store.RecordEnqueueAsync(id, "ApiCallCached", "ERP.GetOrder", null, null); + await store.RecordAttemptAsync(id, nameof(AuditStatus.Attempted), 1, "first failure", 503); + await store.RecordAttemptAsync(id, nameof(AuditStatus.Attempted), 2, "second failure", 503); + await store.RecordAttemptAsync(id, nameof(AuditStatus.Attempted), 3, "third failure", 504); + + var snapshot = await store.GetStatusAsync(id); + Assert.NotNull(snapshot); + Assert.Equal(3, snapshot!.RetryCount); + Assert.Equal("third failure", snapshot.LastError); + Assert.Equal(504, snapshot.HttpStatus); + } + + [Fact] + public async Task PurgeTerminalAsync_RemovesOldTerminalRows_KeepsRecent_KeepsNonTerminal() + { + var (store, dataSource) = CreateStore(nameof(PurgeTerminalAsync_RemovesOldTerminalRows_KeepsRecent_KeepsNonTerminal)); + await using var _store = store; + + // Three rows: + // (a) terminal, old → should be purged + // (b) terminal, fresh → should be kept + // (c) non-terminal, ancient CreatedAt → should be kept (no TerminalAtUtc) + var aId = TrackedOperationId.New(); + var bId = TrackedOperationId.New(); + var cId = TrackedOperationId.New(); + + await store.RecordEnqueueAsync(aId, "ApiCallCached", "A", null, null); + await store.RecordEnqueueAsync(bId, "ApiCallCached", "B", null, null); + await store.RecordEnqueueAsync(cId, "ApiCallCached", "C", null, null); + + await store.RecordTerminalAsync(aId, nameof(AuditStatus.Delivered), null, 200); + await store.RecordTerminalAsync(bId, nameof(AuditStatus.Delivered), null, 200); + + // Backdate the (a) row's TerminalAtUtc to 30 days ago via a direct UPDATE + // — RecordTerminalAsync stamps DateTime.UtcNow which we cannot inject. + // The verifier connection shares the same in-memory store thanks to + // mode=memory&cache=shared. + using (var connection = OpenVerifierConnection(dataSource)) + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = + "UPDATE OperationTracking SET TerminalAtUtc = $old WHERE TrackedOperationId = $id;"; + cmd.Parameters.AddWithValue("$old", DateTime.UtcNow.AddDays(-30).ToString("o")); + cmd.Parameters.AddWithValue("$id", aId.ToString()); + cmd.ExecuteNonQuery(); + } + + // Purge anything terminal older than 7 days. + var threshold = DateTime.UtcNow.AddDays(-7); + await store.PurgeTerminalAsync(threshold); + + Assert.Null(await store.GetStatusAsync(aId)); // purged + Assert.NotNull(await store.GetStatusAsync(bId)); // kept (recent terminal) + Assert.NotNull(await store.GetStatusAsync(cId)); // kept (non-terminal) + } +} diff --git a/tests/ScadaLink.StoreAndForward.Tests/CachedCallAttemptEmissionTests.cs b/tests/ScadaLink.StoreAndForward.Tests/CachedCallAttemptEmissionTests.cs new file mode 100644 index 0000000..05a8233 --- /dev/null +++ b/tests/ScadaLink.StoreAndForward.Tests/CachedCallAttemptEmissionTests.cs @@ -0,0 +1,298 @@ +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging.Abstractions; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.StoreAndForward.Tests; + +/// +/// Audit Log #23 — M3 Bundle E Tasks E4 + E5: the store-and-forward retry +/// loop invokes after every +/// cached-call attempt. The observer is given a +/// derived from the underlying +/// ; the audit bridge then materialises +/// the right CachedCallTelemetry packet (Attempted on every retry, +/// CachedResolve on terminal transitions). Tests run with +/// DefaultRetryInterval=Zero so the timer-driven retry sweep is +/// short-circuited by directly invoking +/// . +/// +public class CachedCallAttemptEmissionTests : IAsyncLifetime, IDisposable +{ + private readonly SqliteConnection _keepAlive; + private readonly StoreAndForwardStorage _storage; + private readonly StoreAndForwardService _service; + private readonly StoreAndForwardOptions _options; + private readonly CapturingObserver _observer; + + public CachedCallAttemptEmissionTests() + { + var dbName = $"E4Tests_{Guid.NewGuid():N}"; + var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared"; + _keepAlive = new SqliteConnection(connStr); + _keepAlive.Open(); + + _storage = new StoreAndForwardStorage(connStr, NullLogger.Instance); + + _options = new StoreAndForwardOptions + { + DefaultRetryInterval = TimeSpan.Zero, + DefaultMaxRetries = 3, + RetryTimerInterval = TimeSpan.FromMinutes(10), + }; + + _observer = new CapturingObserver(); + + _service = new StoreAndForwardService( + _storage, + _options, + NullLogger.Instance, + replication: null, + cachedCallObserver: _observer, + siteId: "site-77"); + } + + public async Task InitializeAsync() => await _storage.InitializeAsync(); + public Task DisposeAsync() => Task.CompletedTask; + public void Dispose() => _keepAlive.Dispose(); + + /// + /// Captures every observer notification so tests can assert on the + /// emitted lifecycle sequence. + /// + private sealed class CapturingObserver : ICachedCallLifecycleObserver + { + public List Notifications { get; } = new(); + public Exception? ThrowOnNotify { get; set; } + + public Task OnAttemptCompletedAsync(CachedCallAttemptContext context, CancellationToken ct = default) + { + if (ThrowOnNotify != null) + { + return Task.FromException(ThrowOnNotify); + } + lock (Notifications) + { + Notifications.Add(context); + } + return Task.CompletedTask; + } + } + + private async Task EnqueueBufferedAsync( + StoreAndForwardCategory category, string target, int maxRetries = 3) + { + // The TrackedOperationId is the S&F message id (Bundle E3 contract). + var trackedId = TrackedOperationId.New(); + await _service.EnqueueAsync( + category, + target, + """{"payload":"x"}""", + originInstanceName: "Plant.Pump42", + maxRetries: maxRetries, + retryInterval: TimeSpan.Zero, + attemptImmediateDelivery: false, + messageId: trackedId.ToString()); + return trackedId; + } + + // ── Task E4: per-attempt observer notifications ── + + [Fact] + public async Task Attempt_FailWithHttp500_EmitsAttemptedTelemetry() + { + // ExternalSystem cached call buffered, retry sweep encounters a + // transient failure on the first attempt. + _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, + _ => throw new HttpRequestException("HTTP 500 from ERP")); + var trackedId = await EnqueueBufferedAsync( + StoreAndForwardCategory.ExternalSystem, "ERP", maxRetries: 5); + + await _service.RetryPendingMessagesAsync(); + + var notification = Assert.Single(_observer.Notifications); + Assert.Equal(trackedId, notification.TrackedOperationId); + Assert.Equal("ApiOutbound", notification.Channel); + Assert.Equal("ERP", notification.Target); + Assert.Equal("site-77", notification.SourceSite); + Assert.Equal(CachedCallAttemptOutcome.TransientFailure, notification.Outcome); + Assert.Equal(1, notification.RetryCount); + Assert.Contains("HTTP 500", notification.LastError); + Assert.Equal("Plant.Pump42", notification.SourceInstanceId); + } + + [Fact] + public async Task Attempt_Success_EmitsDeliveredOutcome() + { + // ExternalSystem cached call buffered, retry sweep delivers the + // message successfully on its first attempt. + _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, + _ => Task.FromResult(true)); + var trackedId = await EnqueueBufferedAsync( + StoreAndForwardCategory.ExternalSystem, "ERP"); + + await _service.RetryPendingMessagesAsync(); + + var notification = Assert.Single(_observer.Notifications); + Assert.Equal(trackedId, notification.TrackedOperationId); + Assert.Equal(CachedCallAttemptOutcome.Delivered, notification.Outcome); + Assert.Null(notification.LastError); + } + + [Fact] + public async Task Attempt_PermanentFailure_EmitsPermanentFailureOutcome() + { + _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, + _ => Task.FromResult(false)); + var trackedId = await EnqueueBufferedAsync( + StoreAndForwardCategory.ExternalSystem, "ERP"); + + await _service.RetryPendingMessagesAsync(); + + var notification = Assert.Single(_observer.Notifications); + Assert.Equal(trackedId, notification.TrackedOperationId); + Assert.Equal(CachedCallAttemptOutcome.PermanentFailure, notification.Outcome); + } + + [Fact] + public async Task Attempt_CachedDbWrite_EmitsDbOutboundChannel() + { + _service.RegisterDeliveryHandler(StoreAndForwardCategory.CachedDbWrite, + _ => Task.FromResult(true)); + var trackedId = await EnqueueBufferedAsync( + StoreAndForwardCategory.CachedDbWrite, "myDb"); + + await _service.RetryPendingMessagesAsync(); + + var notification = Assert.Single(_observer.Notifications); + Assert.Equal(trackedId, notification.TrackedOperationId); + Assert.Equal("DbOutbound", notification.Channel); + Assert.Equal("myDb", notification.Target); + } + + [Fact] + public async Task Attempt_NotificationCategory_NoObserverNotification() + { + // Notifications are NOT cached calls — they're forwarded to central via + // a separate forwarder. The observer must not fire for Notification + // category messages. + _service.RegisterDeliveryHandler(StoreAndForwardCategory.Notification, + _ => Task.FromResult(true)); + await _service.EnqueueAsync( + StoreAndForwardCategory.Notification, + "alerts", + """{"subject":"x"}""", + originInstanceName: "Plant.Pump42", + attemptImmediateDelivery: false); + + await _service.RetryPendingMessagesAsync(); + + Assert.Empty(_observer.Notifications); + } + + [Fact] + public async Task Attempt_MessageIdNotAGuid_NoObserverNotification() + { + // Pre-M3 cached calls (no TrackedOperationId threaded in) use a random + // GUID-N message id from S&F itself. We should still emit (M3 expects + // post-rollout these are tracked) — BUT pre-rollout messages can have + // a non-parseable id, in which case the observer is silently skipped + // to keep S&F bookkeeping intact. + _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, + _ => Task.FromResult(true)); + await _service.EnqueueAsync( + StoreAndForwardCategory.ExternalSystem, + "ERP", + """{}""", + originInstanceName: "Plant.Pump42", + attemptImmediateDelivery: false, + messageId: "not-a-valid-guid-id"); + + await _service.RetryPendingMessagesAsync(); + + Assert.Empty(_observer.Notifications); + } + + // ── Task E5: terminal-state observer notifications ── + + [Fact] + public async Task Terminal_Delivered_EmitsResolveWithDeliveredStatus() + { + // A successful retry produces a single Delivered observer notification + // — the audit bridge maps this to both an Attempted-Delivered audit row + // and the terminal CachedResolve(Delivered) row. The S&F layer fires + // ONE notification per attempt and lets the bridge fan out as needed. + _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, + _ => Task.FromResult(true)); + var trackedId = await EnqueueBufferedAsync( + StoreAndForwardCategory.ExternalSystem, "ERP"); + + await _service.RetryPendingMessagesAsync(); + + var notification = Assert.Single(_observer.Notifications); + Assert.Equal(trackedId, notification.TrackedOperationId); + Assert.Equal(CachedCallAttemptOutcome.Delivered, notification.Outcome); + } + + [Fact] + public async Task Terminal_Parked_OnMaxRetries_EmitsParkedMaxRetries() + { + // Configure handler to throw transient every time. + _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, + _ => throw new HttpRequestException("Connection refused")); + var trackedId = await EnqueueBufferedAsync( + StoreAndForwardCategory.ExternalSystem, "ERP", maxRetries: 2); + + // Two sweeps -> RetryCount climbs to 2 -> parked on the second sweep. + await _service.RetryPendingMessagesAsync(); + await _service.RetryPendingMessagesAsync(); + + Assert.Equal(2, _observer.Notifications.Count); + Assert.Equal(CachedCallAttemptOutcome.TransientFailure, _observer.Notifications[0].Outcome); + Assert.Equal(CachedCallAttemptOutcome.ParkedMaxRetries, _observer.Notifications[1].Outcome); + Assert.Equal(trackedId, _observer.Notifications[1].TrackedOperationId); + } + + [Fact] + public async Task Lifecycle_RetryFail_RetrySucceed_EmitsExpectedSequence() + { + var calls = 0; + _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ => + { + calls++; + if (calls == 1) throw new HttpRequestException("transient"); + return Task.FromResult(true); + }); + var trackedId = await EnqueueBufferedAsync( + StoreAndForwardCategory.ExternalSystem, "ERP", maxRetries: 5); + + await _service.RetryPendingMessagesAsync(); + await _service.RetryPendingMessagesAsync(); + + Assert.Equal(2, _observer.Notifications.Count); + Assert.Equal(CachedCallAttemptOutcome.TransientFailure, _observer.Notifications[0].Outcome); + Assert.Equal(1, _observer.Notifications[0].RetryCount); + Assert.Equal(CachedCallAttemptOutcome.Delivered, _observer.Notifications[1].Outcome); + Assert.Equal(trackedId, _observer.Notifications[1].TrackedOperationId); + } + + // ── Best-effort contract: observer throws must NOT corrupt retry bookkeeping ── + + [Fact] + public async Task Observer_Throws_DoesNotCorruptRetryCount() + { + _observer.ThrowOnNotify = new InvalidOperationException("simulated audit failure"); + _service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, + _ => Task.FromResult(true)); + var trackedId = await EnqueueBufferedAsync( + StoreAndForwardCategory.ExternalSystem, "ERP"); + + // Must not throw — observer is best-effort. + await _service.RetryPendingMessagesAsync(); + + // The message was delivered (handler returned true) so it should be gone. + var msg = await _storage.GetMessageByIdAsync(trackedId.ToString()); + Assert.Null(msg); + } +}