diff --git a/docs/requirements/Component-ConfigurationDatabase.md b/docs/requirements/Component-ConfigurationDatabase.md index 822c93af..7f5a6273 100644 --- a/docs/requirements/Component-ConfigurationDatabase.md +++ b/docs/requirements/Component-ConfigurationDatabase.md @@ -127,7 +127,7 @@ The configuration database stores all central system data, organized by domain a - **Bundle Unlock Attempts**: One row per permitted bundle-unlock passphrase attempt (`ClientKey`, `AttemptedAtUtc`), forming the fleet-wide sliding window behind Transport's `MaxUnlockAttemptsPerIpPerHour` throttle — see `Component-Transport.md` (register row 52). Operational history, not audit: non-partitioned, no DB-role restriction, and rows are pruned opportunistically by the application once they age past twice the window. **Denied** attempts are deliberately not recorded (a denial burns no budget), and no bundle identity is stored — the durable record of a failed unlock is the `BundleImportUnlockFailed` audit row. Central-only: written only by the Transport component, which the host registers for central roles. ### Audit Logging -- **Audit Log Entries**: Append-only audit trail (timestamp, user, action, entity type, entity ID, entity name, state as JSON). Stores only the after-state — change history is reconstructed by comparing consecutive entries. Entries are never modified or deleted. No retention policy — retained indefinitely. Indexed on timestamp, user, entity type, entity ID, and action for efficient filtering. +- **Audit Log Entries**: Append-only audit trail (timestamp, user, action, entity type, entity ID, entity name, state as JSON). Stores only the after-state — change history is reconstructed by comparing consecutive entries. Entries are never modified. Retained indefinitely by default; an optional `ScadaBridge:ConfigAudit:RetentionDays` (default `0` = off, validated `>= 0`) enables a daily, time-sliced batched purge (`ConfigAuditPurgeService`: `PurgeBatchSize` 5000 rows/batch, at most `MaxBatchesPerTick` 200 batches per daily tick, oldest first) of entries older than the window. `AfterStateJson` is capped at 256 K characters — a larger serialized state is replaced by a small valid-JSON truncation marker (`AuditTruncated`, `OriginalLength`, `StateType`, preview) so one row cannot grow without bound. The viewer query orders by `(Timestamp DESC, Id DESC)` so pages are deterministic when many rows share a timestamp (bundle imports). Indexed on timestamp, user, entity type, entity ID, and action for efficient filtering. --- diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/ConfigAuditOptions.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/ConfigAuditOptions.cs new file mode 100644 index 00000000..2cba2c41 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/ConfigAuditOptions.cs @@ -0,0 +1,47 @@ +using Microsoft.Extensions.Options; + +namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance; + +/// +/// Options for the configuration audit trail (AuditLogEntries), bound from +/// ScadaBridge:ConfigAudit. +/// +public class ConfigAuditOptions +{ + /// The configuration section name. + public const string SectionName = "ScadaBridge:ConfigAudit"; + + /// + /// Entries older than this many days are purged daily. 0 (the default) keeps config + /// audit entries indefinitely — retention is strictly opt-in. + /// + public int RetentionDays { get; set; } + + /// Maximum rows deleted per batch (keeps each DELETE short and the log small). + public int PurgeBatchSize { get; set; } = 5000; + + /// Upper bound on batches per purge tick; the remainder is picked up on the next tick. + public int MaxBatchesPerTick { get; set; } = 200; + + /// How often the purge runs. + public TimeSpan PurgeInterval { get; set; } = TimeSpan.FromDays(1); +} + +/// Validates . +public sealed class ConfigAuditOptionsValidator : IValidateOptions +{ + /// + public ValidateOptionsResult Validate(string? name, ConfigAuditOptions options) + { + var failures = new List(); + if (options.RetentionDays < 0) + failures.Add($"{ConfigAuditOptions.SectionName}:RetentionDays must be >= 0 (0 = retain indefinitely)."); + if (options.PurgeBatchSize <= 0) + failures.Add($"{ConfigAuditOptions.SectionName}:PurgeBatchSize must be > 0."); + if (options.MaxBatchesPerTick <= 0) + failures.Add($"{ConfigAuditOptions.SectionName}:MaxBatchesPerTick must be > 0."); + if (options.PurgeInterval <= TimeSpan.Zero) + failures.Add($"{ConfigAuditOptions.SectionName}:PurgeInterval must be > 0."); + return failures.Count == 0 ? ValidateOptionsResult.Success : ValidateOptionsResult.Fail(failures); + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/ConfigAuditPurgeService.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/ConfigAuditPurgeService.cs new file mode 100644 index 00000000..ea946236 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/ConfigAuditPurgeService.cs @@ -0,0 +1,122 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance; + +/// +/// Central hosted service that purges AuditLogEntries (the configuration audit trail) +/// older than . A no-op while +/// RetentionDays is 0 (the default — retain indefinitely). +/// +/// +/// Deletes in bounded batches (, at most +/// per tick) so a first purge over a large +/// backlog is time-sliced across ticks instead of one long-running DELETE. Both central nodes may +/// run it; deleting an already-deleted row is harmless, so no singleton is needed. A failed tick +/// is logged and retried on the next. +/// +public sealed class ConfigAuditPurgeService : BackgroundService +{ + private readonly IServiceScopeFactory _scopeFactory; + private readonly IOptions _options; + private readonly TimeProvider _time; + private readonly ILogger _logger; + + /// Initializes the purge service. + /// Creates a DI scope per tick (the DbContext is scoped). + /// Retention options. + /// Clock. + /// Logger. + public ConfigAuditPurgeService( + IServiceScopeFactory scopeFactory, + IOptions options, + TimeProvider time, + ILogger logger) + { + _scopeFactory = scopeFactory; + _options = options; + _time = time; + _logger = logger; + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var opts = _options.Value; + if (opts.RetentionDays <= 0) + { + _logger.LogInformation("Config audit retention is off (RetentionDays=0); entries are kept indefinitely."); + return; + } + + using var timer = new PeriodicTimer(opts.PurgeInterval, _time); + do + { + try + { + await PurgeOnceAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + _logger.LogError(ex, "Config audit purge tick failed; will retry next interval."); + } + } + while (await WaitNextAsync(timer, stoppingToken)); + } + + private static async Task WaitNextAsync(PeriodicTimer timer, CancellationToken ct) + { + try { return await timer.WaitForNextTickAsync(ct); } + catch (OperationCanceledException) { return false; } + } + + /// + /// Runs one purge pass. Returns the number of rows deleted. Exposed for tests. + /// + /// Cancellation token. + /// Total rows deleted this pass. + public async Task PurgeOnceAsync(CancellationToken cancellationToken = default) + { + var opts = _options.Value; + if (opts.RetentionDays <= 0) + return 0; + + var cutoff = _time.GetUtcNow().AddDays(-opts.RetentionDays); + await using var scope = _scopeFactory.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var total = 0; + for (var batch = 0; batch < opts.MaxBatchesPerTick; batch++) + { + // Oldest-first, bounded: select the batch's keys then delete by key so the DELETE + // stays a short, index-ordered operation on SQL Server. + var ids = await db.AuditLogEntries + .AsNoTracking() + .Where(a => a.Timestamp < cutoff) + .OrderBy(a => a.Timestamp) + .Select(a => a.Id) + .Take(opts.PurgeBatchSize) + .ToListAsync(cancellationToken); + if (ids.Count == 0) + break; + + total += await db.AuditLogEntries + .Where(a => ids.Contains(a.Id)) + .ExecuteDeleteAsync(cancellationToken); + + if (ids.Count < opts.PurgeBatchSize) + break; + } + + if (total > 0) + _logger.LogInformation("Config audit purge removed {Count} entries older than {Cutoff:o}.", total, cutoff); + return total; + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ServiceCollectionExtensions.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ServiceCollectionExtensions.cs index c2f3675a..de890f9e 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ServiceCollectionExtensions.cs @@ -110,6 +110,14 @@ public static class ServiceCollectionExtensions // fresh scope on each tick (mirrors AuditLogPurgeActor / AuditLogIngestActor). services.AddScoped(); + // Opt-in config-audit retention (ScadaBridge:ConfigAudit:RetentionDays, default 0 = off). + // Options binding to configuration is done by the host; without it the defaults apply + // and the purge service is a no-op. + services.TryAddEnumerable(ServiceDescriptor.Singleton< + Microsoft.Extensions.Options.IValidateOptions, ConfigAuditOptionsValidator>()); + services.TryAddSingleton(TimeProvider.System); + services.AddHostedService(); + services.AddDataProtection() .PersistKeysToDbContext(); diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Services/AuditService.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Services/AuditService.cs index 71913cb9..45451306 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Services/AuditService.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Services/AuditService.cs @@ -23,6 +23,15 @@ public class AuditService : IAuditService MaxDepth = 32 }; + /// + /// Upper bound (characters) on a serialized AfterStateJson. A larger payload (e.g. a + /// whole template graph with every script body) is replaced by a small truncation marker so + /// one audit row can never grow without bound. Valid JSON is preserved either way. + /// + public const int MaxAfterStateChars = 256 * 1024; + + private const int TruncatedPreviewChars = 4 * 1024; + /// /// Initializes the audit service with the EF Core context and correlation context. /// @@ -72,7 +81,17 @@ public class AuditService : IAuditService { try { - return JsonSerializer.Serialize(afterState, AuditSerializerOptions); + var json = JsonSerializer.Serialize(afterState, AuditSerializerOptions); + if (json.Length <= MaxAfterStateChars) + return json; + + return JsonSerializer.Serialize(new + { + AuditTruncated = true, + OriginalLength = json.Length, + StateType = afterState.GetType().FullName, + Preview = json[..TruncatedPreviewChars] + }); } catch (Exception ex) when (ex is JsonException or NotSupportedException) { diff --git a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/AuditServiceTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/AuditServiceTests.cs index 00413c5c..237dd1be 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/AuditServiceTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/AuditServiceTests.cs @@ -212,4 +212,20 @@ public class AuditServiceTests : IDisposable Assert.Contains("node", audit.AfterStateJson); Assert.Single(await _context.Templates.Where(t => t.Name == "CyclicAuditTemplate").ToListAsync()); } + + /// An oversized after-state is replaced by a small valid-JSON truncation marker (review 08/G1). + /// A task that represents the asynchronous operation. + [Fact] + public async Task LogAsync_OversizedAfterState_IsTruncatedToMarker() + { + await _auditService.LogAsync("admin", "Update", "Template", "1", "Big", + new { Code = new string('x', AuditService.MaxAfterStateChars + 10) }); + await _context.SaveChangesAsync(); + + var json = _context.AuditLogEntries.Single().AfterStateJson!; + Assert.True(json.Length < 16 * 1024); + using var doc = JsonDocument.Parse(json); + Assert.True(doc.RootElement.GetProperty("AuditTruncated").GetBoolean()); + Assert.True(doc.RootElement.GetProperty("OriginalLength").GetInt32() > AuditService.MaxAfterStateChars); + } } diff --git a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/ConfigAuditPurgeServiceTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/ConfigAuditPurgeServiceTests.cs new file mode 100644 index 00000000..d4b50188 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/ConfigAuditPurgeServiceTests.cs @@ -0,0 +1,78 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit; +using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance; + +namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance; + +/// Tests for the opt-in config-audit retention purge (review 2026-10-21 08/G1). +public class ConfigAuditPurgeServiceTests : IDisposable +{ + private readonly ScadaBridgeDbContext _context = SqliteTestHelper.CreateInMemoryContext(); + + public void Dispose() + { + _context.Database.CloseConnection(); + _context.Dispose(); + } + + private ConfigAuditPurgeService Create(ConfigAuditOptions opts) + { + var services = new ServiceCollection(); + services.AddSingleton(_context); + return new ConfigAuditPurgeService( + services.BuildServiceProvider().GetRequiredService(), + Options.Create(opts), + TimeProvider.System, + NullLogger.Instance); + } + + private async Task SeedAsync(int oldCount, int recentCount) + { + var now = DateTimeOffset.UtcNow; + for (var i = 0; i < oldCount; i++) + _context.AuditLogEntries.Add(new AuditLogEntry("u", "Create", "Template", $"o{i}", $"Old{i}") { Timestamp = now.AddDays(-100).AddMinutes(i) }); + for (var i = 0; i < recentCount; i++) + _context.AuditLogEntries.Add(new AuditLogEntry("u", "Create", "Template", $"r{i}", $"New{i}") { Timestamp = now.AddDays(-1) }); + await _context.SaveChangesAsync(); + } + + [Fact] + public async Task RetentionZero_DeletesNothing() + { + await SeedAsync(5, 2); + var deleted = await Create(new ConfigAuditOptions { RetentionDays = 0 }).PurgeOnceAsync(); + Assert.Equal(0, deleted); + Assert.Equal(7, _context.AuditLogEntries.Count()); + } + + [Fact] + public async Task Retention_DeletesOnlyOlderThanWindow_AcrossBatches() + { + await SeedAsync(25, 3); + var deleted = await Create(new ConfigAuditOptions { RetentionDays = 30, PurgeBatchSize = 10 }).PurgeOnceAsync(); + Assert.Equal(25, deleted); + Assert.Equal(3, _context.AuditLogEntries.Count()); + } + + [Fact] + public async Task Retention_MaxBatchesPerTick_TimeSlicesTheBacklog() + { + await SeedAsync(25, 0); + var svc = Create(new ConfigAuditOptions { RetentionDays = 30, PurgeBatchSize = 10, MaxBatchesPerTick = 2 }); + Assert.Equal(20, await svc.PurgeOnceAsync()); + Assert.Equal(5, await svc.PurgeOnceAsync()); + } + + [Fact] + public void Validator_RejectsNegativeRetention_AcceptsZero() + { + var v = new ConfigAuditOptionsValidator(); + Assert.True(v.Validate(null, new ConfigAuditOptions { RetentionDays = 0 }).Succeeded); + Assert.True(v.Validate(null, new ConfigAuditOptions { RetentionDays = 90 }).Succeeded); + Assert.False(v.Validate(null, new ConfigAuditOptions { RetentionDays = -1 }).Succeeded); + Assert.False(v.Validate(null, new ConfigAuditOptions { PurgeBatchSize = 0 }).Succeeded); + } +}