refactor: rename ScadaLink → ZB.MOM.WW.ScadaBridge (code + projects + namespaces)
Solution + 23 src projects + 26 test projects renamed; folders, csproj, namespaces, and ScadaLinkDbContext/ScadaBridgeDbContext class updated. ActorSystem "scadalink" → "scadabridge", Akka seed-node URLs migrated. SQL roles/logins, LDAP domains, CLI command name, and CLI config dir (~/.scadalink → ~/.scadabridge) also renamed. Build green; 5 Host.Tests fail awaiting SQL login rename in next commit. Pre-existing StaleTagMonitor timing flakes unchanged. Rename script committed at tools/rename-to-scadabridge.sh.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,212 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Immutable;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.AuditLog;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end integration test for the Audit Log (#23) site→central push path
|
||||
/// introduced by the "real ClusterClient-based site audit push client" follow-up.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Exercises the full production chain in one actor system: the real
|
||||
/// <see cref="SqliteAuditWriter"/> site SQLite hot-path, the real
|
||||
/// <see cref="SiteAuditTelemetryActor"/> drain loop, the real
|
||||
/// <see cref="ClusterClientSiteAuditClient"/>, the real
|
||||
/// <see cref="SiteCommunicationActor"/> forward, the real
|
||||
/// <see cref="CentralCommunicationActor"/> routing, and the real
|
||||
/// <c>AuditLogIngestActor</c> ingest — only the cross-cluster ClusterClient
|
||||
/// transport itself is substituted by an in-process <see cref="ClusterClientRelay"/>
|
||||
/// that unwraps <see cref="ClusterClient.Send"/> exactly as a real ClusterClient
|
||||
/// would (a multi-node cluster is out of scope for an in-process test).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The central audit store is an in-memory <see cref="IAuditLogRepository"/> —
|
||||
/// the production <c>AuditLogRepository</c> emits SQL Server-specific T-SQL and
|
||||
/// needs an MSSQL container, which this test deliberately avoids. The test
|
||||
/// asserts both ends of the contract: a central <c>AuditLog</c> row appears AND
|
||||
/// the site SQLite row flips from <see cref="AuditForwardState.Pending"/> to
|
||||
/// <see cref="AuditForwardState.Forwarded"/>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class SiteAuditPushFlowTests : TestKit
|
||||
{
|
||||
/// <summary>
|
||||
/// In-process stand-in for a real Akka ClusterClient: unwraps a
|
||||
/// <see cref="ClusterClient.Send"/> and forwards the inner message to the
|
||||
/// central actor, preserving the original sender so the reply routes back to
|
||||
/// the site's Ask. A real ClusterClient does exactly this across the cluster
|
||||
/// boundary; the in-process relay keeps the test free of a multi-node setup.
|
||||
/// </summary>
|
||||
private sealed class ClusterClientRelay : ReceiveActor
|
||||
{
|
||||
public ClusterClientRelay(IActorRef central)
|
||||
{
|
||||
Receive<ClusterClient.Send>(send => central.Forward(send.Message));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thread-safe in-memory <see cref="IAuditLogRepository"/>. Only
|
||||
/// <see cref="InsertIfNotExistsAsync"/> is exercised by the ingest path; the
|
||||
/// rest throw because they are not reachable from this test.
|
||||
/// </summary>
|
||||
private sealed class InMemoryAuditLogRepository : IAuditLogRepository
|
||||
{
|
||||
private readonly ConcurrentDictionary<Guid, AuditEvent> _rows = new();
|
||||
|
||||
public IReadOnlyCollection<AuditEvent> Rows => _rows.Values.ToList();
|
||||
|
||||
public Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(evt);
|
||||
// First-write-wins idempotency, mirroring the production repository.
|
||||
_rows.TryAdd(evt.EventId, evt);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<AuditEvent>> QueryAsync(
|
||||
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<IReadOnlyList<DateTime>> GetPartitionBoundariesOlderThanAsync(
|
||||
DateTime threshold, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<AuditLogKpiSnapshot> GetKpiSnapshotAsync(
|
||||
TimeSpan window, DateTime? nowUtc = null, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<IReadOnlyList<ExecutionTreeNode>> GetExecutionTreeAsync(
|
||||
Guid executionId, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<IReadOnlyList<string>> GetDistinctSourceNodesAsync(CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
}
|
||||
|
||||
private static AuditEvent NewPendingEvent(Guid id) => new()
|
||||
{
|
||||
EventId = id,
|
||||
OccurredAtUtc = new DateTime(2026, 5, 21, 9, 0, 0, DateTimeKind.Utc),
|
||||
Channel = AuditChannel.ApiOutbound,
|
||||
Kind = AuditKind.ApiCall,
|
||||
Status = AuditStatus.Delivered,
|
||||
SourceSiteId = "site-1",
|
||||
Target = "ext-system-1",
|
||||
PayloadTruncated = false,
|
||||
ForwardState = AuditForwardState.Pending,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public async Task SiteAuditEvent_DrainsToCentral_AndFlipsSiteRowToForwarded()
|
||||
{
|
||||
// ── Central side ──────────────────────────────────────────────────
|
||||
// Real AuditLogIngestActor over an in-memory repository (test-mode ctor).
|
||||
var centralRepo = new InMemoryAuditLogRepository();
|
||||
var ingestActor = Sys.ActorOf(Props.Create(() =>
|
||||
new ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogIngestActor(
|
||||
centralRepo,
|
||||
NullLogger<ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogIngestActor>.Instance)));
|
||||
|
||||
// Real CentralCommunicationActor. Its periodic site-address refresh
|
||||
// resolves an ISiteRepository from this provider; an empty result keeps
|
||||
// the refresh a clean no-op and never touches the audit-ingest path.
|
||||
var siteRepo = Substitute.For<ISiteRepository>();
|
||||
siteRepo.GetAllSitesAsync().Returns(Array.Empty<Site>());
|
||||
var centralServices = new ServiceCollection();
|
||||
centralServices.AddScoped(_ => siteRepo);
|
||||
var centralProvider = centralServices.BuildServiceProvider();
|
||||
|
||||
var centralCommActor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(
|
||||
centralProvider,
|
||||
new DefaultSiteClientFactory(),
|
||||
TimeSpan.FromSeconds(5))));
|
||||
centralCommActor.Tell(new RegisterAuditIngest(ingestActor));
|
||||
|
||||
// ── Site side ─────────────────────────────────────────────────────
|
||||
// Real SqliteAuditWriter on a file-backed SQLite db (the site hot-path
|
||||
// + Pending queue). A temp file so it survives across DI scopes.
|
||||
var dbPath = Path.Combine(Path.GetTempPath(), $"auditpush-{Guid.NewGuid():N}.db");
|
||||
var writerOptions = Options.Create(new SqliteAuditWriterOptions { DatabasePath = dbPath });
|
||||
var nodeIdentity = Substitute.For<ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.INodeIdentityProvider>();
|
||||
nodeIdentity.NodeName.Returns((string?)null);
|
||||
await using var writer = new SqliteAuditWriter(
|
||||
writerOptions, NullLogger<SqliteAuditWriter>.Instance, nodeIdentity);
|
||||
|
||||
// Real SiteCommunicationActor. RegisterCentralClient is given the relay
|
||||
// standing in for the central ClusterClient.
|
||||
var siteCommActor = Sys.ActorOf(Props.Create(() => new SiteCommunicationActor(
|
||||
"site-1",
|
||||
new CommunicationOptions(),
|
||||
CreateTestProbe().Ref))); // deployment-manager proxy is unused here
|
||||
var relay = Sys.ActorOf(Props.Create(() => new ClusterClientRelay(centralCommActor)));
|
||||
siteCommActor.Tell(new RegisterCentralClient(relay));
|
||||
|
||||
// The production site audit push client — the unit under integration.
|
||||
var auditClient = new ClusterClientSiteAuditClient(
|
||||
siteCommActor, TimeSpan.FromSeconds(5));
|
||||
|
||||
// Real SiteAuditTelemetryActor drains the writer's Pending queue and
|
||||
// pushes via the client. Fast intervals so the test completes quickly.
|
||||
var telemetryOptions = Options.Create(new SiteAuditTelemetryOptions
|
||||
{
|
||||
BatchSize = 256,
|
||||
BusyIntervalSeconds = 1,
|
||||
IdleIntervalSeconds = 1,
|
||||
});
|
||||
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
|
||||
writer,
|
||||
auditClient,
|
||||
telemetryOptions,
|
||||
NullLogger<SiteAuditTelemetryActor>.Instance)));
|
||||
|
||||
// ── Act ───────────────────────────────────────────────────────────
|
||||
// Write an audit event onto the site SQLite hot-path. It lands Pending.
|
||||
var eventId = Guid.NewGuid();
|
||||
await writer.WriteAsync(NewPendingEvent(eventId));
|
||||
|
||||
// ── Assert ────────────────────────────────────────────────────────
|
||||
// Within ~10s the drain loop pushes the event to central AND flips the
|
||||
// site row to Forwarded.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
// Central received and persisted the row.
|
||||
Assert.Contains(centralRepo.Rows, r => r.EventId == eventId);
|
||||
|
||||
// The site row reached AuditForwardState.Forwarded specifically —
|
||||
// not merely "no longer Pending" (a Reconciled row would also leave
|
||||
// ReadPendingAsync, so we assert the positive Forwarded state).
|
||||
var forwarded = await writer.ReadForwardedAsync(256, CancellationToken.None);
|
||||
var row = Assert.Single(forwarded, r => r.EventId == eventId);
|
||||
Assert.Equal(AuditForwardState.Forwarded, row.ForwardState);
|
||||
}, TimeSpan.FromSeconds(10), TimeSpan.FromMilliseconds(250));
|
||||
|
||||
// The central-persisted row carries the central-stamped IngestedAtUtc.
|
||||
var ingested = centralRepo.Rows.Single(r => r.EventId == eventId);
|
||||
Assert.NotNull(ingested.IngestedAtUtc);
|
||||
|
||||
// Cleanup the temp SQLite file.
|
||||
try { File.Delete(dbPath); } catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Security;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-22: Audit transactional guarantee — entity change + audit log in same transaction.
|
||||
/// </summary>
|
||||
public class AuditTransactionTests : IClassFixture<ScadaBridgeWebApplicationFactory>
|
||||
{
|
||||
private readonly ScadaBridgeWebApplicationFactory _factory;
|
||||
|
||||
public AuditTransactionTests(ScadaBridgeWebApplicationFactory factory)
|
||||
{
|
||||
_factory = factory;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AuditLog_IsCommittedWithEntityChange_InSameTransaction()
|
||||
{
|
||||
using var scope = _factory.Services.CreateScope();
|
||||
var securityRepo = scope.ServiceProvider.GetRequiredService<ISecurityRepository>();
|
||||
var auditService = scope.ServiceProvider.GetRequiredService<IAuditService>();
|
||||
var dbContext = scope.ServiceProvider.GetRequiredService<ScadaBridgeDbContext>();
|
||||
|
||||
// Add a mapping and an audit log entry in the same unit of work
|
||||
var mapping = new LdapGroupMapping("test-group-audit", "Admin");
|
||||
await securityRepo.AddMappingAsync(mapping);
|
||||
|
||||
await auditService.LogAsync(
|
||||
user: "test-user",
|
||||
action: "Create",
|
||||
entityType: "LdapGroupMapping",
|
||||
entityId: "0", // ID not yet assigned
|
||||
entityName: "test-group-audit",
|
||||
afterState: new { Group = "test-group-audit", Role = "Admin" });
|
||||
|
||||
// Both should be in the change tracker before saving
|
||||
var trackedEntities = dbContext.ChangeTracker.Entries().Count(e => e.State == EntityState.Added);
|
||||
Assert.True(trackedEntities >= 2, "Both entity and audit log should be tracked before SaveChanges");
|
||||
|
||||
// Single SaveChangesAsync commits both
|
||||
await securityRepo.SaveChangesAsync();
|
||||
|
||||
// Verify both were persisted
|
||||
var mappings = await securityRepo.GetAllMappingsAsync();
|
||||
Assert.Contains(mappings, m => m.LdapGroupName == "test-group-audit");
|
||||
|
||||
var auditEntries = await dbContext.AuditLogEntries.ToListAsync();
|
||||
Assert.Contains(auditEntries, a => a.EntityName == "test-group-audit" && a.Action == "Create");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AuditLog_IsNotPersistedWhenSaveNotCalled()
|
||||
{
|
||||
// Create a separate scope so we have a fresh DbContext
|
||||
using var scope1 = _factory.Services.CreateScope();
|
||||
var securityRepo = scope1.ServiceProvider.GetRequiredService<ISecurityRepository>();
|
||||
var auditService = scope1.ServiceProvider.GetRequiredService<IAuditService>();
|
||||
|
||||
// Add entity + audit but do NOT call SaveChangesAsync
|
||||
var mapping = new LdapGroupMapping("orphan-group", "Design");
|
||||
await securityRepo.AddMappingAsync(mapping);
|
||||
await auditService.LogAsync("test", "Create", "LdapGroupMapping", "0", "orphan-group", null);
|
||||
|
||||
// Dispose scope without saving — simulates a failed transaction
|
||||
scope1.Dispose();
|
||||
|
||||
// In a new scope, verify nothing was persisted
|
||||
using var scope2 = _factory.Services.CreateScope();
|
||||
var securityRepo2 = scope2.ServiceProvider.GetRequiredService<ISecurityRepository>();
|
||||
var dbContext2 = scope2.ServiceProvider.GetRequiredService<ScadaBridgeDbContext>();
|
||||
|
||||
var mappings = await securityRepo2.GetAllMappingsAsync();
|
||||
Assert.DoesNotContain(mappings, m => m.LdapGroupName == "orphan-group");
|
||||
|
||||
var auditEntries = await dbContext2.AuditLogEntries.ToListAsync();
|
||||
Assert.DoesNotContain(auditEntries, a => a.EntityName == "orphan-group");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.ScadaBridge.CentralUI.Auth;
|
||||
using ZB.MOM.WW.ScadaBridge.Security;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-22: Auth flow integration tests.
|
||||
/// Tests that require a running LDAP server are marked with Integration trait.
|
||||
/// </summary>
|
||||
public class AuthFlowTests : IClassFixture<ScadaBridgeWebApplicationFactory>
|
||||
{
|
||||
private readonly ScadaBridgeWebApplicationFactory _factory;
|
||||
|
||||
public AuthFlowTests(ScadaBridgeWebApplicationFactory factory)
|
||||
{
|
||||
_factory = factory;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LoginEndpoint_WithEmptyCredentials_RedirectsToLoginWithError()
|
||||
{
|
||||
var client = _factory.CreateClient(new Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactoryClientOptions
|
||||
{
|
||||
AllowAutoRedirect = false
|
||||
});
|
||||
|
||||
var content = new FormUrlEncodedContent(new[]
|
||||
{
|
||||
new KeyValuePair<string, string>("username", ""),
|
||||
new KeyValuePair<string, string>("password", "")
|
||||
});
|
||||
|
||||
var response = await client.PostAsync("/auth/login", content);
|
||||
|
||||
Assert.Equal(HttpStatusCode.Redirect, response.StatusCode);
|
||||
var location = response.Headers.Location?.ToString() ?? "";
|
||||
Assert.Contains("/login", location);
|
||||
Assert.Contains("error", location, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LogoutEndpoint_ClearsCookieAndRedirects()
|
||||
{
|
||||
var client = _factory.CreateClient(new Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactoryClientOptions
|
||||
{
|
||||
AllowAutoRedirect = false
|
||||
});
|
||||
|
||||
var response = await client.PostAsync("/auth/logout", null);
|
||||
|
||||
Assert.Equal(HttpStatusCode.Redirect, response.StatusCode);
|
||||
var location = response.Headers.Location?.ToString() ?? "";
|
||||
Assert.Contains("/login", location);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtTokenService_GenerateAndValidate_RoundTrips()
|
||||
{
|
||||
using var scope = _factory.Services.CreateScope();
|
||||
var jwtService = scope.ServiceProvider.GetRequiredService<JwtTokenService>();
|
||||
|
||||
var token = jwtService.GenerateToken(
|
||||
displayName: "Test User",
|
||||
username: "testuser",
|
||||
roles: new[] { "Admin", "Design" },
|
||||
permittedSiteIds: null);
|
||||
|
||||
Assert.NotNull(token);
|
||||
|
||||
var principal = jwtService.ValidateToken(token);
|
||||
Assert.NotNull(principal);
|
||||
|
||||
var displayName = principal!.FindFirst(JwtTokenService.DisplayNameClaimType)?.Value;
|
||||
var username = principal.FindFirst(JwtTokenService.UsernameClaimType)?.Value;
|
||||
var roles = principal.FindAll(JwtTokenService.RoleClaimType).Select(c => c.Value).ToList();
|
||||
|
||||
Assert.Equal("Test User", displayName);
|
||||
Assert.Equal("testuser", username);
|
||||
Assert.Contains("Admin", roles);
|
||||
Assert.Contains("Design", roles);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtTokenService_WithSiteScopes_IncludesSiteIdClaims()
|
||||
{
|
||||
using var scope = _factory.Services.CreateScope();
|
||||
var jwtService = scope.ServiceProvider.GetRequiredService<JwtTokenService>();
|
||||
|
||||
var token = jwtService.GenerateToken(
|
||||
displayName: "Deployer",
|
||||
username: "deployer1",
|
||||
roles: new[] { "Deployment" },
|
||||
permittedSiteIds: new[] { "1", "3" });
|
||||
|
||||
var principal = jwtService.ValidateToken(token);
|
||||
Assert.NotNull(principal);
|
||||
|
||||
var siteIds = principal!.FindAll(JwtTokenService.SiteIdClaimType).Select(c => c.Value).ToList();
|
||||
Assert.Contains("1", siteIds);
|
||||
Assert.Contains("3", siteIds);
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public async Task LoginEndpoint_WithValidLdapCredentials_SetsCookieAndRedirects()
|
||||
{
|
||||
// Requires GLAuth test LDAP server: docker compose -f infra/docker-compose.yml up -d glauth
|
||||
// GLAuth runs on localhost:3893, baseDN dc=scadabridge,dc=local, all passwords "password"
|
||||
if (!await IsLdapAvailableAsync())
|
||||
{
|
||||
// Skip gracefully if GLAuth not running — not a test failure
|
||||
return;
|
||||
}
|
||||
|
||||
var client = _factory.CreateClient(new Microsoft.AspNetCore.Mvc.Testing.WebApplicationFactoryClientOptions
|
||||
{
|
||||
AllowAutoRedirect = false
|
||||
});
|
||||
|
||||
var content = new FormUrlEncodedContent(new[]
|
||||
{
|
||||
new KeyValuePair<string, string>("username", "admin"),
|
||||
new KeyValuePair<string, string>("password", "password")
|
||||
});
|
||||
|
||||
var response = await client.PostAsync("/auth/login", content);
|
||||
|
||||
Assert.Equal(HttpStatusCode.Redirect, response.StatusCode);
|
||||
var location = response.Headers.Location?.ToString() ?? "";
|
||||
Assert.Equal("/", location);
|
||||
|
||||
// Verify auth cookie was set
|
||||
var setCookieHeader = response.Headers.GetValues("Set-Cookie").FirstOrDefault();
|
||||
Assert.NotNull(setCookieHeader);
|
||||
Assert.Contains("ZB.MOM.WW.ScadaBridge.Auth", setCookieHeader);
|
||||
}
|
||||
|
||||
private static async Task<bool> IsLdapAvailableAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
using var tcp = new System.Net.Sockets.TcpClient();
|
||||
await tcp.ConnectAsync("localhost", 3893).WaitAsync(TimeSpan.FromSeconds(2));
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Security;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-1 (Phase 8): Full-system failover testing — Central.
|
||||
/// Verifies that JWT tokens and deployment state survive central node failover.
|
||||
/// Multi-process failover tests are marked with Integration trait for separate runs.
|
||||
/// </summary>
|
||||
public class CentralFailoverTests
|
||||
{
|
||||
private static JwtTokenService CreateJwtService(string signingKey = "integration-test-signing-key-must-be-at-least-32-chars-long")
|
||||
{
|
||||
var options = Options.Create(new SecurityOptions
|
||||
{
|
||||
JwtSigningKey = signingKey,
|
||||
JwtExpiryMinutes = 15,
|
||||
IdleTimeoutMinutes = 30,
|
||||
JwtRefreshThresholdMinutes = 5
|
||||
});
|
||||
return new JwtTokenService(options, NullLogger<JwtTokenService>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtToken_GeneratedBeforeFailover_ValidatesAfterFailover()
|
||||
{
|
||||
// Simulates: generate token on node A, validate on node B (shared signing key).
|
||||
var jwtServiceA = CreateJwtService();
|
||||
|
||||
var token = jwtServiceA.GenerateToken(
|
||||
displayName: "Failover User",
|
||||
username: "failover_test",
|
||||
roles: new[] { "Admin" },
|
||||
permittedSiteIds: null);
|
||||
|
||||
// Validate with a second instance (same signing key = simulated failover)
|
||||
var jwtServiceB = CreateJwtService();
|
||||
|
||||
var principal = jwtServiceB.ValidateToken(token);
|
||||
Assert.NotNull(principal);
|
||||
|
||||
var username = principal!.FindFirst(JwtTokenService.UsernameClaimType)?.Value;
|
||||
Assert.Equal("failover_test", username);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtToken_WithSiteScopes_SurvivesRevalidation()
|
||||
{
|
||||
var jwtService = CreateJwtService();
|
||||
|
||||
var token = jwtService.GenerateToken(
|
||||
displayName: "Scoped User",
|
||||
username: "scoped_user",
|
||||
roles: new[] { "Deployment" },
|
||||
permittedSiteIds: new[] { "site-1", "site-2", "site-5" });
|
||||
|
||||
var principal = jwtService.ValidateToken(token);
|
||||
Assert.NotNull(principal);
|
||||
|
||||
var siteIds = principal!.FindAll(JwtTokenService.SiteIdClaimType)
|
||||
.Select(c => c.Value).ToList();
|
||||
Assert.Equal(3, siteIds.Count);
|
||||
Assert.Contains("site-1", siteIds);
|
||||
Assert.Contains("site-5", siteIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtToken_DifferentSigningKeys_FailsValidation()
|
||||
{
|
||||
// If two nodes have different signing keys, tokens from one won't validate on the other.
|
||||
var jwtServiceA = CreateJwtService("node-a-signing-key-that-is-long-enough-32chars");
|
||||
var jwtServiceB = CreateJwtService("node-b-signing-key-that-is-long-enough-32chars");
|
||||
|
||||
var token = jwtServiceA.GenerateToken(
|
||||
displayName: "User",
|
||||
username: "user",
|
||||
roles: new[] { "Admin" },
|
||||
permittedSiteIds: null);
|
||||
|
||||
// Token from A should NOT validate on B (different key)
|
||||
var principal = jwtServiceB.ValidateToken(token);
|
||||
Assert.Null(principal);
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public void DeploymentStatus_OptimisticConcurrency_DetectsStaleWrites()
|
||||
{
|
||||
var status1 = new Commons.Messages.Deployment.DeploymentStatusResponse(
|
||||
"dep-1", "instance-1", Commons.Types.Enums.DeploymentStatus.InProgress,
|
||||
null, DateTimeOffset.UtcNow);
|
||||
|
||||
var status2 = new Commons.Messages.Deployment.DeploymentStatusResponse(
|
||||
"dep-1", "instance-1", Commons.Types.Enums.DeploymentStatus.Success,
|
||||
null, DateTimeOffset.UtcNow.AddSeconds(1));
|
||||
|
||||
Assert.True(status2.Timestamp > status1.Timestamp);
|
||||
Assert.Equal(Commons.Types.Enums.DeploymentStatus.Success, status2.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtToken_ExpiredBeforeFailover_RejectedAfterFailover()
|
||||
{
|
||||
var jwtService = CreateJwtService();
|
||||
|
||||
var token = jwtService.GenerateToken(
|
||||
displayName: "Expired User",
|
||||
username: "expired_user",
|
||||
roles: new[] { "Admin" },
|
||||
permittedSiteIds: null);
|
||||
|
||||
var principal = jwtService.ValidateToken(token);
|
||||
Assert.NotNull(principal);
|
||||
|
||||
var expClaim = principal!.FindFirst("exp");
|
||||
Assert.NotNull(expClaim);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtToken_IdleTimeout_Detected()
|
||||
{
|
||||
var jwtService = CreateJwtService();
|
||||
|
||||
var token = jwtService.GenerateToken(
|
||||
displayName: "Idle User",
|
||||
username: "idle_user",
|
||||
roles: new[] { "Viewer" },
|
||||
permittedSiteIds: null);
|
||||
|
||||
var principal = jwtService.ValidateToken(token);
|
||||
Assert.NotNull(principal);
|
||||
|
||||
// Token was just generated — should NOT be idle timed out
|
||||
Assert.False(jwtService.IsIdleTimedOut(principal!));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtToken_ShouldRefresh_DetectsNearExpiry()
|
||||
{
|
||||
var jwtService = CreateJwtService();
|
||||
|
||||
var token = jwtService.GenerateToken(
|
||||
displayName: "User",
|
||||
username: "user",
|
||||
roles: new[] { "Admin" },
|
||||
permittedSiteIds: null);
|
||||
|
||||
var principal = jwtService.ValidateToken(token);
|
||||
Assert.NotNull(principal);
|
||||
|
||||
// Token was just generated with 15min expiry and 5min threshold — NOT near expiry
|
||||
Assert.False(jwtService.ShouldRefresh(principal!));
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public void DeploymentStatus_MultipleInstances_IndependentTracking()
|
||||
{
|
||||
var statuses = new[]
|
||||
{
|
||||
new Commons.Messages.Deployment.DeploymentStatusResponse(
|
||||
"dep-1", "instance-1", Commons.Types.Enums.DeploymentStatus.Success,
|
||||
null, DateTimeOffset.UtcNow),
|
||||
new Commons.Messages.Deployment.DeploymentStatusResponse(
|
||||
"dep-1", "instance-2", Commons.Types.Enums.DeploymentStatus.InProgress,
|
||||
null, DateTimeOffset.UtcNow),
|
||||
new Commons.Messages.Deployment.DeploymentStatusResponse(
|
||||
"dep-1", "instance-3", Commons.Types.Enums.DeploymentStatus.Failed,
|
||||
"Compilation error", DateTimeOffset.UtcNow),
|
||||
};
|
||||
|
||||
Assert.Equal(3, statuses.Length);
|
||||
Assert.All(statuses, s => Assert.Equal("dep-1", s.DeploymentId));
|
||||
Assert.Equal(3, statuses.Select(s => s.InstanceUniqueName).Distinct().Count());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-3 (Phase 8): Dual-node failure recovery.
|
||||
/// Both nodes down, first up forms cluster, rebuilds from persistent storage.
|
||||
/// Tests for both central and site topologies.
|
||||
/// </summary>
|
||||
public class DualNodeRecoveryTests
|
||||
{
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public async Task SiteTopology_BothNodesDown_FirstNodeRebuildsFromSQLite()
|
||||
{
|
||||
// Scenario: both site nodes crash. First node to restart opens the existing
|
||||
// SQLite database and finds all buffered S&F messages intact.
|
||||
var dbPath = Path.Combine(Path.GetTempPath(), $"sf_dual_{Guid.NewGuid():N}.db");
|
||||
var connStr = $"Data Source={dbPath}";
|
||||
|
||||
try
|
||||
{
|
||||
// Setup: populate SQLite with messages (simulating pre-crash state)
|
||||
var storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
|
||||
var messageIds = new List<string>();
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
var msg = new StoreAndForwardMessage
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = $"api-{i % 3}",
|
||||
PayloadJson = $$"""{"index":{{i}}}""",
|
||||
RetryCount = i,
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow.AddMinutes(-i),
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
OriginInstanceName = $"instance-{i % 2}"
|
||||
};
|
||||
await storage.EnqueueAsync(msg);
|
||||
messageIds.Add(msg.Id);
|
||||
}
|
||||
|
||||
// Both nodes down — simulate by creating a fresh storage instance
|
||||
// (new process connecting to same SQLite file)
|
||||
var recoveryStorage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await recoveryStorage.InitializeAsync();
|
||||
|
||||
// Verify all messages are available for retry
|
||||
var pending = await recoveryStorage.GetMessagesForRetryAsync();
|
||||
Assert.Equal(10, pending.Count);
|
||||
|
||||
// Verify messages are ordered by creation time (oldest first)
|
||||
for (var i = 1; i < pending.Count; i++)
|
||||
{
|
||||
Assert.True(pending[i].CreatedAt >= pending[i - 1].CreatedAt);
|
||||
}
|
||||
|
||||
// Verify per-instance message counts
|
||||
var instance0Count = await recoveryStorage.GetMessageCountByOriginInstanceAsync("instance-0");
|
||||
var instance1Count = await recoveryStorage.GetMessageCountByOriginInstanceAsync("instance-1");
|
||||
Assert.Equal(5, instance0Count);
|
||||
Assert.Equal(5, instance1Count);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(dbPath))
|
||||
File.Delete(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public async Task SiteTopology_DualCrash_ParkedMessagesPreserved()
|
||||
{
|
||||
var dbPath = Path.Combine(Path.GetTempPath(), $"sf_dual_parked_{Guid.NewGuid():N}.db");
|
||||
var connStr = $"Data Source={dbPath}";
|
||||
|
||||
try
|
||||
{
|
||||
var storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
|
||||
// Mix of pending and parked messages
|
||||
await storage.EnqueueAsync(new StoreAndForwardMessage
|
||||
{
|
||||
Id = "pending-1",
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "api",
|
||||
PayloadJson = "{}",
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
});
|
||||
|
||||
await storage.EnqueueAsync(new StoreAndForwardMessage
|
||||
{
|
||||
Id = "parked-1",
|
||||
Category = StoreAndForwardCategory.Notification,
|
||||
Target = "alerts",
|
||||
PayloadJson = "{}",
|
||||
MaxRetries = 3,
|
||||
RetryIntervalMs = 10000,
|
||||
CreatedAt = DateTimeOffset.UtcNow.AddHours(-2),
|
||||
RetryCount = 3,
|
||||
Status = StoreAndForwardMessageStatus.Parked,
|
||||
LastError = "SMTP unreachable"
|
||||
});
|
||||
|
||||
// Dual crash recovery
|
||||
var recoveryStorage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await recoveryStorage.InitializeAsync();
|
||||
|
||||
var pendingCount = await recoveryStorage.GetMessageCountByStatusAsync(StoreAndForwardMessageStatus.Pending);
|
||||
var parkedCount = await recoveryStorage.GetMessageCountByStatusAsync(StoreAndForwardMessageStatus.Parked);
|
||||
|
||||
Assert.Equal(1, pendingCount);
|
||||
Assert.Equal(1, parkedCount);
|
||||
|
||||
// Parked message can be retried after recovery
|
||||
var success = await recoveryStorage.RetryParkedMessageAsync("parked-1");
|
||||
Assert.True(success);
|
||||
|
||||
pendingCount = await recoveryStorage.GetMessageCountByStatusAsync(StoreAndForwardMessageStatus.Pending);
|
||||
parkedCount = await recoveryStorage.GetMessageCountByStatusAsync(StoreAndForwardMessageStatus.Parked);
|
||||
Assert.Equal(2, pendingCount);
|
||||
Assert.Equal(0, parkedCount);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(dbPath))
|
||||
File.Delete(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public void CentralTopology_BothNodesDown_FirstNodeFormsSingleNodeCluster()
|
||||
{
|
||||
// Structural verification: Akka.NET cluster config uses min-nr-of-members = 1,
|
||||
// so a single node can form a cluster. The keep-oldest split-brain resolver
|
||||
// with down-if-alone handles the partition scenario.
|
||||
//
|
||||
// When both central nodes crash, the first node to restart:
|
||||
// 1. Forms a single-node cluster (min-nr-of-members = 1)
|
||||
// 2. Connects to SQL Server (which persists all deployment state)
|
||||
// 3. Becomes the active node and accepts traffic
|
||||
//
|
||||
// The second node joins the existing cluster when it starts.
|
||||
|
||||
// Verify the deployment status model supports recovery from SQL Server
|
||||
var statuses = new[]
|
||||
{
|
||||
new Commons.Messages.Deployment.DeploymentStatusResponse(
|
||||
"dep-1", "inst-1", Commons.Types.Enums.DeploymentStatus.Success,
|
||||
null, DateTimeOffset.UtcNow),
|
||||
new Commons.Messages.Deployment.DeploymentStatusResponse(
|
||||
"dep-1", "inst-2", Commons.Types.Enums.DeploymentStatus.InProgress,
|
||||
null, DateTimeOffset.UtcNow),
|
||||
};
|
||||
|
||||
// Each instance has independent status — recovery reads from DB
|
||||
Assert.Equal(DeploymentStatus.Success, statuses[0].Status);
|
||||
Assert.Equal(DeploymentStatus.InProgress, statuses[1].Status);
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public async Task SQLiteStorage_InitializeIdempotent_SafeOnRecovery()
|
||||
{
|
||||
// CREATE TABLE IF NOT EXISTS is idempotent — safe to call on recovery
|
||||
var dbPath = Path.Combine(Path.GetTempPath(), $"sf_idempotent_{Guid.NewGuid():N}.db");
|
||||
var connStr = $"Data Source={dbPath}";
|
||||
|
||||
try
|
||||
{
|
||||
var storage1 = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await storage1.InitializeAsync();
|
||||
|
||||
await storage1.EnqueueAsync(new StoreAndForwardMessage
|
||||
{
|
||||
Id = "test-1",
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "api",
|
||||
PayloadJson = "{}",
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
});
|
||||
|
||||
// Second InitializeAsync on same DB should be safe (no data loss)
|
||||
var storage2 = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await storage2.InitializeAsync();
|
||||
|
||||
var msg = await storage2.GetMessageByIdAsync("test-1");
|
||||
Assert.NotNull(msg);
|
||||
Assert.Equal("api", msg!.Target);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(dbPath))
|
||||
File.Delete(dbPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
using System.Threading.Channels;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for the gRPC streaming pipeline.
|
||||
/// Tests the full in-process flow: subscribe request -> StreamRelayActor creation ->
|
||||
/// domain events via Akka Tell -> Channel relay -> gRPC response stream writes.
|
||||
///
|
||||
/// These tests exercise the real SiteStreamGrpcServer, StreamRelayActor, and Channel
|
||||
/// wiring together with a real Akka actor system, using only mocked gRPC transport
|
||||
/// (IServerStreamWriter + ServerCallContext).
|
||||
///
|
||||
/// Full end-to-end gRPC-over-HTTP/2 tests are performed manually against the Docker
|
||||
/// cluster (docker/deploy.sh + docker/seed-sites.sh + CLI debug-stream).
|
||||
/// </summary>
|
||||
public class GrpcStreamIntegrationTests : TestKit
|
||||
{
|
||||
/// <summary>
|
||||
/// End-to-end pipeline test: subscribe -> relay actor receives domain events ->
|
||||
/// events flow through Channel to gRPC response stream.
|
||||
/// Validates attribute value changes arrive with correct protobuf mapping.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Pipeline_AttributeValueChanged_FlowsToResponseStream()
|
||||
{
|
||||
// Arrange: capture the relay actor created by the gRPC server
|
||||
IActorRef? relayActor = null;
|
||||
var subscriber = Substitute.For<ISiteStreamSubscriber>();
|
||||
subscriber.Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
relayActor = ci.Arg<IActorRef>();
|
||||
return "sub-integ-1";
|
||||
});
|
||||
|
||||
var server = new SiteStreamGrpcServer(
|
||||
subscriber,
|
||||
NullLogger<SiteStreamGrpcServer>.Instance);
|
||||
server.SetReady(Sys);
|
||||
|
||||
var writtenEvents = new List<SiteStreamEvent>();
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
writer.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci => writtenEvents.Add(ci.Arg<SiteStreamEvent>()));
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var context = CreateMockContext(cts.Token);
|
||||
|
||||
var request = new InstanceStreamRequest
|
||||
{
|
||||
CorrelationId = "integ-attr-1",
|
||||
InstanceUniqueName = "SiteA.Pump01"
|
||||
};
|
||||
|
||||
// Act: start the subscription stream
|
||||
var streamTask = Task.Run(() => server.SubscribeInstance(request, writer, context));
|
||||
|
||||
await WaitForConditionAsync(() => relayActor != null);
|
||||
|
||||
// Send domain events to the relay actor (simulating what SiteStreamManager does)
|
||||
var ts1 = new DateTimeOffset(2026, 3, 21, 14, 0, 0, TimeSpan.Zero);
|
||||
var ts2 = new DateTimeOffset(2026, 3, 21, 14, 0, 1, TimeSpan.Zero);
|
||||
|
||||
relayActor!.Tell(new AttributeValueChanged(
|
||||
"SiteA.Pump01", "Modules.Flow", "CurrentGPM", 125.3, "Good", ts1));
|
||||
relayActor.Tell(new AttributeValueChanged(
|
||||
"SiteA.Pump01", "Modules.Pressure", "CurrentPSI", 48.7, "Uncertain", ts2));
|
||||
|
||||
await WaitForConditionAsync(() => writtenEvents.Count >= 2);
|
||||
|
||||
// Cleanup
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
// Assert: both events arrived in order with correct protobuf mapping
|
||||
Assert.Equal(2, writtenEvents.Count);
|
||||
|
||||
var evt1 = writtenEvents[0];
|
||||
Assert.Equal("integ-attr-1", evt1.CorrelationId);
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AttributeChanged, evt1.EventCase);
|
||||
Assert.Equal("SiteA.Pump01", evt1.AttributeChanged.InstanceUniqueName);
|
||||
Assert.Equal("Modules.Flow", evt1.AttributeChanged.AttributePath);
|
||||
Assert.Equal("CurrentGPM", evt1.AttributeChanged.AttributeName);
|
||||
Assert.Equal("125.3", evt1.AttributeChanged.Value);
|
||||
Assert.Equal(Quality.Good, evt1.AttributeChanged.Quality);
|
||||
Assert.Equal(Timestamp.FromDateTimeOffset(ts1), evt1.AttributeChanged.Timestamp);
|
||||
|
||||
var evt2 = writtenEvents[1];
|
||||
Assert.Equal("integ-attr-1", evt2.CorrelationId);
|
||||
Assert.Equal("Modules.Pressure", evt2.AttributeChanged.AttributePath);
|
||||
Assert.Equal(Quality.Uncertain, evt2.AttributeChanged.Quality);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end pipeline test for alarm state changes.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Pipeline_AlarmStateChanged_FlowsToResponseStream()
|
||||
{
|
||||
IActorRef? relayActor = null;
|
||||
var subscriber = Substitute.For<ISiteStreamSubscriber>();
|
||||
subscriber.Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
relayActor = ci.Arg<IActorRef>();
|
||||
return "sub-integ-2";
|
||||
});
|
||||
|
||||
var server = new SiteStreamGrpcServer(
|
||||
subscriber,
|
||||
NullLogger<SiteStreamGrpcServer>.Instance);
|
||||
server.SetReady(Sys);
|
||||
|
||||
var writtenEvents = new List<SiteStreamEvent>();
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
writer.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci => writtenEvents.Add(ci.Arg<SiteStreamEvent>()));
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var context = CreateMockContext(cts.Token);
|
||||
|
||||
var request = new InstanceStreamRequest
|
||||
{
|
||||
CorrelationId = "integ-alarm-1",
|
||||
InstanceUniqueName = "SiteA.Pump01"
|
||||
};
|
||||
|
||||
var streamTask = Task.Run(() => server.SubscribeInstance(request, writer, context));
|
||||
await WaitForConditionAsync(() => relayActor != null);
|
||||
|
||||
var ts = new DateTimeOffset(2026, 3, 21, 14, 5, 0, TimeSpan.Zero);
|
||||
relayActor!.Tell(new AlarmStateChanged(
|
||||
"SiteA.Pump01", "HighPressure",
|
||||
Commons.Types.Enums.AlarmState.Active, 3, ts));
|
||||
|
||||
await WaitForConditionAsync(() => writtenEvents.Count >= 1);
|
||||
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
Assert.Single(writtenEvents);
|
||||
var evt = writtenEvents[0];
|
||||
Assert.Equal("integ-alarm-1", evt.CorrelationId);
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AlarmChanged, evt.EventCase);
|
||||
Assert.Equal("SiteA.Pump01", evt.AlarmChanged.InstanceUniqueName);
|
||||
Assert.Equal("HighPressure", evt.AlarmChanged.AlarmName);
|
||||
Assert.Equal(AlarmStateEnum.AlarmStateActive, evt.AlarmChanged.State);
|
||||
Assert.Equal(3, evt.AlarmChanged.Priority);
|
||||
Assert.Equal(Timestamp.FromDateTimeOffset(ts), evt.AlarmChanged.Timestamp);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that mixed event types (attribute + alarm) flow through the same stream.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Pipeline_MixedEvents_FlowInOrder()
|
||||
{
|
||||
IActorRef? relayActor = null;
|
||||
var subscriber = Substitute.For<ISiteStreamSubscriber>();
|
||||
subscriber.Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
relayActor = ci.Arg<IActorRef>();
|
||||
return "sub-integ-3";
|
||||
});
|
||||
|
||||
var server = new SiteStreamGrpcServer(
|
||||
subscriber,
|
||||
NullLogger<SiteStreamGrpcServer>.Instance);
|
||||
server.SetReady(Sys);
|
||||
|
||||
var writtenEvents = new List<SiteStreamEvent>();
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
writer.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci => writtenEvents.Add(ci.Arg<SiteStreamEvent>()));
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var context = CreateMockContext(cts.Token);
|
||||
|
||||
var request = new InstanceStreamRequest
|
||||
{
|
||||
CorrelationId = "integ-mixed-1",
|
||||
InstanceUniqueName = "SiteB.Motor01"
|
||||
};
|
||||
|
||||
var streamTask = Task.Run(() => server.SubscribeInstance(request, writer, context));
|
||||
await WaitForConditionAsync(() => relayActor != null);
|
||||
|
||||
var ts = DateTimeOffset.UtcNow;
|
||||
|
||||
// Send interleaved attribute and alarm events
|
||||
relayActor!.Tell(new AttributeValueChanged(
|
||||
"SiteB.Motor01", "Speed", "RPM", 1750.0, "Good", ts));
|
||||
relayActor.Tell(new AlarmStateChanged(
|
||||
"SiteB.Motor01", "OverSpeed",
|
||||
Commons.Types.Enums.AlarmState.Active, 4, ts));
|
||||
relayActor.Tell(new AttributeValueChanged(
|
||||
"SiteB.Motor01", "Temperature", "BearingTemp", 85.2, "Good", ts));
|
||||
|
||||
await WaitForConditionAsync(() => writtenEvents.Count >= 3);
|
||||
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
Assert.Equal(3, writtenEvents.Count);
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AttributeChanged, writtenEvents[0].EventCase);
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AlarmChanged, writtenEvents[1].EventCase);
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AttributeChanged, writtenEvents[2].EventCase);
|
||||
Assert.All(writtenEvents, e => Assert.Equal("integ-mixed-1", e.CorrelationId));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that when a stream is cancelled, the subscriber is cleaned up
|
||||
/// and the active stream count returns to zero.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Pipeline_Cancellation_CleansUpRelayActorAndSubscription()
|
||||
{
|
||||
var subscriber = Substitute.For<ISiteStreamSubscriber>();
|
||||
subscriber.Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>())
|
||||
.Returns("sub-integ-4");
|
||||
|
||||
var server = new SiteStreamGrpcServer(
|
||||
subscriber,
|
||||
NullLogger<SiteStreamGrpcServer>.Instance);
|
||||
server.SetReady(Sys);
|
||||
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
var cts = new CancellationTokenSource();
|
||||
var context = CreateMockContext(cts.Token);
|
||||
|
||||
var request = new InstanceStreamRequest
|
||||
{
|
||||
CorrelationId = "integ-cleanup-1",
|
||||
InstanceUniqueName = "SiteC.Valve01"
|
||||
};
|
||||
|
||||
var streamTask = Task.Run(() => server.SubscribeInstance(request, writer, context));
|
||||
await WaitForConditionAsync(() => server.ActiveStreamCount == 1);
|
||||
|
||||
// Cancel the stream
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
// Verify cleanup
|
||||
Assert.Equal(0, server.ActiveStreamCount);
|
||||
subscriber.Received(1).Subscribe("SiteC.Valve01", Arg.Any<IActorRef>());
|
||||
subscriber.Received(1).RemoveSubscriber(Arg.Any<IActorRef>());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a duplicate correlation ID cancels the first stream and
|
||||
/// the second stream continues to receive events.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Pipeline_DuplicateCorrelationId_ReplacesStream()
|
||||
{
|
||||
IActorRef? relayActor2 = null;
|
||||
var callCount = 0;
|
||||
var subscriber = Substitute.For<ISiteStreamSubscriber>();
|
||||
subscriber.Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
callCount++;
|
||||
if (callCount == 2)
|
||||
relayActor2 = ci.Arg<IActorRef>();
|
||||
return $"sub-dup-{callCount}";
|
||||
});
|
||||
|
||||
var server = new SiteStreamGrpcServer(
|
||||
subscriber,
|
||||
NullLogger<SiteStreamGrpcServer>.Instance);
|
||||
server.SetReady(Sys);
|
||||
|
||||
// First stream
|
||||
var writer1 = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
var cts1 = new CancellationTokenSource();
|
||||
var context1 = CreateMockContext(cts1.Token);
|
||||
|
||||
var stream1Task = Task.Run(() => server.SubscribeInstance(
|
||||
new InstanceStreamRequest { CorrelationId = "integ-dup", InstanceUniqueName = "SiteA.Pump01" },
|
||||
writer1, context1));
|
||||
|
||||
await WaitForConditionAsync(() => server.ActiveStreamCount == 1);
|
||||
|
||||
// Second stream with same correlation ID -- should cancel first
|
||||
var writtenEvents2 = new List<SiteStreamEvent>();
|
||||
var writer2 = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
writer2.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci => writtenEvents2.Add(ci.Arg<SiteStreamEvent>()));
|
||||
|
||||
var cts2 = new CancellationTokenSource();
|
||||
var context2 = CreateMockContext(cts2.Token);
|
||||
|
||||
var stream2Task = Task.Run(() => server.SubscribeInstance(
|
||||
new InstanceStreamRequest { CorrelationId = "integ-dup", InstanceUniqueName = "SiteA.Pump01" },
|
||||
writer2, context2));
|
||||
|
||||
// First stream should complete
|
||||
await stream1Task;
|
||||
await WaitForConditionAsync(() => relayActor2 != null);
|
||||
|
||||
// Send event to second relay
|
||||
var ts = DateTimeOffset.UtcNow;
|
||||
relayActor2!.Tell(new AttributeValueChanged(
|
||||
"SiteA.Pump01", "Flow", "GPM", 100.0, "Good", ts));
|
||||
|
||||
await WaitForConditionAsync(() => writtenEvents2.Count >= 1);
|
||||
|
||||
cts2.Cancel();
|
||||
await stream2Task;
|
||||
|
||||
Assert.Single(writtenEvents2);
|
||||
Assert.Equal("integ-dup", writtenEvents2[0].CorrelationId);
|
||||
}
|
||||
|
||||
private static ServerCallContext CreateMockContext(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var context = Substitute.For<ServerCallContext>();
|
||||
context.CancellationToken.Returns(cancellationToken);
|
||||
return context;
|
||||
}
|
||||
|
||||
private static async Task WaitForConditionAsync(Func<bool> condition, int timeoutMs = 5000)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
|
||||
while (!condition() && DateTime.UtcNow < deadline)
|
||||
{
|
||||
await Task.Delay(25);
|
||||
}
|
||||
|
||||
Assert.True(condition(), $"Condition not met within {timeoutMs}ms");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.InboundApi;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.InboundAPI;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-14: End-to-end integration tests for Phase 7 integration surfaces.
|
||||
/// </summary>
|
||||
public class IntegrationSurfaceTests
|
||||
{
|
||||
// ── Inbound API: auth + routing + parameter validation + error codes ──
|
||||
|
||||
[Fact]
|
||||
public async Task InboundAPI_ApiKeyValidator_FullFlow_EndToEnd()
|
||||
{
|
||||
// Validates that ApiKeyValidator correctly chains all checks.
|
||||
var repository = Substitute.For<IInboundApiRepository>();
|
||||
var key = new ApiKey("test-key", "key-value-123") { Id = 1, IsEnabled = true };
|
||||
var method = new ApiMethod("getStatus", "return 1;")
|
||||
{
|
||||
Id = 10,
|
||||
ParameterDefinitions = "[{\"Name\":\"deviceId\",\"Type\":\"String\",\"Required\":true}]",
|
||||
TimeoutSeconds = 30
|
||||
};
|
||||
|
||||
// ConfigurationDatabase-012: the validator fetches every key and matches
|
||||
// the candidate by HMAC hash in constant time (no secret-equality lookup).
|
||||
repository.GetAllApiKeysAsync().Returns(new List<ApiKey> { key });
|
||||
repository.GetMethodByNameAsync("getStatus").Returns(method);
|
||||
repository.GetApprovedKeysForMethodAsync(10).Returns(new List<ApiKey> { key });
|
||||
|
||||
var validator = new ApiKeyValidator(repository);
|
||||
|
||||
// Valid key + approved method
|
||||
var result = await validator.ValidateAsync("key-value-123", "getStatus");
|
||||
Assert.True(result.IsValid);
|
||||
Assert.Equal(method, result.Method);
|
||||
|
||||
// Then validate parameters
|
||||
using var doc = JsonDocument.Parse("{\"deviceId\": \"pump-01\"}");
|
||||
var paramResult = ParameterValidator.Validate(doc.RootElement.Clone(), method.ParameterDefinitions);
|
||||
Assert.True(paramResult.IsValid);
|
||||
Assert.Equal("pump-01", paramResult.Parameters["deviceId"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InboundAPI_ParameterValidation_ExtendedTypes()
|
||||
{
|
||||
// Validates the full extended type system: Boolean, Integer, Float, String, Object, List.
|
||||
var definitions = JsonSerializer.Serialize(new[]
|
||||
{
|
||||
new { Name = "flag", Type = "Boolean", Required = true },
|
||||
new { Name = "count", Type = "Integer", Required = true },
|
||||
new { Name = "ratio", Type = "Float", Required = true },
|
||||
new { Name = "name", Type = "String", Required = true },
|
||||
new { Name = "config", Type = "Object", Required = true },
|
||||
new { Name = "tags", Type = "List", Required = true }
|
||||
});
|
||||
|
||||
var json = "{\"flag\":true,\"count\":42,\"ratio\":3.14,\"name\":\"test\",\"config\":{\"k\":\"v\"},\"tags\":[1,2]}";
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
var result = ParameterValidator.Validate(doc.RootElement.Clone(), definitions);
|
||||
|
||||
Assert.True(result.IsValid);
|
||||
Assert.Equal(true, result.Parameters["flag"]);
|
||||
Assert.Equal((long)42, result.Parameters["count"]);
|
||||
Assert.Equal(3.14, result.Parameters["ratio"]);
|
||||
Assert.Equal("test", result.Parameters["name"]);
|
||||
Assert.NotNull(result.Parameters["config"]);
|
||||
Assert.NotNull(result.Parameters["tags"]);
|
||||
}
|
||||
|
||||
// ── External System: error classification ──
|
||||
|
||||
[Fact]
|
||||
public void ExternalSystem_ErrorClassification_TransientVsPermanent()
|
||||
{
|
||||
// WP-8: Verify the full classification spectrum
|
||||
Assert.True(ExternalSystemGateway.ErrorClassifier.IsTransient(HttpStatusCode.InternalServerError));
|
||||
Assert.True(ExternalSystemGateway.ErrorClassifier.IsTransient(HttpStatusCode.ServiceUnavailable));
|
||||
Assert.True(ExternalSystemGateway.ErrorClassifier.IsTransient(HttpStatusCode.RequestTimeout));
|
||||
Assert.True(ExternalSystemGateway.ErrorClassifier.IsTransient((HttpStatusCode)429));
|
||||
|
||||
Assert.False(ExternalSystemGateway.ErrorClassifier.IsTransient(HttpStatusCode.BadRequest));
|
||||
Assert.False(ExternalSystemGateway.ErrorClassifier.IsTransient(HttpStatusCode.Unauthorized));
|
||||
Assert.False(ExternalSystemGateway.ErrorClassifier.IsTransient(HttpStatusCode.Forbidden));
|
||||
Assert.False(ExternalSystemGateway.ErrorClassifier.IsTransient(HttpStatusCode.NotFound));
|
||||
}
|
||||
|
||||
// ── Notification: mock SMTP delivery ──
|
||||
// NS-019: the site-shaped NotificationDeliveryService that this case exercised
|
||||
// was removed when sites stopped delivering notifications. The central SMTP
|
||||
// delivery path is now covered end-to-end by
|
||||
// ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests.Delivery.EmailNotificationDeliveryAdapterTests;
|
||||
// no equivalent integration-surface assertion is needed here.
|
||||
|
||||
// ── Script Context: integration API wiring ──
|
||||
|
||||
[Fact]
|
||||
public async Task ScriptContext_ExternalSystem_Call_Wired()
|
||||
{
|
||||
// Verify that ExternalSystem.Call is accessible from ScriptRuntimeContext
|
||||
var mockClient = Substitute.For<IExternalSystemClient>();
|
||||
mockClient.CallAsync("api", "getData", null, Arg.Any<CancellationToken>())
|
||||
.Returns(new ExternalCallResult(true, "{\"value\":1}", null));
|
||||
|
||||
var context = CreateMinimalScriptContext(externalSystemClient: mockClient);
|
||||
|
||||
var result = await context.ExternalSystem.Call("api", "getData");
|
||||
|
||||
Assert.True(result.Success);
|
||||
Assert.Equal("{\"value\":1}", result.ResponseJson);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ScriptContext_Notify_Send_Wired()
|
||||
{
|
||||
// Notification Outbox: Notify.Send enqueues into the site Store-and-Forward
|
||||
// Engine and returns the NotificationId handle immediately.
|
||||
var dbName = $"NotifyWired_{Guid.NewGuid():N}";
|
||||
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
|
||||
using var keepAlive = new Microsoft.Data.Sqlite.SqliteConnection(connStr);
|
||||
keepAlive.Open();
|
||||
var storage = new StoreAndForward.StoreAndForwardStorage(
|
||||
connStr, Microsoft.Extensions.Logging.Abstractions.NullLogger<StoreAndForward.StoreAndForwardStorage>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
var saf = new StoreAndForward.StoreAndForwardService(
|
||||
storage, new StoreAndForward.StoreAndForwardOptions(),
|
||||
Microsoft.Extensions.Logging.Abstractions.NullLogger<StoreAndForward.StoreAndForwardService>.Instance);
|
||||
|
||||
var context = CreateMinimalScriptContext(storeAndForward: saf);
|
||||
|
||||
var notificationId = await context.Notify.To("ops").Send("Alert", "Body");
|
||||
|
||||
Assert.False(string.IsNullOrEmpty(notificationId));
|
||||
var buffered = await saf.GetMessageByIdAsync(notificationId);
|
||||
Assert.NotNull(buffered);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ScriptContext_ExternalSystem_NoClient_Throws()
|
||||
{
|
||||
var context = CreateMinimalScriptContext();
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => context.ExternalSystem.Call("api", "method"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ScriptContext_Database_NoGateway_Throws()
|
||||
{
|
||||
var context = CreateMinimalScriptContext();
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => context.Database.Connection("db"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ScriptContext_Notify_NoService_Throws()
|
||||
{
|
||||
// No Store-and-Forward Engine wired → Notify.Send cannot enqueue and throws.
|
||||
var context = CreateMinimalScriptContext();
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => context.Notify.To("list").Send("subj", "body"));
|
||||
}
|
||||
|
||||
private static SiteRuntime.Scripts.ScriptRuntimeContext CreateMinimalScriptContext(
|
||||
IExternalSystemClient? externalSystemClient = null,
|
||||
IDatabaseGateway? databaseGateway = null,
|
||||
StoreAndForward.StoreAndForwardService? storeAndForward = null)
|
||||
{
|
||||
// Create a minimal context — we use Substitute.For<IActorRef> which is fine since
|
||||
// we won't exercise Akka functionality in these tests.
|
||||
var actorRef = Substitute.For<Akka.Actor.IActorRef>();
|
||||
var compilationService = new SiteRuntime.Scripts.ScriptCompilationService(
|
||||
Microsoft.Extensions.Logging.Abstractions.NullLogger<SiteRuntime.Scripts.ScriptCompilationService>.Instance);
|
||||
var sharedLibrary = new SiteRuntime.Scripts.SharedScriptLibrary(
|
||||
compilationService,
|
||||
Microsoft.Extensions.Logging.Abstractions.NullLogger<SiteRuntime.Scripts.SharedScriptLibrary>.Instance);
|
||||
|
||||
return new SiteRuntime.Scripts.ScriptRuntimeContext(
|
||||
actorRef,
|
||||
actorRef,
|
||||
sharedLibrary,
|
||||
currentCallDepth: 0,
|
||||
maxCallDepth: 10,
|
||||
askTimeout: TimeSpan.FromSeconds(5),
|
||||
instanceName: "test-instance",
|
||||
logger: Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance,
|
||||
externalSystemClient: externalSystemClient,
|
||||
databaseGateway: databaseGateway,
|
||||
storeAndForward: storeAndForward,
|
||||
siteId: "test-site");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.AspNetCore.DataProtection;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Diagnostics;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox;
|
||||
using ZB.MOM.WW.ScadaBridge.NotificationOutbox.Delivery;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Task 25: end-to-end integration test for the central Notification Outbox flow. Exercises
|
||||
/// the full ingest -> dispatch -> status pipeline against a real <see cref="NotificationOutboxActor"/>,
|
||||
/// a real <see cref="NotificationOutboxRepository"/> over a real (SQLite in-memory) database, and
|
||||
/// a stub <see cref="INotificationDeliveryAdapter"/>. Two scenarios are covered: a successful
|
||||
/// delivery reaching <see cref="NotificationStatus.Delivered"/>, and a permanently-failed delivery
|
||||
/// reaching <see cref="NotificationStatus.Parked"/> followed by a manual retry back to
|
||||
/// <see cref="NotificationStatus.Pending"/>.
|
||||
/// </summary>
|
||||
public class NotificationOutboxFlowTests : TestKit
|
||||
{
|
||||
/// <summary>
|
||||
/// SQLite-adapted <see cref="ScadaBridgeDbContext"/>: maps <see cref="DateTimeOffset"/> columns
|
||||
/// to sortable ISO 8601 strings so <c>ORDER BY CreatedAt</c> (used by the dispatcher's
|
||||
/// due-batch query) works, and drops the SQL Server rowversion concurrency token.
|
||||
/// </summary>
|
||||
private sealed class SqliteOutboxDbContext : ScadaBridgeDbContext
|
||||
{
|
||||
public SqliteOutboxDbContext(DbContextOptions<ScadaBridgeDbContext> options)
|
||||
: base(options, new EphemeralDataProtectionProvider())
|
||||
{
|
||||
}
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
{
|
||||
base.OnModelCreating(modelBuilder);
|
||||
|
||||
var converter = new ValueConverter<DateTimeOffset, string>(
|
||||
v => v.UtcDateTime.ToString("o"),
|
||||
v => DateTimeOffset.Parse(v));
|
||||
var nullableConverter = new ValueConverter<DateTimeOffset?, string?>(
|
||||
v => v.HasValue ? v.Value.UtcDateTime.ToString("o") : null,
|
||||
v => v != null ? DateTimeOffset.Parse(v) : null);
|
||||
|
||||
foreach (var entityType in modelBuilder.Model.GetEntityTypes())
|
||||
{
|
||||
foreach (var property in entityType.GetProperties())
|
||||
{
|
||||
if (property.ClrType == typeof(DateTimeOffset))
|
||||
{
|
||||
property.SetValueConverter(converter);
|
||||
property.SetColumnType("TEXT");
|
||||
}
|
||||
else if (property.ClrType == typeof(DateTimeOffset?))
|
||||
{
|
||||
property.SetValueConverter(nullableConverter);
|
||||
property.SetColumnType("TEXT");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stub Email delivery adapter that returns a fixed, test-configured <see cref="DeliveryOutcome"/>.
|
||||
/// </summary>
|
||||
private sealed class StubEmailAdapter : INotificationDeliveryAdapter
|
||||
{
|
||||
private readonly DeliveryOutcome _outcome;
|
||||
|
||||
public StubEmailAdapter(DeliveryOutcome outcome) => _outcome = outcome;
|
||||
|
||||
public NotificationType Type => NotificationType.Email;
|
||||
|
||||
public Task<DeliveryOutcome> DeliverAsync(
|
||||
Notification notification, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(_outcome);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Holds the kept-open SQLite connection and the composed service provider for one test.
|
||||
/// The connection must stay open for the lifetime of the test: an in-memory SQLite database
|
||||
/// is destroyed when its last connection closes.
|
||||
/// </summary>
|
||||
private sealed class OutboxHarness : IDisposable
|
||||
{
|
||||
public SqliteConnection Connection { get; }
|
||||
public IServiceProvider Services { get; }
|
||||
|
||||
public OutboxHarness(SqliteConnection connection, IServiceProvider services)
|
||||
{
|
||||
Connection = connection;
|
||||
Services = services;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
(Services as IDisposable)?.Dispose();
|
||||
Connection.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the test harness: a kept-open SQLite in-memory connection with the schema created,
|
||||
/// and a service provider where <see cref="ScadaBridgeDbContext"/> and the repositories are
|
||||
/// scoped over that single connection (the actor opens a fresh DI scope per dispatch sweep).
|
||||
/// </summary>
|
||||
private static OutboxHarness BuildHarness(DeliveryOutcome adapterOutcome)
|
||||
{
|
||||
var connection = new SqliteConnection("DataSource=:memory:");
|
||||
connection.Open();
|
||||
|
||||
var dbOptions = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlite(connection)
|
||||
.ConfigureWarnings(w => w.Ignore(RelationalEventId.PendingModelChangesWarning))
|
||||
.Options;
|
||||
|
||||
// Create the schema once on a throwaway context over the shared connection.
|
||||
using (var schemaContext = new SqliteOutboxDbContext(dbOptions))
|
||||
{
|
||||
schemaContext.Database.EnsureCreated();
|
||||
}
|
||||
|
||||
// The retry policy is resolved from the SMTP configuration; stub the notification
|
||||
// repository so the dispatcher gets a deterministic max-retries / retry-delay pair.
|
||||
var notificationRepository = Substitute.For<INotificationRepository>();
|
||||
notificationRepository
|
||||
.GetAllSmtpConfigurationsAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new[]
|
||||
{
|
||||
new SmtpConfiguration("smtp.example.com", "Basic", "noreply@example.com")
|
||||
{
|
||||
MaxRetries = 3,
|
||||
RetryDelay = TimeSpan.FromMinutes(1),
|
||||
},
|
||||
});
|
||||
|
||||
var services = new ServiceCollection();
|
||||
// Fresh DbContext per DI scope, all over the one kept-open SQLite connection so every
|
||||
// scope sees the same in-memory database.
|
||||
services.AddScoped<ScadaBridgeDbContext>(_ => new SqliteOutboxDbContext(dbOptions));
|
||||
services.AddScoped<INotificationOutboxRepository>(sp =>
|
||||
new NotificationOutboxRepository(sp.GetRequiredService<ScadaBridgeDbContext>()));
|
||||
services.AddScoped(_ => notificationRepository);
|
||||
services.AddScoped<INotificationDeliveryAdapter>(_ => new StubEmailAdapter(adapterOutcome));
|
||||
|
||||
return new OutboxHarness(connection, services.BuildServiceProvider());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a <see cref="NotificationOutboxActor"/>. With <paramref name="fastDispatch"/> the
|
||||
/// dispatch timer runs every 200ms so its own loop drives delivery; otherwise the timer is
|
||||
/// effectively disabled (1h interval) so dispatch only happens on an explicit tick.
|
||||
/// </summary>
|
||||
private IActorRef CreateOutboxActor(IServiceProvider services, bool fastDispatch = true)
|
||||
{
|
||||
var options = new NotificationOutboxOptions
|
||||
{
|
||||
DispatchInterval = fastDispatch ? TimeSpan.FromMilliseconds(200) : TimeSpan.FromHours(1),
|
||||
PurgeInterval = TimeSpan.FromHours(1),
|
||||
};
|
||||
|
||||
return Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
|
||||
services,
|
||||
options,
|
||||
(ICentralAuditWriter)new NoOpCentralAuditWriter(),
|
||||
NullLogger<NotificationOutboxActor>.Instance)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test-only no-op <see cref="ICentralAuditWriter"/>. The integration tests
|
||||
/// in this file pre-date M4 Bundle B's audit-writer injection; they do not
|
||||
/// assert on emission, just need a non-null collaborator.
|
||||
/// </summary>
|
||||
private sealed class NoOpCentralAuditWriter : ICentralAuditWriter
|
||||
{
|
||||
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static NotificationSubmit MakeSubmit(string notificationId)
|
||||
=> new(
|
||||
NotificationId: notificationId,
|
||||
ListName: "ops-team",
|
||||
Subject: "Tank level high",
|
||||
Body: "Tank 3 exceeded 90%.",
|
||||
SourceSiteId: "site-1",
|
||||
SourceInstanceId: "instance-42",
|
||||
SourceScript: "level-alarm",
|
||||
SiteEnqueuedAt: DateTimeOffset.UtcNow);
|
||||
|
||||
/// <summary>Reads a notification row in a fresh DI scope so each poll sees committed state.</summary>
|
||||
private static async Task<Notification?> GetNotificationAsync(
|
||||
IServiceProvider services, string notificationId)
|
||||
{
|
||||
using var scope = services.CreateScope();
|
||||
var repository = scope.ServiceProvider.GetRequiredService<INotificationOutboxRepository>();
|
||||
return await repository.GetByIdAsync(notificationId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Submit_DispatchedSuccessfully_ReachesDeliveredStatus()
|
||||
{
|
||||
using var harness = BuildHarness(DeliveryOutcome.Success("ops@example.com"));
|
||||
var actor = CreateOutboxActor(harness.Services);
|
||||
var notificationId = Guid.NewGuid().ToString();
|
||||
|
||||
// Ingest: the actor persists a Pending row and acks the submitter.
|
||||
actor.Tell(MakeSubmit(notificationId), TestActor);
|
||||
var ack = ExpectMsg<NotificationSubmitAck>(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(notificationId, ack.NotificationId);
|
||||
Assert.True(ack.Accepted, ack.Error);
|
||||
|
||||
// Dispatch: the actor's periodic timer claims the due row and delivers it.
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
var notification = GetNotificationAsync(harness.Services, notificationId)
|
||||
.GetAwaiter().GetResult();
|
||||
Assert.NotNull(notification);
|
||||
Assert.Equal(NotificationStatus.Delivered, notification!.Status);
|
||||
Assert.NotNull(notification.DeliveredAt);
|
||||
Assert.Equal("ops@example.com", notification.ResolvedTargets);
|
||||
Assert.Null(notification.LastError);
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(10),
|
||||
interval: TimeSpan.FromMilliseconds(100));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Submit_PermanentlyFailed_ReachesParked_ThenRetryReturnsToPending()
|
||||
{
|
||||
using var harness = BuildHarness(DeliveryOutcome.Permanent("invalid recipient address"));
|
||||
var dispatchActor = CreateOutboxActor(harness.Services, fastDispatch: true);
|
||||
var notificationId = Guid.NewGuid().ToString();
|
||||
|
||||
// Ingest.
|
||||
dispatchActor.Tell(MakeSubmit(notificationId), TestActor);
|
||||
var ack = ExpectMsg<NotificationSubmitAck>(TimeSpan.FromSeconds(5));
|
||||
Assert.True(ack.Accepted, ack.Error);
|
||||
|
||||
// Dispatch: a permanent failure parks the notification.
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
var notification = GetNotificationAsync(harness.Services, notificationId)
|
||||
.GetAwaiter().GetResult();
|
||||
Assert.NotNull(notification);
|
||||
Assert.Equal(NotificationStatus.Parked, notification!.Status);
|
||||
Assert.Equal("invalid recipient address", notification.LastError);
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(10),
|
||||
interval: TimeSpan.FromMilliseconds(100));
|
||||
|
||||
// Stop the dispatching actor so its periodic timer can never re-claim and re-park the
|
||||
// notification once the retry resets it — keeps the retry assertion deterministic.
|
||||
Watch(dispatchActor);
|
||||
Sys.Stop(dispatchActor);
|
||||
ExpectTerminated(dispatchActor, TimeSpan.FromSeconds(5));
|
||||
|
||||
// Manual retry against a non-dispatching actor: a parked notification is reset for
|
||||
// re-dispatch. With dispatch disabled the row stays Pending for an exact assertion.
|
||||
var retryActor = CreateOutboxActor(harness.Services, fastDispatch: false);
|
||||
var retryRequest = new RetryNotificationRequest(Guid.NewGuid().ToString(), notificationId);
|
||||
retryActor.Tell(retryRequest, TestActor);
|
||||
var retryResponse = ExpectMsg<RetryNotificationResponse>(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(retryRequest.CorrelationId, retryResponse.CorrelationId);
|
||||
Assert.True(retryResponse.Success, retryResponse.ErrorMessage);
|
||||
|
||||
// The retried row is reset to Pending with a cleared retry count.
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
var notification = GetNotificationAsync(harness.Services, notificationId)
|
||||
.GetAwaiter().GetResult();
|
||||
Assert.NotNull(notification);
|
||||
Assert.Equal(NotificationStatus.Pending, notification!.Status);
|
||||
Assert.Equal(0, notification.RetryCount);
|
||||
Assert.Null(notification.NextAttemptAt);
|
||||
Assert.Null(notification.LastError);
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(5),
|
||||
interval: TimeSpan.FromMilliseconds(100));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-8 (Phase 8): Observability validation.
|
||||
/// Verifies structured logs contain SiteId/NodeHostname/NodeRole,
|
||||
/// correlation IDs flow through request chains, and health dashboard shows all metric types.
|
||||
/// </summary>
|
||||
public class ObservabilityTests : IClassFixture<ScadaBridgeWebApplicationFactory>
|
||||
{
|
||||
private readonly ScadaBridgeWebApplicationFactory _factory;
|
||||
|
||||
public ObservabilityTests(ScadaBridgeWebApplicationFactory factory)
|
||||
{
|
||||
_factory = factory;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StructuredLog_SerilogTemplate_IncludesRequiredFields()
|
||||
{
|
||||
// The Serilog output template from Program.cs must include NodeRole and NodeHostname.
|
||||
var template = "[{Timestamp:HH:mm:ss} {Level:u3}] [{NodeRole}/{NodeHostname}] {Message:lj}{NewLine}{Exception}";
|
||||
|
||||
Assert.Contains("{NodeRole}", template);
|
||||
Assert.Contains("{NodeHostname}", template);
|
||||
Assert.Contains("{Timestamp", template);
|
||||
Assert.Contains("{Level", template);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SerilogEnrichment_SiteId_Configured()
|
||||
{
|
||||
// Program.cs enriches all log entries with SiteId, NodeHostname, NodeRole.
|
||||
// These are set from configuration and Serilog's Enrich.WithProperty().
|
||||
// Verify the enrichment properties are the ones we expect.
|
||||
var expectedProperties = new[] { "SiteId", "NodeHostname", "NodeRole" };
|
||||
|
||||
foreach (var prop in expectedProperties)
|
||||
{
|
||||
// Structural check: these property names must be present in the logging pipeline
|
||||
Assert.False(string.IsNullOrEmpty(prop));
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CorrelationId_MessageContracts_AllHaveCorrelationId()
|
||||
{
|
||||
// Verify that key message contracts include a CorrelationId field
|
||||
// for request/response tracing through the system.
|
||||
|
||||
// DeployInstanceCommand has DeploymentId (serves as correlation)
|
||||
var deployCmd = new Commons.Messages.Deployment.DeployInstanceCommand(
|
||||
"dep-1", "inst-1", "rev-1", "{}", "admin", DateTimeOffset.UtcNow);
|
||||
Assert.NotEmpty(deployCmd.DeploymentId);
|
||||
|
||||
// ScriptCallRequest has CorrelationId
|
||||
var scriptCall = new Commons.Messages.ScriptExecution.ScriptCallRequest(
|
||||
"OnTrigger", new Dictionary<string, object?>(), 0, "corr-123");
|
||||
Assert.Equal("corr-123", scriptCall.CorrelationId);
|
||||
|
||||
// ScriptCallResult has CorrelationId
|
||||
var scriptResult = new Commons.Messages.ScriptExecution.ScriptCallResult(
|
||||
"corr-123", true, 42, null);
|
||||
Assert.Equal("corr-123", scriptResult.CorrelationId);
|
||||
|
||||
// Lifecycle commands have CommandId
|
||||
var disableCmd = new Commons.Messages.Lifecycle.DisableInstanceCommand(
|
||||
"cmd-456", "inst-1", DateTimeOffset.UtcNow);
|
||||
Assert.Equal("cmd-456", disableCmd.CommandId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HealthDashboard_AllMetricTypes_RepresentedInReport()
|
||||
{
|
||||
// The SiteHealthReport must carry all metric types for the health dashboard.
|
||||
var report = new SiteHealthReport(
|
||||
SiteId: "site-01",
|
||||
SequenceNumber: 42,
|
||||
ReportTimestamp: DateTimeOffset.UtcNow,
|
||||
DataConnectionStatuses: new Dictionary<string, ConnectionHealth>
|
||||
{
|
||||
["opc-ua-1"] = ConnectionHealth.Connected,
|
||||
["opc-ua-2"] = ConnectionHealth.Disconnected
|
||||
},
|
||||
TagResolutionCounts: new Dictionary<string, TagResolutionStatus>
|
||||
{
|
||||
["opc-ua-1"] = new(75, 72),
|
||||
["opc-ua-2"] = new(50, 0)
|
||||
},
|
||||
ScriptErrorCount: 3,
|
||||
AlarmEvaluationErrorCount: 1,
|
||||
StoreAndForwardBufferDepths: new Dictionary<string, int>
|
||||
{
|
||||
["ext-system"] = 15,
|
||||
["notification"] = 2
|
||||
},
|
||||
DeadLetterCount: 5,
|
||||
DeployedInstanceCount: 0,
|
||||
EnabledInstanceCount: 0,
|
||||
DisabledInstanceCount: 0);
|
||||
|
||||
// Metric type 1: Data connection health
|
||||
Assert.Equal(2, report.DataConnectionStatuses.Count);
|
||||
Assert.Equal(ConnectionHealth.Connected, report.DataConnectionStatuses["opc-ua-1"]);
|
||||
Assert.Equal(ConnectionHealth.Disconnected, report.DataConnectionStatuses["opc-ua-2"]);
|
||||
|
||||
// Metric type 2: Tag resolution
|
||||
Assert.Equal(75, report.TagResolutionCounts["opc-ua-1"].TotalSubscribed);
|
||||
Assert.Equal(72, report.TagResolutionCounts["opc-ua-1"].SuccessfullyResolved);
|
||||
|
||||
// Metric type 3: Script errors
|
||||
Assert.Equal(3, report.ScriptErrorCount);
|
||||
|
||||
// Metric type 4: Alarm evaluation errors
|
||||
Assert.Equal(1, report.AlarmEvaluationErrorCount);
|
||||
|
||||
// Metric type 5: S&F buffer depths
|
||||
Assert.Equal(15, report.StoreAndForwardBufferDepths["ext-system"]);
|
||||
Assert.Equal(2, report.StoreAndForwardBufferDepths["notification"]);
|
||||
|
||||
// Metric type 6: Dead letters
|
||||
Assert.Equal(5, report.DeadLetterCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HealthAggregator_SiteRegistration_MarkedOnline()
|
||||
{
|
||||
var options = Options.Create(new HealthMonitoringOptions
|
||||
{
|
||||
OfflineTimeout = TimeSpan.FromSeconds(60)
|
||||
});
|
||||
|
||||
var aggregator = new CentralHealthAggregator(
|
||||
options, NullLogger<CentralHealthAggregator>.Instance);
|
||||
|
||||
// Register a site
|
||||
aggregator.ProcessReport(new SiteHealthReport(
|
||||
"site-01", 1, DateTimeOffset.UtcNow,
|
||||
new Dictionary<string, ConnectionHealth>(),
|
||||
new Dictionary<string, TagResolutionStatus>(),
|
||||
0, 0, new Dictionary<string, int>(), 0, 0, 0, 0));
|
||||
|
||||
var state = aggregator.GetSiteState("site-01");
|
||||
Assert.NotNull(state);
|
||||
Assert.True(state!.IsOnline);
|
||||
|
||||
// Update with a newer report
|
||||
aggregator.ProcessReport(new SiteHealthReport(
|
||||
"site-01", 2, DateTimeOffset.UtcNow,
|
||||
new Dictionary<string, ConnectionHealth>(),
|
||||
new Dictionary<string, TagResolutionStatus>(),
|
||||
3, 0, new Dictionary<string, int>(), 0, 0, 0, 0));
|
||||
|
||||
state = aggregator.GetSiteState("site-01");
|
||||
Assert.Equal(2, state!.LastSequenceNumber);
|
||||
Assert.Equal(3, state.LatestReport!.ScriptErrorCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HealthReport_SequenceNumbers_Monotonic()
|
||||
{
|
||||
// Sequence numbers must be monotonically increasing per site.
|
||||
// The aggregator should reject stale reports.
|
||||
var options = Options.Create(new HealthMonitoringOptions());
|
||||
var aggregator = new CentralHealthAggregator(
|
||||
options, NullLogger<CentralHealthAggregator>.Instance);
|
||||
|
||||
for (var seq = 1; seq <= 10; seq++)
|
||||
{
|
||||
aggregator.ProcessReport(new SiteHealthReport(
|
||||
"site-01", seq, DateTimeOffset.UtcNow,
|
||||
new Dictionary<string, ConnectionHealth>(),
|
||||
new Dictionary<string, TagResolutionStatus>(),
|
||||
seq, 0, new Dictionary<string, int>(), 0, 0, 0, 0));
|
||||
}
|
||||
|
||||
var state = aggregator.GetSiteState("site-01");
|
||||
Assert.Equal(10, state!.LastSequenceNumber);
|
||||
Assert.Equal(10, state.LatestReport!.ScriptErrorCount);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
using System.Net;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-22: Readiness gating — /health/ready endpoint returns status code.
|
||||
/// </summary>
|
||||
public class ReadinessTests : IClassFixture<ScadaBridgeWebApplicationFactory>
|
||||
{
|
||||
private readonly ScadaBridgeWebApplicationFactory _factory;
|
||||
|
||||
public ReadinessTests(ScadaBridgeWebApplicationFactory factory)
|
||||
{
|
||||
_factory = factory;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HealthReady_ReturnsSuccessStatusCode()
|
||||
{
|
||||
using var client = _factory.CreateClient();
|
||||
|
||||
var response = await client.GetAsync("/health/ready");
|
||||
|
||||
// The endpoint should exist and return 200 OK (or 503 if not ready yet).
|
||||
// For now, just verify the endpoint exists and returns a valid HTTP response.
|
||||
Assert.True(
|
||||
response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.ServiceUnavailable,
|
||||
$"Expected 200 or 503 but got {response.StatusCode}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-7 (Phase 8): Recovery drill test scaffolds.
|
||||
/// Mid-deploy failover, communication drops, and site restart with persisted configs.
|
||||
/// </summary>
|
||||
public class RecoveryDrillTests
|
||||
{
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public void MidDeployFailover_SiteStateQuery_ThenRedeploy()
|
||||
{
|
||||
// Scenario: Deployment in progress, central node fails over.
|
||||
// New central node queries site for current deployment state, then re-issues deploy.
|
||||
|
||||
// Step 1: Deployment started
|
||||
var initialStatus = new DeploymentStatusResponse(
|
||||
"dep-1", "pump-station-1", DeploymentStatus.InProgress,
|
||||
null, DateTimeOffset.UtcNow);
|
||||
|
||||
Assert.Equal(DeploymentStatus.InProgress, initialStatus.Status);
|
||||
|
||||
// Step 2: Central failover — new node queries site state
|
||||
// Site reports current status (InProgress or whatever it actually is)
|
||||
var queriedStatus = new DeploymentStatusResponse(
|
||||
"dep-1", "pump-station-1", DeploymentStatus.InProgress,
|
||||
null, DateTimeOffset.UtcNow.AddSeconds(5));
|
||||
|
||||
Assert.Equal(DeploymentStatus.InProgress, queriedStatus.Status);
|
||||
|
||||
// Step 3: Central re-deploys with same deployment ID + revision hash
|
||||
// Idempotent: same deploymentId + revisionHash = no-op if already applied
|
||||
var redeployCommand = new DeployInstanceCommand(
|
||||
"dep-1", "pump-station-1", "abc123",
|
||||
"""{"attributes":[],"scripts":[],"alarms":[]}""",
|
||||
"admin", DateTimeOffset.UtcNow.AddSeconds(10));
|
||||
|
||||
Assert.Equal("dep-1", redeployCommand.DeploymentId);
|
||||
Assert.Equal("abc123", redeployCommand.RevisionHash);
|
||||
|
||||
// Step 4: Site applies (idempotent — revision hash matches)
|
||||
var completedStatus = new DeploymentStatusResponse(
|
||||
"dep-1", "pump-station-1", DeploymentStatus.Success,
|
||||
null, DateTimeOffset.UtcNow.AddSeconds(15));
|
||||
|
||||
Assert.Equal(DeploymentStatus.Success, completedStatus.Status);
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public async Task CommunicationDrop_DuringArtifactDeployment_BuffersForRetry()
|
||||
{
|
||||
// Scenario: Communication drops while deploying system-wide artifacts.
|
||||
// The deployment command is buffered by S&F and retried when connection restores.
|
||||
var dbPath = Path.Combine(Path.GetTempPath(), $"sf_commdrop_{Guid.NewGuid():N}.db");
|
||||
var connStr = $"Data Source={dbPath}";
|
||||
|
||||
try
|
||||
{
|
||||
var storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
|
||||
var options = new StoreAndForwardOptions
|
||||
{
|
||||
DefaultRetryInterval = TimeSpan.FromSeconds(5),
|
||||
DefaultMaxRetries = 100,
|
||||
};
|
||||
var service = new StoreAndForwardService(storage, options, NullLogger<StoreAndForwardService>.Instance);
|
||||
await service.StartAsync();
|
||||
|
||||
// Register a handler that simulates communication failure
|
||||
var callCount = 0;
|
||||
service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
|
||||
_ =>
|
||||
{
|
||||
callCount++;
|
||||
throw new InvalidOperationException("Connection to site lost");
|
||||
});
|
||||
|
||||
// Attempt delivery — should fail and buffer
|
||||
var result = await service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem,
|
||||
"site-01/artifacts",
|
||||
"""{"deploymentId":"dep-1","artifacts":["shared-script-v2"]}""");
|
||||
|
||||
Assert.True(result.Accepted);
|
||||
Assert.True(result.WasBuffered);
|
||||
Assert.Equal(1, callCount);
|
||||
|
||||
// Verify the message is in the buffer
|
||||
var depths = await service.GetBufferDepthAsync();
|
||||
Assert.True(depths.ContainsKey(StoreAndForwardCategory.ExternalSystem));
|
||||
Assert.Equal(1, depths[StoreAndForwardCategory.ExternalSystem]);
|
||||
|
||||
await service.StopAsync();
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(dbPath))
|
||||
File.Delete(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public async Task SiteRestart_WithPersistedConfigs_RebuildFromSQLite()
|
||||
{
|
||||
// Scenario: Site restarts. Deployed instance configs are persisted in SQLite.
|
||||
// On startup, the Deployment Manager Actor reads configs from SQLite and
|
||||
// recreates Instance Actors.
|
||||
var dbPath = Path.Combine(Path.GetTempPath(), $"sf_restart_{Guid.NewGuid():N}.db");
|
||||
var connStr = $"Data Source={dbPath}";
|
||||
|
||||
try
|
||||
{
|
||||
// Pre-restart: S&F messages in buffer
|
||||
var storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
|
||||
for (var i = 0; i < 3; i++)
|
||||
{
|
||||
await storage.EnqueueAsync(new StoreAndForwardMessage
|
||||
{
|
||||
Id = $"msg-{i}",
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "api-endpoint",
|
||||
PayloadJson = $$"""{"instanceName":"machine-{{i}}","value":42}""",
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
OriginInstanceName = $"machine-{i}"
|
||||
});
|
||||
}
|
||||
|
||||
// Post-restart: new storage instance reads same DB
|
||||
var restartedStorage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await restartedStorage.InitializeAsync();
|
||||
|
||||
var pending = await restartedStorage.GetMessagesForRetryAsync();
|
||||
Assert.Equal(3, pending.Count);
|
||||
|
||||
// Verify each message retains its origin instance
|
||||
Assert.Contains(pending, m => m.OriginInstanceName == "machine-0");
|
||||
Assert.Contains(pending, m => m.OriginInstanceName == "machine-1");
|
||||
Assert.Contains(pending, m => m.OriginInstanceName == "machine-2");
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(dbPath))
|
||||
File.Delete(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeploymentIdempotency_SameRevisionHash_NoOp()
|
||||
{
|
||||
// Verify the deployment model supports idempotency via revision hash.
|
||||
// Two deploy commands with the same deploymentId + revisionHash should
|
||||
// produce the same result (site can detect the duplicate and skip).
|
||||
var cmd1 = new DeployInstanceCommand(
|
||||
"dep-1", "pump-1", "rev-abc123",
|
||||
"""{"attributes":[]}""", "admin", DateTimeOffset.UtcNow);
|
||||
|
||||
var cmd2 = new DeployInstanceCommand(
|
||||
"dep-1", "pump-1", "rev-abc123",
|
||||
"""{"attributes":[]}""", "admin", DateTimeOffset.UtcNow.AddSeconds(30));
|
||||
|
||||
Assert.Equal(cmd1.DeploymentId, cmd2.DeploymentId);
|
||||
Assert.Equal(cmd1.RevisionHash, cmd2.RevisionHash);
|
||||
Assert.Equal(cmd1.InstanceUniqueName, cmd2.InstanceUniqueName);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FlattenedConfigSnapshot_ContainsRevisionHash()
|
||||
{
|
||||
// The FlattenedConfigurationSnapshot includes a revision hash for staleness detection.
|
||||
var snapshot = new FlattenedConfigurationSnapshot(
|
||||
"inst-1", "rev-abc123",
|
||||
"""{"attributes":[],"scripts":[],"alarms":[]}""",
|
||||
DateTimeOffset.UtcNow);
|
||||
|
||||
Assert.Equal("rev-abc123", snapshot.RevisionHash);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
||||
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Shared WebApplicationFactory for integration tests.
|
||||
/// Replaces SQL Server with an in-memory database and skips migrations.
|
||||
/// Removes AkkaHostedService to avoid DNS resolution issues in test environments.
|
||||
/// Uses environment variables for config since Program.cs reads them in the initial ConfigurationBuilder
|
||||
/// before WebApplicationFactory can inject settings.
|
||||
/// </summary>
|
||||
public class ScadaBridgeWebApplicationFactory : WebApplicationFactory<Program>
|
||||
{
|
||||
/// <summary>
|
||||
/// Environment variables that were set by this factory, to be cleaned up on dispose.
|
||||
/// </summary>
|
||||
private readonly Dictionary<string, string?> _previousEnvVars = new();
|
||||
|
||||
public ScadaBridgeWebApplicationFactory()
|
||||
{
|
||||
// The initial ConfigurationBuilder in Program.cs reads env vars with AddEnvironmentVariables().
|
||||
// The env var format uses __ as section separator.
|
||||
var envVars = new Dictionary<string, string>
|
||||
{
|
||||
["DOTNET_ENVIRONMENT"] = "Development",
|
||||
["ScadaBridge__Node__Role"] = "Central",
|
||||
["ScadaBridge__Node__NodeHostname"] = "localhost",
|
||||
["ScadaBridge__Node__RemotingPort"] = "8081",
|
||||
["ScadaBridge__Cluster__SeedNodes__0"] = "akka.tcp://scadabridge@localhost:8081",
|
||||
["ScadaBridge__Cluster__SeedNodes__1"] = "akka.tcp://scadabridge@localhost:8082",
|
||||
["ScadaBridge__Database__ConfigurationDb"] = "Server=localhost;Database=ScadaBridge_Test;TrustServerCertificate=True",
|
||||
["ScadaBridge__Database__MachineDataDb"] = "Server=localhost;Database=ScadaBridge_MachineData_Test;TrustServerCertificate=True",
|
||||
["ScadaBridge__Database__SkipMigrations"] = "true",
|
||||
["ScadaBridge__Security__JwtSigningKey"] = "integration-test-signing-key-must-be-at-least-32-chars-long",
|
||||
["ScadaBridge__Security__LdapServer"] = "localhost",
|
||||
["ScadaBridge__Security__LdapPort"] = "3893",
|
||||
["ScadaBridge__Security__LdapUseTls"] = "false",
|
||||
["ScadaBridge__Security__AllowInsecureLdap"] = "true",
|
||||
["ScadaBridge__Security__LdapSearchBase"] = "dc=scadabridge,dc=local",
|
||||
// GLAuth places users at cn=<name>,ou=<group>,ou=users,dc=... — the
|
||||
// no-service-account fallback DN (uid=<name>,dc=...) does not match,
|
||||
// so a service account is configured to enable search-then-bind:
|
||||
// resolve the user's real DN by (uid=<name>) lookup, then bind it.
|
||||
["ScadaBridge__Security__LdapServiceAccountDn"] = "cn=admin,ou=SCADA-Admins,ou=users,dc=scadabridge,dc=local",
|
||||
["ScadaBridge__Security__LdapServiceAccountPassword"] = "password",
|
||||
};
|
||||
|
||||
foreach (var (key, value) in envVars)
|
||||
{
|
||||
_previousEnvVars[key] = Environment.GetEnvironmentVariable(key);
|
||||
Environment.SetEnvironmentVariable(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
protected override void ConfigureWebHost(IWebHostBuilder builder)
|
||||
{
|
||||
builder.UseEnvironment("Development");
|
||||
|
||||
builder.ConfigureServices(services =>
|
||||
{
|
||||
// Remove ALL DbContext and EF-related service registrations to avoid dual-provider conflict.
|
||||
// AddDbContext<> with UseSqlServer registers many internal services. We must remove them all.
|
||||
var descriptorsToRemove = services
|
||||
.Where(d =>
|
||||
d.ServiceType == typeof(DbContextOptions<ScadaBridgeDbContext>) ||
|
||||
d.ServiceType == typeof(DbContextOptions) ||
|
||||
d.ServiceType == typeof(ScadaBridgeDbContext) ||
|
||||
d.ServiceType.FullName?.Contains("EntityFrameworkCore") == true)
|
||||
.ToList();
|
||||
foreach (var d in descriptorsToRemove)
|
||||
services.Remove(d);
|
||||
|
||||
// Add in-memory database as sole provider
|
||||
services.AddDbContext<ScadaBridgeDbContext>(options =>
|
||||
options.UseInMemoryDatabase($"ScadaBridge_IntegrationTests_{Guid.NewGuid()}"));
|
||||
|
||||
// Remove the factory-registered IHostedService registrations so
|
||||
// Akka.NET remoting / DNS resolution never starts in tests — but
|
||||
// keep the AkkaHostedService SINGLETON resolvable: IClusterNodeProvider
|
||||
// (and other services) depend on it via GetRequiredService.
|
||||
var hostedServiceDescriptors = services
|
||||
.Where(d => d.ServiceType == typeof(IHostedService) && d.ImplementationFactory != null)
|
||||
.ToList();
|
||||
foreach (var d in hostedServiceDescriptors)
|
||||
services.Remove(d);
|
||||
});
|
||||
}
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
base.Dispose(disposing);
|
||||
if (disposing)
|
||||
{
|
||||
foreach (var (key, previousValue) in _previousEnvVars)
|
||||
{
|
||||
Environment.SetEnvironmentVariable(key, previousValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Security;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-5 (Phase 8): Security hardening tests.
|
||||
/// Verifies LDAPS enforcement, JWT key length, secret scrubbing, and API key protection.
|
||||
/// </summary>
|
||||
public class SecurityHardeningTests
|
||||
{
|
||||
private static JwtTokenService CreateJwtService(string signingKey = "integration-test-signing-key-must-be-at-least-32-chars-long")
|
||||
{
|
||||
var options = Options.Create(new SecurityOptions
|
||||
{
|
||||
JwtSigningKey = signingKey,
|
||||
JwtExpiryMinutes = 15,
|
||||
IdleTimeoutMinutes = 30,
|
||||
JwtRefreshThresholdMinutes = 5
|
||||
});
|
||||
return new JwtTokenService(options, NullLogger<JwtTokenService>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SecurityOptions_LdapUseTls_DefaultsToTrue()
|
||||
{
|
||||
// Production requires LDAPS. The default must be true.
|
||||
var options = new SecurityOptions();
|
||||
Assert.True(options.LdapUseTls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SecurityOptions_AllowInsecureLdap_DefaultsToFalse()
|
||||
{
|
||||
var options = new SecurityOptions();
|
||||
Assert.False(options.AllowInsecureLdap);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtSigningKey_MinimumLength_Enforced()
|
||||
{
|
||||
// HMAC-SHA256 requires a key of at least 32 bytes (256 bits).
|
||||
var jwtService = CreateJwtService();
|
||||
|
||||
var token = jwtService.GenerateToken(
|
||||
displayName: "Test",
|
||||
username: "test",
|
||||
roles: new[] { "Admin" },
|
||||
permittedSiteIds: null);
|
||||
|
||||
Assert.NotNull(token);
|
||||
Assert.True(token.Length > 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtSigningKey_ShortKey_FailsValidation()
|
||||
{
|
||||
var shortKey = "tooshort";
|
||||
Assert.True(shortKey.Length < 32,
|
||||
"Test key must be shorter than 32 chars to verify minimum length enforcement");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LogOutputTemplate_DoesNotContainSecrets()
|
||||
{
|
||||
// Verify the Serilog output template does not include secret-bearing properties.
|
||||
var template = "[{Timestamp:HH:mm:ss} {Level:u3}] [{NodeRole}/{NodeHostname}] {Message:lj}{NewLine}{Exception}";
|
||||
|
||||
Assert.DoesNotContain("Password", template, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("ApiKey", template, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("Secret", template, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("SigningKey", template, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("ConnectionString", template, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LogEnrichment_ContainsExpectedProperties()
|
||||
{
|
||||
var enrichmentProperties = new[] { "SiteId", "NodeHostname", "NodeRole" };
|
||||
|
||||
foreach (var prop in enrichmentProperties)
|
||||
{
|
||||
Assert.DoesNotContain("Password", prop, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("Key", prop, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtToken_DoesNotContainSigningKey()
|
||||
{
|
||||
var jwtService = CreateJwtService();
|
||||
|
||||
var token = jwtService.GenerateToken(
|
||||
displayName: "Test",
|
||||
username: "test",
|
||||
roles: new[] { "Admin" },
|
||||
permittedSiteIds: null);
|
||||
|
||||
// JWT tokens are base64-encoded; the signing key should not appear in the payload
|
||||
Assert.DoesNotContain("signing-key", token, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SecurityOptions_JwtExpiryDefaults_AreSecure()
|
||||
{
|
||||
var options = new SecurityOptions();
|
||||
|
||||
Assert.Equal(15, options.JwtExpiryMinutes);
|
||||
Assert.Equal(30, options.IdleTimeoutMinutes);
|
||||
Assert.Equal(5, options.JwtRefreshThresholdMinutes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtToken_TamperedPayload_FailsValidation()
|
||||
{
|
||||
var jwtService = CreateJwtService();
|
||||
|
||||
var token = jwtService.GenerateToken(
|
||||
displayName: "User",
|
||||
username: "user",
|
||||
roles: new[] { "Admin" },
|
||||
permittedSiteIds: null);
|
||||
|
||||
// Tamper with the token payload (second segment)
|
||||
var parts = token.Split('.');
|
||||
Assert.Equal(3, parts.Length);
|
||||
|
||||
// Flip a character in the payload
|
||||
var tamperedPayload = parts[1];
|
||||
if (tamperedPayload.Length > 5)
|
||||
{
|
||||
var chars = tamperedPayload.ToCharArray();
|
||||
chars[5] = chars[5] == 'A' ? 'B' : 'A';
|
||||
tamperedPayload = new string(chars);
|
||||
}
|
||||
var tamperedToken = $"{parts[0]}.{tamperedPayload}.{parts[2]}";
|
||||
|
||||
var principal = jwtService.ValidateToken(tamperedToken);
|
||||
Assert.Null(principal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void JwtRefreshToken_PreservesIdentity()
|
||||
{
|
||||
var jwtService = CreateJwtService();
|
||||
|
||||
var originalToken = jwtService.GenerateToken(
|
||||
displayName: "Original User",
|
||||
username: "orig_user",
|
||||
roles: new[] { "Admin", "Design" },
|
||||
permittedSiteIds: new[] { "site-1" });
|
||||
|
||||
var principal = jwtService.ValidateToken(originalToken);
|
||||
Assert.NotNull(principal);
|
||||
|
||||
// Refresh the token
|
||||
var refreshedToken = jwtService.RefreshToken(
|
||||
principal!,
|
||||
new[] { "Admin", "Design" },
|
||||
new[] { "site-1" });
|
||||
|
||||
Assert.NotNull(refreshedToken);
|
||||
|
||||
var refreshedPrincipal = jwtService.ValidateToken(refreshedToken!);
|
||||
Assert.NotNull(refreshedPrincipal);
|
||||
|
||||
Assert.Equal("Original User", refreshedPrincipal!.FindFirst(JwtTokenService.DisplayNameClaimType)?.Value);
|
||||
Assert.Equal("orig_user", refreshedPrincipal.FindFirst(JwtTokenService.UsernameClaimType)?.Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StartupValidator_RejectsInsecureLdapInProduction()
|
||||
{
|
||||
// The SecurityOptions.AllowInsecureLdap defaults to false.
|
||||
// Only when explicitly set to true (for dev/test) is insecure LDAP allowed.
|
||||
var prodOptions = new SecurityOptions { LdapUseTls = true, AllowInsecureLdap = false };
|
||||
Assert.True(prodOptions.LdapUseTls);
|
||||
Assert.False(prodOptions.AllowInsecureLdap);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-2 (Phase 8): Full-system failover testing — Site.
|
||||
/// Verifies S&F buffer takeover, DCL reconnection structure, alarm re-evaluation,
|
||||
/// and script trigger resumption after site failover.
|
||||
/// </summary>
|
||||
public class SiteFailoverTests
|
||||
{
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public async Task StoreAndForward_BufferSurvivesRestart_MessagesRetained()
|
||||
{
|
||||
// Simulates site failover: messages buffered in SQLite survive process restart.
|
||||
// The standby node picks up the same SQLite file and retries pending messages.
|
||||
var dbPath = Path.Combine(Path.GetTempPath(), $"sf_failover_{Guid.NewGuid():N}.db");
|
||||
var connStr = $"Data Source={dbPath}";
|
||||
|
||||
try
|
||||
{
|
||||
// Phase 1: Buffer messages on "primary" node
|
||||
var storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
|
||||
var message = new StoreAndForwardMessage
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "https://api.example.com/data",
|
||||
PayloadJson = """{"temperature":42.5}""",
|
||||
RetryCount = 2,
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
OriginInstanceName = "pump-station-1"
|
||||
};
|
||||
|
||||
await storage.EnqueueAsync(message);
|
||||
|
||||
// Phase 2: "Standby" node opens the same database (simulating failover)
|
||||
var standbyStorage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await standbyStorage.InitializeAsync();
|
||||
|
||||
var pending = await standbyStorage.GetMessagesForRetryAsync();
|
||||
Assert.Single(pending);
|
||||
Assert.Equal(message.Id, pending[0].Id);
|
||||
Assert.Equal("pump-station-1", pending[0].OriginInstanceName);
|
||||
Assert.Equal(2, pending[0].RetryCount);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(dbPath))
|
||||
File.Delete(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public async Task StoreAndForward_ParkedMessages_SurviveFailover()
|
||||
{
|
||||
var dbPath = Path.Combine(Path.GetTempPath(), $"sf_parked_{Guid.NewGuid():N}.db");
|
||||
var connStr = $"Data Source={dbPath}";
|
||||
|
||||
try
|
||||
{
|
||||
var storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
|
||||
var parkedMsg = new StoreAndForwardMessage
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Category = StoreAndForwardCategory.Notification,
|
||||
Target = "alert-list",
|
||||
PayloadJson = """{"subject":"Critical alarm"}""",
|
||||
RetryCount = 50,
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow.AddHours(-1),
|
||||
LastAttemptAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Parked,
|
||||
LastError = "SMTP connection timeout",
|
||||
OriginInstanceName = "compressor-1"
|
||||
};
|
||||
|
||||
await storage.EnqueueAsync(parkedMsg);
|
||||
|
||||
// Standby opens same DB
|
||||
var standbyStorage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await standbyStorage.InitializeAsync();
|
||||
|
||||
var (parked, count) = await standbyStorage.GetParkedMessagesAsync();
|
||||
Assert.Equal(1, count);
|
||||
Assert.Equal("SMTP connection timeout", parked[0].LastError);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(dbPath))
|
||||
File.Delete(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AlarmReEvaluation_IncomingValue_TriggersNewState()
|
||||
{
|
||||
// Structural verification: AlarmStateChanged carries all data needed for
|
||||
// re-evaluation after failover. When DCL reconnects and pushes new values,
|
||||
// the Alarm Actor evaluates from the incoming value (not stale state).
|
||||
var alarmEvent = new AlarmStateChanged(
|
||||
"pump-station-1",
|
||||
"HighPressureAlarm",
|
||||
AlarmState.Active,
|
||||
1,
|
||||
DateTimeOffset.UtcNow);
|
||||
|
||||
Assert.Equal(AlarmState.Active, alarmEvent.State);
|
||||
Assert.Equal("pump-station-1", alarmEvent.InstanceUniqueName);
|
||||
|
||||
// After failover, a new value triggers re-evaluation
|
||||
var clearedEvent = new AlarmStateChanged(
|
||||
"pump-station-1",
|
||||
"HighPressureAlarm",
|
||||
AlarmState.Normal,
|
||||
1,
|
||||
DateTimeOffset.UtcNow.AddSeconds(5));
|
||||
|
||||
Assert.Equal(AlarmState.Normal, clearedEvent.State);
|
||||
Assert.True(clearedEvent.Timestamp > alarmEvent.Timestamp);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ScriptTriggerResumption_ValueChangeTriggersScript()
|
||||
{
|
||||
// Structural verification: AttributeValueChanged messages from DCL after reconnection
|
||||
// will be routed to Script Actors, which evaluate triggers based on incoming values.
|
||||
// No stale trigger state needed — triggers fire on new values.
|
||||
var valueChange = new AttributeValueChanged(
|
||||
"pump-station-1",
|
||||
"OPC:ns=2;s=Pressure",
|
||||
"Pressure",
|
||||
150.0,
|
||||
"Good",
|
||||
DateTimeOffset.UtcNow);
|
||||
|
||||
Assert.Equal("Pressure", valueChange.AttributeName);
|
||||
Assert.Equal("OPC:ns=2;s=Pressure", valueChange.AttributePath);
|
||||
Assert.Equal(150.0, valueChange.Value);
|
||||
Assert.Equal("Good", valueChange.Quality);
|
||||
}
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
[Fact]
|
||||
public async Task StoreAndForward_BufferDepth_ReportedAfterFailover()
|
||||
{
|
||||
var dbPath = Path.Combine(Path.GetTempPath(), $"sf_depth_{Guid.NewGuid():N}.db");
|
||||
var connStr = $"Data Source={dbPath}";
|
||||
|
||||
try
|
||||
{
|
||||
var storage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await storage.InitializeAsync();
|
||||
|
||||
// Enqueue messages in different categories
|
||||
for (var i = 0; i < 5; i++)
|
||||
{
|
||||
await storage.EnqueueAsync(new StoreAndForwardMessage
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "api",
|
||||
PayloadJson = "{}",
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
});
|
||||
}
|
||||
|
||||
for (var i = 0; i < 3; i++)
|
||||
{
|
||||
await storage.EnqueueAsync(new StoreAndForwardMessage
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Category = StoreAndForwardCategory.Notification,
|
||||
Target = "alerts",
|
||||
PayloadJson = "{}",
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
});
|
||||
}
|
||||
|
||||
// After failover, standby reads buffer depths
|
||||
var standbyStorage = new StoreAndForwardStorage(connStr, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
await standbyStorage.InitializeAsync();
|
||||
|
||||
var depths = await standbyStorage.GetBufferDepthByCategoryAsync();
|
||||
Assert.Equal(5, depths[StoreAndForwardCategory.ExternalSystem]);
|
||||
Assert.Equal(3, depths[StoreAndForwardCategory.Notification]);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(dbPath))
|
||||
File.Delete(dbPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Mvc.Testing;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// WP-22: Startup validation — missing required config fails with clear error.
|
||||
/// Tests the StartupValidator that runs on boot.
|
||||
///
|
||||
/// Note: These tests temporarily set environment variables because Program.cs reads
|
||||
/// configuration from env vars in the initial ConfigurationBuilder (before WebApplicationFactory
|
||||
/// can inject settings). Each test saves/restores env vars to avoid interference.
|
||||
/// </summary>
|
||||
public class StartupValidationTests
|
||||
{
|
||||
[Fact]
|
||||
public void MissingRole_ThrowsInvalidOperationException()
|
||||
{
|
||||
// Set all required config EXCEPT Role
|
||||
using var env = new TempEnvironment(new Dictionary<string, string>
|
||||
{
|
||||
["DOTNET_ENVIRONMENT"] = "Development",
|
||||
["ScadaBridge__Node__NodeHostname"] = "localhost",
|
||||
["ScadaBridge__Node__RemotingPort"] = "8081",
|
||||
["ScadaBridge__Cluster__SeedNodes__0"] = "akka.tcp://scadabridge@localhost:8081",
|
||||
["ScadaBridge__Cluster__SeedNodes__1"] = "akka.tcp://scadabridge@localhost:8082",
|
||||
});
|
||||
|
||||
var factory = new WebApplicationFactory<Program>();
|
||||
|
||||
var ex = Assert.Throws<InvalidOperationException>(() => factory.CreateClient());
|
||||
Assert.Contains("Role", ex.Message, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
factory.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MissingJwtSigningKey_ForCentral_ThrowsInvalidOperationException()
|
||||
{
|
||||
using var env = new TempEnvironment(new Dictionary<string, string>
|
||||
{
|
||||
["DOTNET_ENVIRONMENT"] = "Development",
|
||||
["ScadaBridge__Node__Role"] = "Central",
|
||||
["ScadaBridge__Node__NodeHostname"] = "localhost",
|
||||
["ScadaBridge__Node__RemotingPort"] = "8081",
|
||||
["ScadaBridge__Cluster__SeedNodes__0"] = "akka.tcp://scadabridge@localhost:8081",
|
||||
["ScadaBridge__Cluster__SeedNodes__1"] = "akka.tcp://scadabridge@localhost:8082",
|
||||
["ScadaBridge__Database__ConfigurationDb"] = "Server=x;Database=x",
|
||||
["ScadaBridge__Database__MachineDataDb"] = "Server=x;Database=x",
|
||||
["ScadaBridge__Security__LdapServer"] = "localhost",
|
||||
// Deliberately missing JwtSigningKey
|
||||
});
|
||||
|
||||
var factory = new WebApplicationFactory<Program>();
|
||||
|
||||
var ex = Assert.Throws<InvalidOperationException>(() => factory.CreateClient());
|
||||
Assert.Contains("JwtSigningKey", ex.Message, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
factory.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CentralRole_StartsSuccessfully_WithValidConfig()
|
||||
{
|
||||
using var factory = new ScadaBridgeWebApplicationFactory();
|
||||
using var client = factory.CreateClient();
|
||||
|
||||
Assert.NotNull(client);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper to temporarily set environment variables and restore them on dispose.
|
||||
/// Clears all ScadaBridge__ vars first to ensure a clean slate.
|
||||
/// </summary>
|
||||
private sealed class TempEnvironment : IDisposable
|
||||
{
|
||||
private readonly Dictionary<string, string?> _previousValues = new();
|
||||
|
||||
/// <summary>
|
||||
/// All ScadaBridge env vars that might be set by other tests/factories.
|
||||
/// </summary>
|
||||
private static readonly string[] KnownKeys =
|
||||
{
|
||||
"DOTNET_ENVIRONMENT",
|
||||
"ScadaBridge__Node__Role",
|
||||
"ScadaBridge__Node__NodeHostname",
|
||||
"ScadaBridge__Node__RemotingPort",
|
||||
"ScadaBridge__Node__SiteId",
|
||||
"ScadaBridge__Cluster__SeedNodes__0",
|
||||
"ScadaBridge__Cluster__SeedNodes__1",
|
||||
"ScadaBridge__Database__ConfigurationDb",
|
||||
"ScadaBridge__Database__MachineDataDb",
|
||||
"ScadaBridge__Database__SkipMigrations",
|
||||
"ScadaBridge__Security__JwtSigningKey",
|
||||
"ScadaBridge__Security__LdapServer",
|
||||
"ScadaBridge__Security__LdapPort",
|
||||
"ScadaBridge__Security__LdapUseTls",
|
||||
"ScadaBridge__Security__AllowInsecureLdap",
|
||||
"ScadaBridge__Security__LdapSearchBase",
|
||||
};
|
||||
|
||||
public TempEnvironment(Dictionary<string, string> varsToSet)
|
||||
{
|
||||
// Save and clear all known keys
|
||||
foreach (var key in KnownKeys)
|
||||
{
|
||||
_previousValues[key] = Environment.GetEnvironmentVariable(key);
|
||||
Environment.SetEnvironmentVariable(key, null);
|
||||
}
|
||||
|
||||
// Set the requested vars
|
||||
foreach (var (key, value) in varsToSet)
|
||||
{
|
||||
if (!_previousValues.ContainsKey(key))
|
||||
_previousValues[key] = Environment.GetEnvironmentVariable(key);
|
||||
Environment.SetEnvironmentVariable(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var (key, previousValue) in _previousValues)
|
||||
{
|
||||
Environment.SetEnvironmentVariable(key, previousValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<FrameworkReference Include="Microsoft.AspNetCore.App" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Akka.TestKit.Xunit2" />
|
||||
<PackageReference Include="coverlet.collector" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="NSubstitute" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="xunit.runner.json" CopyToOutputDirectory="PreserveNewest" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.Host/ZB.MOM.WW.ScadaBridge.Host.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.NotificationOutbox/ZB.MOM.WW.ScadaBridge.NotificationOutbox.csproj" />
|
||||
<ProjectReference Include="../../src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
|
||||
"parallelizeAssembly": false,
|
||||
"parallelizeTestCollections": false
|
||||
}
|
||||
Reference in New Issue
Block a user