Joseph Doherty
496 lines
20 KiB
C#
496 lines
20 KiB
C#
using System.Diagnostics.Metrics;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
|
using ZB.MOM.WW.OtOpcUa.Host.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa;
|
|
|
|
/// <summary>
|
|
/// Covers <see cref="BoundedApiKeyStore"/> — the deadline and TTL cache the OPC UA impersonation
|
|
/// callback's key lookup runs under (arch-review 03/S18).
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// The deadline cases are driven through the <b>real</b> <see cref="ApiKeyOpcUaUserAuthenticator"/>
|
|
/// rather than the store directly, because the fail-closed conversion — a throw becoming the
|
|
/// standard opaque denial — is part of what is being asserted, and it lives in the authenticator's
|
|
/// catch-all rather than in the decorator.
|
|
/// </para>
|
|
/// <para>
|
|
/// ⚠️ <b>Deadline cases use a REAL short timeout; TTL cases use the fake clock.</b> The hand-rolled
|
|
/// <see cref="FakeClock"/> overrides <c>GetUtcNow()</c> only, so <c>CreateTimer</c> falls through to
|
|
/// the base implementation and produces a real system timer — advancing the fake clock does not
|
|
/// fire a <c>CancelAfter</c>. A deadline test "improved" to advance the fake clock would pass for
|
|
/// the wrong reason: the inner task simply never completing.
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class BoundedApiKeyStoreTests
|
|
{
|
|
private const string Pepper = "unit-test-pepper";
|
|
private const string Principal = "scadabridge-dcl";
|
|
|
|
/// <summary>
|
|
/// A store that never answers must not hold the activation open: the deadline fires and the
|
|
/// activation is denied.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// The SDK raises impersonation under a <c>SessionManager</c>-wide lock, so a lookup that does not
|
|
/// return stalls every session activation in the server — not just this one. That is the whole of
|
|
/// 03/S18, and it is why the bound is a deadline rather than a cache.
|
|
/// </remarks>
|
|
[Fact]
|
|
public async Task A_blocked_store_denies_without_hanging_the_activation()
|
|
{
|
|
var (presented, inner) = SetUp();
|
|
inner.BlockForever = true;
|
|
var auth = Auth(Bounded(inner, timeoutMs: 150, cacheSeconds: 0));
|
|
|
|
var result = await auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None)
|
|
.WaitAsync(TimeSpan.FromSeconds(5));
|
|
|
|
result.Success.ShouldBeFalse();
|
|
result.Error.ShouldBe(ApiKeyOpcUaUserAuthenticator.DenyMessage);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The positive control for <see cref="A_blocked_store_denies_without_hanging_the_activation"/>:
|
|
/// the same fake, answering normally, still authenticates.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Without it a build whose decorator denied <b>everything</b> would pass every deadline case in
|
|
/// this class, and the bound would look correct while having locked the fleet out.
|
|
/// </remarks>
|
|
[Fact]
|
|
public async Task A_reachable_store_still_authenticates()
|
|
{
|
|
var (presented, inner) = SetUp();
|
|
var auth = Auth(Bounded(inner, timeoutMs: 150, cacheSeconds: 0));
|
|
|
|
var result = await auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None);
|
|
|
|
result.Success.ShouldBeTrue();
|
|
result.Principal.ShouldBe(Principal);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A lookup that completes with a perfectly valid key <b>after</b> the deadline still denies —
|
|
/// nothing races back in past the bound.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_deadline_never_admits()
|
|
{
|
|
var (presented, inner) = SetUp();
|
|
var gate = new TaskCompletionSource();
|
|
inner.Gate = gate.Task;
|
|
var auth = Auth(Bounded(inner, timeoutMs: 100, cacheSeconds: 0));
|
|
|
|
var pending = auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None);
|
|
|
|
// Let the deadline pass, THEN hand the inner store its (valid) answer.
|
|
await Task.Delay(400);
|
|
gate.SetResult();
|
|
|
|
var result = await pending.WaitAsync(TimeSpan.FromSeconds(5));
|
|
result.Success.ShouldBeFalse();
|
|
result.Error.ShouldBe(ApiKeyOpcUaUserAuthenticator.DenyMessage);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A non-positive <c>StoreTimeoutMs</c> disables the deadline, restoring the unbounded behaviour
|
|
/// byte-for-byte — the documented rollback.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_zero_timeout_disables_the_deadline()
|
|
{
|
|
var (presented, inner) = SetUp();
|
|
var gate = new TaskCompletionSource();
|
|
inner.Gate = gate.Task;
|
|
var auth = Auth(Bounded(inner, timeoutMs: 0, cacheSeconds: 0));
|
|
|
|
var pending = auth.AuthenticateUserNameAsync("apikey", presented, CancellationToken.None);
|
|
|
|
// Far longer than any deadline this class configures; an active bound would have denied by now.
|
|
await Task.Delay(300);
|
|
pending.IsCompleted.ShouldBeFalse("a disabled deadline must not cancel a slow lookup");
|
|
|
|
gate.SetResult();
|
|
(await pending.WaitAsync(TimeSpan.FromSeconds(5))).Success.ShouldBeTrue();
|
|
}
|
|
|
|
/// <summary>The anonymous activation path gets the same deadline, and the same denial.</summary>
|
|
/// <remarks>
|
|
/// It runs on every unauthenticated connect, so leaving it unbounded would leave the
|
|
/// server-wide-stall hazard fully open on the busiest activation path.
|
|
/// </remarks>
|
|
[Fact]
|
|
public async Task Anonymous_grant_resolution_is_bounded_too()
|
|
{
|
|
var (_, inner) = SetUp();
|
|
inner.BlockForever = true;
|
|
var auth = Auth(Bounded(inner, timeoutMs: 150, cacheSeconds: 0));
|
|
|
|
var result = await auth.AuthenticateAnonymousAsync(CancellationToken.None)
|
|
.WaitAsync(TimeSpan.FromSeconds(5));
|
|
|
|
result.Success.ShouldBeFalse();
|
|
}
|
|
|
|
// ---------- TTL read-through cache ----------
|
|
|
|
/// <summary>A second activation inside the TTL is served from the cache and never reaches the store.</summary>
|
|
[Fact]
|
|
public async Task A_second_activation_within_the_TTL_does_not_reach_the_store()
|
|
{
|
|
var (_, inner) = SetUp();
|
|
var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
|
|
var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock);
|
|
|
|
(await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
|
|
clock.Advance(TimeSpan.FromSeconds(2));
|
|
(await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
|
|
|
|
inner.Calls.ShouldBe(1);
|
|
}
|
|
|
|
/// <summary>
|
|
/// FALSIFICATION CONTROL. A key revoked in the store is denied once the TTL has passed — the
|
|
/// cache expires rather than merely existing.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// A cache with no expiry at all passes every other case in this class: the hit test, the
|
|
/// read-through test and the fault test are all satisfied by "store once, serve forever". This is
|
|
/// the one that makes <c>StoreCacheSeconds</c> a bound rather than a claim.
|
|
/// </remarks>
|
|
[Fact]
|
|
public async Task A_revoked_key_is_denied_once_the_TTL_has_passed()
|
|
{
|
|
var (_, inner) = SetUp();
|
|
var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
|
|
var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock);
|
|
|
|
(await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default))
|
|
.ShouldNotBeNull("non-vacuity: the key really was usable before the revocation");
|
|
|
|
inner.Revoked = true;
|
|
|
|
// Still inside the TTL — the documented, accepted staleness window.
|
|
clock.Advance(TimeSpan.FromSeconds(3));
|
|
(await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
|
|
|
|
// Past it — the store is re-read and the revocation lands.
|
|
clock.Advance(TimeSpan.FromSeconds(3));
|
|
(await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull();
|
|
inner.Calls.ShouldBe(2);
|
|
}
|
|
|
|
/// <summary>A non-positive TTL disables the cache: every call reads through, today's behaviour.</summary>
|
|
[Fact]
|
|
public async Task TTL_zero_reads_through_every_time()
|
|
{
|
|
var (_, inner) = SetUp();
|
|
var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
|
|
var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 0, clock);
|
|
|
|
for (var i = 0; i < 3; i++)
|
|
await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default);
|
|
|
|
inner.Calls.ShouldBe(3);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A fault is never cached: the next call reaches the store, so an outage cannot outlive itself.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Caching a deny would be fail-closed and is still wrong — it would go on denying valid keys for
|
|
/// a further TTL after the database came back.
|
|
/// </remarks>
|
|
[Fact]
|
|
public async Task A_faulted_store_is_not_cached()
|
|
{
|
|
var (_, inner) = SetUp();
|
|
var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
|
|
var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 60, clock);
|
|
inner.ThrowsRemaining = 1;
|
|
|
|
await Should.ThrowAsync<InvalidOperationException>(
|
|
() => sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default));
|
|
|
|
// Same instant — a cached fault would still be "fresh" here, so the clock cannot rescue this.
|
|
(await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
|
|
inner.Calls.ShouldBe(2);
|
|
}
|
|
|
|
/// <summary>The anonymous grant lookup is cached on the same terms, keyed by principal.</summary>
|
|
[Fact]
|
|
public async Task Anonymous_grants_are_cached_too()
|
|
{
|
|
var (_, inner) = SetUp();
|
|
var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
|
|
var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock);
|
|
|
|
await sut.FindGrantsAsync(OpcUaPrincipals.Anonymous, default);
|
|
await sut.FindGrantsAsync(OpcUaPrincipals.Anonymous, default);
|
|
inner.Calls.ShouldBe(1);
|
|
|
|
clock.Advance(TimeSpan.FromSeconds(6));
|
|
await sut.FindGrantsAsync(OpcUaPrincipals.Anonymous, default);
|
|
inner.Calls.ShouldBe(2);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Past the entry ceiling the cache clears wholesale and keeps answering — a key-id guessing flood
|
|
/// is a bounded memory event, not an unbounded one.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Wholesale rather than per-entry eviction for the reason <c>SessionAclDecisionCache</c> already
|
|
/// documents: choosing WHICH entry to drop needs knowledge the cache does not have, and a
|
|
/// repopulate is cheap and cannot be wrong.
|
|
/// </remarks>
|
|
[Fact]
|
|
public async Task Cache_overflow_clears_wholesale_and_keeps_serving()
|
|
{
|
|
var (_, inner) = SetUp();
|
|
var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
|
|
var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock);
|
|
|
|
// Seed the real key, then flood past the ceiling with ids that do not exist.
|
|
(await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
|
|
for (var i = 0; i < BoundedApiKeyStore.MaxCacheEntries + 1; i++)
|
|
await sut.FindAsync($"ghost-{i}", clock.GetUtcNow().UtcDateTime, default);
|
|
|
|
var callsAfterFlood = inner.Calls;
|
|
|
|
// Still correct after the clear — and the real key was evicted, so it is re-read rather than
|
|
// served from a cache the flood was supposed to have emptied.
|
|
(await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull();
|
|
inner.Calls.ShouldBe(callsAfterFlood + 1);
|
|
}
|
|
|
|
// ---------- metering ----------
|
|
|
|
/// <summary>A deadline that fires meters <c>outcome=timeout</c> — the alarm signal for a sick ConfigDb.</summary>
|
|
[Fact]
|
|
public async Task A_timeout_meters_outcome_timeout()
|
|
{
|
|
var (_, inner) = SetUp();
|
|
inner.BlockForever = true;
|
|
using var meter = new MeterRecorder();
|
|
var sut = Bounded(inner, timeoutMs: 100, cacheSeconds: 0);
|
|
|
|
await Should.ThrowAsync<OperationCanceledException>(
|
|
() => sut.FindAsync(inner.KeyId, DateTime.UtcNow, default).WaitAsync(TimeSpan.FromSeconds(5)));
|
|
|
|
meter.WithOutcome("timeout").ShouldBe(1);
|
|
meter.WithOutcome("cache_hit").ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A cache hit meters <c>outcome=cache_hit</c> — the ratio that tells an operator the TTL is
|
|
/// doing something. A missing stream on a busy fused central is how you notice
|
|
/// <c>StoreCacheSeconds</c> was set to 0 by accident.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_cache_hit_meters_outcome_cache_hit()
|
|
{
|
|
var (_, inner) = SetUp();
|
|
var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero));
|
|
using var meter = new MeterRecorder();
|
|
var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 5, clock);
|
|
|
|
await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default);
|
|
meter.WithOutcome("cache_hit").ShouldBe(0, "the first call is a miss");
|
|
|
|
await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default);
|
|
meter.WithOutcome("cache_hit").ShouldBe(1);
|
|
}
|
|
|
|
/// <summary>A plain read-through — no deadline hit, no cache hit — meters nothing at all.</summary>
|
|
/// <remarks>
|
|
/// The counter names lookups the bound INTERVENED on. Metering every lookup would bury the two
|
|
/// outcomes that matter under the ordinary traffic they have to be spotted in.
|
|
/// </remarks>
|
|
[Fact]
|
|
public async Task A_plain_read_through_meters_neither()
|
|
{
|
|
var (_, inner) = SetUp();
|
|
using var meter = new MeterRecorder();
|
|
var sut = Bounded(inner, timeoutMs: 2000, cacheSeconds: 0);
|
|
|
|
await sut.FindAsync(inner.KeyId, DateTime.UtcNow, default);
|
|
|
|
meter.Total.ShouldBe(0);
|
|
}
|
|
|
|
// ---------- helpers ----------
|
|
|
|
private static BoundedApiKeyStore Bounded(
|
|
IApiKeyStore inner, int timeoutMs, int cacheSeconds, TimeProvider? clock = null) => new(
|
|
inner,
|
|
Options.Create(new ApiKeyOptions
|
|
{
|
|
Username = "apikey",
|
|
Pepper = Pepper,
|
|
StoreTimeoutMs = timeoutMs,
|
|
StoreCacheSeconds = cacheSeconds,
|
|
}),
|
|
NullLogger<BoundedApiKeyStore>.Instance,
|
|
clock ?? TimeProvider.System);
|
|
|
|
private static ApiKeyOpcUaUserAuthenticator Auth(IApiKeyStore store) => new(
|
|
store,
|
|
Options.Create(new ApiKeyOptions { Username = "apikey", Pepper = Pepper }),
|
|
NullLogger<ApiKeyOpcUaUserAuthenticator>.Instance);
|
|
|
|
private static (string Presented, GatedApiKeyStore Store) SetUp()
|
|
{
|
|
var (presented, apiKeyId, hash) = ApiKeySecret.Generate(Pepper);
|
|
var key = new ApiKey
|
|
{
|
|
ApiKeyId = apiKeyId,
|
|
Name = "test key",
|
|
SecretHash = hash,
|
|
Principal = Principal,
|
|
Enabled = true,
|
|
CreatedUtc = DateTime.UtcNow,
|
|
};
|
|
return (presented, new GatedApiKeyStore(key));
|
|
}
|
|
|
|
/// <summary>
|
|
/// An inner store whose answer can be withheld — a hang the EF in-memory provider cannot
|
|
/// represent — and whose calls are counted.
|
|
/// </summary>
|
|
private sealed class GatedApiKeyStore(ApiKey key) : IApiKeyStore
|
|
{
|
|
/// <summary>The id of the key this store holds.</summary>
|
|
public string KeyId => key.ApiKeyId;
|
|
|
|
/// <summary>When set, every call awaits this before answering.</summary>
|
|
public Task? Gate { get; set; }
|
|
|
|
/// <summary>When true, no call ever completes.</summary>
|
|
public bool BlockForever { get; set; }
|
|
|
|
/// <summary>How many calls reached this store (i.e. were not served from the cache).</summary>
|
|
public int Calls => _calls;
|
|
|
|
/// <summary>When true, the key lookup answers "no usable key" — a revocation.</summary>
|
|
public bool Revoked { get; set; }
|
|
|
|
/// <summary>Number of leading calls that throw before the store starts answering.</summary>
|
|
public int ThrowsRemaining { get; set; }
|
|
|
|
private int _calls;
|
|
|
|
public async Task<ApiKeyLookup?> FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct)
|
|
{
|
|
await WaitAsync(ct).ConfigureAwait(false);
|
|
if (Revoked) return null;
|
|
return string.Equals(apiKeyId, key.ApiKeyId, StringComparison.Ordinal)
|
|
? new ApiKeyLookup(key, [])
|
|
: null;
|
|
}
|
|
|
|
public async Task<IReadOnlyList<NodeAcl>> FindGrantsAsync(string principal, CancellationToken ct)
|
|
{
|
|
await WaitAsync(ct).ConfigureAwait(false);
|
|
return [];
|
|
}
|
|
|
|
private async Task WaitAsync(CancellationToken ct)
|
|
{
|
|
Interlocked.Increment(ref _calls);
|
|
|
|
if (ThrowsRemaining > 0)
|
|
{
|
|
ThrowsRemaining--;
|
|
throw new InvalidOperationException("store is down");
|
|
}
|
|
|
|
if (BlockForever) await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
|
|
if (Gate is { } gate) await gate.WaitAsync(ct).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <summary>Collects <c>otopcua.opcua.apikey_store.bounded</c> measurements by their outcome tag.</summary>
|
|
/// <remarks>
|
|
/// ⚠️ Filtered by instrument name only, which is normally the metrics-test leak (a listener sees
|
|
/// a PARALLEL test's measurements too). It is safe HERE for two checkable reasons:
|
|
/// <see cref="BoundedApiKeyStore"/> is the only thing in the product that writes this instrument,
|
|
/// and this assembly declares <c>CollectionBehavior(DisableTestParallelization = true)</c> in
|
|
/// <c>AssemblyInfo.cs</c>, so there is no parallel test to leak from. Removing that attribute
|
|
/// means adding a discriminating tag here.
|
|
/// </remarks>
|
|
private sealed class MeterRecorder : IDisposable
|
|
{
|
|
private readonly MeterListener _listener;
|
|
private readonly List<KeyValuePair<string, object?>[]> _tagSets = [];
|
|
private readonly Lock _gate = new();
|
|
private long _total;
|
|
|
|
public MeterRecorder()
|
|
{
|
|
_listener = new MeterListener
|
|
{
|
|
InstrumentPublished = (instrument, listener) =>
|
|
{
|
|
if (instrument.Meter.Name == OtOpcUaTelemetry.MeterName
|
|
&& instrument.Name == "otopcua.opcua.apikey_store.bounded")
|
|
{
|
|
listener.EnableMeasurementEvents(instrument);
|
|
}
|
|
},
|
|
};
|
|
_listener.SetMeasurementEventCallback<long>((_, value, tags, _) =>
|
|
{
|
|
lock (_gate)
|
|
{
|
|
_total += value;
|
|
_tagSets.Add(tags.ToArray());
|
|
}
|
|
});
|
|
_listener.Start();
|
|
}
|
|
|
|
/// <summary>Every measurement seen, regardless of tags.</summary>
|
|
public long Total
|
|
{
|
|
get { lock (_gate) return _total; }
|
|
}
|
|
|
|
/// <summary>How many measurements carried <c>outcome=<paramref name="outcome"/></c>.</summary>
|
|
/// <param name="outcome">The outcome tag value to count.</param>
|
|
/// <returns>The measurement count.</returns>
|
|
public int WithOutcome(string outcome)
|
|
{
|
|
lock (_gate)
|
|
return _tagSets.Count(set => set.Any(t => t.Key == "outcome" && Equals(t.Value, outcome)));
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public void Dispose() => _listener.Dispose();
|
|
}
|
|
|
|
/// <summary>
|
|
/// A hand-advanced clock. Overrides <c>GetUtcNow()</c> only — see the class remarks for why that
|
|
/// confines it to the TTL cases.
|
|
/// </summary>
|
|
private sealed class FakeClock(DateTimeOffset utcNow) : TimeProvider
|
|
{
|
|
private DateTimeOffset _now = utcNow;
|
|
|
|
/// <inheritdoc />
|
|
public override DateTimeOffset GetUtcNow() => _now;
|
|
|
|
/// <summary>Moves the clock forward.</summary>
|
|
/// <param name="by">How far forward to move.</param>
|
|
public void Advance(TimeSpan by) => _now = _now.Add(by);
|
|
}
|
|
}
|