03686eb5d7
- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).
Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
99 lines
4.6 KiB
C#
99 lines
4.6 KiB
C#
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
|
using Microsoft.Extensions.Options;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.Health;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess;
|
|
using ZB.MOM.WW.OtOpcUa.Host.Health;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Health;
|
|
|
|
/// <summary>
|
|
/// lmxopcua#780 design §10, owner decision §14 #12: the script runner is a READY-tier component that is degraded but
|
|
/// still ready when its circuit is open — a sick runner must never fail readiness (on a redundant pair that would move
|
|
/// ServiceLevel to the peer over a script problem while OPC UA still serves driver data).
|
|
/// </summary>
|
|
public sealed class ScriptRunnerHealthCheckTests
|
|
{
|
|
private static ScriptWorkerShardStatus Shard(string label, bool up, double? downSeconds = null)
|
|
=> new(label, label == "iso", up, downSeconds is { } s ? TimeSpan.FromSeconds(s) : null, up ? 1234 : null, up ? 0 : 3, 0, 0, 0);
|
|
|
|
private static ScriptRunnerStatus Status(ScriptRunnerState state, params ScriptWorkerShardStatus[] shards)
|
|
=> new(state, shards, SuspectCount: 0, QuarantinedCount: 0, PeCacheEnabled: true);
|
|
|
|
[Fact]
|
|
public void In_process_is_healthy()
|
|
=> ScriptRunnerHealthDecision.Evaluate(null).Status.ShouldBe(HealthStatus.Healthy);
|
|
|
|
[Fact]
|
|
public void Ready_is_healthy()
|
|
=> ScriptRunnerHealthDecision.Evaluate(Status(ScriptRunnerState.Ready, Shard("0", true), Shard("1", true))).Status
|
|
.ShouldBe(HealthStatus.Healthy);
|
|
|
|
[Fact]
|
|
public void A_shard_down_is_degraded_and_names_it()
|
|
{
|
|
var (status, description, data) = ScriptRunnerHealthDecision.Evaluate(
|
|
Status(ScriptRunnerState.Degraded, Shard("0", true), Shard("1", false, 4)));
|
|
status.ShouldBe(HealthStatus.Degraded);
|
|
description.ShouldContain("ScriptWorker 1 down");
|
|
data["shard.1"].ShouldBe("down 4 s (3 failure(s))");
|
|
data["state"].ShouldBe("Degraded");
|
|
}
|
|
|
|
/// <summary>★ decision 12: the circuit-open runner is Degraded — never Unhealthy, so /health/ready stays 200.</summary>
|
|
[Fact]
|
|
public void An_open_circuit_is_degraded_never_unhealthy()
|
|
{
|
|
var (status, description, _) = ScriptRunnerHealthDecision.Evaluate(
|
|
Status(ScriptRunnerState.Unavailable, Shard("0", false, 200), Shard("1", false, 200)));
|
|
status.ShouldBe(HealthStatus.Degraded);
|
|
description.ShouldContain("circuit open");
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(false, false)]
|
|
[InlineData(true, true)]
|
|
public async Task It_is_registered_on_the_ready_tier_only_on_driver_nodes(bool hasDriver, bool registered)
|
|
{
|
|
var registrations = new ServiceCollection()
|
|
.AddOtOpcUaHealth(hasAdmin: true, hasDriver: hasDriver)
|
|
.BuildServiceProvider()
|
|
.GetRequiredService<IOptions<HealthCheckServiceOptions>>()
|
|
.Value.Registrations
|
|
.Where(r => r.Name == ScriptRunnerHealthCheck.Name)
|
|
.ToList();
|
|
if (!registered)
|
|
{
|
|
registrations.ShouldBeEmpty();
|
|
return;
|
|
}
|
|
var registration = registrations.ShouldHaveSingleItem();
|
|
registration.Tags.ShouldBe([ZbHealthTags.Ready]);
|
|
// A node without a runner selection (in-process, or a harness) resolves it as Healthy.
|
|
var check = registration.Factory(new ServiceCollection().BuildServiceProvider()).ShouldBeOfType<ScriptRunnerHealthCheck>();
|
|
(await check.CheckHealthAsync(new HealthCheckContext { Registration = registration }, CancellationToken.None))
|
|
.Status.ShouldBe(HealthStatus.Healthy);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(ScriptRunnerState.Ready)]
|
|
[InlineData(ScriptRunnerState.Degraded)]
|
|
[InlineData(ScriptRunnerState.Unavailable)]
|
|
public void Shadow_is_healthy_whatever_its_worker_pool_does(ScriptRunnerState shadowState)
|
|
{
|
|
// lmxopcua#780 P2: under Shadow every published value is in process — the shadow pool must never move readiness.
|
|
var stats = new ZB.MOM.WW.OtOpcUa.Core.Scripting.ScriptShadowStats(10, 7, 3, 1, 2, 1, 0);
|
|
var (status, description, data) = ScriptRunnerHealthDecision.Evaluate(
|
|
null, (Status(shadowState, Shard("0", shadowState == ScriptRunnerState.Ready, 200)), stats));
|
|
status.ShouldBe(HealthStatus.Healthy);
|
|
description.ShouldContain("Shadow");
|
|
description.ShouldContain("3 mismatch(es)");
|
|
data["mode"].ShouldBe("Shadow");
|
|
data["shadow.state"].ShouldBe(shadowState.ToString());
|
|
data["shadow.mismatched"].ShouldBe(3L);
|
|
data["shadow.dropped"].ShouldBe(3L);
|
|
}
|
|
}
|