Joseph Doherty
e1db90f9c9
Two review items from Task 1B.1, both diagnostic: 1. CentralCommandGrpcService's "actor not ready" Warning omitted the peer and the correlation id while the decode-failure Warning six lines above carried both. These are the only two ways an ack fails to reach the coordinator, and an operator has to correlate either against the node's own log. Decoding has already succeeded on this path, so the DECODED correlation id is free. 2. A fused driver,admin,cluster-MAIN node under Mode=Grpc owes BOTH validator surfaces at once (driver keys AND ConfigServe:GrpcListenPort). The two blocks are independent `if`s; chained as `else if` a fused central node would be graded on one surface only and boot with nowhere for any ack to land. Pinned from both sides plus the fully-configured pass.
513 lines
20 KiB
C#
513 lines
20 KiB
C#
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.Options;
|
|
using Shouldly;
|
|
using Xunit;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests;
|
|
|
|
/// <summary>
|
|
/// Every failure this validator catches otherwise surfaces as an <i>absence</i> — a deployment
|
|
/// that never arrives, an alarm ack that does nothing, a node whose ack is dropped without a
|
|
/// trace. Absences are the hardest class of fault to attribute in the field, so the whole point
|
|
/// of these tests is that the host refuses to start instead.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Role-dependent facts run in <see cref="RoleEnvironmentCollection"/> because the validator
|
|
/// resolves roles through <c>RoleParser.ResolveEffectiveRoles</c>, which reads
|
|
/// <c>OTOPCUA_ROLES</c> when <c>Cluster:Roles</c> is empty — a process-wide env var, so the
|
|
/// whole class is serialized against the other role-reading suites.
|
|
/// </remarks>
|
|
[Collection(RoleEnvironmentCollection.Name)]
|
|
public class MeshTransportOptionsValidatorTests
|
|
{
|
|
private static ValidateOptionsResult Validate(MeshTransportOptions o, params string[] roles) =>
|
|
Validate(o, new Dictionary<string, string?>(), roles);
|
|
|
|
/// <summary>
|
|
/// As <see cref="Validate(MeshTransportOptions, string[])"/>, plus arbitrary configuration
|
|
/// keys — the admin-side rule cross-reads <c>ConfigServe:GrpcListenPort</c>, which lives
|
|
/// outside this options class entirely.
|
|
/// </summary>
|
|
private static ValidateOptionsResult Validate(
|
|
MeshTransportOptions o, Dictionary<string, string?> extra, params string[] roles)
|
|
{
|
|
var pairs = new Dictionary<string, string?>(extra);
|
|
for (var i = 0; i < roles.Length; i++)
|
|
{
|
|
pairs[$"Cluster:Roles:{i}"] = roles[i];
|
|
}
|
|
|
|
var configuration = new ConfigurationBuilder().AddInMemoryCollection(pairs).Build();
|
|
|
|
return new MeshTransportOptionsValidator(configuration)
|
|
.Validate(MeshTransportOptions.SectionName, o);
|
|
}
|
|
|
|
private static Dictionary<string, string?> ConfigServePort(string? value) =>
|
|
new() { ["ConfigServe:GrpcListenPort"] = value };
|
|
|
|
private static MeshTransportOptions WellFormedGrpc() => new()
|
|
{
|
|
Mode = MeshTransportOptions.ModeGrpc,
|
|
GrpcPsk = "not-a-real-key",
|
|
CentralGrpcEndpoints = ["http://central-1:4055", "https://central-2:4055"],
|
|
};
|
|
|
|
[Fact]
|
|
public void Default_options_are_valid()
|
|
{
|
|
// The default is Dps — the pre-Phase-2 transport. A node that never sets this section must
|
|
// behave exactly as it did before, which is the whole premise of the dark switch.
|
|
Validate(new MeshTransportOptions()).Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Unknown_mode_fails()
|
|
{
|
|
var result = Validate(new MeshTransportOptions { Mode = "receptionist" });
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("receptionist");
|
|
}
|
|
|
|
[Fact]
|
|
public void Unknown_mode_message_names_all_three_accepted_values()
|
|
{
|
|
// An operator who mistypes the mode gets the full menu, not a partial one — Grpc became a
|
|
// legal value in Phase 0 and an error that omitted it would send them back to ClusterClient.
|
|
var result = Validate(new MeshTransportOptions { Mode = "receptionist" });
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain(MeshTransportOptions.ModeDps);
|
|
result.FailureMessage.ShouldContain(MeshTransportOptions.ModeClusterClient);
|
|
result.FailureMessage.ShouldContain(MeshTransportOptions.ModeGrpc);
|
|
}
|
|
|
|
[Fact]
|
|
public void Mode_matching_is_case_insensitive()
|
|
{
|
|
Validate(new MeshTransportOptions
|
|
{
|
|
Mode = "clusterclient",
|
|
CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"],
|
|
}).Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Non_positive_contact_refresh_fails()
|
|
{
|
|
var result = Validate(new MeshTransportOptions { ContactRefreshSeconds = 0 });
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain(nameof(MeshTransportOptions.ContactRefreshSeconds));
|
|
}
|
|
|
|
[Fact]
|
|
public void ClusterClient_mode_requires_central_contact_points()
|
|
{
|
|
var result = Validate(new MeshTransportOptions
|
|
{
|
|
Mode = MeshTransportOptions.ModeClusterClient,
|
|
CentralContactPoints = [],
|
|
});
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain(nameof(MeshTransportOptions.CentralContactPoints));
|
|
}
|
|
|
|
[Fact]
|
|
public void Dps_mode_does_not_require_central_contact_points()
|
|
{
|
|
// An admin-only node has no reason to carry them, and requiring them under the default mode
|
|
// would make the section mandatory everywhere for a feature that is switched off.
|
|
Validate(new MeshTransportOptions
|
|
{
|
|
Mode = MeshTransportOptions.ModeDps,
|
|
CentralContactPoints = [],
|
|
}).Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void A_contact_point_that_is_not_an_akka_address_fails()
|
|
{
|
|
var result = Validate(new MeshTransportOptions
|
|
{
|
|
Mode = MeshTransportOptions.ModeClusterClient,
|
|
CentralContactPoints = ["central-1:4053"],
|
|
});
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("akka.tcp://");
|
|
}
|
|
|
|
[Fact]
|
|
public void A_contact_point_carrying_an_actor_path_suffix_fails()
|
|
{
|
|
// The sister project's shipped site template listed a full receptionist path, and separately
|
|
// listed the site's OWN remoting port as the second contact — "a permanent failure in the
|
|
// initial-contact rotation". Both are invisible until a command goes missing.
|
|
var result = Validate(new MeshTransportOptions
|
|
{
|
|
Mode = MeshTransportOptions.ModeClusterClient,
|
|
CentralContactPoints = ["akka.tcp://otopcua@central-1:4053/system/receptionist"],
|
|
});
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("/system/receptionist");
|
|
}
|
|
|
|
[Fact]
|
|
public void One_bad_contact_point_among_good_ones_still_fails()
|
|
{
|
|
var result = Validate(new MeshTransportOptions
|
|
{
|
|
Mode = MeshTransportOptions.ModeClusterClient,
|
|
CentralContactPoints =
|
|
[
|
|
"akka.tcp://otopcua@central-1:4053",
|
|
"central-2:4053",
|
|
],
|
|
});
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("central-2:4053");
|
|
}
|
|
|
|
[Fact]
|
|
public void Well_formed_ClusterClient_options_succeed()
|
|
{
|
|
Validate(new MeshTransportOptions
|
|
{
|
|
Mode = MeshTransportOptions.ModeClusterClient,
|
|
CentralContactPoints =
|
|
[
|
|
"akka.tcp://otopcua@central-1:4053",
|
|
"akka.tcp://otopcua@central-2:4053",
|
|
],
|
|
}).Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
// ---- Grpc mode (ClusterClient→gRPC migration, Phase 0) --------------------------------------
|
|
|
|
[Fact]
|
|
public void Grpc_is_an_accepted_mode()
|
|
{
|
|
Validate(WellFormedGrpc(), "driver", "cluster-SITE-A").Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_mode_matching_is_case_insensitive()
|
|
{
|
|
var options = WellFormedGrpc();
|
|
options.Mode = "grpc";
|
|
|
|
Validate(options, "driver", "cluster-SITE-A").Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_driver_node_without_a_psk_fails()
|
|
{
|
|
var options = WellFormedGrpc();
|
|
options.GrpcPsk = null;
|
|
|
|
var result = Validate(options, "driver", "cluster-SITE-A");
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("MeshTransport:GrpcPsk");
|
|
result.FailureMessage.ShouldContain("MeshTransport__GrpcPsk");
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_driver_node_with_an_empty_psk_fails()
|
|
{
|
|
// Empty is the shape an unset environment variable actually produces, and it is exactly as
|
|
// fatal as null — a blank bearer authenticates nothing.
|
|
var options = WellFormedGrpc();
|
|
options.GrpcPsk = " ";
|
|
|
|
Validate(options, "driver", "cluster-SITE-A").Failed.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void The_psk_value_is_never_echoed_in_a_failure_message()
|
|
{
|
|
// The whole point of a validator that reads a secret is that its diagnostics are safe to
|
|
// paste into a ticket. Naming the key is the contract; printing it is a leak.
|
|
var options = WellFormedGrpc();
|
|
options.GrpcPsk = "super-secret-mesh-psk";
|
|
options.CentralGrpcEndpoints = ["central-1:4055"];
|
|
|
|
var result = Validate(options, "driver", "cluster-SITE-A");
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldNotContain("super-secret-mesh-psk");
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_driver_node_without_central_endpoints_fails()
|
|
{
|
|
var options = WellFormedGrpc();
|
|
options.CentralGrpcEndpoints = [];
|
|
|
|
var result = Validate(options, "driver", "cluster-SITE-A");
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("MeshTransport:CentralGrpcEndpoints");
|
|
}
|
|
|
|
[Fact]
|
|
public void A_central_endpoint_that_is_not_an_http_url_fails()
|
|
{
|
|
var options = WellFormedGrpc();
|
|
options.CentralGrpcEndpoints = ["central-1:4055"];
|
|
|
|
var result = Validate(options, "driver", "cluster-SITE-A");
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("central-1:4055");
|
|
result.FailureMessage.ShouldContain("http://");
|
|
}
|
|
|
|
[Fact]
|
|
public void One_bad_central_endpoint_among_good_ones_still_fails()
|
|
{
|
|
var options = WellFormedGrpc();
|
|
options.CentralGrpcEndpoints = ["http://central-1:4055", "akka.tcp://otopcua@central-2:4053"];
|
|
|
|
var result = Validate(options, "driver", "cluster-SITE-A");
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("akka.tcp://otopcua@central-2:4053");
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_driver_node_with_a_non_positive_dispatch_timeout_fails()
|
|
{
|
|
var options = WellFormedGrpc();
|
|
options.DispatchTimeoutSeconds = 0;
|
|
|
|
var result = Validate(options, "driver", "cluster-SITE-A");
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("MeshTransport:DispatchTimeoutSeconds");
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_driver_node_with_a_non_positive_ack_timeout_fails()
|
|
{
|
|
var options = WellFormedGrpc();
|
|
options.AckTimeoutSeconds = -1;
|
|
|
|
var result = Validate(options, "driver", "cluster-SITE-A");
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("MeshTransport:AckTimeoutSeconds");
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_admin_only_node_needs_none_of_the_node_side_keys()
|
|
{
|
|
// The node-side surface is what a DRIVER uses to dial central and post its ack. An admin-only
|
|
// central node dials nodes instead and resolves per-cluster keys at runtime, so requiring
|
|
// these here would make an admin node carry a PSK and an endpoint list it never reads.
|
|
// (The ConfigServe port IS required of an admin node under Grpc — supplied here so this test
|
|
// stays about the node-side keys and nothing else.)
|
|
Validate(
|
|
new MeshTransportOptions { Mode = MeshTransportOptions.ModeGrpc },
|
|
ConfigServePort("4055"),
|
|
"admin", "cluster-MAIN").Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_admin_node_without_a_config_serve_listener_fails()
|
|
{
|
|
// Central hosts CentralCommandService on the ConfigServe h2c listener, so an unset port means
|
|
// the service is never mapped and every node's ack has nowhere to land. Nothing errors at
|
|
// runtime — the deployment simply times out naming nodes that applied perfectly, which is the
|
|
// exact absence this validator exists to convert into a refusal to start.
|
|
var result = Validate(
|
|
new MeshTransportOptions { Mode = MeshTransportOptions.ModeGrpc },
|
|
"admin", "cluster-MAIN");
|
|
|
|
result.Failed.ShouldBeTrue();
|
|
result.FailureMessage.ShouldContain("ConfigServe:GrpcListenPort");
|
|
result.FailureMessage.ShouldContain("ConfigServe__GrpcListenPort");
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_admin_node_with_a_zero_config_serve_port_fails()
|
|
{
|
|
// 0 is the documented "disabled" value, not merely a missing key — it must fail exactly as an
|
|
// absent one does, because it binds nothing just as surely.
|
|
Validate(
|
|
new MeshTransportOptions { Mode = MeshTransportOptions.ModeGrpc },
|
|
ConfigServePort("0"),
|
|
"admin", "cluster-MAIN").Failed.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_admin_node_with_an_unparseable_config_serve_port_fails()
|
|
{
|
|
// A typo'd port binds nothing, so it belongs on the same side of the line as an absent one.
|
|
Validate(
|
|
new MeshTransportOptions { Mode = MeshTransportOptions.ModeGrpc },
|
|
ConfigServePort("40o55"),
|
|
"admin", "cluster-MAIN").Failed.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_admin_node_with_a_config_serve_listener_passes()
|
|
{
|
|
Validate(
|
|
new MeshTransportOptions { Mode = MeshTransportOptions.ModeGrpc },
|
|
ConfigServePort("4055"),
|
|
"admin", "cluster-MAIN").Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_driver_only_node_needs_no_config_serve_listener()
|
|
{
|
|
// A driver node dials central; it serves no central command endpoint at all, and requiring an
|
|
// admin-side listener port of it would demand configuration it never reads.
|
|
Validate(WellFormedGrpc(), "driver", "cluster-SITE-A").Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Dps_admin_node_needs_no_config_serve_listener()
|
|
{
|
|
// The no-new-obligations invariant, admin side: a central node still on the default transport
|
|
// publishes on DistributedPubSub and hosts no command endpoint.
|
|
Validate(
|
|
new MeshTransportOptions { Mode = MeshTransportOptions.ModeDps },
|
|
"admin", "cluster-MAIN").Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void ClusterClient_admin_node_needs_no_config_serve_listener()
|
|
{
|
|
// Contact points are required by the mode itself, independently of role — supplied so this
|
|
// test fails only if the ConfigServe rule wrongly fires outside Grpc mode.
|
|
Validate(
|
|
new MeshTransportOptions
|
|
{
|
|
Mode = MeshTransportOptions.ModeClusterClient,
|
|
CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"],
|
|
},
|
|
"admin", "cluster-MAIN").Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void The_admin_role_is_also_read_from_OTOPCUA_ROLES()
|
|
{
|
|
// Same roles-source divergence as the driver case (Gitea #532), on the side that hosts the
|
|
// endpoint: a central node configured only through the env var is still the node whose
|
|
// listener every ack depends on.
|
|
using (new EnvVarScope("OTOPCUA_ROLES", "admin,cluster-MAIN"))
|
|
{
|
|
Validate(new MeshTransportOptions { Mode = MeshTransportOptions.ModeGrpc })
|
|
.Failed.ShouldBeTrue();
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_role_less_node_needs_none_of_the_node_side_keys()
|
|
{
|
|
// Every in-process test host and every single-mesh node has no declared role. Requiring the
|
|
// Grpc surface of them would turn a dark-switch value into a tree-wide config obligation.
|
|
Validate(new MeshTransportOptions { Mode = MeshTransportOptions.ModeGrpc }).Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Roles_come_from_OTOPCUA_ROLES_when_Cluster_Roles_is_empty()
|
|
{
|
|
// Roles-source divergence (Gitea #532): a node configured only through the env var is still a
|
|
// driver node in Akka. A validator reading Cluster:Roles alone would exempt the exact node
|
|
// that has to dial central, and its acks would be rejected unauthenticated at runtime.
|
|
using (new EnvVarScope("OTOPCUA_ROLES", "driver,cluster-SITE-A"))
|
|
{
|
|
var options = WellFormedGrpc();
|
|
options.GrpcPsk = null;
|
|
|
|
Validate(options).Failed.ShouldBeTrue();
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Dps_mode_requires_none_of_the_new_Grpc_keys()
|
|
{
|
|
// The no-new-obligations invariant: adding a third mode must not make any existing
|
|
// deployment's configuration invalid. A Dps driver node that sets nothing new still passes.
|
|
Validate(
|
|
new MeshTransportOptions { Mode = MeshTransportOptions.ModeDps },
|
|
"driver", "cluster-SITE-A").Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void ClusterClient_mode_requires_none_of_the_new_Grpc_keys()
|
|
{
|
|
Validate(
|
|
new MeshTransportOptions
|
|
{
|
|
Mode = MeshTransportOptions.ModeClusterClient,
|
|
CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"],
|
|
},
|
|
"driver", "cluster-SITE-A").Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_fused_admin_and_driver_node_must_satisfy_BOTH_surfaces()
|
|
{
|
|
// The central pair is fused (admin,driver,cluster-MAIN), so it is the one node shape that owes
|
|
// both halves at once — and the two blocks are independent `if`s, not an `if`/`else if`. Were
|
|
// they chained, a fused node would be graded on the driver surface only and its missing
|
|
// ConfigServe listener would pass validation, leaving central with no place for any node's ack
|
|
// to land. Asserted from BOTH sides so a future chaining in either order is caught.
|
|
var missingNodeSide = new MeshTransportOptions { Mode = MeshTransportOptions.ModeGrpc };
|
|
var withPort = Validate(missingNodeSide, ConfigServePort("4055"), "driver", "admin", "cluster-MAIN");
|
|
withPort.Failed.ShouldBeTrue();
|
|
withPort.FailureMessage.ShouldContain("MeshTransport:GrpcPsk");
|
|
withPort.FailureMessage.ShouldContain("MeshTransport:CentralGrpcEndpoints");
|
|
|
|
var missingAdminSide = Validate(WellFormedGrpc(), "driver", "admin", "cluster-MAIN");
|
|
missingAdminSide.Failed.ShouldBeTrue();
|
|
missingAdminSide.FailureMessage.ShouldContain("ConfigServe:GrpcListenPort");
|
|
|
|
// And the fully-configured fused node passes — the rule is "both", not "neither".
|
|
Validate(WellFormedGrpc(), ConfigServePort("4055"), "driver", "admin", "cluster-MAIN")
|
|
.Succeeded.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void ClusterPsks_defaults_to_an_empty_case_insensitive_map()
|
|
{
|
|
// Production leaves the map empty and the runtime provider falls back to the Secrets store;
|
|
// the rig populates it. Case-insensitivity matters because a ClusterId reaches this lookup
|
|
// from three sources — a role suffix, a DB column and an env key — that do not agree on case.
|
|
var options = new MeshTransportOptions();
|
|
|
|
options.ClusterPsks.ShouldBeEmpty();
|
|
options.ClusterPsks["MAIN"] = "rig-key";
|
|
options.ClusterPsks.ContainsKey("main").ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void ClusterPsks_stays_case_insensitive_after_binding_configuration()
|
|
{
|
|
// Asserting the comparer on a fresh instance proves nothing about the shape that reaches
|
|
// production: the options binder could construct its own Dictionary and drop the comparer
|
|
// silently, and the symptom would be a rig lookup that misses on a ClusterId spelled the way
|
|
// the DB spells it. Bind for real and check the comparer survived.
|
|
var configuration = new ConfigurationBuilder()
|
|
.AddInMemoryCollection(new Dictionary<string, string?>
|
|
{
|
|
["MeshTransport:ClusterPsks:SITE-A"] = "rig-key",
|
|
})
|
|
.Build();
|
|
|
|
var options = new MeshTransportOptions();
|
|
configuration.GetSection(MeshTransportOptions.SectionName).Bind(options);
|
|
|
|
options.ClusterPsks["site-a"].ShouldBe("rig-key");
|
|
}
|
|
}
|