Add client-side failover to CLI tool for redundancy testing

All commands gain --failover-urls (-F) to specify alternate endpoints.
Short-lived commands try each URL in order on initial connect. The
subscribe command monitors KeepAlive and automatically reconnects to
the next available server, re-creating the subscription on failover.
Verified with live service start/stop: primary down triggers failover
to secondary, primary restart allows failback.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

docs/CliTool.md

@@ -37,6 +37,29 @@ Example:
 dotnet run -- write -u opc.tcp://localhost:4840 -n "ns=2;s=MyNode" -v 42 -U operator -P op123
 ```
 
+## Failover Options
+
+All commands accept the `-F` / `--failover-urls` flag for automatic redundancy failover:
+
+| Flag | Description |
+|------|-------------|
+| `-F` / `--failover-urls` | Comma-separated list of alternate OPC UA endpoint URLs to try if the primary is unreachable |
+
+When failover URLs are provided, the CLI tries the primary URL first, then each failover URL in order until one connects. For long-running commands (`subscribe`), the CLI monitors the session and automatically reconnects to the next available server if the current one drops.
+
+Examples:
+
+```bash
+# Connect with failover
+dotnet run -- connect -u opc.tcp://localhost:4840/LmxOpcUa -F opc.tcp://localhost:4841/LmxOpcUa
+
+# Subscribe with automatic failover on disconnect
+dotnet run -- subscribe -u opc.tcp://localhost:4840/LmxOpcUa -F opc.tcp://localhost:4841/LmxOpcUa -n "ns=1;s=MyNode"
+
+# Read redundancy state with failover
+dotnet run -- redundancy -u opc.tcp://localhost:4840/LmxOpcUa -F opc.tcp://localhost:4841/LmxOpcUa
+```
+
 ## Transport Security Options
 
 All commands accept the `-S` / `--security` flag to select the transport security mode:

docs/Redundancy.md

@@ -162,6 +162,21 @@ Application URI:  urn:localhost:LmxOpcUa:instance1
 
 The command also supports `--username`/`--password` and `--security` options for authenticated or encrypted connections.
 
+### Client Failover with `-F`
+
+All CLI commands support the `-F` / `--failover-urls` flag for automatic client-side failover. When provided, the CLI tries the primary endpoint first and falls back to the listed URLs if the primary is unreachable.
+
+```bash
+# Connect with failover — uses secondary if primary is down
+dotnet run -- connect -u opc.tcp://localhost:4840/LmxOpcUa -F opc.tcp://localhost:4841/LmxOpcUa
+
+# Subscribe with live failover — reconnects to secondary if primary drops mid-stream
+dotnet run -- subscribe -u opc.tcp://localhost:4840/LmxOpcUa -F opc.tcp://localhost:4841/LmxOpcUa \
+  -n "ns=1;s=TestMachine_001.MachineID"
+```
+
+For long-running commands (`subscribe`), the CLI monitors the session KeepAlive and automatically reconnects to the next available server when the current session drops. The subscription is re-created on the new server.
+
 ## Troubleshooting
 
 **Mismatched ServerUris between instances** -- Both instances must list the exact same set of ApplicationUri values in `Redundancy.ServerUris`. If they differ, clients may not discover the full redundant set. Check the startup log for the `Redundancy.ServerUris` line on each instance.

tools/opcuacli-dotnet/Commands/AlarmsCommand.cs

@@ -24,6 +24,9 @@ public class AlarmsCommand : ICommand
     [CommandOption("security", 'S', Description = "Transport security: none, sign, encrypt (default: none)")]
     public string Security { get; init; } = "none";
 
+    [CommandOption("failover-urls", 'F', Description = "Comma-separated failover endpoint URLs for redundancy")]
+    public string? FailoverUrls { get; init; }
+
     /// <summary>
     /// Gets the node to subscribe to for event notifications, typically a source object or the server node.
     /// </summary>
@@ -48,7 +51,9 @@ public class AlarmsCommand : ICommand
     /// <param name="console">The CLI console used for cancellation and alarm-event output.</param>
     public async ValueTask ExecuteAsync(IConsole console)
     {
-        using var session = await OpcUaHelper.ConnectAsync(Url, Username, Password, Security);
+        var urls = FailoverUrlParser.Parse(Url, FailoverUrls);
+        using var failover = new OpcUaFailoverHelper(urls, Username, Password, Security);
+        using var session = await failover.ConnectAsync();
 
         var nodeId = string.IsNullOrEmpty(NodeId)
             ? ObjectIds.Server

tools/opcuacli-dotnet/Commands/BrowseCommand.cs

@@ -24,6 +24,9 @@ public class BrowseCommand : ICommand
     [CommandOption("security", 'S', Description = "Transport security: none, sign, encrypt (default: none)")]
     public string Security { get; init; } = "none";
 
+    [CommandOption("failover-urls", 'F', Description = "Comma-separated failover endpoint URLs for redundancy")]
+    public string? FailoverUrls { get; init; }
+
     /// <summary>
     /// Gets the optional node identifier to browse from; defaults to the OPC UA Objects folder.
     /// </summary>
@@ -48,7 +51,9 @@ public class BrowseCommand : ICommand
     /// <param name="console">The console used to emit browse output.</param>
     public async ValueTask ExecuteAsync(IConsole console)
     {
-        using var session = await OpcUaHelper.ConnectAsync(Url, Username, Password, Security);
+        var urls = FailoverUrlParser.Parse(Url, FailoverUrls);
+        using var failover = new OpcUaFailoverHelper(urls, Username, Password, Security);
+        using var session = await failover.ConnectAsync();
 
         var startNode = string.IsNullOrEmpty(NodeId)
             ? ObjectIds.ObjectsFolder

tools/opcuacli-dotnet/Commands/ConnectCommand.cs

@@ -22,13 +22,18 @@ public class ConnectCommand : ICommand
     [CommandOption("security", 'S', Description = "Transport security: none, sign, encrypt (default: none)")]
     public string Security { get; init; } = "none";
 
+    [CommandOption("failover-urls", 'F', Description = "Comma-separated failover endpoint URLs for redundancy")]
+    public string? FailoverUrls { get; init; }
+
     /// <summary>
     /// Connects to the OPC UA endpoint and prints the resolved server metadata.
     /// </summary>
     /// <param name="console">The console used to report connection results.</param>
     public async ValueTask ExecuteAsync(IConsole console)
     {
-        using var session = await OpcUaHelper.ConnectAsync(Url, Username, Password, Security);
+        var urls = FailoverUrlParser.Parse(Url, FailoverUrls);
+        using var failover = new OpcUaFailoverHelper(urls, Username, Password, Security);
+        using var session = await failover.ConnectAsync();
         await console.Output.WriteLineAsync($"Connected to: {session.Endpoint.EndpointUrl}");
         await console.Output.WriteLineAsync($"Server: {session.Endpoint.Server!.ApplicationName}");
         await console.Output.WriteLineAsync($"Security Mode: {session.Endpoint.SecurityMode}");

tools/opcuacli-dotnet/Commands/HistoryReadCommand.cs

@@ -24,6 +24,9 @@ public class HistoryReadCommand : ICommand
     [CommandOption("security", 'S', Description = "Transport security: none, sign, encrypt (default: none)")]
     public string Security { get; init; } = "none";
 
+    [CommandOption("failover-urls", 'F', Description = "Comma-separated failover endpoint URLs for redundancy")]
+    public string? FailoverUrls { get; init; }
+
     /// <summary>
     /// Gets the node identifier for the historized variable to query.
     /// </summary>
@@ -66,7 +69,9 @@ public class HistoryReadCommand : ICommand
     /// <param name="console">The CLI console used for output, errors, and cancellation handling.</param>
     public async ValueTask ExecuteAsync(IConsole console)
     {
-        using var session = await OpcUaHelper.ConnectAsync(Url, Username, Password, Security);
+        var urls = FailoverUrlParser.Parse(Url, FailoverUrls);
+        using var failover = new OpcUaFailoverHelper(urls, Username, Password, Security);
+        using var session = await failover.ConnectAsync();
 
         var nodeId = new NodeId(NodeId);
         var start = string.IsNullOrEmpty(StartTime) ? DateTime.UtcNow.AddHours(-24) : DateTime.Parse(StartTime).ToUniversalTime();

tools/opcuacli-dotnet/Commands/ReadCommand.cs

@@ -24,6 +24,9 @@ public class ReadCommand : ICommand
     [CommandOption("security", 'S', Description = "Transport security: none, sign, encrypt (default: none)")]
     public string Security { get; init; } = "none";
 
+    [CommandOption("failover-urls", 'F', Description = "Comma-separated failover endpoint URLs for redundancy")]
+    public string? FailoverUrls { get; init; }
+
     /// <summary>
     /// Gets the node identifier whose value should be read.
     /// </summary>
@@ -36,7 +39,9 @@ public class ReadCommand : ICommand
     /// <param name="console">The console used to report the read result.</param>
     public async ValueTask ExecuteAsync(IConsole console)
     {
-        using var session = await OpcUaHelper.ConnectAsync(Url, Username, Password, Security);
+        var urls = FailoverUrlParser.Parse(Url, FailoverUrls);
+        using var failover = new OpcUaFailoverHelper(urls, Username, Password, Security);
+        using var session = await failover.ConnectAsync();
 
         var node = new NodeId(NodeId);
         var value = await session.ReadValueAsync(node);

tools/opcuacli-dotnet/Commands/RedundancyCommand.cs

@@ -21,9 +21,14 @@ public class RedundancyCommand : ICommand
     [CommandOption("security", 'S', Description = "Transport security: none, sign, encrypt (default: none)")]
     public string Security { get; init; } = "none";
 
+    [CommandOption("failover-urls", 'F', Description = "Comma-separated failover endpoint URLs for redundancy")]
+    public string? FailoverUrls { get; init; }
+
     public async ValueTask ExecuteAsync(IConsole console)
     {
-        using var session = await OpcUaHelper.ConnectAsync(Url, Username, Password, Security);
+        var urls = FailoverUrlParser.Parse(Url, FailoverUrls);
+        using var failover = new OpcUaFailoverHelper(urls, Username, Password, Security);
+        using var session = await failover.ConnectAsync();
 
         // Read RedundancySupport
         var redundancySupportValue = await session.ReadValueAsync(VariableIds.Server_ServerRedundancy_RedundancySupport);

tools/opcuacli-dotnet/Commands/SubscribeCommand.cs

@@ -9,9 +9,6 @@ namespace OpcUaCli.Commands;
 [Command("subscribe", Description = "Monitor a node for value changes")]
 public class SubscribeCommand : ICommand
 {
-    /// <summary>
-    /// Gets the OPC UA endpoint URL to connect to before subscribing.
-    /// </summary>
     [CommandOption("url", 'u', Description = "OPC UA server endpoint URL", IsRequired = true)]
     public string Url { get; init; } = default!;
 
@@ -24,26 +21,108 @@ public class SubscribeCommand : ICommand
     [CommandOption("security", 'S', Description = "Transport security: none, sign, encrypt (default: none)")]
     public string Security { get; init; } = "none";
 
-    /// <summary>
+    [CommandOption("failover-urls", 'F', Description = "Comma-separated failover endpoint URLs for redundancy")]
-    /// Gets the node identifier to monitor for value changes.
+    public string? FailoverUrls { get; init; }
-    /// </summary>
+
     [CommandOption("node", 'n', Description = "Node ID to monitor", IsRequired = true)]
     public string NodeId { get; init; } = default!;
 
-    /// <summary>
-    /// Gets the sampling and publishing interval, in milliseconds, for the monitored item.
-    /// </summary>
     [CommandOption("interval", 'i', Description = "Polling interval in milliseconds")]
     public int Interval { get; init; } = 1000;
 
-    /// <summary>
-    /// Connects to the OPC UA endpoint and streams monitored-item notifications until cancellation.
-    /// </summary>
-    /// <param name="console">The console used to display subscription updates.</param>
     public async ValueTask ExecuteAsync(IConsole console)
     {
-        using var session = await OpcUaHelper.ConnectAsync(Url, Username, Password, Security);
+        var urls = FailoverUrlParser.Parse(Url, FailoverUrls);
+        var hasFailover = urls.Length > 1;
 
+        if (hasFailover)
+        {
+            await RunWithFailoverAsync(console, urls);
+        }
+        else
+        {
+            await RunSimpleAsync(console);
+        }
+    }
+
+    private async Task RunSimpleAsync(IConsole console)
+    {
+        using var session = await OpcUaHelper.ConnectAsync(Url, Username, Password, Security);
+        var (subscription, item) = await CreateSubscriptionAsync(session);
+        await console.Output.WriteLineAsync($"Subscribed to {NodeId} (interval: {Interval}ms). Press Ctrl+C to stop.");
+
+        var ct = console.RegisterCancellationHandler();
+        await MonitorLoopAsync(session, subscription, item, ct);
+        await console.Output.WriteLineAsync("Unsubscribed.");
+    }
+
+    private async Task RunWithFailoverAsync(IConsole console, string[] urls)
+    {
+        using var failover = new OpcUaFailoverHelper(urls, Username, Password, Security);
+        var session = await failover.ConnectAsync();
+
+        Subscription? subscription = null;
+        MonitoredItem? item = null;
+        var subLock = new object();
+
+        (subscription, item) = await CreateSubscriptionAsync(session);
+        await console.Output.WriteLineAsync(
+            $"Subscribed to {NodeId} (interval: {Interval}ms, failover enabled). Press Ctrl+C to stop.");
+
+        // Install failover handler
+        failover.OnFailover += (oldUrl, newUrl) =>
+        {
+            Console.WriteLine($"  [failover] Switched from {oldUrl} to {newUrl}");
+        };
+
+        failover.InstallKeepAliveHandler(async newSession =>
+        {
+            try
+            {
+                var (newSub, newItem) = await CreateSubscriptionAsync(newSession);
+                lock (subLock)
+                {
+                    subscription = newSub;
+                    item = newItem;
+                }
+                Console.WriteLine($"  [failover] Re-subscribed to {NodeId} on {failover.CurrentEndpointUrl}");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"  [failover] Failed to re-subscribe: {ex.Message}");
+            }
+        });
+
+        var ct = console.RegisterCancellationHandler();
+
+        int tick = 0;
+        while (!ct.IsCancellationRequested)
+        {
+            await Task.Delay(2000, ct).ContinueWith(_ => { });
+            tick++;
+
+            Session? currentSession;
+            MonitoredItem? currentItem;
+            Subscription? currentSub;
+            lock (subLock)
+            {
+                currentSession = failover.Session;
+                currentSub = subscription;
+                currentItem = item;
+            }
+
+            Console.WriteLine(
+                $"  [tick {tick}] Server={failover.CurrentEndpointUrl}, Connected={currentSession?.Connected}, " +
+                $"Sub.Id={currentSub?.Id}, " +
+                $"LastValue={((currentItem?.LastValue as MonitoredItemNotification)?.Value?.Value)} " +
+                $"({((currentItem?.LastValue as MonitoredItemNotification)?.Value?.StatusCode)})");
+        }
+
+        await console.Output.WriteLineAsync("Unsubscribed.");
+    }
+
+    private async Task<(Subscription, MonitoredItem)> CreateSubscriptionAsync(Session session)
+    {
         var subscription = new Subscription(session.DefaultSubscription)
         {
             PublishingInterval = Interval,
@@ -57,7 +136,7 @@ public class SubscribeCommand : ICommand
             SamplingInterval = Interval
         };
 
-        item.Notification += (monitoredItem, e) =>
+        item.Notification += (_, e) =>
         {
             if (e.NotificationValue is MonitoredItemNotification notification)
             {
@@ -70,11 +149,11 @@ public class SubscribeCommand : ICommand
         session.AddSubscription(subscription);
         await subscription.CreateAsync();
 
-        await console.Output.WriteLineAsync(
+        return (subscription, item);
-            $"Subscribed to {NodeId} (interval: {Interval}ms). Press Ctrl+C to stop.");
+    }
-
-        var ct = console.RegisterCancellationHandler();
 
+    private static async Task MonitorLoopAsync(Session session, Subscription subscription, MonitoredItem item, CancellationToken ct)
+    {
         int tick = 0;
         while (!ct.IsCancellationRequested)
         {
@@ -87,7 +166,5 @@ public class SubscribeCommand : ICommand
                 $"ItemStatus={item.Status?.Id}, " +
                 $"LastNotification={((item.LastValue as MonitoredItemNotification)?.Value?.Value)} ({((item.LastValue as MonitoredItemNotification)?.Value?.StatusCode)})");
         }
-
-        await console.Output.WriteLineAsync("Unsubscribed.");
     }
 }

tools/opcuacli-dotnet/Commands/WriteCommand.cs

@@ -24,6 +24,9 @@ public class WriteCommand : ICommand
     [CommandOption("security", 'S', Description = "Transport security: none, sign, encrypt (default: none)")]
     public string Security { get; init; } = "none";
 
+    [CommandOption("failover-urls", 'F', Description = "Comma-separated failover endpoint URLs for redundancy")]
+    public string? FailoverUrls { get; init; }
+
     /// <summary>
     /// Gets the node identifier that should receive the write.
     /// </summary>
@@ -42,7 +45,9 @@ public class WriteCommand : ICommand
     /// <param name="console">The console used to report the write result.</param>
     public async ValueTask ExecuteAsync(IConsole console)
     {
-        using var session = await OpcUaHelper.ConnectAsync(Url, Username, Password, Security);
+        var urls = FailoverUrlParser.Parse(Url, FailoverUrls);
+        using var failover = new OpcUaFailoverHelper(urls, Username, Password, Security);
+        using var session = await failover.ConnectAsync();
 
         var node = new NodeId(NodeId);
         var current = await session.ReadValueAsync(node);

tools/opcuacli-dotnet/OpcUaFailoverHelper.cs

@@ -0,0 +1,156 @@
+using Opc.Ua;
+using Opc.Ua.Client;
+
+namespace OpcUaCli;
+
+/// <summary>
+/// Manages OPC UA client sessions with automatic failover across a set of redundant server endpoints.
+/// </summary>
+public sealed class OpcUaFailoverHelper : IDisposable
+{
+    private readonly string[] _endpointUrls;
+    private readonly string? _username;
+    private readonly string? _password;
+    private readonly string _security;
+    private Session? _session;
+    private int _currentIndex;
+    private bool _disposed;
+
+    /// <summary>
+    /// Gets the active session, or null if not connected.
+    /// </summary>
+    public Session? Session => _session;
+
+    /// <summary>
+    /// Gets the endpoint URL the session is currently connected to.
+    /// </summary>
+    public string? CurrentEndpointUrl => _currentIndex < _endpointUrls.Length ? _endpointUrls[_currentIndex] : null;
+
+    /// <summary>
+    /// Raised when a failover occurs, providing the old and new endpoint URLs.
+    /// </summary>
+    public event Action<string?, string>? OnFailover;
+
+    public OpcUaFailoverHelper(string[] endpointUrls, string? username = null, string? password = null, string security = "none")
+    {
+        if (endpointUrls.Length == 0)
+            throw new ArgumentException("At least one endpoint URL is required.");
+        _endpointUrls = endpointUrls;
+        _username = username;
+        _password = password;
+        _security = security;
+        _currentIndex = 0;
+    }
+
+    /// <summary>
+    /// Connects to the first reachable server in the endpoint list.
+    /// </summary>
+    public async Task<Session> ConnectAsync()
+    {
+        for (int attempt = 0; attempt < _endpointUrls.Length; attempt++)
+        {
+            var idx = (_currentIndex + attempt) % _endpointUrls.Length;
+            var url = _endpointUrls[idx];
+            try
+            {
+                Console.WriteLine($"  [failover] Connecting to {url}...");
+                _session = await OpcUaHelper.ConnectAsync(url, _username, _password, _security);
+                _currentIndex = idx;
+                Console.WriteLine($"  [failover] Connected to {url}");
+                return _session;
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"  [failover] Failed to connect to {url}: {ex.Message}");
+            }
+        }
+
+        throw new InvalidOperationException("All redundant servers are unreachable.");
+    }
+
+    /// <summary>
+    /// Attempts to fail over to the next available server in the set.
+    /// Closes the old session if still open.
+    /// </summary>
+    /// <returns>The new session.</returns>
+    public async Task<Session> FailoverAsync()
+    {
+        var oldUrl = CurrentEndpointUrl;
+
+        // Close old session
+        if (_session != null)
+        {
+            try { _session.Close(); } catch { }
+            _session.Dispose();
+            _session = null;
+        }
+
+        // Try the next server first, then cycle through all
+        _currentIndex = (_currentIndex + 1) % _endpointUrls.Length;
+
+        var newSession = await ConnectAsync();
+        OnFailover?.Invoke(oldUrl, CurrentEndpointUrl!);
+        return newSession;
+    }
+
+    /// <summary>
+    /// Installs a KeepAlive handler that triggers automatic failover when the session drops.
+    /// Returns a task that completes when the session is lost and failover is needed.
+    /// </summary>
+    public void InstallKeepAliveHandler(Action<Session> onReconnected)
+    {
+        if (_session == null) return;
+
+        _session.KeepAlive += async (session, e) =>
+        {
+            if (e.Status == null || ServiceResult.IsGood(e.Status))
+                return;
+
+            Console.WriteLine($"  [failover] Session lost (status={e.Status}). Attempting failover...");
+
+            try
+            {
+                var newSession = await FailoverAsync();
+                onReconnected(newSession);
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"  [failover] All servers unreachable: {ex.Message}");
+            }
+        };
+    }
+
+    public void Dispose()
+    {
+        if (_disposed) return;
+        _disposed = true;
+        if (_session != null)
+        {
+            try { _session.Close(); } catch { }
+            _session.Dispose();
+        }
+    }
+}
+
+/// <summary>
+/// Parses the --failover-urls option into an array of endpoint URLs.
+/// If failover URLs are provided, the primary URL is prepended to form the full set.
+/// If not provided, returns only the primary URL (no failover).
+/// </summary>
+public static class FailoverUrlParser
+{
+    public static string[] Parse(string primaryUrl, string? failoverUrls)
+    {
+        if (string.IsNullOrWhiteSpace(failoverUrls))
+            return new[] { primaryUrl };
+
+        var urls = new List<string> { primaryUrl };
+        foreach (var url in failoverUrls.Split(',', StringSplitOptions.RemoveEmptyEntries))
+        {
+            var trimmed = url.Trim();
+            if (!string.IsNullOrEmpty(trimmed) && trimmed != primaryUrl)
+                urls.Add(trimmed);
+        }
+        return urls.ToArray();
+    }
+}