perf(mqtt): publish a write batch concurrently, serialised per topic; writes feed health (R06:M-8)

WriteAsync published one awaited QoS 1 round-trip per request, so 50 writes to a
stalled broker cost 50 publish deadlines. Different topics now publish concurrently
(at most 8 in flight); same-topic writes stay in request order. A caller cancel still
fills every slot. A successful publish advances LastSuccessfulRead. MiniBroker now
frames packets properly and records/PUBACKs PUBLISHes.

Refs #744
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs

@@ -104,6 +104,12 @@ public sealed class MqttDriver
     /// </summary>
     private const string DiscoveryFolderName = "Mqtt";
 
+    /// <summary>
+    ///     Ceiling on PUBLISHes one write batch keeps in flight (R06:M-8). Bounded so a large batch cannot
+    ///     flood a broker's inflight window; different topics share it, one topic's writes stay sequential.
+    /// </summary>
+    internal const int MaxConcurrentPublishes = 8;
+
     /// <summary>How often the host-connectivity poll samples <see cref="MqttConnection.State"/>.</summary>
     private static readonly TimeSpan HostProbeInterval = TimeSpan.FromSeconds(1);
 
@@ -209,6 +215,9 @@ public sealed class MqttDriver
 
     private DriverHealth _health = new(DriverState.Unknown, null, null);
 
+    /// <summary>UTC ticks of the last successful write PUBLISH; 0 = none. Answer evidence for health.</summary>
+    private long _lastPublishUtcTicks;
+
     /// <summary>Guards the generation bump against the generation-checked health write.</summary>
     private readonly object _healthLock = new();
 
@@ -446,7 +455,7 @@ public sealed class MqttDriver
             return stored;
         }
 
-        var lastMessage = connection.LastMessageUtc ?? stored.LastSuccessfulRead;
+        var lastMessage = Latest(connection.LastMessageUtc ?? stored.LastSuccessfulRead, LastPublishUtc);
         return connection.State switch
         {
             MqttConnectionState.Connected => new DriverHealth(DriverState.Healthy, lastMessage, stored.LastError),
@@ -708,9 +717,12 @@ public sealed class MqttDriver
     {
         ArgumentNullException.ThrowIfNull(writes);
 
-        var results = new WriteResult[writes.Count];
+        var results = new WriteResult?[writes.Count];
         var connection = _connection;
 
+        // Pass 1 — everything that needs no broker: resolve, encode, refuse. What survives is grouped by
+        // destination topic in REQUEST order.
+        var byTopic = new Dictionary<string, List<(int Index, MqttEncodedWrite Encoded, string Reference)>>(StringComparer.Ordinal);
         for (var i = 0; i < writes.Count; i++)
         {
             var w = writes[i];
@@ -738,22 +750,70 @@ public sealed class MqttDriver
                 continue;
             }
 
+            if (!byTopic.TryGetValue(encoded.Topic, out var queue))
+            {
+                byTopic[encoded.Topic] = queue = [];
+            }
+
+            queue.Add((i, encoded, w.FullReference));
+        }
+
+        // Pass 2 — publish (R06:M-8). Different topics go out concurrently, at most MaxConcurrentPublishes
+        // in flight, so a stalled broker costs a batch ~one publish deadline instead of one per request.
+        // Writes to the SAME topic stay strictly sequential in request order: the device applies them in
+        // arrival order, and the last one must win.
+        if (byTopic.Count > 0)
+        {
+            using var slots = new SemaphoreSlim(MaxConcurrentPublishes, MaxConcurrentPublishes);
+            await Task.WhenAll(byTopic.Values.Select(queue => PublishQueueAsync(connection!, queue, slots, results, cancellationToken)))
+                .ConfigureAwait(false);
+        }
+
+        // Every slot is filled — a caller cancellation leaves the unattempted ones BadCommunicationError
+        // rather than throwing, so the batch shape holds.
+        var filled = new WriteResult[writes.Count];
+        for (var i = 0; i < filled.Length; i++)
+        {
+            filled[i] = results[i] ?? new WriteResult(StatusBadCommunicationError);
+        }
+
+        return filled;
+    }
+
+    /// <summary>Publishes one topic's writes in request order, each under a shared concurrency slot.</summary>
+    private async Task PublishQueueAsync(
+        MqttConnection connection,
+        List<(int Index, MqttEncodedWrite Encoded, string Reference)> queue,
+        SemaphoreSlim slots,
+        WriteResult?[] results,
+        CancellationToken cancellationToken)
+    {
+        foreach (var (index, encoded, reference) in queue)
+        {
+            try
+            {
+                await slots.WaitAsync(cancellationToken).ConfigureAwait(false);
+            }
+            catch (OperationCanceledException)
+            {
+                return;   // caller-cancelled: the remaining slots are filled by the caller
+            }
+
             try
             {
                 await connection
                     .PublishAsync(encoded.Topic, encoded.Payload, encoded.Qos, retain: false, cancellationToken)
                     .ConfigureAwait(false);
-                results[i] = new WriteResult(0u);
+                results[index] = new WriteResult(0u);
+
+                // A PUBACK (or a sent QoS 0 command) is the broker answering — evidence the session works,
+                // so it feeds LastSuccessfulRead alongside inbound messages.
+                Interlocked.Exchange(ref _lastPublishUtcTicks, DateTime.UtcNow.Ticks);
             }
             catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
             {
-                // Caller-cancelled: fill the remainder rather than throwing, so the batch shape holds.
-                for (var j = i; j < writes.Count; j++)
-                {
-                    results[j] = new WriteResult(StatusBadCommunicationError);
-                }
-
-                return results;
+                results[index] = new WriteResult(StatusBadCommunicationError);
+                return;
             }
             catch (Exception ex)
             {
@@ -761,13 +821,15 @@ public sealed class MqttDriver
                     ex,
                     "MQTT driver '{DriverId}': write to '{Reference}' could not be published on '{Topic}'.",
                     _driverInstanceId,
-                    w.FullReference,
+                    reference,
                     encoded.Topic);
-                results[i] = new WriteResult(StatusBadCommunicationError);
+                results[index] = new WriteResult(StatusBadCommunicationError);
+            }
+            finally
+            {
+                slots.Release();
             }
         }
-
-        return results;
     }
 
     /// <summary>Maps an encode refusal to the status an OPC UA client sees.</summary>
@@ -1528,6 +1590,20 @@ public sealed class MqttDriver
         OnHostStatusChanged?.Invoke(this, new HostStatusChangedEventArgs(HostName, previous, next));
     }
 
+    /// <summary>The last successful write publish, or null.</summary>
+    private DateTime? LastPublishUtc
+    {
+        get
+        {
+            var ticks = Interlocked.Read(ref _lastPublishUtcTicks);
+            return ticks == 0 ? null : new DateTime(ticks, DateTimeKind.Utc);
+        }
+    }
+
+    /// <summary>The later of two optional instants.</summary>
+    private static DateTime? Latest(DateTime? a, DateTime? b)
+        => a is null ? b : b is null ? a : (a > b ? a : b);
+
     /// <summary>Barrier-protected read of the multi-threaded health field.</summary>
     private DriverHealth ReadHealth() => Volatile.Read(ref _health);