Merge branch 'worktree-agent-a7084b23177344196' into arch-review-remediation
This commit is contained in:
@@ -758,10 +758,17 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
/// </summary>
|
||||
/// <param name="sinceUtc">Lower bound timestamp (UTC) for event occurrence.</param>
|
||||
/// <param name="batchSize">Maximum number of rows to return.</param>
|
||||
/// <param name="afterId">
|
||||
/// Composite-keyset tiebreak: the EventId of the last row already consumed at
|
||||
/// <paramref name="sinceUtc"/>. Non-null switches the predicate from the inclusive
|
||||
/// <c>OccurredAtUtc >= $since</c> to the strict composite
|
||||
/// <c>(OccurredAtUtc, EventId) > ($since, $afterId)</c>, matching the query's own
|
||||
/// ORDER BY so a batch cannot stall on rows sharing one instant.
|
||||
/// </param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to a read-only list of audit events since the given timestamp.</returns>
|
||||
public Task<IReadOnlyList<AuditEvent>> ReadPendingSinceAsync(
|
||||
DateTime sinceUtc, int batchSize, CancellationToken ct = default)
|
||||
DateTime sinceUtc, int batchSize, string? afterId = null, CancellationToken ct = default)
|
||||
{
|
||||
if (batchSize <= 0)
|
||||
{
|
||||
@@ -779,13 +786,19 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
|
||||
using var cmd = _readConnection.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
var cursorPredicate = afterId is null
|
||||
? "fs.OccurredAtUtc >= $since"
|
||||
// Composite keyset, lexicographic on (OccurredAtUtc, EventId) — the same
|
||||
// ordering the query applies, so it is a strict "everything after this row".
|
||||
: "(fs.OccurredAtUtc > $since OR (fs.OccurredAtUtc = $since AND ae.EventId > $afterId))";
|
||||
|
||||
cmd.CommandText = $"""
|
||||
SELECT ae.EventId, ae.OccurredAtUtc, ae.Actor, ae.Action, ae.Outcome,
|
||||
ae.Category, ae.Target, ae.SourceNode, ae.CorrelationId, ae.DetailsJson
|
||||
FROM audit_event ae
|
||||
JOIN audit_forward_state fs ON fs.EventId = ae.EventId
|
||||
WHERE fs.ForwardState IN ($pending, $forwarded)
|
||||
AND fs.OccurredAtUtc >= $since
|
||||
AND {cursorPredicate}
|
||||
ORDER BY fs.OccurredAtUtc ASC, ae.EventId ASC
|
||||
LIMIT $limit;
|
||||
""";
|
||||
@@ -796,12 +809,63 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
// that encoding so we can index-scan against it.
|
||||
cmd.Parameters.AddWithValue("$since", EnsureUtc(sinceUtc).ToString(
|
||||
"o", System.Globalization.CultureInfo.InvariantCulture));
|
||||
if (afterId is not null)
|
||||
{
|
||||
// EventIds are stored as Guid.ToString() ("D"), so compare in that form.
|
||||
cmd.Parameters.AddWithValue("$afterId", NormalizeEventId(afterId));
|
||||
}
|
||||
cmd.Parameters.AddWithValue("$limit", batchSize);
|
||||
|
||||
return Task.FromResult(ReadRows(cmd, batchSize));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalises a wire-supplied event id to the exact textual form stored in
|
||||
/// <c>audit_event.EventId</c> so the keyset comparison is apples-to-apples. An
|
||||
/// unparseable value is passed through verbatim rather than throwing — a malformed
|
||||
/// cursor must degrade to "serves a bit too much", never to a failed pull.
|
||||
/// </summary>
|
||||
private static string NormalizeEventId(string afterId) =>
|
||||
Guid.TryParse(afterId, out var g) ? g.ToString() : afterId;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<int> MarkReconciledUpToAsync(
|
||||
DateTime sinceUtc, string? afterId, CancellationToken ct = default)
|
||||
{
|
||||
lock (_writeLock)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
|
||||
// Everything at or before central's cursor is proven received. With no
|
||||
// afterId the read contract is inclusive (>= $since), so only rows STRICTLY
|
||||
// older than the cursor instant are proven — the boundary instant may be
|
||||
// half-consumed and must stay servable.
|
||||
var predicate = afterId is null
|
||||
? "fs.OccurredAtUtc < $since"
|
||||
: "(fs.OccurredAtUtc < $since OR (fs.OccurredAtUtc = $since AND fs.EventId <= $afterId))";
|
||||
|
||||
using var cmd = _connection.CreateCommand();
|
||||
cmd.CommandText = $"""
|
||||
UPDATE audit_forward_state AS fs
|
||||
SET ForwardState = $reconciled
|
||||
WHERE fs.ForwardState IN ($pending, $forwarded)
|
||||
AND {predicate};
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("$reconciled", AuditForwardState.Reconciled.ToString());
|
||||
cmd.Parameters.AddWithValue("$pending", AuditForwardState.Pending.ToString());
|
||||
cmd.Parameters.AddWithValue("$forwarded", AuditForwardState.Forwarded.ToString());
|
||||
cmd.Parameters.AddWithValue("$since", EnsureUtc(sinceUtc).ToString(
|
||||
"o", System.Globalization.CultureInfo.InvariantCulture));
|
||||
if (afterId is not null)
|
||||
{
|
||||
cmd.Parameters.AddWithValue("$afterId", NormalizeEventId(afterId));
|
||||
}
|
||||
|
||||
return Task.FromResult(cmd.ExecuteNonQuery());
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task MarkReconciledAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
|
||||
{
|
||||
|
||||
@@ -109,10 +109,47 @@ public interface ISiteAuditQueue
|
||||
/// </remarks>
|
||||
/// <param name="sinceUtc">Lower bound timestamp (UTC).</param>
|
||||
/// <param name="batchSize">Maximum number of rows to return.</param>
|
||||
/// <param name="afterId">
|
||||
/// Composite-keyset tiebreak cursor (WP2.3), mirroring
|
||||
/// <see cref="IOperationTrackingStore.ReadChangedSinceAsync"/>: when non-null it is the
|
||||
/// <see cref="AuditEvent.EventId"/> ("D" GUID form) of the last row already consumed at
|
||||
/// <paramref name="sinceUtc"/>, and only rows strictly after the composite
|
||||
/// <c>(OccurredAtUtc, EventId)</c> pair are returned — so a burst sharing one exact
|
||||
/// instant drains via the id tiebreak instead of pinning the inclusive-timestamp cursor.
|
||||
/// Null (the first pull, or a central that never sets it) keeps the inclusive
|
||||
/// <c>>=</c> contract.
|
||||
/// </param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to audit events at or after <paramref name="sinceUtc"/> in pending or forwarded state, up to <paramref name="batchSize"/>.</returns>
|
||||
Task<IReadOnlyList<AuditEvent>> ReadPendingSinceAsync(
|
||||
DateTime sinceUtc, int batchSize, CancellationToken ct = default);
|
||||
DateTime sinceUtc, int batchSize, string? afterId = null, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Reconciliation-pull commit surface, cursor form (WP2.3): flips every row at or before
|
||||
/// the composite cursor <c>(<paramref name="sinceUtc"/>, <paramref name="afterId"/>)</c>
|
||||
/// from <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState.Pending"/>/
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState.Forwarded"/> to
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState.Reconciled"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The cursor is central's proof of receipt: it only advances past rows central has
|
||||
/// actually ingested. Flipping on THAT — rather than on having merely served the rows —
|
||||
/// is what makes the pull at-least-once: a fault between the response leaving the site
|
||||
/// and central committing it leaves the rows unflipped, so the next pull re-serves them
|
||||
/// (central dedups on <see cref="AuditEvent.EventId"/>).
|
||||
/// <para>
|
||||
/// With <paramref name="afterId"/> null the cursor is a bare timestamp under the legacy
|
||||
/// inclusive <c>>=</c> read contract, so only rows STRICTLY older than
|
||||
/// <paramref name="sinceUtc"/> are proven received; rows at the boundary instant are left
|
||||
/// alone. Idempotent; already-Reconciled rows are untouched.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="sinceUtc">The cursor timestamp central has consumed up to (UTC).</param>
|
||||
/// <param name="afterId">The last consumed <see cref="AuditEvent.EventId"/> at that instant, or null.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to the number of rows flipped.</returns>
|
||||
Task<int> MarkReconciledUpToAsync(
|
||||
DateTime sinceUtc, string? afterId, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Reconciliation-pull commit surface: flips the supplied EventIds to
|
||||
|
||||
@@ -1,5 +1,19 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
||||
|
||||
/// <summary>
|
||||
/// Asks one instance for a point-in-time <see cref="DebugViewSnapshot"/>.
|
||||
/// </summary>
|
||||
/// <param name="InstanceUniqueName">Unique name of the instance to snapshot.</param>
|
||||
/// <param name="CorrelationId">Correlation id echoed on the reply.</param>
|
||||
/// <param name="AlarmsOnly">
|
||||
/// When <c>true</c> the site builds ONLY the alarm half of the snapshot and returns an
|
||||
/// empty <see cref="DebugViewSnapshot.AttributeValues"/> list (wire efficiency, WP2.3).
|
||||
/// Set by the central per-site live alarm cache, whose seed/reconcile fan-out discards
|
||||
/// every attribute row anyway. Defaults to <c>false</c> so the Debug View and every other
|
||||
/// caller keep the full snapshot; additive on the wire
|
||||
/// (<c>DebugSnapshotRequestDto.alarms_only</c>, field 3).
|
||||
/// </param>
|
||||
public record DebugSnapshotRequest(
|
||||
string InstanceUniqueName,
|
||||
string CorrelationId);
|
||||
string CorrelationId,
|
||||
bool AlarmsOnly = false);
|
||||
|
||||
@@ -64,6 +64,35 @@ public static class ScadaBridgeTelemetry
|
||||
Meter.CreateCounter<long>("scadabridge.site.alarm_cache.reconnects", unit: "1",
|
||||
description: "Live-alarm aggregator site-wide gRPC stream reconnects (NodeA↔NodeB flip or reconcile-driven reopen).");
|
||||
|
||||
/// <summary>
|
||||
/// Incremented for each live delta evicted from a per-site live-alarm aggregator's
|
||||
/// bounded pre-seed buffer (drop-oldest, WP2.3). Non-zero means a seed/reconcile
|
||||
/// fan-out ran long enough for the delta storm behind it to exceed the cap — the
|
||||
/// dropped transitions are recovered by the fan-out's authoritative snapshot, but a
|
||||
/// sustained climb points at a slow site.
|
||||
/// </summary>
|
||||
private static readonly Counter<long> _liveAlarmBufferDrops =
|
||||
Meter.CreateCounter<long>("scadabridge.site.alarm_cache.buffer_dropped", unit: "1",
|
||||
description: "Live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer (drop-oldest).");
|
||||
|
||||
/// <summary>
|
||||
/// Incremented for each debug event evicted from a central debug session's bounded
|
||||
/// pre-snapshot buffer (drop-oldest, WP2.3). The Debug View is lossy-under-backpressure
|
||||
/// by design; this makes the loss measurable instead of unbounded memory growth.
|
||||
/// </summary>
|
||||
private static readonly Counter<long> _debugPreSnapshotDrops =
|
||||
Meter.CreateCounter<long>("scadabridge.central.debug_view.presnapshot_dropped", unit: "1",
|
||||
description: "Debug events evicted from a central debug session's bounded pre-snapshot buffer (drop-oldest).");
|
||||
|
||||
/// <summary>
|
||||
/// Incremented for each event evicted from a site-hosted gRPC stream's bounded send
|
||||
/// channel, tagged by stream kind (<c>instance</c> = Debug View, <c>site-alarms</c> =
|
||||
/// the site-wide alarm feed behind the operator Alarm Summary).
|
||||
/// </summary>
|
||||
private static readonly Counter<long> _siteStreamEventDrops =
|
||||
Meter.CreateCounter<long>("scadabridge.site.stream.events_dropped", unit: "1",
|
||||
description: "Events evicted from a site gRPC stream's bounded send channel, tagged by stream kind.");
|
||||
|
||||
// ---------------- Observable gauges ----------------
|
||||
|
||||
/// <summary>Current count of open site connections, mutated via <see cref="Interlocked"/>.</summary>
|
||||
@@ -136,6 +165,19 @@ public static class ScadaBridgeTelemetry
|
||||
/// <summary>Records that a per-site live-alarm aggregator re-established its site-wide gRPC stream.</summary>
|
||||
public static void RecordLiveAlarmStreamReconnect() => _liveAlarmStreamReconnects.Add(1);
|
||||
|
||||
/// <summary>Records live deltas evicted from a live-alarm aggregator's bounded pre-seed buffer.</summary>
|
||||
/// <param name="count">Number of deltas evicted.</param>
|
||||
public static void RecordLiveAlarmBufferDrop(long count = 1) => _liveAlarmBufferDrops.Add(count);
|
||||
|
||||
/// <summary>Records debug events evicted from a debug session's bounded pre-snapshot buffer.</summary>
|
||||
/// <param name="count">Number of events evicted.</param>
|
||||
public static void RecordDebugPreSnapshotDrop(long count = 1) => _debugPreSnapshotDrops.Add(count);
|
||||
|
||||
/// <summary>Records an event evicted from a site gRPC stream's bounded send channel.</summary>
|
||||
/// <param name="streamKind">Stream kind tag (<c>instance</c> or <c>site-alarms</c>).</param>
|
||||
public static void RecordSiteStreamEventDropped(string streamKind) =>
|
||||
_siteStreamEventDrops.Add(1, new KeyValuePair<string, object?>("stream", streamKind));
|
||||
|
||||
/// <summary>
|
||||
/// Registers the provider the StoreAndForward queue-depth gauge reads on each observation.
|
||||
/// A later task supplies a provider that reads the real StoreAndForward depth. A null
|
||||
|
||||
@@ -2,6 +2,7 @@ using Akka.Actor;
|
||||
using Akka.Event;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
@@ -45,9 +46,19 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
|
||||
private const int MaxRetries = 3;
|
||||
private const string ReconnectTimerKey = "grpc-reconnect";
|
||||
private const string StabilityTimerKey = "grpc-stability";
|
||||
private const string SnapshotTimerKey = "debug-snapshot-deadline";
|
||||
/// <summary>Delay between gRPC reconnection attempts.</summary>
|
||||
internal static TimeSpan ReconnectDelay { get; set; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// Hard deadline on the initial <see cref="DebugViewSnapshot"/> (WP2.3). The site builds
|
||||
/// it in milliseconds; if none arrives inside this window the site never answered (the
|
||||
/// Ask was lost, the singleton moved mid-request, the instance actor is wedged) and the
|
||||
/// session must FAIL rather than sit in the buffering phase accumulating live events
|
||||
/// behind a snapshot that is never coming. Settable for tests.
|
||||
/// </summary>
|
||||
internal static TimeSpan SnapshotTimeout { get; set; } = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary>
|
||||
/// How long a freshly-opened gRPC stream must stay up before its retry budget
|
||||
/// is considered "recovered" and <see cref="_retryCount"/> is reset to 0.
|
||||
@@ -85,17 +96,35 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
|
||||
/// Ordered buffer of live gRPC events (<see cref="AttributeValueChanged"/>/
|
||||
/// <see cref="AlarmStateChanged"/>) that arrived before the snapshot was delivered.
|
||||
/// Flushed (with per-entity dedup against the snapshot) when the snapshot arrives,
|
||||
/// then never used again. Mutated only on the actor thread.
|
||||
/// then never used again. Bounded by <see cref="MaxPreSnapshotBuffer"/> with drop-oldest
|
||||
/// eviction (WP2.3): a snapshot that never arrives used to buffer without limit on the
|
||||
/// central node. Mutated only on the actor thread.
|
||||
/// </summary>
|
||||
private readonly List<object> _preSnapshotBuffer = new();
|
||||
private readonly Queue<object> _preSnapshotBuffer = new();
|
||||
|
||||
/// <summary>
|
||||
/// Defensive log threshold: if the pre-snapshot buffer grows past this many events
|
||||
/// during a slow snapshot we log once (events are NOT dropped — the window is short).
|
||||
/// Defensive log threshold: the first warning fires when the pre-snapshot buffer grows
|
||||
/// past this many events during a slow snapshot, before the hard cap starts evicting.
|
||||
/// </summary>
|
||||
private const int BufferWarnThreshold = 10_000;
|
||||
private bool _bufferWarned;
|
||||
|
||||
/// <summary>
|
||||
/// Hard cap on the pre-snapshot buffer. Beyond it the OLDEST event is evicted — the
|
||||
/// snapshot that ends the buffering phase is authoritative for anything that old, so
|
||||
/// keeping the newest events is what preserves the post-snapshot delta chain.
|
||||
/// </summary>
|
||||
private const int MaxPreSnapshotBuffer = 20_000;
|
||||
|
||||
/// <summary>Events evicted from the pre-snapshot buffer in this session. Actor-thread only.</summary>
|
||||
private long _preSnapshotDropped;
|
||||
|
||||
/// <summary>
|
||||
/// Total pre-snapshot events dropped across all debug sessions on this node — the raw
|
||||
/// counter behind <c>scadabridge.central.debug_view.presnapshot_dropped</c>.
|
||||
/// </summary>
|
||||
internal static long TotalPreSnapshotDropped;
|
||||
|
||||
/// <summary>Timer scheduler for reconnect and stability window timers.</summary>
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
|
||||
@@ -176,6 +205,9 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
|
||||
_instanceUniqueName, snapshot.AttributeValues.Count, snapshot.AlarmStates.Count,
|
||||
_preSnapshotBuffer.Count);
|
||||
|
||||
// The snapshot arrived — stand the hard deadline down.
|
||||
Timers.Cancel(SnapshotTimerKey);
|
||||
|
||||
// Deliver the snapshot, then flush the gap-window buffer (deduped), then
|
||||
// switch to pass-through. Order matters: snapshot first, buffered events next.
|
||||
_onEvent(snapshot);
|
||||
@@ -183,14 +215,40 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
|
||||
_snapshotDelivered = true;
|
||||
});
|
||||
|
||||
// Domain events arriving via Self.Tell from gRPC callback.
|
||||
// Receiving an event must NOT reset _retryCount — a
|
||||
// flapping stream that delivers a single event between failures would
|
||||
// otherwise never trip MaxRetries. The retry budget is recovered only by
|
||||
// GrpcStreamStable (a stream that has stayed up for StabilityWindow).
|
||||
// Before the snapshot has been delivered, BUFFER (in arrival order)
|
||||
// rather than deliver — these may be gap-window events. After the snapshot has
|
||||
// been flushed, pass through directly (same handler, phase-dependent behavior).
|
||||
// Hard snapshot deadline (WP2.3). Nothing else ends a session stuck in the
|
||||
// buffering phase: the site's reply was lost, so no gRPC error fires, the stream
|
||||
// keeps delivering events, and (with the wrapper above) they no longer even reset
|
||||
// the orphan timeout. Fail the session so the consumer is told and can reopen.
|
||||
Receive<DebugSnapshotDeadline>(_ =>
|
||||
{
|
||||
if (_stopped || _snapshotDelivered) return;
|
||||
_log.Error(
|
||||
"No debug snapshot for {0} within {1}s ({2} event(s) buffered, {3} dropped); failing the session",
|
||||
_instanceUniqueName, SnapshotTimeout.TotalSeconds,
|
||||
_preSnapshotBuffer.Count, _preSnapshotDropped);
|
||||
CleanupGrpc();
|
||||
SendUnsubscribe();
|
||||
_stopped = true;
|
||||
_preSnapshotBuffer.Clear();
|
||||
_onTerminated();
|
||||
Context.Stop(Self);
|
||||
});
|
||||
|
||||
// Domain events arriving via Self.Tell from the gRPC callback, wrapped so they do
|
||||
// NOT influence the receive timeout (WP2.3): the orphan safety net exists to end a
|
||||
// session whose CONSUMER is gone, and a busy site's event flood used to keep that
|
||||
// net permanently reset — an abandoned session on a chatty instance never timed out.
|
||||
// Receiving an event must not reset _retryCount either: a flapping stream that
|
||||
// delivers a single event between failures would otherwise never trip MaxRetries.
|
||||
// The retry budget is recovered only by GrpcStreamStable (a stream that has stayed
|
||||
// up for StabilityWindow). Before the snapshot has been delivered, BUFFER (in arrival
|
||||
// order) rather than deliver — these may be gap-window events; after the snapshot has
|
||||
// been flushed, pass through directly (phase-dependent behavior).
|
||||
Receive<LiveDebugStreamEvent>(wrapped => HandleStreamEvent(wrapped.Event));
|
||||
|
||||
// Unwrapped forms are still accepted (a direct Tell from a test or a future
|
||||
// in-process producer); those DO influence the receive timeout, which is correct —
|
||||
// they are not the high-volume stream path.
|
||||
Receive<AttributeValueChanged>(changed => HandleStreamEvent(changed));
|
||||
Receive<AlarmStateChanged>(changed => HandleStreamEvent(changed));
|
||||
|
||||
@@ -286,15 +344,30 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
|
||||
return;
|
||||
}
|
||||
|
||||
_preSnapshotBuffer.Add(evt);
|
||||
if (!_bufferWarned && _preSnapshotBuffer.Count > BufferWarnThreshold)
|
||||
if (!_bufferWarned && _preSnapshotBuffer.Count + 1 > BufferWarnThreshold)
|
||||
{
|
||||
_bufferWarned = true;
|
||||
_log.Warning(
|
||||
"Pre-snapshot debug-event buffer for {0} exceeded {1} events while awaiting the snapshot; " +
|
||||
"events are still retained (not dropped).",
|
||||
_instanceUniqueName, BufferWarnThreshold);
|
||||
"Pre-snapshot debug-event buffer for {0} exceeded {1} events while awaiting the snapshot " +
|
||||
"(hard cap {2}, drop-oldest beyond it).",
|
||||
_instanceUniqueName, BufferWarnThreshold, MaxPreSnapshotBuffer);
|
||||
}
|
||||
|
||||
while (_preSnapshotBuffer.Count >= MaxPreSnapshotBuffer)
|
||||
{
|
||||
_preSnapshotBuffer.Dequeue();
|
||||
_preSnapshotDropped++;
|
||||
Interlocked.Increment(ref TotalPreSnapshotDropped);
|
||||
ScadaBridgeTelemetry.RecordDebugPreSnapshotDrop();
|
||||
if (_preSnapshotDropped == 1 || _preSnapshotDropped % 500 == 0)
|
||||
{
|
||||
_log.Warning(
|
||||
"Pre-snapshot debug-event buffer for {0} is at its {1}-event cap; {2} event(s) evicted so far",
|
||||
_instanceUniqueName, MaxPreSnapshotBuffer, _preSnapshotDropped);
|
||||
}
|
||||
}
|
||||
|
||||
_preSnapshotBuffer.Enqueue(evt);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -352,8 +425,9 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
|
||||
|
||||
if (dropped > 0 || flushed > 0)
|
||||
{
|
||||
_log.Debug("Flushed {0} buffered debug event(s) for {1}, dropped {2} as already-in-snapshot",
|
||||
flushed, _instanceUniqueName, dropped);
|
||||
_log.Debug("Flushed {0} buffered debug event(s) for {1}, dropped {2} as already-in-snapshot" +
|
||||
" ({3} previously evicted at the buffer cap)",
|
||||
flushed, _instanceUniqueName, dropped, _preSnapshotDropped);
|
||||
}
|
||||
|
||||
_preSnapshotBuffer.Clear();
|
||||
@@ -429,6 +503,10 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
|
||||
var request = new SubscribeDebugViewRequest(_instanceUniqueName, _correlationId);
|
||||
var envelope = new SiteEnvelope(_siteIdentifier, request);
|
||||
_centralCommunicationActor.Tell(envelope, Self);
|
||||
|
||||
// Arm the hard snapshot deadline alongside the request.
|
||||
if (SnapshotTimeout > TimeSpan.Zero)
|
||||
Timers.StartSingleTimer(SnapshotTimerKey, new DebugSnapshotDeadline(), SnapshotTimeout);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -468,7 +546,8 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
|
||||
await client.SubscribeAsync(
|
||||
_correlationId,
|
||||
_instanceUniqueName,
|
||||
evt => self.Tell(evt),
|
||||
// Wrapped: stream traffic must not reset the orphan receive timeout.
|
||||
evt => self.Tell(new LiveDebugStreamEvent(evt)),
|
||||
ex => self.Tell(new GrpcStreamError(ex, generation)),
|
||||
() => self.Tell(new GrpcStreamCompleted(generation)),
|
||||
ct);
|
||||
@@ -579,6 +658,19 @@ public class DebugStreamBridgeActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
public record StopDebugStream;
|
||||
|
||||
/// <summary>
|
||||
/// Envelope for a live gRPC stream event (<c>AttributeValueChanged</c>/
|
||||
/// <c>AlarmStateChanged</c>). Implements <see cref="INotInfluenceReceiveTimeout"/> so a busy
|
||||
/// site's event flood cannot keep resetting the orphan-session receive timeout — the timeout
|
||||
/// measures consumer/session liveness, not site chatter (WP2.3).
|
||||
/// </summary>
|
||||
internal record LiveDebugStreamEvent(object Event) : INotInfluenceReceiveTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// Internal message: the hard deadline for the initial <c>DebugViewSnapshot</c> expired.
|
||||
/// </summary>
|
||||
internal record DebugSnapshotDeadline;
|
||||
|
||||
/// <summary>
|
||||
/// Internal message indicating a gRPC stream error occurred, stamped with the stream
|
||||
/// generation it came from so a late error out of a cancelled stream can be ignored.
|
||||
|
||||
@@ -31,10 +31,14 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Failover + drift:</b> a gRPC error flips NodeA↔NodeB with the same retry budget +
|
||||
/// stability window as <see cref="DebugStreamBridgeActor"/>, and each reconnect triggers
|
||||
/// a RE-SEED (never silently serve stale). A periodic reconcile snapshot
|
||||
/// (<see cref="_reconcileInterval"/>, default 60s) corrects instance-set drift and any
|
||||
/// missed delta.
|
||||
/// stability window as <see cref="DebugStreamBridgeActor"/>. A re-seed runs <b>once per
|
||||
/// successful (re)connect</b> — driven by the stream's connected callback, not by each
|
||||
/// reconnect ATTEMPT (WP2.3: a site outage used to fan a full snapshot out per retry, all
|
||||
/// of them against the very site that is unreachable). A periodic reconcile snapshot
|
||||
/// (<see cref="_reconcileInterval"/>, default 60s, jittered per site so N aggregators do
|
||||
/// not fan out in lockstep) remains the drift/backstop: it corrects instance-set drift and
|
||||
/// any missed delta, but it is <em>skipped</em> when a fan-out already completed inside the
|
||||
/// window, and it publishes to viewers only when the snapshot actually changed the cache.
|
||||
/// </para>
|
||||
/// All state is mutated only on the actor thread: gRPC callbacks and fan-out results are
|
||||
/// marshalled back via <c>Self.Tell</c>, so the cache needs no internal lock. The
|
||||
@@ -47,18 +51,20 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
private readonly string _siteIdentifier;
|
||||
private readonly string _correlationId;
|
||||
private readonly Func<CancellationToken, Task<IReadOnlyList<AlarmStateChanged>>> _seedFn;
|
||||
private readonly Action<IReadOnlyList<AlarmStateChanged>> _publish;
|
||||
private readonly Action<IReadOnlyList<AlarmStateChanged>, bool> _publish;
|
||||
private readonly SiteStreamGrpcClientFactory _grpcFactory;
|
||||
private readonly string _grpcNodeAAddress;
|
||||
private readonly string _grpcNodeBAddress;
|
||||
private readonly TimeSpan _reconcileInterval;
|
||||
private readonly TimeSpan _publishCoalesce;
|
||||
private readonly double _reconcileJitterFraction;
|
||||
|
||||
private const int MaxRetries = 3;
|
||||
private const string ReconnectTimerKey = "alarm-grpc-reconnect";
|
||||
private const string StabilityTimerKey = "alarm-grpc-stability";
|
||||
private const string ReconcileTimerKey = "alarm-reconcile";
|
||||
private const string PublishTimerKey = "alarm-publish-coalesce";
|
||||
private const string SeedRetryTimerKey = "alarm-seed-retry";
|
||||
|
||||
/// <summary>True while a coalesced publish is armed (dirty deltas awaiting one tick). Actor-thread only.</summary>
|
||||
private bool _publishPending;
|
||||
@@ -83,9 +89,10 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
/// lifetime). Reconcile snapshots keep serving in the meantime; the next reconcile tick
|
||||
/// self-heals the stream by resetting the retry budget and reopening it, so neither a
|
||||
/// sustained site outage nor a routine 4h stream expiry permanently drops the live feed.
|
||||
/// Actor-thread only.
|
||||
/// Starts <c>true</c>: until the site accepts the first subscription there is no live
|
||||
/// feed, and the owning cache must not advertise one. Actor-thread only.
|
||||
/// </summary>
|
||||
private bool _streamDown;
|
||||
private bool _streamDown = true;
|
||||
|
||||
/// <summary>
|
||||
/// Why the stream is down: <c>true</c> = the retry budget was exhausted, so the
|
||||
@@ -107,8 +114,57 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
/// <summary>True while a seed/reconcile snapshot fan-out is in flight (deltas buffer). Actor-thread only.</summary>
|
||||
private bool _fanoutInFlight;
|
||||
|
||||
/// <summary>Ordered buffer of live deltas that arrived while a fan-out was in flight. Actor-thread only.</summary>
|
||||
private readonly List<AlarmStateChanged> _buffer = new();
|
||||
/// <summary>
|
||||
/// Ordered buffer of live deltas that arrived while a fan-out was in flight, capped at
|
||||
/// <see cref="MaxBufferedDeltas"/> with drop-oldest eviction (WP2.3 — a stuck fan-out
|
||||
/// used to buffer without bound). Dropping the OLDEST is the right eviction here: the
|
||||
/// fan-out that follows rebuilds the cache authoritatively, so an evicted delta is
|
||||
/// superseded rather than lost, while the newest transitions — the ones the snapshot may
|
||||
/// predate — are the ones kept. Actor-thread only.
|
||||
/// </summary>
|
||||
private readonly Queue<AlarmStateChanged> _buffer = new();
|
||||
|
||||
/// <summary>Total deltas evicted from <see cref="_buffer"/> over this actor's life. Actor-thread only.</summary>
|
||||
private long _bufferDropped;
|
||||
|
||||
/// <summary>
|
||||
/// Hard cap on the pre-fan-out delta buffer. Beyond this the oldest entry is evicted and
|
||||
/// counted; the counter is logged (first drop + every 500th) and exported as
|
||||
/// <c>scadabridge.site.alarm_cache.buffer_dropped</c>.
|
||||
/// </summary>
|
||||
private const int MaxBufferedDeltas = 20_000;
|
||||
|
||||
/// <summary>
|
||||
/// True when the next successful (re)connect must run a seed fan-out: set whenever the
|
||||
/// stream is lost (fault or graceful completion) and cleared by the connect that consumes
|
||||
/// it. This is what makes the re-seed happen once per successful reconnect rather than
|
||||
/// once per reconnect ATTEMPT — the old code fanned a full snapshot out on every retry,
|
||||
/// against a site that was by definition unreachable. Actor-thread only.
|
||||
/// </summary>
|
||||
private bool _seedOnConnect;
|
||||
|
||||
/// <summary>
|
||||
/// Set whenever a fan-out finishes (success or failure); consumed by the next reconcile
|
||||
/// tick, which skips its own fan-out when it finds the flag set. That makes the
|
||||
/// connect-driven seed and the periodic backstop mutually exclusive — the pair used to
|
||||
/// run BOTH, which is the "full unconditional snapshot every 60s" waste — while bounding
|
||||
/// staleness at two intervals (a tick can be skipped at most once in a row).
|
||||
/// Actor-thread only.
|
||||
/// </summary>
|
||||
private bool _fanoutSinceLastTick;
|
||||
|
||||
/// <summary>
|
||||
/// True between issuing a stream open and its first outcome (connected / error /
|
||||
/// completed), so a reconcile tick never stacks a second open on an in-flight one.
|
||||
/// Actor-thread only.
|
||||
/// </summary>
|
||||
private bool _openInFlight;
|
||||
|
||||
/// <summary>Consecutive whole-fan-out failures, driving the seed-retry backoff. Actor-thread only.</summary>
|
||||
private int _consecutiveSeedFailures;
|
||||
|
||||
/// <summary>Upper bound on the seed-retry backoff, as a multiple of the reconcile interval.</summary>
|
||||
private const int MaxSeedRetryBackoffMultiplier = 8;
|
||||
|
||||
/// <summary>
|
||||
/// A failover re-seed was requested while a fan-out was already in flight; it must run
|
||||
@@ -144,6 +200,10 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
/// <param name="publish">
|
||||
/// Publishes a fresh immutable snapshot of the cache to the owning service, which
|
||||
/// stores it and raises viewer <c>onChanged</c> callbacks. Invoked on the actor thread.
|
||||
/// The second argument is the live-stream liveness flag: <c>false</c> means the site-wide
|
||||
/// gRPC stream is currently down (fault-exhausted or gracefully completed), so the cache
|
||||
/// is only as fresh as the last reconcile and the page must fall back to polling
|
||||
/// (WP2.3 carried residual — <c>IsLive</c> used to stay true across a dead stream).
|
||||
/// </param>
|
||||
/// <param name="grpcFactory">Factory caching one gRPC client per (site, endpoint).</param>
|
||||
/// <param name="grpcNodeAAddress">gRPC address of the site's node A.</param>
|
||||
@@ -158,18 +218,24 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
/// <param name="stabilityWindow">
|
||||
/// How long a fresh gRPC stream must stay up before its retry budget recovers (production 60s).
|
||||
/// </param>
|
||||
/// <param name="reconcileJitterFraction">
|
||||
/// Fraction of <paramref name="reconcileInterval"/> added as a per-tick random offset so
|
||||
/// N per-site aggregators on one central node do not fan out in lockstep (production 0.2 =
|
||||
/// up to +20%). Zero disables jitter, which is what tests want for determinism.
|
||||
/// </param>
|
||||
public SiteAlarmAggregatorActor(
|
||||
string siteIdentifier,
|
||||
string correlationId,
|
||||
Func<CancellationToken, Task<IReadOnlyList<AlarmStateChanged>>> seedFn,
|
||||
Action<IReadOnlyList<AlarmStateChanged>> publish,
|
||||
Action<IReadOnlyList<AlarmStateChanged>, bool> publish,
|
||||
SiteStreamGrpcClientFactory grpcFactory,
|
||||
string grpcNodeAAddress,
|
||||
string grpcNodeBAddress,
|
||||
TimeSpan reconcileInterval,
|
||||
TimeSpan publishCoalesce,
|
||||
TimeSpan reconnectDelay,
|
||||
TimeSpan stabilityWindow)
|
||||
TimeSpan stabilityWindow,
|
||||
double reconcileJitterFraction = 0.0)
|
||||
{
|
||||
_siteIdentifier = siteIdentifier;
|
||||
_correlationId = correlationId;
|
||||
@@ -182,6 +248,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
_publishCoalesce = publishCoalesce;
|
||||
_reconnectDelay = reconnectDelay;
|
||||
_stabilityWindow = stabilityWindow;
|
||||
_reconcileJitterFraction = Math.Clamp(reconcileJitterFraction, 0.0, 1.0);
|
||||
|
||||
// Live delta from the site-wide alarm stream (marshalled in via Self.Tell).
|
||||
// A received delta must NOT reset the retry budget (a flapping stream that
|
||||
@@ -198,6 +265,22 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
// Periodic reconcile tick (and the re-seed kicked after a reconnect).
|
||||
Receive<RunReconcile>(_ => OnReconcileTick());
|
||||
|
||||
// Backoff-scheduled retry of a fan-out that failed as a whole.
|
||||
Receive<RetrySeed>(_ =>
|
||||
{
|
||||
if (_stopped) return;
|
||||
StartFanout(isInitial: false);
|
||||
});
|
||||
|
||||
// The site-wide stream is confirmed established (response headers received). This —
|
||||
// not the reconnect attempt — is the "successful (re)connect" that earns one re-seed.
|
||||
Receive<GrpcAlarmStreamConnected>(msg =>
|
||||
{
|
||||
if (_stopped) return;
|
||||
if (msg.Generation != _streamGeneration) return;
|
||||
OnStreamConnected();
|
||||
});
|
||||
|
||||
// Coalesced-publish tick: one publish for a batch of dirtying deltas (N6).
|
||||
Receive<PublishCoalesced>(_ =>
|
||||
{
|
||||
@@ -270,11 +353,31 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
// in the seed window are captured (buffered) rather than lost.
|
||||
OpenGrpcStream();
|
||||
|
||||
// Kick the initial seed fan-out.
|
||||
// Kick the initial seed fan-out. The connect callback for this first stream must
|
||||
// NOT seed again on top of it, so the flag starts cleared.
|
||||
_seedOnConnect = false;
|
||||
StartFanout(isInitial: true);
|
||||
|
||||
// Periodic reconcile backstop.
|
||||
Timers.StartPeriodicTimer(ReconcileTimerKey, new RunReconcile(), _reconcileInterval, _reconcileInterval);
|
||||
// Periodic reconcile backstop. Single-shot and re-armed with fresh jitter each
|
||||
// tick (a periodic timer would lock every site to the same phase forever).
|
||||
ArmReconcileTimer();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Arms the next reconcile tick at <see cref="_reconcileInterval"/> plus a random
|
||||
/// offset of up to <see cref="_reconcileJitterFraction"/> of it, so N per-site
|
||||
/// aggregators started together (a central failover restarts them all at once) spread
|
||||
/// their fan-outs instead of stampeding the same 60s boundary.
|
||||
/// </summary>
|
||||
private void ArmReconcileTimer()
|
||||
{
|
||||
var delay = _reconcileInterval;
|
||||
if (_reconcileJitterFraction > 0 && _reconcileInterval > TimeSpan.Zero)
|
||||
{
|
||||
var jitterTicks = (long)(_reconcileInterval.Ticks * _reconcileJitterFraction * Random.Shared.NextDouble());
|
||||
delay += TimeSpan.FromTicks(jitterTicks);
|
||||
}
|
||||
Timers.StartSingleTimer(ReconcileTimerKey, new RunReconcile(), delay);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -294,16 +397,24 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
// ── Reconcile tick ──────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Periodic reconcile: always re-run the snapshot fan-out (corrects drift + missed
|
||||
/// deltas), and if the live stream was previously given up, self-heal it by resetting
|
||||
/// the retry budget and reopening — so a sustained outage never permanently kills the
|
||||
/// live feed.
|
||||
/// Periodic reconcile backstop: re-run the snapshot fan-out (corrects instance-set drift
|
||||
/// + any missed delta) UNLESS one already completed inside this window — a reconnect-driven
|
||||
/// seed makes the tick redundant, and running both was the "full unconditional snapshot
|
||||
/// every 60s" waste (WP2.3). If the live stream was previously given up, self-heal it by
|
||||
/// resetting the retry budget and reopening, so a sustained outage never permanently kills
|
||||
/// the live feed.
|
||||
/// </summary>
|
||||
private void OnReconcileTick()
|
||||
{
|
||||
if (_stopped) return;
|
||||
StartFanout(isInitial: false);
|
||||
if (_streamDown)
|
||||
|
||||
// Re-arm first: every exit path below must leave the backstop running.
|
||||
ArmReconcileTimer();
|
||||
|
||||
// A down stream is reopened in preference to fanning out: the reopen's connect
|
||||
// callback seeds by itself, so doing both would double the work.
|
||||
var reopening = false;
|
||||
if (_streamDown && !_openInFlight)
|
||||
{
|
||||
_log.Info("Site-alarm gRPC stream for {0} was down; reopening on reconcile tick", _siteIdentifier);
|
||||
if (_retryBudgetExhausted)
|
||||
@@ -313,8 +424,24 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
// Telemetry: a reconcile-driven reopen after the stream was given up is a reconnect.
|
||||
ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
|
||||
_seedOnConnect = true;
|
||||
OpenGrpcStream();
|
||||
reopening = true;
|
||||
}
|
||||
|
||||
if (_fanoutSinceLastTick)
|
||||
{
|
||||
// A connect-driven seed (or the initial seed) already refreshed the cache inside
|
||||
// this window. Clear the flag so the NEXT tick fans out regardless — staleness
|
||||
// stays bounded at two intervals even if reconnects keep arriving.
|
||||
_fanoutSinceLastTick = false;
|
||||
_log.Debug("Site-alarm reconcile for {0} skipped; a fan-out already ran this window",
|
||||
_siteIdentifier);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!reopening)
|
||||
StartFanout(isInitial: false);
|
||||
}
|
||||
|
||||
// ── Seed / reconcile fan-out ────────────────────────────────────────────────
|
||||
@@ -366,6 +493,12 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
// Rebuild the cache authoritatively from the fresh snapshot (this is what makes
|
||||
// reconcile able to DROP rows for instances/alarms that disappeared — a merge
|
||||
// could never remove a stale row since the live stream sends no "removed" event).
|
||||
// The previous cache is kept alongside so the reconcile can publish as a DIFF:
|
||||
// an unchanged snapshot must not wake every viewer's render path once a minute.
|
||||
var previous = _cache.Count == 0
|
||||
? null
|
||||
: new Dictionary<string, AlarmStateChanged>(_cache);
|
||||
|
||||
_cache.Clear();
|
||||
foreach (var alarm in msg.Alarms)
|
||||
{
|
||||
@@ -378,19 +511,24 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
// buffered delta whose key is in the fresh snapshot with an equal-or-newer
|
||||
// timestamp is already reflected → drop; a strictly-newer (or new-key) delta is
|
||||
// applied. Inclusive-on-snapshot boundary matches DebugStreamBridgeActor.
|
||||
FlushBuffer();
|
||||
var flushChanged = FlushBuffer();
|
||||
|
||||
_fanoutInFlight = false;
|
||||
_fanoutSinceLastTick = true;
|
||||
_consecutiveSeedFailures = 0;
|
||||
Timers.Cancel(SeedRetryTimerKey);
|
||||
var firstSeed = !_seeded;
|
||||
_seeded = true;
|
||||
|
||||
_log.Debug("Site-alarm {0} {1} complete: {2} alarm row(s)",
|
||||
_siteIdentifier, msg.IsInitial ? "seed" : "reconcile", _cache.Count);
|
||||
|
||||
// The fresh snapshot already carries the buffered deltas; drop any armed coalesce
|
||||
// tick so we publish once, immediately.
|
||||
// tick so we publish once, immediately — but only when something actually moved.
|
||||
Timers.Cancel(PublishTimerKey);
|
||||
_publishPending = false;
|
||||
Publish();
|
||||
if (firstSeed || flushChanged || DiffersFrom(previous))
|
||||
Publish();
|
||||
|
||||
// A failover re-seed requested while this fan-out was in flight runs now (N7.1).
|
||||
if (_reseedQueued)
|
||||
@@ -400,22 +538,44 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when the just-installed snapshot differs from <paramref name="previous"/> in any
|
||||
/// key or any row value — the reconcile's "is this publish worth a viewer fan-out?" test.
|
||||
/// A null previous cache (nothing published yet) always counts as different.
|
||||
/// </summary>
|
||||
private bool DiffersFrom(Dictionary<string, AlarmStateChanged>? previous)
|
||||
{
|
||||
if (previous is null || previous.Count != _cache.Count) return true;
|
||||
|
||||
foreach (var (key, current) in _cache)
|
||||
{
|
||||
if (!previous.TryGetValue(key, out var old)) return true;
|
||||
// AlarmStateChanged is a record: structural equality covers state, level,
|
||||
// timestamp and every native-alarm enrichment field.
|
||||
if (!Equals(old, current)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void OnSeedFailed(SeedFailed msg)
|
||||
{
|
||||
if (_stopped) return;
|
||||
|
||||
_consecutiveSeedFailures++;
|
||||
|
||||
_log.Warning(msg.Exception,
|
||||
"Site-alarm {0} {1} fan-out failed; keeping current cache and relying on the next reconcile",
|
||||
_siteIdentifier, msg.IsInitial ? "seed" : "reconcile");
|
||||
"Site-alarm {0} {1} fan-out failed ({2} consecutive); keeping current cache and retrying with backoff",
|
||||
_siteIdentifier, msg.IsInitial ? "seed" : "reconcile", _consecutiveSeedFailures);
|
||||
|
||||
// Don't lose deltas captured during the failed window — apply them pass-through
|
||||
// into the (possibly stale/empty) cache. The next reconcile re-seeds authoritatively.
|
||||
_fanoutInFlight = false;
|
||||
FlushBuffer(dedupAgainstSeed: false);
|
||||
_fanoutSinceLastTick = true;
|
||||
var flushChanged = FlushBuffer(dedupAgainstSeed: false);
|
||||
|
||||
// Only publish if we already had a seed (so IsLive doesn't flip true on a
|
||||
// failed initial seed — the page keeps its poll fallback until we truly seed).
|
||||
if (_seeded)
|
||||
if (_seeded && flushChanged)
|
||||
{
|
||||
Timers.Cancel(PublishTimerKey);
|
||||
_publishPending = false;
|
||||
@@ -427,27 +587,54 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
_reseedQueued = false;
|
||||
StartFanout(isInitial: false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Backoff retry (WP2.3): the seed leg used to have none — a site returning errors
|
||||
// was re-fanned at full reconcile cadence forever. Retry at reconnectDelay doubling
|
||||
// up to MaxSeedRetryBackoffMultiplier × the reconcile interval; the periodic
|
||||
// reconcile remains the floor once the backoff saturates.
|
||||
ScheduleSeedRetry();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Arms the backoff retry for a failed fan-out: <c>reconnectDelay × 2^(failures-1)</c>,
|
||||
/// capped at <see cref="MaxSeedRetryBackoffMultiplier"/> × the reconcile interval. Once
|
||||
/// the cap is reached the periodic reconcile tick is doing the same work anyway, so no
|
||||
/// extra timer is armed.
|
||||
/// </summary>
|
||||
private void ScheduleSeedRetry()
|
||||
{
|
||||
var cap = _reconcileInterval > TimeSpan.Zero
|
||||
? TimeSpan.FromTicks(_reconcileInterval.Ticks * MaxSeedRetryBackoffMultiplier)
|
||||
: TimeSpan.FromMinutes(8);
|
||||
|
||||
var shift = Math.Min(_consecutiveSeedFailures - 1, 16);
|
||||
var delayTicks = _reconnectDelay.Ticks * (1L << shift);
|
||||
var delay = TimeSpan.FromTicks(Math.Min(delayTicks, cap.Ticks));
|
||||
if (delay <= TimeSpan.Zero) return;
|
||||
|
||||
Timers.StartSingleTimer(SeedRetryTimerKey, new RetrySeed(), delay);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flushes the pre-fan-out buffer in arrival order. When <paramref name="dedupAgainstSeed"/>
|
||||
/// is true (normal completion) a buffered delta already reflected in the just-installed
|
||||
/// snapshot (same key, timestamp <= cache entry) is dropped; otherwise every buffered
|
||||
/// delta is applied pass-through.
|
||||
/// delta is applied pass-through. Returns <c>true</c> when at least one delta changed the
|
||||
/// cache (the reconcile's publish-only-on-change test must count these too).
|
||||
/// </summary>
|
||||
private void FlushBuffer(bool dedupAgainstSeed = true)
|
||||
private bool FlushBuffer(bool dedupAgainstSeed = true)
|
||||
{
|
||||
if (_buffer.Count == 0) return;
|
||||
if (_buffer.Count == 0) return false;
|
||||
|
||||
var changed = false;
|
||||
foreach (var delta in _buffer)
|
||||
{
|
||||
if (dedupAgainstSeed)
|
||||
ApplyDelta(delta, requireStrictlyNewer: true);
|
||||
else
|
||||
ApplyDelta(delta, requireStrictlyNewer: false);
|
||||
changed |= ApplyDelta(delta, requireStrictlyNewer: dedupAgainstSeed);
|
||||
}
|
||||
_buffer.Clear();
|
||||
return changed;
|
||||
}
|
||||
|
||||
// ── Live delta handling ─────────────────────────────────────────────────────
|
||||
@@ -458,15 +645,32 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
|
||||
if (_fanoutInFlight)
|
||||
{
|
||||
_buffer.Add(delta);
|
||||
if (!_bufferWarned && _buffer.Count > BufferWarnThreshold)
|
||||
if (!_bufferWarned && _buffer.Count + 1 > BufferWarnThreshold)
|
||||
{
|
||||
_bufferWarned = true;
|
||||
_log.Warning(
|
||||
"Site-alarm pre-seed buffer for {0} exceeded {1} deltas while a fan-out was in flight " +
|
||||
"(deltas retained, not dropped).",
|
||||
_siteIdentifier, BufferWarnThreshold);
|
||||
"(hard cap {2}, drop-oldest beyond it).",
|
||||
_siteIdentifier, BufferWarnThreshold, MaxBufferedDeltas);
|
||||
}
|
||||
|
||||
// Hard cap with drop-oldest: an unbounded buffer behind a stuck fan-out was a
|
||||
// straight memory leak. The evicted rows are superseded by the snapshot that
|
||||
// ends the fan-out, so the drop costs freshness, never correctness.
|
||||
while (_buffer.Count >= MaxBufferedDeltas)
|
||||
{
|
||||
_buffer.Dequeue();
|
||||
_bufferDropped++;
|
||||
ScadaBridgeTelemetry.RecordLiveAlarmBufferDrop();
|
||||
if (_bufferDropped == 1 || _bufferDropped % 500 == 0)
|
||||
{
|
||||
_log.Warning(
|
||||
"Site-alarm pre-seed buffer for {0} is at its {1}-delta cap; {2} delta(s) evicted so far",
|
||||
_siteIdentifier, MaxBufferedDeltas, _bufferDropped);
|
||||
}
|
||||
}
|
||||
|
||||
_buffer.Enqueue(delta);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -516,7 +720,10 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
var snapshot = _cache.Values.ToList();
|
||||
try
|
||||
{
|
||||
_publish(snapshot);
|
||||
// Liveness rides every publish so the owning service's IsLive tracks the STREAM,
|
||||
// not merely "a snapshot was published once". A down stream means the cache is
|
||||
// only as fresh as the last reconcile and the page must keep polling.
|
||||
_publish(snapshot, !_streamDown);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -530,7 +737,8 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
if (_stopped) return;
|
||||
|
||||
_streamDown = false;
|
||||
// The stream is not live again until the server actually accepts it — liveness flips
|
||||
// on GrpcAlarmStreamConnected, not on the attempt (WP2.3 carried residual).
|
||||
var endpoint = _useNodeA ? _grpcNodeAAddress : _grpcNodeBAddress;
|
||||
_log.Info("Opening site-alarm gRPC stream for {0} to {1}", _siteIdentifier, endpoint);
|
||||
|
||||
@@ -540,6 +748,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
|
||||
Timers.StartSingleTimer(StabilityTimerKey, new GrpcAlarmStreamStable(), _stabilityWindow);
|
||||
|
||||
_openInFlight = true;
|
||||
var generation = ++_streamGeneration;
|
||||
var client = _grpcFactory.GetOrCreate(_siteIdentifier, endpoint);
|
||||
var self = Self;
|
||||
@@ -556,7 +765,8 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
alarm => self.Tell(alarm),
|
||||
ex => self.Tell(new GrpcAlarmStreamError(ex, generation)),
|
||||
() => self.Tell(new GrpcAlarmStreamCompleted(generation)),
|
||||
ct);
|
||||
ct,
|
||||
() => self.Tell(new GrpcAlarmStreamConnected(generation)));
|
||||
}, ct).ContinueWith(t =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
@@ -567,6 +777,32 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
}, TaskContinuationOptions.ExecuteSynchronously);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The site accepted the subscription — the only point at which the stream is genuinely
|
||||
/// live. Consumes the pending re-seed (so exactly ONE fan-out runs per successful
|
||||
/// (re)connect, instead of one per reconnect attempt against a site that is by definition
|
||||
/// unreachable) and republishes so <c>IsLive</c> recovers.
|
||||
/// </summary>
|
||||
private void OnStreamConnected()
|
||||
{
|
||||
_openInFlight = false;
|
||||
|
||||
var wasDown = _streamDown;
|
||||
_streamDown = false;
|
||||
|
||||
if (_seedOnConnect)
|
||||
{
|
||||
_seedOnConnect = false;
|
||||
_log.Info("Site-alarm gRPC stream for {0} (re)connected; running one re-seed", _siteIdentifier);
|
||||
StartFanout(isInitial: false);
|
||||
}
|
||||
|
||||
// Liveness recovered — republish so viewers stop falling back to polling even if the
|
||||
// seed that follows finds nothing changed.
|
||||
if (wasDown && _seeded)
|
||||
Publish();
|
||||
}
|
||||
|
||||
private void HandleGrpcError()
|
||||
{
|
||||
if (_stopped) return;
|
||||
@@ -574,6 +810,19 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
// Stream failed before the stability window — retry budget NOT recovered.
|
||||
Timers.Cancel(StabilityTimerKey);
|
||||
|
||||
_openInFlight = false;
|
||||
var wasLive = !_streamDown;
|
||||
_streamDown = true;
|
||||
|
||||
// The next successful connect owes exactly one re-seed. Setting it here (rather than
|
||||
// fanning out now) is the fix for the per-attempt re-fan-out: a site outage used to
|
||||
// cost one whole-site snapshot per retry.
|
||||
_seedOnConnect = true;
|
||||
|
||||
// Liveness lost — tell viewers immediately so a dead stream stops reading as live.
|
||||
if (wasLive && _seeded)
|
||||
Publish();
|
||||
|
||||
_retryCount++;
|
||||
|
||||
if (_retryCount > MaxRetries)
|
||||
@@ -585,7 +834,6 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
_log.Error("Site-alarm gRPC stream for {0} exceeded max retries ({1}); leaving stream down, " +
|
||||
"reconcile snapshots continue and the next reconcile tick will retry the stream",
|
||||
_siteIdentifier, MaxRetries);
|
||||
_streamDown = true;
|
||||
_retryBudgetExhausted = true;
|
||||
CleanupGrpc();
|
||||
return;
|
||||
@@ -602,11 +850,9 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
// Telemetry: a NodeA↔NodeB failover flip is a reconnect + re-seed.
|
||||
ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
|
||||
|
||||
// A failover flip must RE-SEED (never silently serve stale) — kick a reconcile
|
||||
// fan-out alongside the reconnect. Buffering during the fan-out keeps the new
|
||||
// stream's deltas coherent with the fresh snapshot.
|
||||
StartFanout(isInitial: false);
|
||||
|
||||
// The re-seed is owed to the CONNECT (see _seedOnConnect above), not to this
|
||||
// attempt: a snapshot fan-out issued while the site is unreachable degrades to
|
||||
// empty rows and would have to be redone on reconnect anyway.
|
||||
if (_retryCount == 1)
|
||||
Self.Tell(new ReconnectAlarmStream());
|
||||
else
|
||||
@@ -629,8 +875,17 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
_log.Info("Site-alarm gRPC stream for {0} completed gracefully (server end of stream); " +
|
||||
"reopening on the next reconcile tick", _siteIdentifier);
|
||||
|
||||
_openInFlight = false;
|
||||
var wasLive = !_streamDown;
|
||||
_streamDown = true;
|
||||
// The reopen owes one re-seed, run when it actually connects.
|
||||
_seedOnConnect = true;
|
||||
CleanupGrpc();
|
||||
|
||||
// A completed stream is NOT live: without this the cache kept reporting live for up
|
||||
// to a full reconcile interval after the site closed the feed (WP2.3 residual).
|
||||
if (wasLive && _seeded)
|
||||
Publish();
|
||||
}
|
||||
|
||||
private void CleanupGrpc()
|
||||
@@ -676,6 +931,14 @@ internal sealed record SeedFailed(Exception Exception, bool IsInitial);
|
||||
/// <summary>Internal: periodic reconcile tick (and the re-seed kicked after a reconnect).</summary>
|
||||
internal sealed record RunReconcile;
|
||||
|
||||
/// <summary>Internal: backoff-scheduled retry of a fan-out that failed as a whole.</summary>
|
||||
internal sealed record RetrySeed;
|
||||
|
||||
/// <summary>Internal: the site accepted the site-wide alarm subscription (response headers
|
||||
/// received), stamped with its stream generation so a late connect from a cancelled stream
|
||||
/// is ignored. This — not the reconnect attempt — is what earns a re-seed.</summary>
|
||||
internal sealed record GrpcAlarmStreamConnected(int Generation);
|
||||
|
||||
/// <summary>Internal: coalesced-publish tick — flush the dirty cache to viewers once (N6).</summary>
|
||||
internal sealed record PublishCoalesced;
|
||||
|
||||
|
||||
@@ -107,6 +107,21 @@ public class CommunicationOptions
|
||||
/// <summary>Maximum number of concurrent gRPC streaming subscriptions per site node.</summary>
|
||||
public int GrpcMaxConcurrentStreams { get; set; } = 100;
|
||||
|
||||
/// <summary>
|
||||
/// Send-channel capacity for a per-instance Debug View stream (<c>SubscribeInstance</c>).
|
||||
/// Lossy by design: the Debug View is a diagnostic surface and drops its oldest events
|
||||
/// under backpressure rather than stalling the site's event hub.
|
||||
/// </summary>
|
||||
public int GrpcInstanceStreamChannelCapacity { get; set; } = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Send-channel capacity for the site-wide alarm stream (<c>SubscribeSite</c>). Larger
|
||||
/// than the Debug View's (WP2.3): that feed backs the operator Alarm Summary, where a
|
||||
/// silently dropped transition is a missed alarm rather than a missed diagnostic frame,
|
||||
/// and an alarm burst arriving during a WAN stall must survive the stall.
|
||||
/// </summary>
|
||||
public int GrpcSiteAlarmStreamChannelCapacity { get; set; } = 20_000;
|
||||
|
||||
/// <summary>Akka.Remote transport heartbeat interval.</summary>
|
||||
public TimeSpan TransportHeartbeatInterval { get; set; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
@@ -185,4 +200,12 @@ public class CommunicationOptions
|
||||
/// reconcile publishes are always immediate. Default 250 ms.
|
||||
/// </summary>
|
||||
public TimeSpan LiveAlarmCachePublishCoalesce { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Random jitter added to each per-site reconcile tick, as a fraction of
|
||||
/// <see cref="LiveAlarmCacheReconcileInterval"/> (WP2.3). Without it every aggregator
|
||||
/// started by one central failover fans its whole-site snapshot out on the same 60s
|
||||
/// boundary forever. Zero disables jitter.
|
||||
/// </summary>
|
||||
public double LiveAlarmCacheReconcileJitterFraction { get; set; } = 0.2;
|
||||
}
|
||||
|
||||
@@ -1620,7 +1620,10 @@ public static class SiteCommandDtoMapper
|
||||
return new DebugSnapshotRequestDto
|
||||
{
|
||||
InstanceUniqueName = request.InstanceUniqueName,
|
||||
CorrelationId = request.CorrelationId
|
||||
CorrelationId = request.CorrelationId,
|
||||
// Additive field 3 — proto3 false default means an older site that does
|
||||
// not know the field simply returns the full snapshot (correct, just fatter).
|
||||
AlarmsOnly = request.AlarmsOnly
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1631,7 +1634,7 @@ public static class SiteCommandDtoMapper
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
|
||||
return new DebugSnapshotRequest(dto.InstanceUniqueName, dto.CorrelationId);
|
||||
return new DebugSnapshotRequest(dto.InstanceUniqueName, dto.CorrelationId, dto.AlarmsOnly);
|
||||
}
|
||||
|
||||
/// <summary>Projects a <see cref="SubscribeDebugViewRequest"/> onto the wire.</summary>
|
||||
|
||||
@@ -245,13 +245,21 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
|
||||
/// <paramref name="onError"/>; see <see cref="ConsumeStreamAsync"/>.
|
||||
/// </param>
|
||||
/// <param name="ct">Cancellation token to stop the subscription.</param>
|
||||
/// <param name="onConnected">
|
||||
/// Optional callback invoked once when the site has ACCEPTED the subscription (response
|
||||
/// headers received — the site writes them as soon as its relay actor is subscribed, so
|
||||
/// no event can be missed after this point). The per-site aggregator uses it to run
|
||||
/// exactly one re-seed per successful (re)connect instead of one per reconnect attempt.
|
||||
/// Never invoked more than once per call, and never after <paramref name="onError"/>.
|
||||
/// </param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public virtual async Task SubscribeSiteAsync(
|
||||
string correlationId,
|
||||
Action<AlarmStateChanged> onAlarmEvent,
|
||||
Action<Exception> onError,
|
||||
Action onCompleted,
|
||||
CancellationToken ct)
|
||||
CancellationToken ct,
|
||||
Action? onConnected = null)
|
||||
{
|
||||
if (_client is null)
|
||||
throw new InvalidOperationException("Cannot subscribe on a test-only client.");
|
||||
@@ -275,7 +283,8 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
|
||||
onAlarmEvent(alarm);
|
||||
},
|
||||
onError,
|
||||
onCompleted);
|
||||
onCompleted,
|
||||
onConnected);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -301,6 +310,13 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
|
||||
/// <param name="onEvent">Invoked per wire event.</param>
|
||||
/// <param name="onError">Invoked once if the stream faulted.</param>
|
||||
/// <param name="onCompleted">Invoked once if the server ended the stream with OK.</param>
|
||||
/// <param name="onConnected">
|
||||
/// Optional; invoked once when the server's response headers arrive — i.e. the site has
|
||||
/// accepted the subscription and its relay actor is attached. Bounded by
|
||||
/// <see cref="ConnectedHeaderTimeout"/> so a peer that defers headers (a pre-WP2.3 site,
|
||||
/// which only flushes them with its first event) still reports connected instead of
|
||||
/// leaving the caller waiting for a signal that may never come on a quiet site.
|
||||
/// </param>
|
||||
/// <returns>A task that completes when the stream has ended and its outcome been reported.</returns>
|
||||
internal async Task ConsumeStreamAsync(
|
||||
string correlationId,
|
||||
@@ -308,13 +324,20 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
|
||||
Func<AsyncServerStreamingCall<SiteStreamEvent>> openCall,
|
||||
Action<SiteStreamEvent> onEvent,
|
||||
Action<Exception> onError,
|
||||
Action onCompleted)
|
||||
Action onCompleted,
|
||||
Action? onConnected = null)
|
||||
{
|
||||
var completedGracefully = false;
|
||||
try
|
||||
{
|
||||
using (var call = openCall())
|
||||
{
|
||||
if (onConnected is not null)
|
||||
{
|
||||
await AwaitHeadersAsync(call, cts.Token).ConfigureAwait(false);
|
||||
onConnected();
|
||||
}
|
||||
|
||||
await foreach (var evt in call.ResponseStream.ReadAllAsync(cts.Token))
|
||||
{
|
||||
onEvent(evt);
|
||||
@@ -348,6 +371,38 @@ public class SiteStreamGrpcClient : IAsyncDisposable, IDisposable
|
||||
onCompleted();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// How long to wait for response headers before treating the stream as connected anyway.
|
||||
/// A peer that only flushes headers with its first message would otherwise hold the
|
||||
/// connected signal — and with it the aggregator's re-seed — for as long as the site
|
||||
/// happens to be quiet.
|
||||
/// </summary>
|
||||
internal static TimeSpan ConnectedHeaderTimeout { get; set; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>
|
||||
/// Awaits the call's response headers, bounded by <see cref="ConnectedHeaderTimeout"/>.
|
||||
/// A fault propagates (the caller reports it through <c>onError</c> like any other stream
|
||||
/// fault); a timeout returns normally. On timeout the abandoned headers task is observed
|
||||
/// so a later fault on it can never surface as an unobserved task exception.
|
||||
/// </summary>
|
||||
private static async Task AwaitHeadersAsync(
|
||||
AsyncServerStreamingCall<SiteStreamEvent> call, CancellationToken ct)
|
||||
{
|
||||
var headers = call.ResponseHeadersAsync;
|
||||
try
|
||||
{
|
||||
await headers.WaitAsync(ConnectedHeaderTimeout, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
_ = headers.ContinueWith(
|
||||
t => _ = t.Exception,
|
||||
CancellationToken.None,
|
||||
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cancels an active subscription by correlation ID.
|
||||
/// </summary>
|
||||
|
||||
@@ -27,6 +27,8 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
private readonly ConcurrentDictionary<string, StreamEntry> _activeStreams = new();
|
||||
private readonly int _maxConcurrentStreams;
|
||||
private readonly TimeSpan _maxStreamLifetime;
|
||||
private readonly int _instanceChannelCapacity;
|
||||
private readonly int _siteAlarmChannelCapacity;
|
||||
private volatile bool _ready;
|
||||
// Flipped by CancelAllStreams() when the host enters
|
||||
// CoordinatedShutdown so SubscribeInstance refuses new streams with
|
||||
@@ -72,10 +74,17 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
ISiteStreamSubscriber streamSubscriber,
|
||||
ILogger<SiteStreamGrpcServer> logger,
|
||||
int maxConcurrentStreams = 100)
|
||||
: this(streamSubscriber, logger, maxConcurrentStreams, TimeSpan.FromHours(4))
|
||||
: this(streamSubscriber, logger, maxConcurrentStreams, TimeSpan.FromHours(4),
|
||||
DefaultInstanceChannelCapacity, DefaultSiteAlarmChannelCapacity)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Fallback Debug View send-channel capacity when no options are bound.</summary>
|
||||
internal const int DefaultInstanceChannelCapacity = 1000;
|
||||
|
||||
/// <summary>Fallback site-wide alarm send-channel capacity when no options are bound.</summary>
|
||||
internal const int DefaultSiteAlarmChannelCapacity = 20_000;
|
||||
|
||||
/// <summary>
|
||||
/// DI constructor — binds <see cref="CommunicationOptions.GrpcMaxConcurrentStreams"/>
|
||||
/// and <see cref="CommunicationOptions.GrpcMaxStreamLifetime"/> so the documented
|
||||
@@ -91,7 +100,9 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
IOptions<CommunicationOptions> options)
|
||||
: this(streamSubscriber, logger,
|
||||
options.Value.GrpcMaxConcurrentStreams,
|
||||
options.Value.GrpcMaxStreamLifetime)
|
||||
options.Value.GrpcMaxStreamLifetime,
|
||||
options.Value.GrpcInstanceStreamChannelCapacity,
|
||||
options.Value.GrpcSiteAlarmStreamChannelCapacity)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -99,12 +110,16 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
ISiteStreamSubscriber streamSubscriber,
|
||||
ILogger<SiteStreamGrpcServer> logger,
|
||||
int maxConcurrentStreams,
|
||||
TimeSpan maxStreamLifetime)
|
||||
TimeSpan maxStreamLifetime,
|
||||
int instanceChannelCapacity,
|
||||
int siteAlarmChannelCapacity)
|
||||
{
|
||||
_streamSubscriber = streamSubscriber;
|
||||
_logger = logger;
|
||||
_maxConcurrentStreams = maxConcurrentStreams;
|
||||
_maxStreamLifetime = maxStreamLifetime;
|
||||
_instanceChannelCapacity = Math.Max(1, instanceChannelCapacity);
|
||||
_siteAlarmChannelCapacity = Math.Max(1, siteAlarmChannelCapacity);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -209,6 +224,21 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
/// <summary>Effective per-stream session lifetime. Exposed for tests.</summary>
|
||||
internal TimeSpan MaxStreamLifetime => _maxStreamLifetime;
|
||||
|
||||
/// <summary>Effective Debug View send-channel capacity. Exposed for tests.</summary>
|
||||
internal int InstanceChannelCapacity => _instanceChannelCapacity;
|
||||
|
||||
/// <summary>Effective site-wide alarm send-channel capacity. Exposed for tests.</summary>
|
||||
internal int SiteAlarmChannelCapacity => _siteAlarmChannelCapacity;
|
||||
|
||||
/// <summary>
|
||||
/// Total events evicted from stream send channels on this node since start (both stream
|
||||
/// kinds). Exported as <c>scadabridge.site.stream.events_dropped</c>; exposed here so a
|
||||
/// test — and an operator via diagnostics — can read the raw count.
|
||||
/// </summary>
|
||||
public long DroppedStreamEventCount => Interlocked.Read(ref _droppedStreamEvents);
|
||||
|
||||
private long _droppedStreamEvents;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task SubscribeInstance(
|
||||
InstanceStreamRequest request,
|
||||
@@ -219,7 +249,9 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
responseStream,
|
||||
context,
|
||||
relay => _streamSubscriber.Subscribe(request.InstanceUniqueName, relay),
|
||||
request.InstanceUniqueName);
|
||||
request.InstanceUniqueName,
|
||||
_instanceChannelCapacity,
|
||||
streamKind: "instance");
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task SubscribeSite(
|
||||
@@ -234,7 +266,12 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
// already drops IsConfiguredPlaceholder rows and maps only the
|
||||
// enriched AlarmStateUpdate, so it is reused unchanged.
|
||||
_streamSubscriber.SubscribeSiteAlarms,
|
||||
"site-wide alarms");
|
||||
"site-wide alarms",
|
||||
// Its OWN, much larger channel (WP2.3): sharing the Debug View's 1000-slot
|
||||
// DropOldest meant an alarm burst during a WAN stall silently evicted operator-
|
||||
// visible transitions to make room for diagnostics traffic.
|
||||
_siteAlarmChannelCapacity,
|
||||
streamKind: "site-alarms");
|
||||
|
||||
/// <summary>
|
||||
/// Shared streaming pipeline behind <see cref="SubscribeInstance"/> and
|
||||
@@ -250,12 +287,16 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
/// <param name="context">The server call context (carries the client cancellation token).</param>
|
||||
/// <param name="subscribe">Subscribes the relay actor to the hub, returning a subscription id.</param>
|
||||
/// <param name="description">Human-readable subscription description for logging.</param>
|
||||
/// <param name="channelCapacity">Send-channel capacity for this stream kind.</param>
|
||||
/// <param name="streamKind">Telemetry tag for this stream kind (<c>instance</c>/<c>site-alarms</c>).</param>
|
||||
private async Task RunSubscriptionStreamAsync(
|
||||
string correlationId,
|
||||
IServerStreamWriter<SiteStreamEvent> responseStream,
|
||||
ServerCallContext context,
|
||||
Func<IActorRef, string> subscribe,
|
||||
string description)
|
||||
string description,
|
||||
int channelCapacity,
|
||||
string streamKind)
|
||||
{
|
||||
if (!_ready)
|
||||
throw new RpcException(new GrpcStatus(StatusCode.Unavailable, "Server not ready"));
|
||||
@@ -302,16 +343,20 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
|
||||
long dropped = 0;
|
||||
var channel = Channel.CreateBounded<SiteStreamEvent>(
|
||||
new BoundedChannelOptions(1000) { FullMode = BoundedChannelFullMode.DropOldest },
|
||||
new BoundedChannelOptions(channelCapacity) { FullMode = BoundedChannelFullMode.DropOldest },
|
||||
_ =>
|
||||
{
|
||||
// Lossy-under-backpressure is the debug-view spec; make the real
|
||||
// loss visible: first eviction + every 500th thereafter.
|
||||
// loss visible: first eviction + every 500th thereafter, plus a
|
||||
// per-kind counter on the node (WP2.3).
|
||||
var n = Interlocked.Increment(ref dropped);
|
||||
Interlocked.Increment(ref _droppedStreamEvents);
|
||||
ScadaBridgeTelemetry.RecordSiteStreamEventDropped(streamKind);
|
||||
if (n == 1 || n % 500 == 0)
|
||||
_logger.LogWarning(
|
||||
"Debug stream {CorrelationId} backpressure: {Dropped} oldest event(s) evicted so far",
|
||||
correlationId, n);
|
||||
"Stream {CorrelationId} ({StreamKind}) backpressure: {Dropped} oldest event(s) evicted so far " +
|
||||
"of a {Capacity}-slot channel",
|
||||
correlationId, streamKind, n, channelCapacity);
|
||||
});
|
||||
|
||||
var actorSeq = Interlocked.Increment(ref _actorCounter);
|
||||
@@ -348,6 +393,25 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
"Stream {CorrelationId} started for {Description} (subscription {SubscriptionId})",
|
||||
correlationId, description, subscriptionId);
|
||||
|
||||
// Flush response headers NOW, while the relay is attached and before the first event
|
||||
// (WP2.3). This is the client's "the site accepted my subscription" signal — the
|
||||
// per-site aggregator hangs its once-per-successful-reconnect re-seed off it. Without
|
||||
// an explicit flush, ASP.NET Core defers headers to the first written message, so a
|
||||
// quiet site would never report connected. Best-effort: a client that vanished between
|
||||
// Subscribe and here fails the write, and the read loop below handles the teardown.
|
||||
try
|
||||
{
|
||||
await context.WriteResponseHeadersAsync(Metadata.Empty);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Deliberately catch-all: an unsent header must never leak the relay actor and
|
||||
// the _activeStreams entry, which the finally below owns. Any real transport
|
||||
// failure resurfaces on the read/write loop immediately after.
|
||||
_logger.LogDebug(ex,
|
||||
"Could not flush response headers for stream {CorrelationId}; continuing.", correlationId);
|
||||
}
|
||||
|
||||
// Telemetry follow-on: the connection is now fully established (Subscribe
|
||||
// succeeded, so no leak via the catch above). Count it up here and balance
|
||||
// it in the finally below so the scadabridge.site.connection.up gauge is
|
||||
@@ -519,17 +583,46 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
? DateTime.SpecifyKind(request.SinceUtc.ToDateTime(), DateTimeKind.Utc)
|
||||
: DateTime.MinValue;
|
||||
|
||||
// Composite-keyset cursor (WP2.3), mirroring PullSiteCalls: proto3
|
||||
// defaults an unset string to "", which the queue treats as "no cursor"
|
||||
// (legacy inclusive >= behaviour), so an older central is unaffected.
|
||||
var afterId = string.IsNullOrEmpty(request.AfterId) ? null : request.AfterId;
|
||||
|
||||
// AT-LEAST-ONCE (WP2.3). The incoming cursor is central's receipt for
|
||||
// everything at or before it — that is the ONLY proof the site accepts.
|
||||
// Rows are flipped to Reconciled here, at the START of the NEXT pull,
|
||||
// instead of right after the previous response was projected: a fault
|
||||
// between the response leaving the site and central committing it used
|
||||
// to lose the rows outright, because they were already Reconciled and
|
||||
// ReadPendingSinceAsync would never serve them again. The flip runs
|
||||
// before the read so the rows it retires do not consume this batch's
|
||||
// budget. Best-effort — a failure here only costs a re-ship, which
|
||||
// central dedups on EventId.
|
||||
if (since > DateTime.MinValue)
|
||||
{
|
||||
try
|
||||
{
|
||||
await queue.MarkReconciledUpToAsync(since, afterId, context.CancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"MarkReconciledUpToAsync failed for cursor since={Since} afterId={AfterId}; rows stay pending for the next pull.",
|
||||
since, afterId);
|
||||
}
|
||||
}
|
||||
|
||||
IReadOnlyList<AuditEvent> events;
|
||||
try
|
||||
{
|
||||
events = await queue.ReadPendingSinceAsync(
|
||||
since, request.BatchSize, context.CancellationToken);
|
||||
since, request.BatchSize, afterId, context.CancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"ReadPendingSinceAsync failed for since={Since} batch={Batch}; returning empty response.",
|
||||
since, request.BatchSize);
|
||||
"ReadPendingSinceAsync failed for since={Since} batch={Batch} afterId={AfterId}; returning empty response.",
|
||||
since, request.BatchSize, afterId);
|
||||
return new PullAuditEventsResponse();
|
||||
}
|
||||
|
||||
@@ -537,7 +630,8 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
{
|
||||
// batch_size saturated → tell central to issue a follow-up pull
|
||||
// with an advanced cursor. The site doesn't compute the cursor —
|
||||
// central walks it forward from the last returned OccurredAtUtc.
|
||||
// central walks it forward from the last returned OccurredAtUtc
|
||||
// (plus, once it sets after_id, that row's EventId).
|
||||
MoreAvailable = events.Count >= request.BatchSize,
|
||||
};
|
||||
foreach (var evt in events)
|
||||
@@ -545,31 +639,6 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
|
||||
response.Events.Add(AuditEventDtoMapper.ToDto(evt));
|
||||
}
|
||||
|
||||
// Flip to Reconciled AFTER projecting the response so a fault below the
|
||||
// try/catch (mid-response, mid-flip) leaves the rows in Pending/Forwarded
|
||||
// and central pulls them again next cycle. The flip itself is
|
||||
// best-effort — its failure is a warning, not a fault, because central
|
||||
// will dedup on EventId on the next pull.
|
||||
var ids = new List<Guid>(events.Count);
|
||||
foreach (var evt in events)
|
||||
{
|
||||
ids.Add(evt.EventId);
|
||||
}
|
||||
|
||||
if (ids.Count > 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
await queue.MarkReconciledAsync(ids, context.CancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"MarkReconciledAsync failed after PullAuditEvents response of {Count} rows; rows stay Pending for retry.",
|
||||
ids.Count);
|
||||
}
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
|
||||
@@ -553,6 +553,14 @@ message EventLogQueryResponseDto {
|
||||
message DebugSnapshotRequestDto {
|
||||
string instance_unique_name = 1;
|
||||
string correlation_id = 2;
|
||||
// Alarms-only projection (WP2.3 wire efficiency): when true the site builds
|
||||
// and returns ONLY the alarm rows of the debug snapshot, leaving
|
||||
// attribute_values empty. Used by the central per-site live alarm cache,
|
||||
// whose seed/reconcile fan-out discards every attribute row anyway — a full
|
||||
// snapshot ships the whole attribute surface of every enabled instance once
|
||||
// per reconcile for nothing. proto3 defaults it to false, so an older
|
||||
// central that never sets it keeps the full-snapshot behaviour. Additive-only.
|
||||
bool alarms_only = 3;
|
||||
}
|
||||
|
||||
message SubscribeDebugViewRequestDto {
|
||||
|
||||
@@ -164,13 +164,26 @@ message CachedTelemetryBatch { repeated CachedTelemetryPacket packets = 1; }
|
||||
|
||||
// Audit Log (#23) M6 reconciliation pull: central→site request for any
|
||||
// site-local AuditLog rows with OccurredAtUtc >= since_utc that have not yet
|
||||
// been ingested centrally (ForwardState in {Pending, Forwarded}). The site
|
||||
// flips returned rows to Reconciled after the response is on the wire.
|
||||
// been ingested centrally (ForwardState in {Pending, Forwarded}). Rows are NOT
|
||||
// flipped to Reconciled when they are served — only when a LATER pull's cursor
|
||||
// proves central consumed them (see after_id), so a fault between the response
|
||||
// leaving the site and central committing it re-serves the rows instead of
|
||||
// silently losing them (at-least-once).
|
||||
// more_available signals batch_size was saturated so the caller knows to
|
||||
// issue a follow-up pull with an advanced since_utc cursor.
|
||||
message PullAuditEventsRequest {
|
||||
google.protobuf.Timestamp since_utc = 1;
|
||||
int32 batch_size = 2;
|
||||
// Composite-keyset cursor (WP2.3), mirroring PullSiteCallsRequest.after_id:
|
||||
// the EventId ("D" GUID form) of the last row central has already CONSUMED at
|
||||
// since_utc. When set, the site returns only rows strictly after the composite
|
||||
// (OccurredAtUtc, EventId) pair — un-pinning a batch that would otherwise stall
|
||||
// when more than batch_size rows share one since_utc instant — AND treats the
|
||||
// cursor as proof of receipt: everything at or before it is flipped to
|
||||
// Reconciled. Empty (the proto3 string default) preserves the legacy inclusive
|
||||
// >= behaviour, under which only rows strictly older than since_utc are proven
|
||||
// received. Additive-only.
|
||||
string after_id = 3;
|
||||
}
|
||||
|
||||
message PullAuditEventsResponse {
|
||||
|
||||
@@ -132,7 +132,7 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
public bool IsLive(int siteId)
|
||||
{
|
||||
lock (_lock)
|
||||
return _sites.TryGetValue(siteId, out var entry) && entry.HasPublished;
|
||||
return _sites.TryGetValue(siteId, out var entry) && entry.HasPublished && entry.StreamLive;
|
||||
}
|
||||
|
||||
// ── Subscriber teardown ─────────────────────────────────────────────────────
|
||||
@@ -213,8 +213,8 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
|
||||
Func<CancellationToken, Task<IReadOnlyList<AlarmStateChanged>>> seedFn =
|
||||
ct => FanOutSnapshotsAsync(entry.SiteId, ct);
|
||||
Action<IReadOnlyList<AlarmStateChanged>> publish =
|
||||
snapshot => OnPublish(entry, snapshot);
|
||||
Action<IReadOnlyList<AlarmStateChanged>, bool> publish =
|
||||
(snapshot, streamLive) => OnPublish(entry, snapshot, streamLive);
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
@@ -236,7 +236,8 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
_options.LiveAlarmCacheReconcileInterval,
|
||||
_options.LiveAlarmCachePublishCoalesce,
|
||||
TimeSpan.FromSeconds(5), // reconnect delay — former ReconnectDelay static default
|
||||
TimeSpan.FromSeconds(60))); // stability window — former StabilityWindow static default
|
||||
TimeSpan.FromSeconds(60), // stability window — former StabilityWindow static default
|
||||
_options.LiveAlarmCacheReconcileJitterFraction));
|
||||
|
||||
entry.Actor = system.ActorOf(props, $"site-alarm-aggregator-{entry.SiteId}-{Guid.NewGuid():N}");
|
||||
|
||||
@@ -378,7 +379,12 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
await gate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var request = new DebugSnapshotRequest(instanceUniqueName, Guid.NewGuid().ToString("N"));
|
||||
// Alarms-only (WP2.3): this fan-out discards snapshot.AttributeValues, so
|
||||
// asking the site to build and ship them is pure wire waste. The flag is
|
||||
// additive on the wire; a pre-WP2.3 site ignores it and returns the full
|
||||
// snapshot, which this loop reads exactly as before.
|
||||
var request = new DebugSnapshotRequest(
|
||||
instanceUniqueName, Guid.NewGuid().ToString("N"), AlarmsOnly: true);
|
||||
var snapshot = await _communicationService.RequestDebugSnapshotAsync(siteIdentifier, request, ct);
|
||||
if (snapshot.InstanceNotFound)
|
||||
return;
|
||||
@@ -402,7 +408,7 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
|
||||
// ── Publish (actor thread → viewers) ────────────────────────────────────────
|
||||
|
||||
private void OnPublish(SiteEntry entry, IReadOnlyList<AlarmStateChanged> snapshot)
|
||||
private void OnPublish(SiteEntry entry, IReadOnlyList<AlarmStateChanged> snapshot, bool streamLive)
|
||||
{
|
||||
Subscription[] subscribers;
|
||||
lock (_lock)
|
||||
@@ -410,6 +416,10 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
// Store the fresh immutable snapshot (readers get it lock-free-ish via GetCurrentAlarms).
|
||||
entry.Current = snapshot;
|
||||
entry.HasPublished = true;
|
||||
// Liveness is the STREAM's, not the cache's: a completed or given-up stream must
|
||||
// stop reporting live between reopen ticks, or the page grafts a freezing snapshot
|
||||
// over fresh poll data (WP2.3 carried residual).
|
||||
entry.StreamLive = streamLive;
|
||||
subscribers = entry.Subscribers.ToArray();
|
||||
}
|
||||
|
||||
@@ -462,6 +472,7 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
|
||||
entry.Actor = null;
|
||||
entry.HasPublished = false;
|
||||
entry.StreamLive = false;
|
||||
entry.Current = Empty;
|
||||
|
||||
if (entry.Subscribers.Count > 0 && !entry.Starting)
|
||||
@@ -498,6 +509,13 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
/// <summary>True once the aggregator has seeded and published at least once.</summary>
|
||||
public bool HasPublished { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Liveness of the aggregator's site-wide gRPC stream as of the last publish. Both
|
||||
/// this and <see cref="HasPublished"/> must hold for <c>IsLive</c>: a published cache
|
||||
/// behind a dead stream is stale, not live.
|
||||
/// </summary>
|
||||
public bool StreamLive { get; set; }
|
||||
|
||||
public Timer? LingerTimer { get; set; }
|
||||
public int LingerVersion { get; set; }
|
||||
|
||||
|
||||
@@ -260,241 +260,242 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
"cmlkZ2Uuc2l0ZWNvbW1hbmQudjEuRXZlbnRMb2dFbnRyeUR0bxIaChJjb250",
|
||||
"aW51YXRpb25fdG9rZW4YBCABKAkSEAoIaGFzX21vcmUYBSABKAgSDwoHc3Vj",
|
||||
"Y2VzcxgGIAEoCBIVCg1lcnJvcl9tZXNzYWdlGAcgASgJEi0KCXRpbWVzdGFt",
|
||||
"cBgIIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiTwoXRGVidWdT",
|
||||
"cBgIIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiZAoXRGVidWdT",
|
||||
"bmFwc2hvdFJlcXVlc3REdG8SHAoUaW5zdGFuY2VfdW5pcXVlX25hbWUYASAB",
|
||||
"KAkSFgoOY29ycmVsYXRpb25faWQYAiABKAkiVAocU3Vic2NyaWJlRGVidWdW",
|
||||
"aWV3UmVxdWVzdER0bxIcChRpbnN0YW5jZV91bmlxdWVfbmFtZRgBIAEoCRIW",
|
||||
"Cg5jb3JyZWxhdGlvbl9pZBgCIAEoCSJWCh5VbnN1YnNjcmliZURlYnVnVmll",
|
||||
"d1JlcXVlc3REdG8SHAoUaW5zdGFuY2VfdW5pcXVlX25hbWUYASABKAkSFgoO",
|
||||
"Y29ycmVsYXRpb25faWQYAiABKAkiHAoaVW5zdWJzY3JpYmVEZWJ1Z1ZpZXdB",
|
||||
"Y2tEdG8i1AEKFkFsYXJtQ29uZGl0aW9uU3RhdGVEdG8SDgoGYWN0aXZlGAEg",
|
||||
"ASgIEhQKDGFja25vd2xlZGdlZBgCIAEoCBItCgljb25maXJtZWQYAyABKAsy",
|
||||
"Gi5nb29nbGUucHJvdG9idWYuQm9vbFZhbHVlEj8KBnNoZWx2ZRgEIAEoDjIv",
|
||||
"LnNjYWRhYnJpZGdlLnNpdGVjb21tYW5kLnYxLkFsYXJtU2hlbHZlU3RhdGVE",
|
||||
"dG8SEgoKc3VwcHJlc3NlZBgFIAEoCBIQCghzZXZlcml0eRgGIAEoBSLdAQoW",
|
||||
"RGVidWdBdHRyaWJ1dGVWYWx1ZUR0bxIcChRpbnN0YW5jZV91bmlxdWVfbmFt",
|
||||
"ZRgBIAEoCRIWCg5hdHRyaWJ1dGVfcGF0aBgCIAEoCRIWCg5hdHRyaWJ1dGVf",
|
||||
"bmFtZRgDIAEoCRI1CgV2YWx1ZRgEIAEoCzImLnNjYWRhYnJpZGdlLnNpdGVj",
|
||||
"b21tYW5kLnYxLkxvb3NlVmFsdWUSDwoHcXVhbGl0eRgFIAEoCRItCgl0aW1l",
|
||||
"c3RhbXAYBiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wIq8FChJE",
|
||||
"ZWJ1Z0FsYXJtU3RhdGVEdG8SHAoUaW5zdGFuY2VfdW5pcXVlX25hbWUYASAB",
|
||||
"KAkSEgoKYWxhcm1fbmFtZRgCIAEoCRI4CgVzdGF0ZRgDIAEoDjIpLnNjYWRh",
|
||||
"YnJpZGdlLnNpdGVjb21tYW5kLnYxLkFsYXJtU3RhdGVEdG8SEAoIcHJpb3Jp",
|
||||
"dHkYBCABKAUSLQoJdGltZXN0YW1wGAUgASgLMhouZ29vZ2xlLnByb3RvYnVm",
|
||||
"LlRpbWVzdGFtcBI4CgVsZXZlbBgGIAEoDjIpLnNjYWRhYnJpZGdlLnNpdGVj",
|
||||
"b21tYW5kLnYxLkFsYXJtTGV2ZWxEdG8SDwoHbWVzc2FnZRgHIAEoCRI2CgRr",
|
||||
"aW5kGAggASgOMiguc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuQWxhcm1L",
|
||||
"aW5kRHRvEkUKCWNvbmRpdGlvbhgJIAEoCzIyLnNjYWRhYnJpZGdlLnNpdGVj",
|
||||
"b21tYW5kLnYxLkFsYXJtQ29uZGl0aW9uU3RhdGVEdG8SGAoQc291cmNlX3Jl",
|
||||
"ZmVyZW5jZRgKIAEoCRIXCg9hbGFybV90eXBlX25hbWUYCyABKAkSEAoIY2F0",
|
||||
"ZWdvcnkYDCABKAkSFQoNb3BlcmF0b3JfdXNlchgNIAEoCRIYChBvcGVyYXRv",
|
||||
"cl9jb21tZW50GA4gASgJEjcKE29yaWdpbmFsX3JhaXNlX3RpbWUYDyABKAsy",
|
||||
"Gi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhUKDWN1cnJlbnRfdmFsdWUY",
|
||||
"ECABKAkSEwoLbGltaXRfdmFsdWUYESABKAkSJAocbmF0aXZlX3NvdXJjZV9j",
|
||||
"YW5vbmljYWxfbmFtZRgSIAEoCRIhChlpc19jb25maWd1cmVkX3BsYWNlaG9s",
|
||||
"ZGVyGBMgASgIIpwCChREZWJ1Z1ZpZXdTbmFwc2hvdER0bxIcChRpbnN0YW5j",
|
||||
"ZV91bmlxdWVfbmFtZRgBIAEoCRJMChBhdHRyaWJ1dGVfdmFsdWVzGAIgAygL",
|
||||
"MjIuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuRGVidWdBdHRyaWJ1dGVW",
|
||||
"YWx1ZUR0bxJECgxhbGFybV9zdGF0ZXMYAyADKAsyLi5zY2FkYWJyaWRnZS5z",
|
||||
"aXRlY29tbWFuZC52MS5EZWJ1Z0FsYXJtU3RhdGVEdG8SNgoSc25hcHNob3Rf",
|
||||
"dGltZXN0YW1wGAQgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIa",
|
||||
"ChJpbnN0YW5jZV9ub3RfZm91bmQYBSABKAgi8AIKDFF1ZXJ5UmVxdWVzdBJO",
|
||||
"Cg9ldmVudF9sb2dfcXVlcnkYASABKAsyMy5zY2FkYWJyaWRnZS5zaXRlY29t",
|
||||
"bWFuZC52MS5FdmVudExvZ1F1ZXJ5UmVxdWVzdER0b0gAEk0KDmRlYnVnX3Nu",
|
||||
"YXBzaG90GAIgASgLMjMuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuRGVi",
|
||||
"dWdTbmFwc2hvdFJlcXVlc3REdG9IABJYChRzdWJzY3JpYmVfZGVidWdfdmll",
|
||||
"dxgDIAEoCzI4LnNjYWRhYnJpZGdlLnNpdGVjb21tYW5kLnYxLlN1YnNjcmli",
|
||||
"ZURlYnVnVmlld1JlcXVlc3REdG9IABJcChZ1bnN1YnNjcmliZV9kZWJ1Z192",
|
||||
"aWV3GAQgASgLMjouc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuVW5zdWJz",
|
||||
"Y3JpYmVEZWJ1Z1ZpZXdSZXF1ZXN0RHRvSABCCQoHY29tbWFuZCKRAgoKUXVl",
|
||||
"cnlSZXBseRJPCg9ldmVudF9sb2dfcXVlcnkYASABKAsyNC5zY2FkYWJyaWRn",
|
||||
"ZS5zaXRlY29tbWFuZC52MS5FdmVudExvZ1F1ZXJ5UmVzcG9uc2VEdG9IABJP",
|
||||
"ChNkZWJ1Z192aWV3X3NuYXBzaG90GAIgASgLMjAuc2NhZGFicmlkZ2Uuc2l0",
|
||||
"ZWNvbW1hbmQudjEuRGVidWdWaWV3U25hcHNob3REdG9IABJYChZ1bnN1YnNj",
|
||||
"cmliZV9kZWJ1Z192aWV3GAMgASgLMjYuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1h",
|
||||
"bmQudjEuVW5zdWJzY3JpYmVEZWJ1Z1ZpZXdBY2tEdG9IAEIHCgVyZXBseSKe",
|
||||
"AQocUGFya2VkTWVzc2FnZVF1ZXJ5UmVxdWVzdER0bxIWCg5jb3JyZWxhdGlv",
|
||||
"bl9pZBgBIAEoCRIPCgdzaXRlX2lkGAIgASgJEhMKC3BhZ2VfbnVtYmVyGAMg",
|
||||
"ASgFEhEKCXBhZ2Vfc2l6ZRgEIAEoBRItCgl0aW1lc3RhbXAYBSABKAsyGi5n",
|
||||
"b29nbGUucHJvdG9idWYuVGltZXN0YW1wIvICChVQYXJrZWRNZXNzYWdlRW50",
|
||||
"cnlEdG8SEgoKbWVzc2FnZV9pZBgBIAEoCRIVCg10YXJnZXRfc3lzdGVtGAIg",
|
||||
"ASgJEhMKC21ldGhvZF9uYW1lGAMgASgJEhUKDWVycm9yX21lc3NhZ2UYBCAB",
|
||||
"KAkSFQoNYXR0ZW1wdF9jb3VudBgFIAEoBRI2ChJvcmlnaW5hbF90aW1lc3Rh",
|
||||
"bXAYBiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEjoKFmxhc3Rf",
|
||||
"YXR0ZW1wdF90aW1lc3RhbXAYByABKAsyGi5nb29nbGUucHJvdG9idWYuVGlt",
|
||||
"ZXN0YW1wEhQKDG1heF9hdHRlbXB0cxgIIAEoBRJICghjYXRlZ29yeRgJIAEo",
|
||||
"DjI2LnNjYWRhYnJpZGdlLnNpdGVjb21tYW5kLnYxLlN0b3JlQW5kRm9yd2Fy",
|
||||
"ZENhdGVnb3J5RHRvEhcKD29yaWdpbl9pbnN0YW5jZRgKIAEoCSKhAgodUGFy",
|
||||
"a2VkTWVzc2FnZVF1ZXJ5UmVzcG9uc2VEdG8SFgoOY29ycmVsYXRpb25faWQY",
|
||||
"ASABKAkSDwoHc2l0ZV9pZBgCIAEoCRJDCghtZXNzYWdlcxgDIAMoCzIxLnNj",
|
||||
"YWRhYnJpZGdlLnNpdGVjb21tYW5kLnYxLlBhcmtlZE1lc3NhZ2VFbnRyeUR0",
|
||||
"bxITCgt0b3RhbF9jb3VudBgEIAEoBRITCgtwYWdlX251bWJlchgFIAEoBRIR",
|
||||
"CglwYWdlX3NpemUYBiABKAUSDwoHc3VjY2VzcxgHIAEoCBIVCg1lcnJvcl9t",
|
||||
"ZXNzYWdlGAggASgJEi0KCXRpbWVzdGFtcBgJIAEoCzIaLmdvb2dsZS5wcm90",
|
||||
"b2J1Zi5UaW1lc3RhbXAiigEKHFBhcmtlZE1lc3NhZ2VSZXRyeVJlcXVlc3RE",
|
||||
"dG8SFgoOY29ycmVsYXRpb25faWQYASABKAkSDwoHc2l0ZV9pZBgCIAEoCRIS",
|
||||
"CgptZXNzYWdlX2lkGAMgASgJEi0KCXRpbWVzdGFtcBgEIAEoCzIaLmdvb2ds",
|
||||
"ZS5wcm90b2J1Zi5UaW1lc3RhbXAiXwodUGFya2VkTWVzc2FnZVJldHJ5UmVz",
|
||||
"cG9uc2VEdG8SFgoOY29ycmVsYXRpb25faWQYASABKAkSDwoHc3VjY2VzcxgC",
|
||||
"IAEoCBIVCg1lcnJvcl9tZXNzYWdlGAMgASgJIowBCh5QYXJrZWRNZXNzYWdl",
|
||||
"RGlzY2FyZFJlcXVlc3REdG8SFgoOY29ycmVsYXRpb25faWQYASABKAkSDwoH",
|
||||
"c2l0ZV9pZBgCIAEoCRISCgptZXNzYWdlX2lkGAMgASgJEi0KCXRpbWVzdGFt",
|
||||
"cBgEIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiYQofUGFya2Vk",
|
||||
"TWVzc2FnZURpc2NhcmRSZXNwb25zZUR0bxIWCg5jb3JyZWxhdGlvbl9pZBgB",
|
||||
"IAEoCRIPCgdzdWNjZXNzGAIgASgIEhUKDWVycm9yX21lc3NhZ2UYAyABKAki",
|
||||
"TwoXUmV0cnlQYXJrZWRPcGVyYXRpb25EdG8SFgoOY29ycmVsYXRpb25faWQY",
|
||||
"ASABKAkSHAoUdHJhY2tlZF9vcGVyYXRpb25faWQYAiABKAkiUQoZRGlzY2Fy",
|
||||
"ZFBhcmtlZE9wZXJhdGlvbkR0bxIWCg5jb3JyZWxhdGlvbl9pZBgBIAEoCRIc",
|
||||
"ChR0cmFja2VkX29wZXJhdGlvbl9pZBgCIAEoCSJdChtQYXJrZWRPcGVyYXRp",
|
||||
"b25BY3Rpb25BY2tEdG8SFgoOY29ycmVsYXRpb25faWQYASABKAkSDwoHYXBw",
|
||||
"bGllZBgCIAEoCBIVCg1lcnJvcl9tZXNzYWdlGAMgASgJIt4DCg1QYXJrZWRS",
|
||||
"ZXF1ZXN0ElgKFHBhcmtlZF9tZXNzYWdlX3F1ZXJ5GAEgASgLMjguc2NhZGFi",
|
||||
"cmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUGFya2VkTWVzc2FnZVF1ZXJ5UmVxdWVz",
|
||||
"dER0b0gAElgKFHBhcmtlZF9tZXNzYWdlX3JldHJ5GAIgASgLMjguc2NhZGFi",
|
||||
"cmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUGFya2VkTWVzc2FnZVJldHJ5UmVxdWVz",
|
||||
"dER0b0gAElwKFnBhcmtlZF9tZXNzYWdlX2Rpc2NhcmQYAyABKAsyOi5zY2Fk",
|
||||
"YWJyaWRnZS5zaXRlY29tbWFuZC52MS5QYXJrZWRNZXNzYWdlRGlzY2FyZFJl",
|
||||
"cXVlc3REdG9IABJVChZyZXRyeV9wYXJrZWRfb3BlcmF0aW9uGAQgASgLMjMu",
|
||||
"c2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUmV0cnlQYXJrZWRPcGVyYXRp",
|
||||
"b25EdG9IABJZChhkaXNjYXJkX3BhcmtlZF9vcGVyYXRpb24YBSABKAsyNS5z",
|
||||
"Y2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5EaXNjYXJkUGFya2VkT3BlcmF0",
|
||||
"aW9uRHRvSABCCQoHY29tbWFuZCKHAwoLUGFya2VkUmVwbHkSWQoUcGFya2Vk",
|
||||
"X21lc3NhZ2VfcXVlcnkYASABKAsyOS5zY2FkYWJyaWRnZS5zaXRlY29tbWFu",
|
||||
"ZC52MS5QYXJrZWRNZXNzYWdlUXVlcnlSZXNwb25zZUR0b0gAElkKFHBhcmtl",
|
||||
"ZF9tZXNzYWdlX3JldHJ5GAIgASgLMjkuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1h",
|
||||
"bmQudjEuUGFya2VkTWVzc2FnZVJldHJ5UmVzcG9uc2VEdG9IABJdChZwYXJr",
|
||||
"ZWRfbWVzc2FnZV9kaXNjYXJkGAMgASgLMjsuc2NhZGFicmlkZ2Uuc2l0ZWNv",
|
||||
"bW1hbmQudjEuUGFya2VkTWVzc2FnZURpc2NhcmRSZXNwb25zZUR0b0gAEloK",
|
||||
"F3BhcmtlZF9vcGVyYXRpb25fYWN0aW9uGAQgASgLMjcuc2NhZGFicmlkZ2Uu",
|
||||
"c2l0ZWNvbW1hbmQudjEuUGFya2VkT3BlcmF0aW9uQWN0aW9uQWNrRHRvSABC",
|
||||
"BwoFcmVwbHki7QEKFVJvdXRlVG9DYWxsUmVxdWVzdER0bxIWCg5jb3JyZWxh",
|
||||
"dGlvbl9pZBgBIAEoCRIcChRpbnN0YW5jZV91bmlxdWVfbmFtZRgCIAEoCRIT",
|
||||
"CgtzY3JpcHRfbmFtZRgDIAEoCRI9CgpwYXJhbWV0ZXJzGAQgASgLMikuc2Nh",
|
||||
"ZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuTG9vc2VWYWx1ZU1hcBItCgl0aW1l",
|
||||
"c3RhbXAYBSABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhsKE3Bh",
|
||||
"cmVudF9leGVjdXRpb25faWQYBiABKAkixQEKFlJvdXRlVG9DYWxsUmVzcG9u",
|
||||
"c2VEdG8SFgoOY29ycmVsYXRpb25faWQYASABKAkSDwoHc3VjY2VzcxgCIAEo",
|
||||
"CBI8CgxyZXR1cm5fdmFsdWUYAyABKAsyJi5zY2FkYWJyaWRnZS5zaXRlY29t",
|
||||
"bWFuZC52MS5Mb29zZVZhbHVlEhUKDWVycm9yX21lc3NhZ2UYBCABKAkSLQoJ",
|
||||
"dGltZXN0YW1wGAUgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcCK7",
|
||||
"AQoeUm91dGVUb0dldEF0dHJpYnV0ZXNSZXF1ZXN0RHRvEhYKDmNvcnJlbGF0",
|
||||
"aW9uX2lkGAEgASgJEhwKFGluc3RhbmNlX3VuaXF1ZV9uYW1lGAIgASgJEhcK",
|
||||
"D2F0dHJpYnV0ZV9uYW1lcxgDIAMoCRItCgl0aW1lc3RhbXAYBCABKAsyGi5n",
|
||||
"b29nbGUucHJvdG9idWYuVGltZXN0YW1wEhsKE3BhcmVudF9leGVjdXRpb25f",
|
||||
"aWQYBSABKAkiywEKH1JvdXRlVG9HZXRBdHRyaWJ1dGVzUmVzcG9uc2VEdG8S",
|
||||
"FgoOY29ycmVsYXRpb25faWQYASABKAkSOQoGdmFsdWVzGAIgASgLMikuc2Nh",
|
||||
"ZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuTG9vc2VWYWx1ZU1hcBIPCgdzdWNj",
|
||||
"ZXNzGAMgASgIEhUKDWVycm9yX21lc3NhZ2UYBCABKAkSLQoJdGltZXN0YW1w",
|
||||
"GAUgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcCLFAgoeUm91dGVU",
|
||||
"b1NldEF0dHJpYnV0ZXNSZXF1ZXN0RHRvEhYKDmNvcnJlbGF0aW9uX2lkGAEg",
|
||||
"ASgJEhwKFGluc3RhbmNlX3VuaXF1ZV9uYW1lGAIgASgJEmkKEGF0dHJpYnV0",
|
||||
"ZV92YWx1ZXMYAyADKAsyTy5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5S",
|
||||
"b3V0ZVRvU2V0QXR0cmlidXRlc1JlcXVlc3REdG8uQXR0cmlidXRlVmFsdWVz",
|
||||
"RW50cnkSLQoJdGltZXN0YW1wGAQgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRp",
|
||||
"bWVzdGFtcBIbChNwYXJlbnRfZXhlY3V0aW9uX2lkGAUgASgJGjYKFEF0dHJp",
|
||||
"YnV0ZVZhbHVlc0VudHJ5EgsKA2tleRgBIAEoCRINCgV2YWx1ZRgCIAEoCToC",
|
||||
"OAEikAEKH1JvdXRlVG9TZXRBdHRyaWJ1dGVzUmVzcG9uc2VEdG8SFgoOY29y",
|
||||
"cmVsYXRpb25faWQYASABKAkSDwoHc3VjY2VzcxgCIAEoCBIVCg1lcnJvcl9t",
|
||||
"ZXNzYWdlGAMgASgJEi0KCXRpbWVzdGFtcBgEIAEoCzIaLmdvb2dsZS5wcm90",
|
||||
"b2J1Zi5UaW1lc3RhbXAiwwIKIVJvdXRlVG9XYWl0Rm9yQXR0cmlidXRlUmVx",
|
||||
"dWVzdER0bxIWCg5jb3JyZWxhdGlvbl9pZBgBIAEoCRIcChRpbnN0YW5jZV91",
|
||||
"bmlxdWVfbmFtZRgCIAEoCRIWCg5hdHRyaWJ1dGVfbmFtZRgDIAEoCRI6ChR0",
|
||||
"YXJnZXRfdmFsdWVfZW5jb2RlZBgEIAEoCzIcLmdvb2dsZS5wcm90b2J1Zi5T",
|
||||
"dHJpbmdWYWx1ZRIqCgd0aW1lb3V0GAUgASgLMhkuZ29vZ2xlLnByb3RvYnVm",
|
||||
"LkR1cmF0aW9uEi0KCXRpbWVzdGFtcBgGIAEoCzIaLmdvb2dsZS5wcm90b2J1",
|
||||
"Zi5UaW1lc3RhbXASGwoTcGFyZW50X2V4ZWN1dGlvbl9pZBgHIAEoCRIcChRy",
|
||||
"ZXF1aXJlX2dvb2RfcXVhbGl0eRgIIAEoCCL/AQoiUm91dGVUb1dhaXRGb3JB",
|
||||
"dHRyaWJ1dGVSZXNwb25zZUR0bxIWCg5jb3JyZWxhdGlvbl9pZBgBIAEoCRIP",
|
||||
"CgdtYXRjaGVkGAIgASgIEjUKBXZhbHVlGAMgASgLMiYuc2NhZGFicmlkZ2Uu",
|
||||
"c2l0ZWNvbW1hbmQudjEuTG9vc2VWYWx1ZRIPCgdxdWFsaXR5GAQgASgJEhEK",
|
||||
"CXRpbWVkX291dBgFIAEoCBIPCgdzdWNjZXNzGAYgASgIEhUKDWVycm9yX21l",
|
||||
"c3NhZ2UYByABKAkSLQoJdGltZXN0YW1wGAggASgLMhouZ29vZ2xlLnByb3Rv",
|
||||
"YnVmLlRpbWVzdGFtcCKJAwoMUm91dGVSZXF1ZXN0EkoKDXJvdXRlX3RvX2Nh",
|
||||
"bGwYASABKAsyMS5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5Sb3V0ZVRv",
|
||||
"Q2FsbFJlcXVlc3REdG9IABJdChdyb3V0ZV90b19nZXRfYXR0cmlidXRlcxgC",
|
||||
"IAEoCzI6LnNjYWRhYnJpZGdlLnNpdGVjb21tYW5kLnYxLlJvdXRlVG9HZXRB",
|
||||
"dHRyaWJ1dGVzUmVxdWVzdER0b0gAEl0KF3JvdXRlX3RvX3NldF9hdHRyaWJ1",
|
||||
"dGVzGAMgASgLMjouc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUm91dGVU",
|
||||
"b1NldEF0dHJpYnV0ZXNSZXF1ZXN0RHRvSAASZAobcm91dGVfdG9fd2FpdF9m",
|
||||
"b3JfYXR0cmlidXRlGAQgASgLMj0uc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQu",
|
||||
"djEuUm91dGVUb1dhaXRGb3JBdHRyaWJ1dGVSZXF1ZXN0RHRvSABCCQoHY29t",
|
||||
"bWFuZCKJAwoKUm91dGVSZXBseRJLCg1yb3V0ZV90b19jYWxsGAEgASgLMjIu",
|
||||
"c2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUm91dGVUb0NhbGxSZXNwb25z",
|
||||
"ZUR0b0gAEl4KF3JvdXRlX3RvX2dldF9hdHRyaWJ1dGVzGAIgASgLMjsuc2Nh",
|
||||
"ZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUm91dGVUb0dldEF0dHJpYnV0ZXNS",
|
||||
"ZXNwb25zZUR0b0gAEl4KF3JvdXRlX3RvX3NldF9hdHRyaWJ1dGVzGAMgASgL",
|
||||
"Mjsuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUm91dGVUb1NldEF0dHJp",
|
||||
"YnV0ZXNSZXNwb25zZUR0b0gAEmUKG3JvdXRlX3RvX3dhaXRfZm9yX2F0dHJp",
|
||||
"YnV0ZRgEIAEoCzI+LnNjYWRhYnJpZGdlLnNpdGVjb21tYW5kLnYxLlJvdXRl",
|
||||
"VG9XYWl0Rm9yQXR0cmlidXRlUmVzcG9uc2VEdG9IAEIHCgVyZXBseSJBChZU",
|
||||
"cmlnZ2VyU2l0ZUZhaWxvdmVyRHRvEhYKDmNvcnJlbGF0aW9uX2lkGAEgASgJ",
|
||||
"Eg8KB3NpdGVfaWQYAiABKAkibQoSU2l0ZUZhaWxvdmVyQWNrRHRvEhYKDmNv",
|
||||
"cnJlbGF0aW9uX2lkGAEgASgJEhAKCGFjY2VwdGVkGAIgASgIEhYKDnRhcmdl",
|
||||
"dF9hZGRyZXNzGAMgASgJEhUKDWVycm9yX21lc3NhZ2UYBCABKAkqywEKE0Rl",
|
||||
"cGxveW1lbnRTdGF0dXNEdG8SJQohREVQTE9ZTUVOVF9TVEFUVVNfRFRPX1VO",
|
||||
"U1BFQ0lGSUVEEAASIQodREVQTE9ZTUVOVF9TVEFUVVNfRFRPX1BFTkRJTkcQ",
|
||||
"ARIlCiFERVBMT1lNRU5UX1NUQVRVU19EVE9fSU5fUFJPR1JFU1MQAhIhCh1E",
|
||||
"RVBMT1lNRU5UX1NUQVRVU19EVE9fU1VDQ0VTUxADEiAKHERFUExPWU1FTlRf",
|
||||
"U1RBVFVTX0RUT19GQUlMRUQQBCrEAQoSQnJvd3NlTm9kZUNsYXNzRHRvEiUK",
|
||||
"IUJST1dTRV9OT0RFX0NMQVNTX0RUT19VTlNQRUNJRklFRBAAEiAKHEJST1dT",
|
||||
"RV9OT0RFX0NMQVNTX0RUT19PQkpFQ1QQARIiCh5CUk9XU0VfTk9ERV9DTEFT",
|
||||
"U19EVE9fVkFSSUFCTEUQAhIgChxCUk9XU0VfTk9ERV9DTEFTU19EVE9fTUVU",
|
||||
"SE9EEAMSHwobQlJPV1NFX05PREVfQ0xBU1NfRFRPX09USEVSEAQqoQIKFEJy",
|
||||
"b3dzZUZhaWx1cmVLaW5kRHRvEicKI0JST1dTRV9GQUlMVVJFX0tJTkRfRFRP",
|
||||
"X1VOU1BFQ0lGSUVEEAASMAosQlJPV1NFX0ZBSUxVUkVfS0lORF9EVE9fQ09O",
|
||||
"TkVDVElPTl9OT1RfRk9VTkQQARI0CjBCUk9XU0VfRkFJTFVSRV9LSU5EX0RU",
|
||||
"T19DT05ORUNUSU9OX05PVF9DT05ORUNURUQQAhIpCiVCUk9XU0VfRkFJTFVS",
|
||||
"RV9LSU5EX0RUT19OT1RfQlJPV1NBQkxFEAMSIwofQlJPV1NFX0ZBSUxVUkVf",
|
||||
"S0lORF9EVE9fVElNRU9VVBAEEigKJEJST1dTRV9GQUlMVVJFX0tJTkRfRFRP",
|
||||
"X1NFUlZFUl9FUlJPUhAFKqoCChtSZWFkVGFnVmFsdWVzRmFpbHVyZUtpbmRE",
|
||||
"dG8SMAosUkVBRF9UQUdfVkFMVUVTX0ZBSUxVUkVfS0lORF9EVE9fVU5TUEVD",
|
||||
"SUZJRUQQABI5CjVSRUFEX1RBR19WQUxVRVNfRkFJTFVSRV9LSU5EX0RUT19D",
|
||||
"T05ORUNUSU9OX05PVF9GT1VORBABEj0KOVJFQURfVEFHX1ZBTFVFU19GQUlM",
|
||||
"VVJFX0tJTkRfRFRPX0NPTk5FQ1RJT05fTk9UX0NPTk5FQ1RFRBACEiwKKFJF",
|
||||
"QURfVEFHX1ZBTFVFU19GQUlMVVJFX0tJTkRfRFRPX1RJTUVPVVQQAxIxCi1S",
|
||||
"RUFEX1RBR19WQUxVRVNfRkFJTFVSRV9LSU5EX0RUT19TRVJWRVJfRVJST1IQ",
|
||||
"BCqTAgoUVmVyaWZ5RmFpbHVyZUtpbmREdG8SJwojVkVSSUZZX0ZBSUxVUkVf",
|
||||
"S0lORF9EVE9fVU5TUEVDSUZJRUQQABInCiNWRVJJRllfRkFJTFVSRV9LSU5E",
|
||||
"X0RUT19VTlJFQUNIQUJMRRABEicKI1ZFUklGWV9GQUlMVVJFX0tJTkRfRFRP",
|
||||
"X0FVVEhfRkFJTEVEEAISMQotVkVSSUZZX0ZBSUxVUkVfS0lORF9EVE9fVU5U",
|
||||
"UlVTVEVEX0NFUlRJRklDQVRFEAMSIwofVkVSSUZZX0ZBSUxVUkVfS0lORF9E",
|
||||
"VE9fVElNRU9VVBAEEigKJFZFUklGWV9GQUlMVVJFX0tJTkRfRFRPX1NFUlZF",
|
||||
"Ul9FUlJPUhAFKuUBChpTdG9yZUFuZEZvcndhcmRDYXRlZ29yeUR0bxIuCipT",
|
||||
"VE9SRV9BTkRfRk9SV0FSRF9DQVRFR09SWV9EVE9fVU5TUEVDSUZJRUQQABIy",
|
||||
"Ci5TVE9SRV9BTkRfRk9SV0FSRF9DQVRFR09SWV9EVE9fRVhURVJOQUxfU1lT",
|
||||
"VEVNEAESLworU1RPUkVfQU5EX0ZPUldBUkRfQ0FURUdPUllfRFRPX05PVElG",
|
||||
"SUNBVElPThACEjIKLlNUT1JFX0FORF9GT1JXQVJEX0NBVEVHT1JZX0RUT19D",
|
||||
"QUNIRURfREJfV1JJVEUQAypoCg1BbGFybVN0YXRlRHRvEh8KG0FMQVJNX1NU",
|
||||
"QVRFX0RUT19VTlNQRUNJRklFRBAAEhoKFkFMQVJNX1NUQVRFX0RUT19BQ1RJ",
|
||||
"VkUQARIaChZBTEFSTV9TVEFURV9EVE9fTk9STUFMEAIquQEKDUFsYXJtTGV2",
|
||||
"ZWxEdG8SHwobQUxBUk1fTEVWRUxfRFRPX1VOU1BFQ0lGSUVEEAASGAoUQUxB",
|
||||
"Uk1fTEVWRUxfRFRPX05PTkUQARIXChNBTEFSTV9MRVZFTF9EVE9fTE9XEAIS",
|
||||
"GwoXQUxBUk1fTEVWRUxfRFRPX0xPV19MT1cQAxIYChRBTEFSTV9MRVZFTF9E",
|
||||
"VE9fSElHSBAEEh0KGUFMQVJNX0xFVkVMX0RUT19ISUdIX0hJR0gQBSqSAQoM",
|
||||
"QWxhcm1LaW5kRHRvEh4KGkFMQVJNX0tJTkRfRFRPX1VOU1BFQ0lGSUVEEAAS",
|
||||
"GwoXQUxBUk1fS0lORF9EVE9fQ09NUFVURUQQARIgChxBTEFSTV9LSU5EX0RU",
|
||||
"T19OQVRJVkVfT1BDX1VBEAISIwofQUxBUk1fS0lORF9EVE9fTkFUSVZFX01Y",
|
||||
"X0FDQ0VTUxADKugBChNBbGFybVNoZWx2ZVN0YXRlRHRvEiYKIkFMQVJNX1NI",
|
||||
"RUxWRV9TVEFURV9EVE9fVU5TUEVDSUZJRUQQABIkCiBBTEFSTV9TSEVMVkVf",
|
||||
"U1RBVEVfRFRPX1VOU0hFTFZFRBABEisKJ0FMQVJNX1NIRUxWRV9TVEFURV9E",
|
||||
"VE9fT05FX1NIT1RfU0hFTFZFRBACEigKJEFMQVJNX1NIRUxWRV9TVEFURV9E",
|
||||
"VE9fVElNRURfU0hFTFZFRBADEiwKKEFMQVJNX1NIRUxWRV9TVEFURV9EVE9f",
|
||||
"UEVSTUFORU5UX1NIRUxWRUQQBDKEBQoSU2l0ZUNvbW1hbmRTZXJ2aWNlEmwK",
|
||||
"EEV4ZWN1dGVMaWZlY3ljbGUSLC5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52",
|
||||
"MS5MaWZlY3ljbGVSZXF1ZXN0Giouc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQu",
|
||||
"djEuTGlmZWN5Y2xlUmVwbHkSYAoMRXhlY3V0ZU9wY1VhEiguc2NhZGFicmlk",
|
||||
"Z2Uuc2l0ZWNvbW1hbmQudjEuT3BjVWFSZXF1ZXN0GiYuc2NhZGFicmlkZ2Uu",
|
||||
"c2l0ZWNvbW1hbmQudjEuT3BjVWFSZXBseRJgCgxFeGVjdXRlUXVlcnkSKC5z",
|
||||
"Y2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5RdWVyeVJlcXVlc3QaJi5zY2Fk",
|
||||
"YWJyaWRnZS5zaXRlY29tbWFuZC52MS5RdWVyeVJlcGx5EmMKDUV4ZWN1dGVQ",
|
||||
"YXJrZWQSKS5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5QYXJrZWRSZXF1",
|
||||
"ZXN0Gicuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUGFya2VkUmVwbHkS",
|
||||
"YAoMRXhlY3V0ZVJvdXRlEiguc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEu",
|
||||
"Um91dGVSZXF1ZXN0GiYuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUm91",
|
||||
"dGVSZXBseRJ1Cg9UcmlnZ2VyRmFpbG92ZXISMi5zY2FkYWJyaWRnZS5zaXRl",
|
||||
"Y29tbWFuZC52MS5UcmlnZ2VyU2l0ZUZhaWxvdmVyRHRvGi4uc2NhZGFicmlk",
|
||||
"Z2Uuc2l0ZWNvbW1hbmQudjEuU2l0ZUZhaWxvdmVyQWNrRHRvQiuqAihaQi5N",
|
||||
"T00uV1cuU2NhZGFCcmlkZ2UuQ29tbXVuaWNhdGlvbi5HcnBjYgZwcm90bzM="));
|
||||
"KAkSFgoOY29ycmVsYXRpb25faWQYAiABKAkSEwoLYWxhcm1zX29ubHkYAyAB",
|
||||
"KAgiVAocU3Vic2NyaWJlRGVidWdWaWV3UmVxdWVzdER0bxIcChRpbnN0YW5j",
|
||||
"ZV91bmlxdWVfbmFtZRgBIAEoCRIWCg5jb3JyZWxhdGlvbl9pZBgCIAEoCSJW",
|
||||
"Ch5VbnN1YnNjcmliZURlYnVnVmlld1JlcXVlc3REdG8SHAoUaW5zdGFuY2Vf",
|
||||
"dW5pcXVlX25hbWUYASABKAkSFgoOY29ycmVsYXRpb25faWQYAiABKAkiHAoa",
|
||||
"VW5zdWJzY3JpYmVEZWJ1Z1ZpZXdBY2tEdG8i1AEKFkFsYXJtQ29uZGl0aW9u",
|
||||
"U3RhdGVEdG8SDgoGYWN0aXZlGAEgASgIEhQKDGFja25vd2xlZGdlZBgCIAEo",
|
||||
"CBItCgljb25maXJtZWQYAyABKAsyGi5nb29nbGUucHJvdG9idWYuQm9vbFZh",
|
||||
"bHVlEj8KBnNoZWx2ZRgEIAEoDjIvLnNjYWRhYnJpZGdlLnNpdGVjb21tYW5k",
|
||||
"LnYxLkFsYXJtU2hlbHZlU3RhdGVEdG8SEgoKc3VwcHJlc3NlZBgFIAEoCBIQ",
|
||||
"CghzZXZlcml0eRgGIAEoBSLdAQoWRGVidWdBdHRyaWJ1dGVWYWx1ZUR0bxIc",
|
||||
"ChRpbnN0YW5jZV91bmlxdWVfbmFtZRgBIAEoCRIWCg5hdHRyaWJ1dGVfcGF0",
|
||||
"aBgCIAEoCRIWCg5hdHRyaWJ1dGVfbmFtZRgDIAEoCRI1CgV2YWx1ZRgEIAEo",
|
||||
"CzImLnNjYWRhYnJpZGdlLnNpdGVjb21tYW5kLnYxLkxvb3NlVmFsdWUSDwoH",
|
||||
"cXVhbGl0eRgFIAEoCRItCgl0aW1lc3RhbXAYBiABKAsyGi5nb29nbGUucHJv",
|
||||
"dG9idWYuVGltZXN0YW1wIq8FChJEZWJ1Z0FsYXJtU3RhdGVEdG8SHAoUaW5z",
|
||||
"dGFuY2VfdW5pcXVlX25hbWUYASABKAkSEgoKYWxhcm1fbmFtZRgCIAEoCRI4",
|
||||
"CgVzdGF0ZRgDIAEoDjIpLnNjYWRhYnJpZGdlLnNpdGVjb21tYW5kLnYxLkFs",
|
||||
"YXJtU3RhdGVEdG8SEAoIcHJpb3JpdHkYBCABKAUSLQoJdGltZXN0YW1wGAUg",
|
||||
"ASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBI4CgVsZXZlbBgGIAEo",
|
||||
"DjIpLnNjYWRhYnJpZGdlLnNpdGVjb21tYW5kLnYxLkFsYXJtTGV2ZWxEdG8S",
|
||||
"DwoHbWVzc2FnZRgHIAEoCRI2CgRraW5kGAggASgOMiguc2NhZGFicmlkZ2Uu",
|
||||
"c2l0ZWNvbW1hbmQudjEuQWxhcm1LaW5kRHRvEkUKCWNvbmRpdGlvbhgJIAEo",
|
||||
"CzIyLnNjYWRhYnJpZGdlLnNpdGVjb21tYW5kLnYxLkFsYXJtQ29uZGl0aW9u",
|
||||
"U3RhdGVEdG8SGAoQc291cmNlX3JlZmVyZW5jZRgKIAEoCRIXCg9hbGFybV90",
|
||||
"eXBlX25hbWUYCyABKAkSEAoIY2F0ZWdvcnkYDCABKAkSFQoNb3BlcmF0b3Jf",
|
||||
"dXNlchgNIAEoCRIYChBvcGVyYXRvcl9jb21tZW50GA4gASgJEjcKE29yaWdp",
|
||||
"bmFsX3JhaXNlX3RpbWUYDyABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0",
|
||||
"YW1wEhUKDWN1cnJlbnRfdmFsdWUYECABKAkSEwoLbGltaXRfdmFsdWUYESAB",
|
||||
"KAkSJAocbmF0aXZlX3NvdXJjZV9jYW5vbmljYWxfbmFtZRgSIAEoCRIhChlp",
|
||||
"c19jb25maWd1cmVkX3BsYWNlaG9sZGVyGBMgASgIIpwCChREZWJ1Z1ZpZXdT",
|
||||
"bmFwc2hvdER0bxIcChRpbnN0YW5jZV91bmlxdWVfbmFtZRgBIAEoCRJMChBh",
|
||||
"dHRyaWJ1dGVfdmFsdWVzGAIgAygLMjIuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1h",
|
||||
"bmQudjEuRGVidWdBdHRyaWJ1dGVWYWx1ZUR0bxJECgxhbGFybV9zdGF0ZXMY",
|
||||
"AyADKAsyLi5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5EZWJ1Z0FsYXJt",
|
||||
"U3RhdGVEdG8SNgoSc25hcHNob3RfdGltZXN0YW1wGAQgASgLMhouZ29vZ2xl",
|
||||
"LnByb3RvYnVmLlRpbWVzdGFtcBIaChJpbnN0YW5jZV9ub3RfZm91bmQYBSAB",
|
||||
"KAgi8AIKDFF1ZXJ5UmVxdWVzdBJOCg9ldmVudF9sb2dfcXVlcnkYASABKAsy",
|
||||
"My5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5FdmVudExvZ1F1ZXJ5UmVx",
|
||||
"dWVzdER0b0gAEk0KDmRlYnVnX3NuYXBzaG90GAIgASgLMjMuc2NhZGFicmlk",
|
||||
"Z2Uuc2l0ZWNvbW1hbmQudjEuRGVidWdTbmFwc2hvdFJlcXVlc3REdG9IABJY",
|
||||
"ChRzdWJzY3JpYmVfZGVidWdfdmlldxgDIAEoCzI4LnNjYWRhYnJpZGdlLnNp",
|
||||
"dGVjb21tYW5kLnYxLlN1YnNjcmliZURlYnVnVmlld1JlcXVlc3REdG9IABJc",
|
||||
"ChZ1bnN1YnNjcmliZV9kZWJ1Z192aWV3GAQgASgLMjouc2NhZGFicmlkZ2Uu",
|
||||
"c2l0ZWNvbW1hbmQudjEuVW5zdWJzY3JpYmVEZWJ1Z1ZpZXdSZXF1ZXN0RHRv",
|
||||
"SABCCQoHY29tbWFuZCKRAgoKUXVlcnlSZXBseRJPCg9ldmVudF9sb2dfcXVl",
|
||||
"cnkYASABKAsyNC5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5FdmVudExv",
|
||||
"Z1F1ZXJ5UmVzcG9uc2VEdG9IABJPChNkZWJ1Z192aWV3X3NuYXBzaG90GAIg",
|
||||
"ASgLMjAuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuRGVidWdWaWV3U25h",
|
||||
"cHNob3REdG9IABJYChZ1bnN1YnNjcmliZV9kZWJ1Z192aWV3GAMgASgLMjYu",
|
||||
"c2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuVW5zdWJzY3JpYmVEZWJ1Z1Zp",
|
||||
"ZXdBY2tEdG9IAEIHCgVyZXBseSKeAQocUGFya2VkTWVzc2FnZVF1ZXJ5UmVx",
|
||||
"dWVzdER0bxIWCg5jb3JyZWxhdGlvbl9pZBgBIAEoCRIPCgdzaXRlX2lkGAIg",
|
||||
"ASgJEhMKC3BhZ2VfbnVtYmVyGAMgASgFEhEKCXBhZ2Vfc2l6ZRgEIAEoBRIt",
|
||||
"Cgl0aW1lc3RhbXAYBSABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1w",
|
||||
"IvICChVQYXJrZWRNZXNzYWdlRW50cnlEdG8SEgoKbWVzc2FnZV9pZBgBIAEo",
|
||||
"CRIVCg10YXJnZXRfc3lzdGVtGAIgASgJEhMKC21ldGhvZF9uYW1lGAMgASgJ",
|
||||
"EhUKDWVycm9yX21lc3NhZ2UYBCABKAkSFQoNYXR0ZW1wdF9jb3VudBgFIAEo",
|
||||
"BRI2ChJvcmlnaW5hbF90aW1lc3RhbXAYBiABKAsyGi5nb29nbGUucHJvdG9i",
|
||||
"dWYuVGltZXN0YW1wEjoKFmxhc3RfYXR0ZW1wdF90aW1lc3RhbXAYByABKAsy",
|
||||
"Gi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhQKDG1heF9hdHRlbXB0cxgI",
|
||||
"IAEoBRJICghjYXRlZ29yeRgJIAEoDjI2LnNjYWRhYnJpZGdlLnNpdGVjb21t",
|
||||
"YW5kLnYxLlN0b3JlQW5kRm9yd2FyZENhdGVnb3J5RHRvEhcKD29yaWdpbl9p",
|
||||
"bnN0YW5jZRgKIAEoCSKhAgodUGFya2VkTWVzc2FnZVF1ZXJ5UmVzcG9uc2VE",
|
||||
"dG8SFgoOY29ycmVsYXRpb25faWQYASABKAkSDwoHc2l0ZV9pZBgCIAEoCRJD",
|
||||
"CghtZXNzYWdlcxgDIAMoCzIxLnNjYWRhYnJpZGdlLnNpdGVjb21tYW5kLnYx",
|
||||
"LlBhcmtlZE1lc3NhZ2VFbnRyeUR0bxITCgt0b3RhbF9jb3VudBgEIAEoBRIT",
|
||||
"CgtwYWdlX251bWJlchgFIAEoBRIRCglwYWdlX3NpemUYBiABKAUSDwoHc3Vj",
|
||||
"Y2VzcxgHIAEoCBIVCg1lcnJvcl9tZXNzYWdlGAggASgJEi0KCXRpbWVzdGFt",
|
||||
"cBgJIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiigEKHFBhcmtl",
|
||||
"ZE1lc3NhZ2VSZXRyeVJlcXVlc3REdG8SFgoOY29ycmVsYXRpb25faWQYASAB",
|
||||
"KAkSDwoHc2l0ZV9pZBgCIAEoCRISCgptZXNzYWdlX2lkGAMgASgJEi0KCXRp",
|
||||
"bWVzdGFtcBgEIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiXwod",
|
||||
"UGFya2VkTWVzc2FnZVJldHJ5UmVzcG9uc2VEdG8SFgoOY29ycmVsYXRpb25f",
|
||||
"aWQYASABKAkSDwoHc3VjY2VzcxgCIAEoCBIVCg1lcnJvcl9tZXNzYWdlGAMg",
|
||||
"ASgJIowBCh5QYXJrZWRNZXNzYWdlRGlzY2FyZFJlcXVlc3REdG8SFgoOY29y",
|
||||
"cmVsYXRpb25faWQYASABKAkSDwoHc2l0ZV9pZBgCIAEoCRISCgptZXNzYWdl",
|
||||
"X2lkGAMgASgJEi0KCXRpbWVzdGFtcBgEIAEoCzIaLmdvb2dsZS5wcm90b2J1",
|
||||
"Zi5UaW1lc3RhbXAiYQofUGFya2VkTWVzc2FnZURpc2NhcmRSZXNwb25zZUR0",
|
||||
"bxIWCg5jb3JyZWxhdGlvbl9pZBgBIAEoCRIPCgdzdWNjZXNzGAIgASgIEhUK",
|
||||
"DWVycm9yX21lc3NhZ2UYAyABKAkiTwoXUmV0cnlQYXJrZWRPcGVyYXRpb25E",
|
||||
"dG8SFgoOY29ycmVsYXRpb25faWQYASABKAkSHAoUdHJhY2tlZF9vcGVyYXRp",
|
||||
"b25faWQYAiABKAkiUQoZRGlzY2FyZFBhcmtlZE9wZXJhdGlvbkR0bxIWCg5j",
|
||||
"b3JyZWxhdGlvbl9pZBgBIAEoCRIcChR0cmFja2VkX29wZXJhdGlvbl9pZBgC",
|
||||
"IAEoCSJdChtQYXJrZWRPcGVyYXRpb25BY3Rpb25BY2tEdG8SFgoOY29ycmVs",
|
||||
"YXRpb25faWQYASABKAkSDwoHYXBwbGllZBgCIAEoCBIVCg1lcnJvcl9tZXNz",
|
||||
"YWdlGAMgASgJIt4DCg1QYXJrZWRSZXF1ZXN0ElgKFHBhcmtlZF9tZXNzYWdl",
|
||||
"X3F1ZXJ5GAEgASgLMjguc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUGFy",
|
||||
"a2VkTWVzc2FnZVF1ZXJ5UmVxdWVzdER0b0gAElgKFHBhcmtlZF9tZXNzYWdl",
|
||||
"X3JldHJ5GAIgASgLMjguc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUGFy",
|
||||
"a2VkTWVzc2FnZVJldHJ5UmVxdWVzdER0b0gAElwKFnBhcmtlZF9tZXNzYWdl",
|
||||
"X2Rpc2NhcmQYAyABKAsyOi5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5Q",
|
||||
"YXJrZWRNZXNzYWdlRGlzY2FyZFJlcXVlc3REdG9IABJVChZyZXRyeV9wYXJr",
|
||||
"ZWRfb3BlcmF0aW9uGAQgASgLMjMuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQu",
|
||||
"djEuUmV0cnlQYXJrZWRPcGVyYXRpb25EdG9IABJZChhkaXNjYXJkX3Bhcmtl",
|
||||
"ZF9vcGVyYXRpb24YBSABKAsyNS5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52",
|
||||
"MS5EaXNjYXJkUGFya2VkT3BlcmF0aW9uRHRvSABCCQoHY29tbWFuZCKHAwoL",
|
||||
"UGFya2VkUmVwbHkSWQoUcGFya2VkX21lc3NhZ2VfcXVlcnkYASABKAsyOS5z",
|
||||
"Y2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5QYXJrZWRNZXNzYWdlUXVlcnlS",
|
||||
"ZXNwb25zZUR0b0gAElkKFHBhcmtlZF9tZXNzYWdlX3JldHJ5GAIgASgLMjku",
|
||||
"c2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUGFya2VkTWVzc2FnZVJldHJ5",
|
||||
"UmVzcG9uc2VEdG9IABJdChZwYXJrZWRfbWVzc2FnZV9kaXNjYXJkGAMgASgL",
|
||||
"Mjsuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUGFya2VkTWVzc2FnZURp",
|
||||
"c2NhcmRSZXNwb25zZUR0b0gAEloKF3BhcmtlZF9vcGVyYXRpb25fYWN0aW9u",
|
||||
"GAQgASgLMjcuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUGFya2VkT3Bl",
|
||||
"cmF0aW9uQWN0aW9uQWNrRHRvSABCBwoFcmVwbHki7QEKFVJvdXRlVG9DYWxs",
|
||||
"UmVxdWVzdER0bxIWCg5jb3JyZWxhdGlvbl9pZBgBIAEoCRIcChRpbnN0YW5j",
|
||||
"ZV91bmlxdWVfbmFtZRgCIAEoCRITCgtzY3JpcHRfbmFtZRgDIAEoCRI9Cgpw",
|
||||
"YXJhbWV0ZXJzGAQgASgLMikuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEu",
|
||||
"TG9vc2VWYWx1ZU1hcBItCgl0aW1lc3RhbXAYBSABKAsyGi5nb29nbGUucHJv",
|
||||
"dG9idWYuVGltZXN0YW1wEhsKE3BhcmVudF9leGVjdXRpb25faWQYBiABKAki",
|
||||
"xQEKFlJvdXRlVG9DYWxsUmVzcG9uc2VEdG8SFgoOY29ycmVsYXRpb25faWQY",
|
||||
"ASABKAkSDwoHc3VjY2VzcxgCIAEoCBI8CgxyZXR1cm5fdmFsdWUYAyABKAsy",
|
||||
"Ji5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5Mb29zZVZhbHVlEhUKDWVy",
|
||||
"cm9yX21lc3NhZ2UYBCABKAkSLQoJdGltZXN0YW1wGAUgASgLMhouZ29vZ2xl",
|
||||
"LnByb3RvYnVmLlRpbWVzdGFtcCK7AQoeUm91dGVUb0dldEF0dHJpYnV0ZXNS",
|
||||
"ZXF1ZXN0RHRvEhYKDmNvcnJlbGF0aW9uX2lkGAEgASgJEhwKFGluc3RhbmNl",
|
||||
"X3VuaXF1ZV9uYW1lGAIgASgJEhcKD2F0dHJpYnV0ZV9uYW1lcxgDIAMoCRIt",
|
||||
"Cgl0aW1lc3RhbXAYBCABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1w",
|
||||
"EhsKE3BhcmVudF9leGVjdXRpb25faWQYBSABKAkiywEKH1JvdXRlVG9HZXRB",
|
||||
"dHRyaWJ1dGVzUmVzcG9uc2VEdG8SFgoOY29ycmVsYXRpb25faWQYASABKAkS",
|
||||
"OQoGdmFsdWVzGAIgASgLMikuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEu",
|
||||
"TG9vc2VWYWx1ZU1hcBIPCgdzdWNjZXNzGAMgASgIEhUKDWVycm9yX21lc3Nh",
|
||||
"Z2UYBCABKAkSLQoJdGltZXN0YW1wGAUgASgLMhouZ29vZ2xlLnByb3RvYnVm",
|
||||
"LlRpbWVzdGFtcCLFAgoeUm91dGVUb1NldEF0dHJpYnV0ZXNSZXF1ZXN0RHRv",
|
||||
"EhYKDmNvcnJlbGF0aW9uX2lkGAEgASgJEhwKFGluc3RhbmNlX3VuaXF1ZV9u",
|
||||
"YW1lGAIgASgJEmkKEGF0dHJpYnV0ZV92YWx1ZXMYAyADKAsyTy5zY2FkYWJy",
|
||||
"aWRnZS5zaXRlY29tbWFuZC52MS5Sb3V0ZVRvU2V0QXR0cmlidXRlc1JlcXVl",
|
||||
"c3REdG8uQXR0cmlidXRlVmFsdWVzRW50cnkSLQoJdGltZXN0YW1wGAQgASgL",
|
||||
"MhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIbChNwYXJlbnRfZXhlY3V0",
|
||||
"aW9uX2lkGAUgASgJGjYKFEF0dHJpYnV0ZVZhbHVlc0VudHJ5EgsKA2tleRgB",
|
||||
"IAEoCRINCgV2YWx1ZRgCIAEoCToCOAEikAEKH1JvdXRlVG9TZXRBdHRyaWJ1",
|
||||
"dGVzUmVzcG9uc2VEdG8SFgoOY29ycmVsYXRpb25faWQYASABKAkSDwoHc3Vj",
|
||||
"Y2VzcxgCIAEoCBIVCg1lcnJvcl9tZXNzYWdlGAMgASgJEi0KCXRpbWVzdGFt",
|
||||
"cBgEIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiwwIKIVJvdXRl",
|
||||
"VG9XYWl0Rm9yQXR0cmlidXRlUmVxdWVzdER0bxIWCg5jb3JyZWxhdGlvbl9p",
|
||||
"ZBgBIAEoCRIcChRpbnN0YW5jZV91bmlxdWVfbmFtZRgCIAEoCRIWCg5hdHRy",
|
||||
"aWJ1dGVfbmFtZRgDIAEoCRI6ChR0YXJnZXRfdmFsdWVfZW5jb2RlZBgEIAEo",
|
||||
"CzIcLmdvb2dsZS5wcm90b2J1Zi5TdHJpbmdWYWx1ZRIqCgd0aW1lb3V0GAUg",
|
||||
"ASgLMhkuZ29vZ2xlLnByb3RvYnVmLkR1cmF0aW9uEi0KCXRpbWVzdGFtcBgG",
|
||||
"IAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASGwoTcGFyZW50X2V4",
|
||||
"ZWN1dGlvbl9pZBgHIAEoCRIcChRyZXF1aXJlX2dvb2RfcXVhbGl0eRgIIAEo",
|
||||
"CCL/AQoiUm91dGVUb1dhaXRGb3JBdHRyaWJ1dGVSZXNwb25zZUR0bxIWCg5j",
|
||||
"b3JyZWxhdGlvbl9pZBgBIAEoCRIPCgdtYXRjaGVkGAIgASgIEjUKBXZhbHVl",
|
||||
"GAMgASgLMiYuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuTG9vc2VWYWx1",
|
||||
"ZRIPCgdxdWFsaXR5GAQgASgJEhEKCXRpbWVkX291dBgFIAEoCBIPCgdzdWNj",
|
||||
"ZXNzGAYgASgIEhUKDWVycm9yX21lc3NhZ2UYByABKAkSLQoJdGltZXN0YW1w",
|
||||
"GAggASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcCKJAwoMUm91dGVS",
|
||||
"ZXF1ZXN0EkoKDXJvdXRlX3RvX2NhbGwYASABKAsyMS5zY2FkYWJyaWRnZS5z",
|
||||
"aXRlY29tbWFuZC52MS5Sb3V0ZVRvQ2FsbFJlcXVlc3REdG9IABJdChdyb3V0",
|
||||
"ZV90b19nZXRfYXR0cmlidXRlcxgCIAEoCzI6LnNjYWRhYnJpZGdlLnNpdGVj",
|
||||
"b21tYW5kLnYxLlJvdXRlVG9HZXRBdHRyaWJ1dGVzUmVxdWVzdER0b0gAEl0K",
|
||||
"F3JvdXRlX3RvX3NldF9hdHRyaWJ1dGVzGAMgASgLMjouc2NhZGFicmlkZ2Uu",
|
||||
"c2l0ZWNvbW1hbmQudjEuUm91dGVUb1NldEF0dHJpYnV0ZXNSZXF1ZXN0RHRv",
|
||||
"SAASZAobcm91dGVfdG9fd2FpdF9mb3JfYXR0cmlidXRlGAQgASgLMj0uc2Nh",
|
||||
"ZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUm91dGVUb1dhaXRGb3JBdHRyaWJ1",
|
||||
"dGVSZXF1ZXN0RHRvSABCCQoHY29tbWFuZCKJAwoKUm91dGVSZXBseRJLCg1y",
|
||||
"b3V0ZV90b19jYWxsGAEgASgLMjIuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQu",
|
||||
"djEuUm91dGVUb0NhbGxSZXNwb25zZUR0b0gAEl4KF3JvdXRlX3RvX2dldF9h",
|
||||
"dHRyaWJ1dGVzGAIgASgLMjsuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEu",
|
||||
"Um91dGVUb0dldEF0dHJpYnV0ZXNSZXNwb25zZUR0b0gAEl4KF3JvdXRlX3Rv",
|
||||
"X3NldF9hdHRyaWJ1dGVzGAMgASgLMjsuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1h",
|
||||
"bmQudjEuUm91dGVUb1NldEF0dHJpYnV0ZXNSZXNwb25zZUR0b0gAEmUKG3Jv",
|
||||
"dXRlX3RvX3dhaXRfZm9yX2F0dHJpYnV0ZRgEIAEoCzI+LnNjYWRhYnJpZGdl",
|
||||
"LnNpdGVjb21tYW5kLnYxLlJvdXRlVG9XYWl0Rm9yQXR0cmlidXRlUmVzcG9u",
|
||||
"c2VEdG9IAEIHCgVyZXBseSJBChZUcmlnZ2VyU2l0ZUZhaWxvdmVyRHRvEhYK",
|
||||
"DmNvcnJlbGF0aW9uX2lkGAEgASgJEg8KB3NpdGVfaWQYAiABKAkibQoSU2l0",
|
||||
"ZUZhaWxvdmVyQWNrRHRvEhYKDmNvcnJlbGF0aW9uX2lkGAEgASgJEhAKCGFj",
|
||||
"Y2VwdGVkGAIgASgIEhYKDnRhcmdldF9hZGRyZXNzGAMgASgJEhUKDWVycm9y",
|
||||
"X21lc3NhZ2UYBCABKAkqywEKE0RlcGxveW1lbnRTdGF0dXNEdG8SJQohREVQ",
|
||||
"TE9ZTUVOVF9TVEFUVVNfRFRPX1VOU1BFQ0lGSUVEEAASIQodREVQTE9ZTUVO",
|
||||
"VF9TVEFUVVNfRFRPX1BFTkRJTkcQARIlCiFERVBMT1lNRU5UX1NUQVRVU19E",
|
||||
"VE9fSU5fUFJPR1JFU1MQAhIhCh1ERVBMT1lNRU5UX1NUQVRVU19EVE9fU1VD",
|
||||
"Q0VTUxADEiAKHERFUExPWU1FTlRfU1RBVFVTX0RUT19GQUlMRUQQBCrEAQoS",
|
||||
"QnJvd3NlTm9kZUNsYXNzRHRvEiUKIUJST1dTRV9OT0RFX0NMQVNTX0RUT19V",
|
||||
"TlNQRUNJRklFRBAAEiAKHEJST1dTRV9OT0RFX0NMQVNTX0RUT19PQkpFQ1QQ",
|
||||
"ARIiCh5CUk9XU0VfTk9ERV9DTEFTU19EVE9fVkFSSUFCTEUQAhIgChxCUk9X",
|
||||
"U0VfTk9ERV9DTEFTU19EVE9fTUVUSE9EEAMSHwobQlJPV1NFX05PREVfQ0xB",
|
||||
"U1NfRFRPX09USEVSEAQqoQIKFEJyb3dzZUZhaWx1cmVLaW5kRHRvEicKI0JS",
|
||||
"T1dTRV9GQUlMVVJFX0tJTkRfRFRPX1VOU1BFQ0lGSUVEEAASMAosQlJPV1NF",
|
||||
"X0ZBSUxVUkVfS0lORF9EVE9fQ09OTkVDVElPTl9OT1RfRk9VTkQQARI0CjBC",
|
||||
"Uk9XU0VfRkFJTFVSRV9LSU5EX0RUT19DT05ORUNUSU9OX05PVF9DT05ORUNU",
|
||||
"RUQQAhIpCiVCUk9XU0VfRkFJTFVSRV9LSU5EX0RUT19OT1RfQlJPV1NBQkxF",
|
||||
"EAMSIwofQlJPV1NFX0ZBSUxVUkVfS0lORF9EVE9fVElNRU9VVBAEEigKJEJS",
|
||||
"T1dTRV9GQUlMVVJFX0tJTkRfRFRPX1NFUlZFUl9FUlJPUhAFKqoCChtSZWFk",
|
||||
"VGFnVmFsdWVzRmFpbHVyZUtpbmREdG8SMAosUkVBRF9UQUdfVkFMVUVTX0ZB",
|
||||
"SUxVUkVfS0lORF9EVE9fVU5TUEVDSUZJRUQQABI5CjVSRUFEX1RBR19WQUxV",
|
||||
"RVNfRkFJTFVSRV9LSU5EX0RUT19DT05ORUNUSU9OX05PVF9GT1VORBABEj0K",
|
||||
"OVJFQURfVEFHX1ZBTFVFU19GQUlMVVJFX0tJTkRfRFRPX0NPTk5FQ1RJT05f",
|
||||
"Tk9UX0NPTk5FQ1RFRBACEiwKKFJFQURfVEFHX1ZBTFVFU19GQUlMVVJFX0tJ",
|
||||
"TkRfRFRPX1RJTUVPVVQQAxIxCi1SRUFEX1RBR19WQUxVRVNfRkFJTFVSRV9L",
|
||||
"SU5EX0RUT19TRVJWRVJfRVJST1IQBCqTAgoUVmVyaWZ5RmFpbHVyZUtpbmRE",
|
||||
"dG8SJwojVkVSSUZZX0ZBSUxVUkVfS0lORF9EVE9fVU5TUEVDSUZJRUQQABIn",
|
||||
"CiNWRVJJRllfRkFJTFVSRV9LSU5EX0RUT19VTlJFQUNIQUJMRRABEicKI1ZF",
|
||||
"UklGWV9GQUlMVVJFX0tJTkRfRFRPX0FVVEhfRkFJTEVEEAISMQotVkVSSUZZ",
|
||||
"X0ZBSUxVUkVfS0lORF9EVE9fVU5UUlVTVEVEX0NFUlRJRklDQVRFEAMSIwof",
|
||||
"VkVSSUZZX0ZBSUxVUkVfS0lORF9EVE9fVElNRU9VVBAEEigKJFZFUklGWV9G",
|
||||
"QUlMVVJFX0tJTkRfRFRPX1NFUlZFUl9FUlJPUhAFKuUBChpTdG9yZUFuZEZv",
|
||||
"cndhcmRDYXRlZ29yeUR0bxIuCipTVE9SRV9BTkRfRk9SV0FSRF9DQVRFR09S",
|
||||
"WV9EVE9fVU5TUEVDSUZJRUQQABIyCi5TVE9SRV9BTkRfRk9SV0FSRF9DQVRF",
|
||||
"R09SWV9EVE9fRVhURVJOQUxfU1lTVEVNEAESLworU1RPUkVfQU5EX0ZPUldB",
|
||||
"UkRfQ0FURUdPUllfRFRPX05PVElGSUNBVElPThACEjIKLlNUT1JFX0FORF9G",
|
||||
"T1JXQVJEX0NBVEVHT1JZX0RUT19DQUNIRURfREJfV1JJVEUQAypoCg1BbGFy",
|
||||
"bVN0YXRlRHRvEh8KG0FMQVJNX1NUQVRFX0RUT19VTlNQRUNJRklFRBAAEhoK",
|
||||
"FkFMQVJNX1NUQVRFX0RUT19BQ1RJVkUQARIaChZBTEFSTV9TVEFURV9EVE9f",
|
||||
"Tk9STUFMEAIquQEKDUFsYXJtTGV2ZWxEdG8SHwobQUxBUk1fTEVWRUxfRFRP",
|
||||
"X1VOU1BFQ0lGSUVEEAASGAoUQUxBUk1fTEVWRUxfRFRPX05PTkUQARIXChNB",
|
||||
"TEFSTV9MRVZFTF9EVE9fTE9XEAISGwoXQUxBUk1fTEVWRUxfRFRPX0xPV19M",
|
||||
"T1cQAxIYChRBTEFSTV9MRVZFTF9EVE9fSElHSBAEEh0KGUFMQVJNX0xFVkVM",
|
||||
"X0RUT19ISUdIX0hJR0gQBSqSAQoMQWxhcm1LaW5kRHRvEh4KGkFMQVJNX0tJ",
|
||||
"TkRfRFRPX1VOU1BFQ0lGSUVEEAASGwoXQUxBUk1fS0lORF9EVE9fQ09NUFVU",
|
||||
"RUQQARIgChxBTEFSTV9LSU5EX0RUT19OQVRJVkVfT1BDX1VBEAISIwofQUxB",
|
||||
"Uk1fS0lORF9EVE9fTkFUSVZFX01YX0FDQ0VTUxADKugBChNBbGFybVNoZWx2",
|
||||
"ZVN0YXRlRHRvEiYKIkFMQVJNX1NIRUxWRV9TVEFURV9EVE9fVU5TUEVDSUZJ",
|
||||
"RUQQABIkCiBBTEFSTV9TSEVMVkVfU1RBVEVfRFRPX1VOU0hFTFZFRBABEisK",
|
||||
"J0FMQVJNX1NIRUxWRV9TVEFURV9EVE9fT05FX1NIT1RfU0hFTFZFRBACEigK",
|
||||
"JEFMQVJNX1NIRUxWRV9TVEFURV9EVE9fVElNRURfU0hFTFZFRBADEiwKKEFM",
|
||||
"QVJNX1NIRUxWRV9TVEFURV9EVE9fUEVSTUFORU5UX1NIRUxWRUQQBDKEBQoS",
|
||||
"U2l0ZUNvbW1hbmRTZXJ2aWNlEmwKEEV4ZWN1dGVMaWZlY3ljbGUSLC5zY2Fk",
|
||||
"YWJyaWRnZS5zaXRlY29tbWFuZC52MS5MaWZlY3ljbGVSZXF1ZXN0Giouc2Nh",
|
||||
"ZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuTGlmZWN5Y2xlUmVwbHkSYAoMRXhl",
|
||||
"Y3V0ZU9wY1VhEiguc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuT3BjVWFS",
|
||||
"ZXF1ZXN0GiYuc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuT3BjVWFSZXBs",
|
||||
"eRJgCgxFeGVjdXRlUXVlcnkSKC5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52",
|
||||
"MS5RdWVyeVJlcXVlc3QaJi5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5R",
|
||||
"dWVyeVJlcGx5EmMKDUV4ZWN1dGVQYXJrZWQSKS5zY2FkYWJyaWRnZS5zaXRl",
|
||||
"Y29tbWFuZC52MS5QYXJrZWRSZXF1ZXN0Gicuc2NhZGFicmlkZ2Uuc2l0ZWNv",
|
||||
"bW1hbmQudjEuUGFya2VkUmVwbHkSYAoMRXhlY3V0ZVJvdXRlEiguc2NhZGFi",
|
||||
"cmlkZ2Uuc2l0ZWNvbW1hbmQudjEuUm91dGVSZXF1ZXN0GiYuc2NhZGFicmlk",
|
||||
"Z2Uuc2l0ZWNvbW1hbmQudjEuUm91dGVSZXBseRJ1Cg9UcmlnZ2VyRmFpbG92",
|
||||
"ZXISMi5zY2FkYWJyaWRnZS5zaXRlY29tbWFuZC52MS5UcmlnZ2VyU2l0ZUZh",
|
||||
"aWxvdmVyRHRvGi4uc2NhZGFicmlkZ2Uuc2l0ZWNvbW1hbmQudjEuU2l0ZUZh",
|
||||
"aWxvdmVyQWNrRHRvQiuqAihaQi5NT00uV1cuU2NhZGFCcmlkZ2UuQ29tbXVu",
|
||||
"aWNhdGlvbi5HcnBjYgZwcm90bzM="));
|
||||
descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
|
||||
new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.TimestampReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.DurationReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.WrappersReflection.Descriptor, },
|
||||
new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.DeploymentStatusDto), typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.BrowseNodeClassDto), typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.BrowseFailureKindDto), typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ReadTagValuesFailureKindDto), typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.VerifyFailureKindDto), typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.StoreAndForwardCategoryDto), typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.AlarmStateDto), typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.AlarmLevelDto), typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.AlarmKindDto), typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.AlarmShelveStateDto), }, null, new pbr::GeneratedClrTypeInfo[] {
|
||||
@@ -554,7 +555,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.EventLogQueryRequestDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.EventLogQueryRequestDto.Parser, new[]{ "CorrelationId", "SiteId", "From", "To", "EventType", "Severity", "InstanceId", "KeywordFilter", "ContinuationToken", "PageSize", "Timestamp" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.EventLogEntryDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.EventLogEntryDto.Parser, new[]{ "Id", "Timestamp", "EventType", "Severity", "InstanceId", "Source", "Message", "Details" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.EventLogQueryResponseDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.EventLogQueryResponseDto.Parser, new[]{ "CorrelationId", "SiteId", "Entries", "ContinuationToken", "HasMore", "Success", "ErrorMessage", "Timestamp" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.DebugSnapshotRequestDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.DebugSnapshotRequestDto.Parser, new[]{ "InstanceUniqueName", "CorrelationId" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.DebugSnapshotRequestDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.DebugSnapshotRequestDto.Parser, new[]{ "InstanceUniqueName", "CorrelationId", "AlarmsOnly" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SubscribeDebugViewRequestDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SubscribeDebugViewRequestDto.Parser, new[]{ "InstanceUniqueName", "CorrelationId" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.UnsubscribeDebugViewRequestDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.UnsubscribeDebugViewRequestDto.Parser, new[]{ "InstanceUniqueName", "CorrelationId" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.UnsubscribeDebugViewAckDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.UnsubscribeDebugViewAckDto.Parser, null, null, null, null, null),
|
||||
@@ -19509,6 +19510,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
public DebugSnapshotRequestDto(DebugSnapshotRequestDto other) : this() {
|
||||
instanceUniqueName_ = other.instanceUniqueName_;
|
||||
correlationId_ = other.correlationId_;
|
||||
alarmsOnly_ = other.alarmsOnly_;
|
||||
_unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -19542,6 +19544,27 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Field number for the "alarms_only" field.</summary>
|
||||
public const int AlarmsOnlyFieldNumber = 3;
|
||||
private bool alarmsOnly_;
|
||||
/// <summary>
|
||||
/// Alarms-only projection (WP2.3 wire efficiency): when true the site builds
|
||||
/// and returns ONLY the alarm rows of the debug snapshot, leaving
|
||||
/// attribute_values empty. Used by the central per-site live alarm cache,
|
||||
/// whose seed/reconcile fan-out discards every attribute row anyway — a full
|
||||
/// snapshot ships the whole attribute surface of every enabled instance once
|
||||
/// per reconcile for nothing. proto3 defaults it to false, so an older
|
||||
/// central that never sets it keeps the full-snapshot behaviour. Additive-only.
|
||||
/// </summary>
|
||||
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
|
||||
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
|
||||
public bool AlarmsOnly {
|
||||
get { return alarmsOnly_; }
|
||||
set {
|
||||
alarmsOnly_ = value;
|
||||
}
|
||||
}
|
||||
|
||||
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
|
||||
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
|
||||
public override bool Equals(object other) {
|
||||
@@ -19559,6 +19582,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
}
|
||||
if (InstanceUniqueName != other.InstanceUniqueName) return false;
|
||||
if (CorrelationId != other.CorrelationId) return false;
|
||||
if (AlarmsOnly != other.AlarmsOnly) return false;
|
||||
return Equals(_unknownFields, other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -19568,6 +19592,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
int hash = 1;
|
||||
if (InstanceUniqueName.Length != 0) hash ^= InstanceUniqueName.GetHashCode();
|
||||
if (CorrelationId.Length != 0) hash ^= CorrelationId.GetHashCode();
|
||||
if (AlarmsOnly != false) hash ^= AlarmsOnly.GetHashCode();
|
||||
if (_unknownFields != null) {
|
||||
hash ^= _unknownFields.GetHashCode();
|
||||
}
|
||||
@@ -19594,6 +19619,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
output.WriteRawTag(18);
|
||||
output.WriteString(CorrelationId);
|
||||
}
|
||||
if (AlarmsOnly != false) {
|
||||
output.WriteRawTag(24);
|
||||
output.WriteBool(AlarmsOnly);
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
_unknownFields.WriteTo(output);
|
||||
}
|
||||
@@ -19612,6 +19641,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
output.WriteRawTag(18);
|
||||
output.WriteString(CorrelationId);
|
||||
}
|
||||
if (AlarmsOnly != false) {
|
||||
output.WriteRawTag(24);
|
||||
output.WriteBool(AlarmsOnly);
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
_unknownFields.WriteTo(ref output);
|
||||
}
|
||||
@@ -19628,6 +19661,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
if (CorrelationId.Length != 0) {
|
||||
size += 1 + pb::CodedOutputStream.ComputeStringSize(CorrelationId);
|
||||
}
|
||||
if (AlarmsOnly != false) {
|
||||
size += 1 + 1;
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
size += _unknownFields.CalculateSize();
|
||||
}
|
||||
@@ -19646,6 +19682,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
if (other.CorrelationId.Length != 0) {
|
||||
CorrelationId = other.CorrelationId;
|
||||
}
|
||||
if (other.AlarmsOnly != false) {
|
||||
AlarmsOnly = other.AlarmsOnly;
|
||||
}
|
||||
_unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -19673,6 +19712,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
CorrelationId = input.ReadString();
|
||||
break;
|
||||
}
|
||||
case 24: {
|
||||
AlarmsOnly = input.ReadBool();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -19700,6 +19743,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
CorrelationId = input.ReadString();
|
||||
break;
|
||||
}
|
||||
case 24: {
|
||||
AlarmsOnly = input.ReadBool();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,36 +81,36 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
"dEV2ZW50RHRvEjcKC29wZXJhdGlvbmFsGAIgASgLMiIuc2l0ZXN0cmVhbS5T",
|
||||
"aXRlQ2FsbE9wZXJhdGlvbmFsRHRvIkoKFENhY2hlZFRlbGVtZXRyeUJhdGNo",
|
||||
"EjIKB3BhY2tldHMYASADKAsyIS5zaXRlc3RyZWFtLkNhY2hlZFRlbGVtZXRy",
|
||||
"eVBhY2tldCJbChZQdWxsQXVkaXRFdmVudHNSZXF1ZXN0Ei0KCXNpbmNlX3V0",
|
||||
"eVBhY2tldCJtChZQdWxsQXVkaXRFdmVudHNSZXF1ZXN0Ei0KCXNpbmNlX3V0",
|
||||
"YxgBIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASEgoKYmF0Y2hf",
|
||||
"c2l6ZRgCIAEoBSJcChdQdWxsQXVkaXRFdmVudHNSZXNwb25zZRIpCgZldmVu",
|
||||
"dHMYASADKAsyGS5zaXRlc3RyZWFtLkF1ZGl0RXZlbnREdG8SFgoObW9yZV9h",
|
||||
"dmFpbGFibGUYAiABKAgiawoUUHVsbFNpdGVDYWxsc1JlcXVlc3QSLQoJc2lu",
|
||||
"Y2VfdXRjGAEgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBISCgpi",
|
||||
"YXRjaF9zaXplGAIgASgFEhAKCGFmdGVyX2lkGAMgASgJImkKFVB1bGxTaXRl",
|
||||
"Q2FsbHNSZXNwb25zZRI4CgxvcGVyYXRpb25hbHMYASADKAsyIi5zaXRlc3Ry",
|
||||
"ZWFtLlNpdGVDYWxsT3BlcmF0aW9uYWxEdG8SFgoObW9yZV9hdmFpbGFibGUY",
|
||||
"AiABKAgqXAoHUXVhbGl0eRIXChNRVUFMSVRZX1VOU1BFQ0lGSUVEEAASEAoM",
|
||||
"UVVBTElUWV9HT09EEAESFQoRUVVBTElUWV9VTkNFUlRBSU4QAhIPCgtRVUFM",
|
||||
"SVRZX0JBRBADKl0KDkFsYXJtU3RhdGVFbnVtEhsKF0FMQVJNX1NUQVRFX1VO",
|
||||
"U1BFQ0lGSUVEEAASFgoSQUxBUk1fU1RBVEVfTk9STUFMEAESFgoSQUxBUk1f",
|
||||
"U1RBVEVfQUNUSVZFEAIqhQEKDkFsYXJtTGV2ZWxFbnVtEhQKEEFMQVJNX0xF",
|
||||
"VkVMX05PTkUQABITCg9BTEFSTV9MRVZFTF9MT1cQARIXChNBTEFSTV9MRVZF",
|
||||
"TF9MT1dfTE9XEAISFAoQQUxBUk1fTEVWRUxfSElHSBADEhkKFUFMQVJNX0xF",
|
||||
"VkVMX0hJR0hfSElHSBAEMoYEChFTaXRlU3RyZWFtU2VydmljZRJVChFTdWJz",
|
||||
"Y3JpYmVJbnN0YW5jZRIhLnNpdGVzdHJlYW0uSW5zdGFuY2VTdHJlYW1SZXF1",
|
||||
"ZXN0Ghsuc2l0ZXN0cmVhbS5TaXRlU3RyZWFtRXZlbnQwARJNCg1TdWJzY3Jp",
|
||||
"YmVTaXRlEh0uc2l0ZXN0cmVhbS5TaXRlU3RyZWFtUmVxdWVzdBobLnNpdGVz",
|
||||
"dHJlYW0uU2l0ZVN0cmVhbUV2ZW50MAESRwoRSW5nZXN0QXVkaXRFdmVudHMS",
|
||||
"Gy5zaXRlc3RyZWFtLkF1ZGl0RXZlbnRCYXRjaBoVLnNpdGVzdHJlYW0uSW5n",
|
||||
"ZXN0QWNrElAKFUluZ2VzdENhY2hlZFRlbGVtZXRyeRIgLnNpdGVzdHJlYW0u",
|
||||
"Q2FjaGVkVGVsZW1ldHJ5QmF0Y2gaFS5zaXRlc3RyZWFtLkluZ2VzdEFjaxJa",
|
||||
"Cg9QdWxsQXVkaXRFdmVudHMSIi5zaXRlc3RyZWFtLlB1bGxBdWRpdEV2ZW50",
|
||||
"c1JlcXVlc3QaIy5zaXRlc3RyZWFtLlB1bGxBdWRpdEV2ZW50c1Jlc3BvbnNl",
|
||||
"ElQKDVB1bGxTaXRlQ2FsbHMSIC5zaXRlc3RyZWFtLlB1bGxTaXRlQ2FsbHNS",
|
||||
"ZXF1ZXN0GiEuc2l0ZXN0cmVhbS5QdWxsU2l0ZUNhbGxzUmVzcG9uc2VCK6oC",
|
||||
"KFpCLk1PTS5XVy5TY2FkYUJyaWRnZS5Db21tdW5pY2F0aW9uLkdycGNiBnBy",
|
||||
"b3RvMw=="));
|
||||
"c2l6ZRgCIAEoBRIQCghhZnRlcl9pZBgDIAEoCSJcChdQdWxsQXVkaXRFdmVu",
|
||||
"dHNSZXNwb25zZRIpCgZldmVudHMYASADKAsyGS5zaXRlc3RyZWFtLkF1ZGl0",
|
||||
"RXZlbnREdG8SFgoObW9yZV9hdmFpbGFibGUYAiABKAgiawoUUHVsbFNpdGVD",
|
||||
"YWxsc1JlcXVlc3QSLQoJc2luY2VfdXRjGAEgASgLMhouZ29vZ2xlLnByb3Rv",
|
||||
"YnVmLlRpbWVzdGFtcBISCgpiYXRjaF9zaXplGAIgASgFEhAKCGFmdGVyX2lk",
|
||||
"GAMgASgJImkKFVB1bGxTaXRlQ2FsbHNSZXNwb25zZRI4CgxvcGVyYXRpb25h",
|
||||
"bHMYASADKAsyIi5zaXRlc3RyZWFtLlNpdGVDYWxsT3BlcmF0aW9uYWxEdG8S",
|
||||
"FgoObW9yZV9hdmFpbGFibGUYAiABKAgqXAoHUXVhbGl0eRIXChNRVUFMSVRZ",
|
||||
"X1VOU1BFQ0lGSUVEEAASEAoMUVVBTElUWV9HT09EEAESFQoRUVVBTElUWV9V",
|
||||
"TkNFUlRBSU4QAhIPCgtRVUFMSVRZX0JBRBADKl0KDkFsYXJtU3RhdGVFbnVt",
|
||||
"EhsKF0FMQVJNX1NUQVRFX1VOU1BFQ0lGSUVEEAASFgoSQUxBUk1fU1RBVEVf",
|
||||
"Tk9STUFMEAESFgoSQUxBUk1fU1RBVEVfQUNUSVZFEAIqhQEKDkFsYXJtTGV2",
|
||||
"ZWxFbnVtEhQKEEFMQVJNX0xFVkVMX05PTkUQABITCg9BTEFSTV9MRVZFTF9M",
|
||||
"T1cQARIXChNBTEFSTV9MRVZFTF9MT1dfTE9XEAISFAoQQUxBUk1fTEVWRUxf",
|
||||
"SElHSBADEhkKFUFMQVJNX0xFVkVMX0hJR0hfSElHSBAEMoYEChFTaXRlU3Ry",
|
||||
"ZWFtU2VydmljZRJVChFTdWJzY3JpYmVJbnN0YW5jZRIhLnNpdGVzdHJlYW0u",
|
||||
"SW5zdGFuY2VTdHJlYW1SZXF1ZXN0Ghsuc2l0ZXN0cmVhbS5TaXRlU3RyZWFt",
|
||||
"RXZlbnQwARJNCg1TdWJzY3JpYmVTaXRlEh0uc2l0ZXN0cmVhbS5TaXRlU3Ry",
|
||||
"ZWFtUmVxdWVzdBobLnNpdGVzdHJlYW0uU2l0ZVN0cmVhbUV2ZW50MAESRwoR",
|
||||
"SW5nZXN0QXVkaXRFdmVudHMSGy5zaXRlc3RyZWFtLkF1ZGl0RXZlbnRCYXRj",
|
||||
"aBoVLnNpdGVzdHJlYW0uSW5nZXN0QWNrElAKFUluZ2VzdENhY2hlZFRlbGVt",
|
||||
"ZXRyeRIgLnNpdGVzdHJlYW0uQ2FjaGVkVGVsZW1ldHJ5QmF0Y2gaFS5zaXRl",
|
||||
"c3RyZWFtLkluZ2VzdEFjaxJaCg9QdWxsQXVkaXRFdmVudHMSIi5zaXRlc3Ry",
|
||||
"ZWFtLlB1bGxBdWRpdEV2ZW50c1JlcXVlc3QaIy5zaXRlc3RyZWFtLlB1bGxB",
|
||||
"dWRpdEV2ZW50c1Jlc3BvbnNlElQKDVB1bGxTaXRlQ2FsbHMSIC5zaXRlc3Ry",
|
||||
"ZWFtLlB1bGxTaXRlQ2FsbHNSZXF1ZXN0GiEuc2l0ZXN0cmVhbS5QdWxsU2l0",
|
||||
"ZUNhbGxzUmVzcG9uc2VCK6oCKFpCLk1PTS5XVy5TY2FkYUJyaWRnZS5Db21t",
|
||||
"dW5pY2F0aW9uLkdycGNiBnByb3RvMw=="));
|
||||
descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
|
||||
new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.TimestampReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.WrappersReflection.Descriptor, },
|
||||
new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.Quality), typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.AlarmStateEnum), typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.AlarmLevelEnum), }, null, new pbr::GeneratedClrTypeInfo[] {
|
||||
@@ -125,7 +125,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteCallOperationalDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteCallOperationalDto.Parser, new[]{ "TrackedOperationId", "Channel", "Target", "SourceSite", "Status", "RetryCount", "LastError", "HttpStatus", "CreatedAtUtc", "UpdatedAtUtc", "TerminalAtUtc", "SourceNode" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.CachedTelemetryPacket), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.CachedTelemetryPacket.Parser, new[]{ "AuditEvent", "Operational" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.CachedTelemetryBatch), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.CachedTelemetryBatch.Parser, new[]{ "Packets" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullAuditEventsRequest), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullAuditEventsRequest.Parser, new[]{ "SinceUtc", "BatchSize" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullAuditEventsRequest), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullAuditEventsRequest.Parser, new[]{ "SinceUtc", "BatchSize", "AfterId" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullAuditEventsResponse), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullAuditEventsResponse.Parser, new[]{ "Events", "MoreAvailable" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullSiteCallsRequest), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullSiteCallsRequest.Parser, new[]{ "SinceUtc", "BatchSize", "AfterId" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullSiteCallsResponse), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.PullSiteCallsResponse.Parser, new[]{ "Operationals", "MoreAvailable" }, null, null, null, null)
|
||||
@@ -4942,8 +4942,11 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
/// <summary>
|
||||
/// Audit Log (#23) M6 reconciliation pull: central→site request for any
|
||||
/// site-local AuditLog rows with OccurredAtUtc >= since_utc that have not yet
|
||||
/// been ingested centrally (ForwardState in {Pending, Forwarded}). The site
|
||||
/// flips returned rows to Reconciled after the response is on the wire.
|
||||
/// been ingested centrally (ForwardState in {Pending, Forwarded}). Rows are NOT
|
||||
/// flipped to Reconciled when they are served — only when a LATER pull's cursor
|
||||
/// proves central consumed them (see after_id), so a fault between the response
|
||||
/// leaving the site and central committing it re-serves the rows instead of
|
||||
/// silently losing them (at-least-once).
|
||||
/// more_available signals batch_size was saturated so the caller knows to
|
||||
/// issue a follow-up pull with an advanced since_utc cursor.
|
||||
/// </summary>
|
||||
@@ -4984,6 +4987,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
public PullAuditEventsRequest(PullAuditEventsRequest other) : this() {
|
||||
sinceUtc_ = other.sinceUtc_ != null ? other.sinceUtc_.Clone() : null;
|
||||
batchSize_ = other.batchSize_;
|
||||
afterId_ = other.afterId_;
|
||||
_unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -5017,6 +5021,29 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Field number for the "after_id" field.</summary>
|
||||
public const int AfterIdFieldNumber = 3;
|
||||
private string afterId_ = "";
|
||||
/// <summary>
|
||||
/// Composite-keyset cursor (WP2.3), mirroring PullSiteCallsRequest.after_id:
|
||||
/// the EventId ("D" GUID form) of the last row central has already CONSUMED at
|
||||
/// since_utc. When set, the site returns only rows strictly after the composite
|
||||
/// (OccurredAtUtc, EventId) pair — un-pinning a batch that would otherwise stall
|
||||
/// when more than batch_size rows share one since_utc instant — AND treats the
|
||||
/// cursor as proof of receipt: everything at or before it is flipped to
|
||||
/// Reconciled. Empty (the proto3 string default) preserves the legacy inclusive
|
||||
/// >= behaviour, under which only rows strictly older than since_utc are proven
|
||||
/// received. Additive-only.
|
||||
/// </summary>
|
||||
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
|
||||
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
|
||||
public string AfterId {
|
||||
get { return afterId_; }
|
||||
set {
|
||||
afterId_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
|
||||
}
|
||||
}
|
||||
|
||||
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
|
||||
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
|
||||
public override bool Equals(object other) {
|
||||
@@ -5034,6 +5061,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
}
|
||||
if (!object.Equals(SinceUtc, other.SinceUtc)) return false;
|
||||
if (BatchSize != other.BatchSize) return false;
|
||||
if (AfterId != other.AfterId) return false;
|
||||
return Equals(_unknownFields, other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -5043,6 +5071,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
int hash = 1;
|
||||
if (sinceUtc_ != null) hash ^= SinceUtc.GetHashCode();
|
||||
if (BatchSize != 0) hash ^= BatchSize.GetHashCode();
|
||||
if (AfterId.Length != 0) hash ^= AfterId.GetHashCode();
|
||||
if (_unknownFields != null) {
|
||||
hash ^= _unknownFields.GetHashCode();
|
||||
}
|
||||
@@ -5069,6 +5098,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
output.WriteRawTag(16);
|
||||
output.WriteInt32(BatchSize);
|
||||
}
|
||||
if (AfterId.Length != 0) {
|
||||
output.WriteRawTag(26);
|
||||
output.WriteString(AfterId);
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
_unknownFields.WriteTo(output);
|
||||
}
|
||||
@@ -5087,6 +5120,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
output.WriteRawTag(16);
|
||||
output.WriteInt32(BatchSize);
|
||||
}
|
||||
if (AfterId.Length != 0) {
|
||||
output.WriteRawTag(26);
|
||||
output.WriteString(AfterId);
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
_unknownFields.WriteTo(ref output);
|
||||
}
|
||||
@@ -5103,6 +5140,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
if (BatchSize != 0) {
|
||||
size += 1 + pb::CodedOutputStream.ComputeInt32Size(BatchSize);
|
||||
}
|
||||
if (AfterId.Length != 0) {
|
||||
size += 1 + pb::CodedOutputStream.ComputeStringSize(AfterId);
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
size += _unknownFields.CalculateSize();
|
||||
}
|
||||
@@ -5124,6 +5164,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
if (other.BatchSize != 0) {
|
||||
BatchSize = other.BatchSize;
|
||||
}
|
||||
if (other.AfterId.Length != 0) {
|
||||
AfterId = other.AfterId;
|
||||
}
|
||||
_unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -5154,6 +5197,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
BatchSize = input.ReadInt32();
|
||||
break;
|
||||
}
|
||||
case 26: {
|
||||
AfterId = input.ReadString();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -5184,6 +5231,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
BatchSize = input.ReadInt32();
|
||||
break;
|
||||
}
|
||||
case 26: {
|
||||
AfterId = input.ReadString();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1372,13 +1372,21 @@ public class InstanceActor : ReceiveActor
|
||||
private void HandleDebugSnapshot(DebugSnapshotRequest request)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var attributeValues = _attributes.Select(kvp => new AttributeValueChanged(
|
||||
_instanceUniqueName,
|
||||
kvp.Key,
|
||||
kvp.Key,
|
||||
kvp.Value,
|
||||
_attributeQualities.GetValueOrDefault(kvp.Key, "Good"),
|
||||
_attributeTimestamps.GetValueOrDefault(kvp.Key, now))).ToList();
|
||||
|
||||
// Alarms-only projection (WP2.3): the central live-alarm cache seeds/reconciles
|
||||
// through this same query surface and throws every attribute row away. Building
|
||||
// and shipping them is pure waste — one instance with a few hundred attributes
|
||||
// costs more on the wire than the whole alarm set of the site. The flag is
|
||||
// additive and defaults false, so the Debug View is untouched.
|
||||
var attributeValues = request.AlarmsOnly
|
||||
? new List<AttributeValueChanged>()
|
||||
: _attributes.Select(kvp => new AttributeValueChanged(
|
||||
_instanceUniqueName,
|
||||
kvp.Key,
|
||||
kvp.Key,
|
||||
kvp.Value,
|
||||
_attributeQualities.GetValueOrDefault(kvp.Key, "Good"),
|
||||
_attributeTimestamps.GetValueOrDefault(kvp.Key, now))).ToList();
|
||||
|
||||
var snapshot = new DebugViewSnapshot(
|
||||
_instanceUniqueName,
|
||||
|
||||
Reference in New Issue
Block a user