fix(historian): respect gateway ceilings and history value contracts
v2-ci / build (pull_request) Successful in 5m10s
v2-ci / unit-tests (pull_request) Failing after 28m34s

This commit is contained in:
Joseph Doherty
2026-10-13 03:51:04 -04:00
parent 66a2b0c3e9
commit 97668d26d3
15 changed files with 474 additions and 29 deletions
@@ -152,6 +152,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
catch (Exception ex)
{
RecordReadFailure(ex);
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
throw;
}
}
@@ -239,7 +240,15 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
for (var attempt = 0; attempt < MaxCrowdedReReads && preWindow <= _maxPreWindowRows; attempt++)
{
var budget = (int)Math.Min(int.MaxValue, (long)cap + room + RawAggregateReader.PreWindowAllowance);
if (_ceiling.Learned is var known and > 0 && budget > known) break; // over a strict ceiling: refused anyway
// #695: doubling can leap over the ceiling while the cluster still fits. Spend the remaining room once.
if (_ceiling.Learned is var known and > 0 && budget > known)
{
var available = known - cap - RawAggregateReader.PreWindowAllowance;
if (available <= 0) break;
room = Math.Min(room, available);
budget = cap + room + RawAggregateReader.PreWindowAllowance;
}
if (budget <= pass.Budget) break; // no more room, never retry the same full read
pass = await PassAsync(budget).ConfigureAwait(false);
if (pass.Result.Complete) return pass.Result.Samples;
preWindow = Math.Max(preWindow, pass.Result.PreWindowRows);
@@ -365,6 +374,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
catch (Exception ex)
{
RecordReadFailure(ex);
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
throw;
}
}
@@ -408,6 +418,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
catch (Exception ex)
{
RecordReadFailure(ex);
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
throw;
}
}
@@ -466,6 +477,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
catch (Exception ex)
{
RecordReadFailure(ex);
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
throw;
}
}
@@ -486,6 +498,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
catch (Exception ex)
{
RecordReadFailure(ex);
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
throw;
}
}
@@ -581,6 +594,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
catch (Exception ex)
{
RecordReadFailure(ex);
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
throw;
}
}
@@ -655,6 +669,15 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
return result;
}
/// <summary>Classifies actual gRPC/package transport failures without treating refusals as outages (#697).</summary>
private static bool IsTransportFailure(Exception ex) => ex switch
{
ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayUnavailableException => true,
RpcException rpc => rpc.StatusCode is Grpc.Core.StatusCode.Unavailable or Grpc.Core.StatusCode.DeadlineExceeded,
TimeoutException => true,
_ => ex.InnerException is { } inner && IsTransportFailure(inner),
};
/// <summary>
/// Records a failed read: bumps the health counters and logs a generic, redaction-safe
/// debug line (no tag, host, or value). The exception itself is rethrown by the caller.