Compare commits
4 Commits
abcip-udt-
...
identifica
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2acea08ced | ||
| 49f6c9484e | |||
|
|
d06cc01a48 | ||
| 5536e96b46 |
@@ -36,7 +36,10 @@ else if (_equipment.Count > 0)
|
|||||||
<td>@e.SAPID</td>
|
<td>@e.SAPID</td>
|
||||||
<td>@e.Manufacturer / @e.Model</td>
|
<td>@e.Manufacturer / @e.Model</td>
|
||||||
<td>@e.SerialNumber</td>
|
<td>@e.SerialNumber</td>
|
||||||
<td><button class="btn btn-sm btn-outline-danger" @onclick="() => DeleteAsync(e.EquipmentRowId)">Remove</button></td>
|
<td>
|
||||||
|
<button class="btn btn-sm btn-outline-secondary me-1" @onclick="() => StartEdit(e)">Edit</button>
|
||||||
|
<button class="btn btn-sm btn-outline-danger" @onclick="() => DeleteAsync(e.EquipmentRowId)">Remove</button>
|
||||||
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
}
|
}
|
||||||
</tbody>
|
</tbody>
|
||||||
@@ -47,8 +50,8 @@ else if (_equipment.Count > 0)
|
|||||||
{
|
{
|
||||||
<div class="card mt-3">
|
<div class="card mt-3">
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<h5>New equipment</h5>
|
<h5>@(_editMode ? "Edit equipment" : "New equipment")</h5>
|
||||||
<EditForm Model="_draft" OnValidSubmit="SaveAsync" FormName="new-equipment">
|
<EditForm Model="_draft" OnValidSubmit="SaveAsync" FormName="equipment-form">
|
||||||
<DataAnnotationsValidator/>
|
<DataAnnotationsValidator/>
|
||||||
<div class="row g-3">
|
<div class="row g-3">
|
||||||
<div class="col-md-4">
|
<div class="col-md-4">
|
||||||
@@ -78,24 +81,13 @@ else if (_equipment.Count > 0)
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<h6 class="mt-4">OPC 40010 Identification</h6>
|
<IdentificationFields Equipment="_draft"/>
|
||||||
<div class="row g-3">
|
|
||||||
<div class="col-md-4"><label class="form-label">Manufacturer</label><InputText @bind-Value="_draft.Manufacturer" class="form-control"/></div>
|
|
||||||
<div class="col-md-4"><label class="form-label">Model</label><InputText @bind-Value="_draft.Model" class="form-control"/></div>
|
|
||||||
<div class="col-md-4"><label class="form-label">Serial number</label><InputText @bind-Value="_draft.SerialNumber" class="form-control"/></div>
|
|
||||||
<div class="col-md-4"><label class="form-label">Hardware rev</label><InputText @bind-Value="_draft.HardwareRevision" class="form-control"/></div>
|
|
||||||
<div class="col-md-4"><label class="form-label">Software rev</label><InputText @bind-Value="_draft.SoftwareRevision" class="form-control"/></div>
|
|
||||||
<div class="col-md-4">
|
|
||||||
<label class="form-label">Year of construction</label>
|
|
||||||
<InputNumber @bind-Value="_draft.YearOfConstruction" class="form-control"/>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
@if (_error is not null) { <div class="alert alert-danger mt-3">@_error</div> }
|
@if (_error is not null) { <div class="alert alert-danger mt-3">@_error</div> }
|
||||||
|
|
||||||
<div class="mt-3">
|
<div class="mt-3">
|
||||||
<button type="submit" class="btn btn-primary btn-sm">Save</button>
|
<button type="submit" class="btn btn-primary btn-sm">Save</button>
|
||||||
<button type="button" class="btn btn-secondary btn-sm ms-2" @onclick="() => _showForm = false">Cancel</button>
|
<button type="button" class="btn btn-secondary btn-sm ms-2" @onclick="Cancel">Cancel</button>
|
||||||
</div>
|
</div>
|
||||||
</EditForm>
|
</EditForm>
|
||||||
</div>
|
</div>
|
||||||
@@ -106,6 +98,7 @@ else if (_equipment.Count > 0)
|
|||||||
[Parameter] public long GenerationId { get; set; }
|
[Parameter] public long GenerationId { get; set; }
|
||||||
private List<Equipment>? _equipment;
|
private List<Equipment>? _equipment;
|
||||||
private bool _showForm;
|
private bool _showForm;
|
||||||
|
private bool _editMode;
|
||||||
private Equipment _draft = NewBlankDraft();
|
private Equipment _draft = NewBlankDraft();
|
||||||
private string? _error;
|
private string? _error;
|
||||||
|
|
||||||
@@ -125,20 +118,68 @@ else if (_equipment.Count > 0)
|
|||||||
private void StartAdd()
|
private void StartAdd()
|
||||||
{
|
{
|
||||||
_draft = NewBlankDraft();
|
_draft = NewBlankDraft();
|
||||||
|
_editMode = false;
|
||||||
_error = null;
|
_error = null;
|
||||||
_showForm = true;
|
_showForm = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void StartEdit(Equipment row)
|
||||||
|
{
|
||||||
|
// Shallow-clone so Cancel doesn't mutate the list-displayed row with in-flight form edits.
|
||||||
|
_draft = new Equipment
|
||||||
|
{
|
||||||
|
EquipmentRowId = row.EquipmentRowId,
|
||||||
|
GenerationId = row.GenerationId,
|
||||||
|
EquipmentId = row.EquipmentId,
|
||||||
|
EquipmentUuid = row.EquipmentUuid,
|
||||||
|
DriverInstanceId = row.DriverInstanceId,
|
||||||
|
DeviceId = row.DeviceId,
|
||||||
|
UnsLineId = row.UnsLineId,
|
||||||
|
Name = row.Name,
|
||||||
|
MachineCode = row.MachineCode,
|
||||||
|
ZTag = row.ZTag,
|
||||||
|
SAPID = row.SAPID,
|
||||||
|
Manufacturer = row.Manufacturer,
|
||||||
|
Model = row.Model,
|
||||||
|
SerialNumber = row.SerialNumber,
|
||||||
|
HardwareRevision = row.HardwareRevision,
|
||||||
|
SoftwareRevision = row.SoftwareRevision,
|
||||||
|
YearOfConstruction = row.YearOfConstruction,
|
||||||
|
AssetLocation = row.AssetLocation,
|
||||||
|
ManufacturerUri = row.ManufacturerUri,
|
||||||
|
DeviceManualUri = row.DeviceManualUri,
|
||||||
|
EquipmentClassRef = row.EquipmentClassRef,
|
||||||
|
Enabled = row.Enabled,
|
||||||
|
};
|
||||||
|
_editMode = true;
|
||||||
|
_error = null;
|
||||||
|
_showForm = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void Cancel()
|
||||||
|
{
|
||||||
|
_showForm = false;
|
||||||
|
_editMode = false;
|
||||||
|
}
|
||||||
|
|
||||||
private async Task SaveAsync()
|
private async Task SaveAsync()
|
||||||
{
|
{
|
||||||
_error = null;
|
_error = null;
|
||||||
_draft.EquipmentUuid = Guid.NewGuid();
|
|
||||||
_draft.EquipmentId = DraftValidator.DeriveEquipmentId(_draft.EquipmentUuid);
|
|
||||||
_draft.GenerationId = GenerationId;
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await EquipmentSvc.CreateAsync(GenerationId, _draft, CancellationToken.None);
|
if (_editMode)
|
||||||
|
{
|
||||||
|
await EquipmentSvc.UpdateAsync(_draft, CancellationToken.None);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
_draft.EquipmentUuid = Guid.NewGuid();
|
||||||
|
_draft.EquipmentId = DraftValidator.DeriveEquipmentId(_draft.EquipmentUuid);
|
||||||
|
_draft.GenerationId = GenerationId;
|
||||||
|
await EquipmentSvc.CreateAsync(GenerationId, _draft, CancellationToken.None);
|
||||||
|
}
|
||||||
_showForm = false;
|
_showForm = false;
|
||||||
|
_editMode = false;
|
||||||
await ReloadAsync();
|
await ReloadAsync();
|
||||||
}
|
}
|
||||||
catch (Exception ex) { _error = ex.Message; }
|
catch (Exception ex) { _error = ex.Message; }
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
@using ZB.MOM.WW.OtOpcUa.Configuration.Entities
|
||||||
|
|
||||||
|
@* Reusable OPC 40010 Machinery Identification editor. Binds to an Equipment row and renders the
|
||||||
|
nine decision #139 fields in a consistent 3-column Bootstrap grid. Used by EquipmentTab's
|
||||||
|
create + edit forms so the same UI renders regardless of which flow opened it. *@
|
||||||
|
|
||||||
|
<h6 class="mt-4">OPC 40010 Identification</h6>
|
||||||
|
<div class="row g-3">
|
||||||
|
<div class="col-md-4">
|
||||||
|
<label class="form-label">Manufacturer</label>
|
||||||
|
<InputText @bind-Value="Equipment!.Manufacturer" class="form-control"/>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<label class="form-label">Model</label>
|
||||||
|
<InputText @bind-Value="Equipment!.Model" class="form-control"/>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<label class="form-label">Serial number</label>
|
||||||
|
<InputText @bind-Value="Equipment!.SerialNumber" class="form-control"/>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<label class="form-label">Hardware rev</label>
|
||||||
|
<InputText @bind-Value="Equipment!.HardwareRevision" class="form-control"/>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<label class="form-label">Software rev</label>
|
||||||
|
<InputText @bind-Value="Equipment!.SoftwareRevision" class="form-control"/>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<label class="form-label">Year of construction</label>
|
||||||
|
<InputNumber @bind-Value="Equipment!.YearOfConstruction" class="form-control"/>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<label class="form-label">Asset location</label>
|
||||||
|
<InputText @bind-Value="Equipment!.AssetLocation" class="form-control"/>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<label class="form-label">Manufacturer URI</label>
|
||||||
|
<InputText @bind-Value="Equipment!.ManufacturerUri" class="form-control" placeholder="https://…"/>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<label class="form-label">Device manual URI</label>
|
||||||
|
<InputText @bind-Value="Equipment!.DeviceManualUri" class="form-control" placeholder="https://…"/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
@code {
|
||||||
|
[Parameter, EditorRequired] public Equipment? Equipment { get; set; }
|
||||||
|
}
|
||||||
@@ -56,6 +56,16 @@ else
|
|||||||
</div></div></div>
|
</div></div></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
@if (_rows.Any(HostStatusService.IsFlagged))
|
||||||
|
{
|
||||||
|
var flaggedCount = _rows.Count(HostStatusService.IsFlagged);
|
||||||
|
<div class="alert alert-danger small mb-3">
|
||||||
|
<strong>@flaggedCount host@(flaggedCount == 1 ? "" : "s")</strong>
|
||||||
|
reporting ≥ @HostStatusService.FailureFlagThreshold consecutive failures — circuit breaker
|
||||||
|
may trip soon. Inspect the resilience columns below to locate.
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
|
||||||
@foreach (var cluster in _rows.GroupBy(r => r.ClusterId ?? "(unassigned)").OrderBy(g => g.Key))
|
@foreach (var cluster in _rows.GroupBy(r => r.ClusterId ?? "(unassigned)").OrderBy(g => g.Key))
|
||||||
{
|
{
|
||||||
<h2 class="h5 mt-4">Cluster: <code>@cluster.Key</code></h2>
|
<h2 class="h5 mt-4">Cluster: <code>@cluster.Key</code></h2>
|
||||||
@@ -66,6 +76,9 @@ else
|
|||||||
<th>Driver</th>
|
<th>Driver</th>
|
||||||
<th>Host</th>
|
<th>Host</th>
|
||||||
<th>State</th>
|
<th>State</th>
|
||||||
|
<th class="text-end" title="Consecutive failures — resets when a call succeeds or the breaker closes">Fail#</th>
|
||||||
|
<th class="text-end" title="In-flight capability calls (bulkhead-depth proxy)">In-flight</th>
|
||||||
|
<th>Breaker opened</th>
|
||||||
<th>Last transition</th>
|
<th>Last transition</th>
|
||||||
<th>Last seen</th>
|
<th>Last seen</th>
|
||||||
<th>Detail</th>
|
<th>Detail</th>
|
||||||
@@ -84,10 +97,21 @@ else
|
|||||||
{
|
{
|
||||||
<span class="badge bg-warning text-dark ms-1">Stale</span>
|
<span class="badge bg-warning text-dark ms-1">Stale</span>
|
||||||
}
|
}
|
||||||
|
@if (HostStatusService.IsFlagged(r))
|
||||||
|
{
|
||||||
|
<span class="badge bg-danger ms-1" title="≥ @HostStatusService.FailureFlagThreshold consecutive failures">Flagged</span>
|
||||||
|
}
|
||||||
|
</td>
|
||||||
|
<td class="text-end small @(HostStatusService.IsFlagged(r) ? "text-danger fw-bold" : "")">
|
||||||
|
@r.ConsecutiveFailures
|
||||||
|
</td>
|
||||||
|
<td class="text-end small">@r.CurrentBulkheadDepth</td>
|
||||||
|
<td class="small">
|
||||||
|
@(r.LastCircuitBreakerOpenUtc is null ? "—" : FormatAge(r.LastCircuitBreakerOpenUtc.Value))
|
||||||
</td>
|
</td>
|
||||||
<td class="small">@FormatAge(r.StateChangedUtc)</td>
|
<td class="small">@FormatAge(r.StateChangedUtc)</td>
|
||||||
<td class="small @(HostStatusService.IsStale(r) ? "text-warning" : "")">@FormatAge(r.LastSeenUtc)</td>
|
<td class="small @(HostStatusService.IsStale(r) ? "text-warning" : "")">@FormatAge(r.LastSeenUtc)</td>
|
||||||
<td class="text-truncate small" style="max-width: 320px;" title="@r.Detail">@r.Detail</td>
|
<td class="text-truncate small" style="max-width: 240px;" title="@r.Detail">@r.Detail</td>
|
||||||
</tr>
|
</tr>
|
||||||
}
|
}
|
||||||
</tbody>
|
</tbody>
|
||||||
|
|||||||
@@ -7,8 +7,9 @@ namespace ZB.MOM.WW.OtOpcUa.Admin.Services;
|
|||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// One row per <see cref="DriverHostStatus"/> record, enriched with the owning
|
/// One row per <see cref="DriverHostStatus"/> record, enriched with the owning
|
||||||
/// <c>ClusterNode.ClusterId</c> when available (left-join). The Admin <c>/hosts</c> page
|
/// <c>ClusterNode.ClusterId</c> (left-join) + the per-<c>(DriverInstanceId, HostName)</c>
|
||||||
/// groups by cluster and renders a per-node → per-driver → per-host tree.
|
/// <see cref="DriverInstanceResilienceStatus"/> counters (also left-join) so the Admin
|
||||||
|
/// <c>/hosts</c> page renders the resilience surface inline with host state.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed record HostStatusRow(
|
public sealed record HostStatusRow(
|
||||||
string NodeId,
|
string NodeId,
|
||||||
@@ -18,7 +19,11 @@ public sealed record HostStatusRow(
|
|||||||
DriverHostState State,
|
DriverHostState State,
|
||||||
DateTime StateChangedUtc,
|
DateTime StateChangedUtc,
|
||||||
DateTime LastSeenUtc,
|
DateTime LastSeenUtc,
|
||||||
string? Detail);
|
string? Detail,
|
||||||
|
int ConsecutiveFailures,
|
||||||
|
DateTime? LastCircuitBreakerOpenUtc,
|
||||||
|
int CurrentBulkheadDepth,
|
||||||
|
DateTime? LastRecycleUtc);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Read-side service for the Admin UI's per-host drill-down. Loads
|
/// Read-side service for the Admin UI's per-host drill-down. Loads
|
||||||
@@ -36,15 +41,26 @@ public sealed class HostStatusService(OtOpcUaConfigDbContext db)
|
|||||||
{
|
{
|
||||||
public static readonly TimeSpan StaleThreshold = TimeSpan.FromSeconds(30);
|
public static readonly TimeSpan StaleThreshold = TimeSpan.FromSeconds(30);
|
||||||
|
|
||||||
|
/// <summary>Consecutive-failure threshold at which <see cref="IsFlagged"/> returns <c>true</c>
|
||||||
|
/// so the Admin UI can paint a red badge. Matches Phase 6.1 decision #143's conservative
|
||||||
|
/// half-of-breaker-threshold convention — flags before the breaker actually opens.</summary>
|
||||||
|
public const int FailureFlagThreshold = 3;
|
||||||
|
|
||||||
public async Task<IReadOnlyList<HostStatusRow>> ListAsync(CancellationToken ct = default)
|
public async Task<IReadOnlyList<HostStatusRow>> ListAsync(CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
// LEFT JOIN on NodeId so a row persists even when its owning ClusterNode row hasn't
|
// Two LEFT JOINs:
|
||||||
// been created yet (first-boot bootstrap case — keeps the UI from losing sight of
|
// 1. ClusterNodes on NodeId — row persists even when its owning ClusterNode row
|
||||||
// the reporting server).
|
// hasn't been created yet (first-boot bootstrap case).
|
||||||
|
// 2. DriverInstanceResilienceStatuses on (DriverInstanceId, HostName) — resilience
|
||||||
|
// counters haven't been sampled yet for brand-new hosts, so a missing row means
|
||||||
|
// zero failures + never-opened breaker.
|
||||||
var rows = await (from s in db.DriverHostStatuses.AsNoTracking()
|
var rows = await (from s in db.DriverHostStatuses.AsNoTracking()
|
||||||
join n in db.ClusterNodes.AsNoTracking()
|
join n in db.ClusterNodes.AsNoTracking()
|
||||||
on s.NodeId equals n.NodeId into nodeJoin
|
on s.NodeId equals n.NodeId into nodeJoin
|
||||||
from n in nodeJoin.DefaultIfEmpty()
|
from n in nodeJoin.DefaultIfEmpty()
|
||||||
|
join r in db.DriverInstanceResilienceStatuses.AsNoTracking()
|
||||||
|
on new { s.DriverInstanceId, s.HostName } equals new { r.DriverInstanceId, r.HostName } into resilJoin
|
||||||
|
from r in resilJoin.DefaultIfEmpty()
|
||||||
orderby s.NodeId, s.DriverInstanceId, s.HostName
|
orderby s.NodeId, s.DriverInstanceId, s.HostName
|
||||||
select new HostStatusRow(
|
select new HostStatusRow(
|
||||||
s.NodeId,
|
s.NodeId,
|
||||||
@@ -54,10 +70,21 @@ public sealed class HostStatusService(OtOpcUaConfigDbContext db)
|
|||||||
s.State,
|
s.State,
|
||||||
s.StateChangedUtc,
|
s.StateChangedUtc,
|
||||||
s.LastSeenUtc,
|
s.LastSeenUtc,
|
||||||
s.Detail)).ToListAsync(ct);
|
s.Detail,
|
||||||
|
r != null ? r.ConsecutiveFailures : 0,
|
||||||
|
r != null ? r.LastCircuitBreakerOpenUtc : null,
|
||||||
|
r != null ? r.CurrentBulkheadDepth : 0,
|
||||||
|
r != null ? r.LastRecycleUtc : null)).ToListAsync(ct);
|
||||||
return rows;
|
return rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static bool IsStale(HostStatusRow row) =>
|
public static bool IsStale(HostStatusRow row) =>
|
||||||
DateTime.UtcNow - row.LastSeenUtc > StaleThreshold;
|
DateTime.UtcNow - row.LastSeenUtc > StaleThreshold;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Red-badge predicate — <c>true</c> when the host has accumulated enough consecutive
|
||||||
|
/// failures that an operator should take notice before the breaker trips.
|
||||||
|
/// </summary>
|
||||||
|
public static bool IsFlagged(HostStatusRow row) =>
|
||||||
|
row.ConsecutiveFailures >= FailureFlagThreshold;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,11 +24,21 @@ public sealed class DriverResiliencePipelineBuilder
|
|||||||
{
|
{
|
||||||
private readonly ConcurrentDictionary<PipelineKey, ResiliencePipeline> _pipelines = new();
|
private readonly ConcurrentDictionary<PipelineKey, ResiliencePipeline> _pipelines = new();
|
||||||
private readonly TimeProvider _timeProvider;
|
private readonly TimeProvider _timeProvider;
|
||||||
|
private readonly DriverResilienceStatusTracker? _statusTracker;
|
||||||
|
|
||||||
/// <summary>Construct with the ambient clock (use <see cref="TimeProvider.System"/> in prod).</summary>
|
/// <summary>Construct with the ambient clock (use <see cref="TimeProvider.System"/> in prod).</summary>
|
||||||
public DriverResiliencePipelineBuilder(TimeProvider? timeProvider = null)
|
/// <param name="timeProvider">Clock source for pipeline timeouts + breaker sampling. Defaults to system.</param>
|
||||||
|
/// <param name="statusTracker">When non-null, every built pipeline wires Polly telemetry into
|
||||||
|
/// the tracker — retries increment <c>ConsecutiveFailures</c>, breaker-open stamps
|
||||||
|
/// <c>LastBreakerOpenUtc</c>, breaker-close resets failures. Feeds Admin <c>/hosts</c> +
|
||||||
|
/// the Polly bulkhead-depth column. Absent tracker means no telemetry (unit tests +
|
||||||
|
/// deployments that don't care about resilience observability).</param>
|
||||||
|
public DriverResiliencePipelineBuilder(
|
||||||
|
TimeProvider? timeProvider = null,
|
||||||
|
DriverResilienceStatusTracker? statusTracker = null)
|
||||||
{
|
{
|
||||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||||
|
_statusTracker = statusTracker;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -54,8 +64,9 @@ public sealed class DriverResiliencePipelineBuilder
|
|||||||
ArgumentException.ThrowIfNullOrWhiteSpace(hostName);
|
ArgumentException.ThrowIfNullOrWhiteSpace(hostName);
|
||||||
|
|
||||||
var key = new PipelineKey(driverInstanceId, hostName, capability);
|
var key = new PipelineKey(driverInstanceId, hostName, capability);
|
||||||
return _pipelines.GetOrAdd(key, static (_, state) => Build(state.capability, state.options, state.timeProvider),
|
return _pipelines.GetOrAdd(key, static (k, state) => Build(
|
||||||
(capability, options, timeProvider: _timeProvider));
|
k.DriverInstanceId, k.HostName, state.capability, state.options, state.timeProvider, state.tracker),
|
||||||
|
(capability, options, timeProvider: _timeProvider, tracker: _statusTracker));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>Drop cached pipelines for one driver instance (e.g. on ResilienceConfig change). Test + Admin-reload use.</summary>
|
/// <summary>Drop cached pipelines for one driver instance (e.g. on ResilienceConfig change). Test + Admin-reload use.</summary>
|
||||||
@@ -74,9 +85,12 @@ public sealed class DriverResiliencePipelineBuilder
|
|||||||
public int CachedPipelineCount => _pipelines.Count;
|
public int CachedPipelineCount => _pipelines.Count;
|
||||||
|
|
||||||
private static ResiliencePipeline Build(
|
private static ResiliencePipeline Build(
|
||||||
|
string driverInstanceId,
|
||||||
|
string hostName,
|
||||||
DriverCapability capability,
|
DriverCapability capability,
|
||||||
DriverResilienceOptions options,
|
DriverResilienceOptions options,
|
||||||
TimeProvider timeProvider)
|
TimeProvider timeProvider,
|
||||||
|
DriverResilienceStatusTracker? tracker)
|
||||||
{
|
{
|
||||||
var policy = options.Resolve(capability);
|
var policy = options.Resolve(capability);
|
||||||
var builder = new ResiliencePipelineBuilder { TimeProvider = timeProvider };
|
var builder = new ResiliencePipelineBuilder { TimeProvider = timeProvider };
|
||||||
@@ -88,7 +102,7 @@ public sealed class DriverResiliencePipelineBuilder
|
|||||||
|
|
||||||
if (policy.RetryCount > 0)
|
if (policy.RetryCount > 0)
|
||||||
{
|
{
|
||||||
builder.AddRetry(new RetryStrategyOptions
|
var retryOptions = new RetryStrategyOptions
|
||||||
{
|
{
|
||||||
MaxRetryAttempts = policy.RetryCount,
|
MaxRetryAttempts = policy.RetryCount,
|
||||||
BackoffType = DelayBackoffType.Exponential,
|
BackoffType = DelayBackoffType.Exponential,
|
||||||
@@ -96,19 +110,44 @@ public sealed class DriverResiliencePipelineBuilder
|
|||||||
Delay = TimeSpan.FromMilliseconds(100),
|
Delay = TimeSpan.FromMilliseconds(100),
|
||||||
MaxDelay = TimeSpan.FromSeconds(5),
|
MaxDelay = TimeSpan.FromSeconds(5),
|
||||||
ShouldHandle = new PredicateBuilder().Handle<Exception>(ex => ex is not OperationCanceledException),
|
ShouldHandle = new PredicateBuilder().Handle<Exception>(ex => ex is not OperationCanceledException),
|
||||||
});
|
};
|
||||||
|
if (tracker is not null)
|
||||||
|
{
|
||||||
|
retryOptions.OnRetry = args =>
|
||||||
|
{
|
||||||
|
tracker.RecordFailure(driverInstanceId, hostName, timeProvider.GetUtcNow().UtcDateTime);
|
||||||
|
return default;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
builder.AddRetry(retryOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (policy.BreakerFailureThreshold > 0)
|
if (policy.BreakerFailureThreshold > 0)
|
||||||
{
|
{
|
||||||
builder.AddCircuitBreaker(new CircuitBreakerStrategyOptions
|
var breakerOptions = new CircuitBreakerStrategyOptions
|
||||||
{
|
{
|
||||||
FailureRatio = 1.0,
|
FailureRatio = 1.0,
|
||||||
MinimumThroughput = policy.BreakerFailureThreshold,
|
MinimumThroughput = policy.BreakerFailureThreshold,
|
||||||
SamplingDuration = TimeSpan.FromSeconds(30),
|
SamplingDuration = TimeSpan.FromSeconds(30),
|
||||||
BreakDuration = TimeSpan.FromSeconds(15),
|
BreakDuration = TimeSpan.FromSeconds(15),
|
||||||
ShouldHandle = new PredicateBuilder().Handle<Exception>(ex => ex is not OperationCanceledException),
|
ShouldHandle = new PredicateBuilder().Handle<Exception>(ex => ex is not OperationCanceledException),
|
||||||
});
|
};
|
||||||
|
if (tracker is not null)
|
||||||
|
{
|
||||||
|
breakerOptions.OnOpened = args =>
|
||||||
|
{
|
||||||
|
tracker.RecordBreakerOpen(driverInstanceId, hostName, timeProvider.GetUtcNow().UtcDateTime);
|
||||||
|
return default;
|
||||||
|
};
|
||||||
|
breakerOptions.OnClosed = args =>
|
||||||
|
{
|
||||||
|
// Closing the breaker means the target recovered — reset the consecutive-
|
||||||
|
// failure counter so Admin UI stops flashing red for this host.
|
||||||
|
tracker.RecordSuccess(driverInstanceId, hostName, timeProvider.GetUtcNow().UtcDateTime);
|
||||||
|
return default;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
builder.AddCircuitBreaker(breakerOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
return builder.Build();
|
return builder.Build();
|
||||||
|
|||||||
@@ -219,4 +219,67 @@ public sealed class DriverResiliencePipelineBuilderTests
|
|||||||
|
|
||||||
attempts.ShouldBeLessThanOrEqualTo(1);
|
attempts.ShouldBeLessThanOrEqualTo(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Tracker_RecordsFailure_OnEveryRetry()
|
||||||
|
{
|
||||||
|
var tracker = new DriverResilienceStatusTracker();
|
||||||
|
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
||||||
|
var pipeline = builder.GetOrCreate("drv-trk", "host-x", DriverCapability.Read, TierAOptions);
|
||||||
|
|
||||||
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||||
|
await pipeline.ExecuteAsync(async _ =>
|
||||||
|
{
|
||||||
|
await Task.Yield();
|
||||||
|
throw new InvalidOperationException("always fails");
|
||||||
|
}));
|
||||||
|
|
||||||
|
var snap = tracker.TryGet("drv-trk", "host-x");
|
||||||
|
snap.ShouldNotBeNull();
|
||||||
|
var retryCount = TierAOptions.Resolve(DriverCapability.Read).RetryCount;
|
||||||
|
snap!.ConsecutiveFailures.ShouldBe(retryCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Tracker_StampsBreakerOpen_WhenBreakerTrips()
|
||||||
|
{
|
||||||
|
var tracker = new DriverResilienceStatusTracker();
|
||||||
|
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
||||||
|
var pipeline = builder.GetOrCreate("drv-trk", "host-b", DriverCapability.Write, TierAOptions);
|
||||||
|
|
||||||
|
var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold;
|
||||||
|
for (var i = 0; i < threshold; i++)
|
||||||
|
{
|
||||||
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||||
|
await pipeline.ExecuteAsync(async _ =>
|
||||||
|
{
|
||||||
|
await Task.Yield();
|
||||||
|
throw new InvalidOperationException("boom");
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
var snap = tracker.TryGet("drv-trk", "host-b");
|
||||||
|
snap.ShouldNotBeNull();
|
||||||
|
snap!.LastBreakerOpenUtc.ShouldNotBeNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Tracker_IsolatesCounters_PerHost()
|
||||||
|
{
|
||||||
|
var tracker = new DriverResilienceStatusTracker();
|
||||||
|
var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
|
||||||
|
var dead = builder.GetOrCreate("drv-trk", "dead", DriverCapability.Read, TierAOptions);
|
||||||
|
var live = builder.GetOrCreate("drv-trk", "live", DriverCapability.Read, TierAOptions);
|
||||||
|
|
||||||
|
await Should.ThrowAsync<InvalidOperationException>(async () =>
|
||||||
|
await dead.ExecuteAsync(async _ =>
|
||||||
|
{
|
||||||
|
await Task.Yield();
|
||||||
|
throw new InvalidOperationException("dead");
|
||||||
|
}));
|
||||||
|
await live.ExecuteAsync(async _ => await Task.Yield());
|
||||||
|
|
||||||
|
tracker.TryGet("drv-trk", "dead")!.ConsecutiveFailures.ShouldBeGreaterThan(0);
|
||||||
|
tracker.TryGet("drv-trk", "live").ShouldBeNull();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user