Phase 6.1 Stream E (data layer) — DriverInstanceResilienceStatus entity + DriverResilienceStatusTracker + EF migration

Ships the data + runtime layer of Stream E. The SignalR hub and Blazor /hosts
page refresh (E.2-E.3) are follow-up work paired with the visual-compliance
review per Phase 6.4 patterns — documented as a deferred follow-up below.

Configuration:
- New entity DriverInstanceResilienceStatus with:
  DriverInstanceId, HostName (composite PK),
  LastCircuitBreakerOpenUtc, ConsecutiveFailures, CurrentBulkheadDepth,
  LastRecycleUtc, BaselineFootprintBytes, CurrentFootprintBytes,
  LastSampledUtc.
- Separate from DriverHostStatus (per-host connectivity view) so a Running
  host that has tripped its breaker or is nearing its memory ceiling shows up
  distinctly on Admin /hosts. Admin page left-joins both for display.
- OtOpcUaConfigDbContext + Fluent-API config + IX_DriverResilience_LastSampled
  index for the stale-sample filter query.
- EF migration: 20260419124034_AddDriverInstanceResilienceStatus.

Core.Resilience:
- DriverResilienceStatusTracker — process-singleton in-memory tracker keyed on
  (DriverInstanceId, HostName). CapabilityInvoker + MemoryTracking +
  MemoryRecycle callers record failure/success/breaker-open/recycle/footprint
  events; a HostedService (Stream E.2 follow-up) samples this tracker every
  5 s and persists to the DB. Pure in-memory keeps tests fast + the core
  free of EF/SQL dependencies.

Tests:
- DriverResilienceStatusTrackerTests (9 new, all pass): tryget-before-write
  returns null; failures accumulate; success resets; breaker/recycle/footprint
  fields populate; per-host isolation; snapshot returns all pairs; concurrent
  writes don't lose counts.
- SchemaComplianceTests: expected-tables list updated to include the new
  DriverInstanceResilienceStatus table.

Full solution dotnet test: 1042 passing (baseline 906, +136 for Phase 6.1 so
far across Streams A/B/C/D/E.1). Pre-existing Client.CLI Subscribe flake
unchanged.

Deferred to follow-up PR (E.2/E.3):
- ResilienceStatusPublisher HostedService that samples DriverResilienceStatusTracker
  every 5 s + upserts DriverInstanceResilienceStatus rows.
- Admin FleetStatusHub SignalR hub pushing LastCircuitBreakerOpenUtc /
  CurrentBulkheadDepth / LastRecycleUtc on change.
- Admin /hosts Blazor column additions (red badge when
  ConsecutiveFailures > breakerThreshold / 2). Visual-compliance reviewer
  signoff alongside Phase 6.4 admin-ui patterns.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
2026-04-19 08:47:43 -04:00
parent 8d81715079
commit cbcaf6593a
8 changed files with 1650 additions and 0 deletions

View File

@@ -0,0 +1,44 @@
namespace ZB.MOM.WW.OtOpcUa.Configuration.Entities;
/// <summary>
/// Runtime resilience counters the CapabilityInvoker + MemoryTracking + MemoryRecycle
/// surfaces for each <c>(DriverInstanceId, HostName)</c> pair. Separate from
/// <see cref="DriverHostStatus"/> (which owns per-host <i>connectivity</i> state) so a
/// host that's Running but has tripped its breaker or is approaching its memory ceiling
/// shows up distinctly on Admin <c>/hosts</c>.
/// </summary>
/// <remarks>
/// Per <c>docs/v2/implementation/phase-6-1-resilience-and-observability.md</c> §Stream E.1.
/// The Admin UI left-joins this table on DriverHostStatus for display; rows are written
/// by the runtime via a HostedService that samples the tracker at a configurable
/// interval (default 5 s) — writes are non-critical, a missed sample is tolerated.
/// </remarks>
public sealed class DriverInstanceResilienceStatus
{
public required string DriverInstanceId { get; set; }
public required string HostName { get; set; }
/// <summary>Most recent time the circuit breaker for this (instance, host) opened; null if never.</summary>
public DateTime? LastCircuitBreakerOpenUtc { get; set; }
/// <summary>Rolling count of consecutive Polly pipeline failures for this (instance, host).</summary>
public int ConsecutiveFailures { get; set; }
/// <summary>Current Polly bulkhead depth (in-flight calls) for this (instance, host).</summary>
public int CurrentBulkheadDepth { get; set; }
/// <summary>Most recent process recycle time (Tier C only; null for in-process tiers).</summary>
public DateTime? LastRecycleUtc { get; set; }
/// <summary>
/// Post-init memory baseline captured by <c>MemoryTracking</c> (median of first
/// BaselineWindow samples). Zero while still warming up.
/// </summary>
public long BaselineFootprintBytes { get; set; }
/// <summary>Most recent footprint sample the tracker saw (steady-state read).</summary>
public long CurrentFootprintBytes { get; set; }
/// <summary>Row last-write timestamp — advances on every sampling tick.</summary>
public DateTime LastSampledUtc { get; set; }
}

View File

@@ -0,0 +1,46 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations
{
/// <inheritdoc />
public partial class AddDriverInstanceResilienceStatus : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "DriverInstanceResilienceStatus",
columns: table => new
{
DriverInstanceId = table.Column<string>(type: "nvarchar(64)", maxLength: 64, nullable: false),
HostName = table.Column<string>(type: "nvarchar(256)", maxLength: 256, nullable: false),
LastCircuitBreakerOpenUtc = table.Column<DateTime>(type: "datetime2(3)", nullable: true),
ConsecutiveFailures = table.Column<int>(type: "int", nullable: false),
CurrentBulkheadDepth = table.Column<int>(type: "int", nullable: false),
LastRecycleUtc = table.Column<DateTime>(type: "datetime2(3)", nullable: true),
BaselineFootprintBytes = table.Column<long>(type: "bigint", nullable: false),
CurrentFootprintBytes = table.Column<long>(type: "bigint", nullable: false),
LastSampledUtc = table.Column<DateTime>(type: "datetime2(3)", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_DriverInstanceResilienceStatus", x => new { x.DriverInstanceId, x.HostName });
});
migrationBuilder.CreateIndex(
name: "IX_DriverResilience_LastSampled",
table: "DriverInstanceResilienceStatus",
column: "LastSampledUtc");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "DriverInstanceResilienceStatus");
}
}
}

View File

@@ -434,6 +434,45 @@ namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations
});
});
modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.DriverInstanceResilienceStatus", b =>
{
b.Property<string>("DriverInstanceId")
.HasMaxLength(64)
.HasColumnType("nvarchar(64)");
b.Property<string>("HostName")
.HasMaxLength(256)
.HasColumnType("nvarchar(256)");
b.Property<long>("BaselineFootprintBytes")
.HasColumnType("bigint");
b.Property<int>("ConsecutiveFailures")
.HasColumnType("int");
b.Property<int>("CurrentBulkheadDepth")
.HasColumnType("int");
b.Property<long>("CurrentFootprintBytes")
.HasColumnType("bigint");
b.Property<DateTime?>("LastCircuitBreakerOpenUtc")
.HasColumnType("datetime2(3)");
b.Property<DateTime?>("LastRecycleUtc")
.HasColumnType("datetime2(3)");
b.Property<DateTime>("LastSampledUtc")
.HasColumnType("datetime2(3)");
b.HasKey("DriverInstanceId", "HostName");
b.HasIndex("LastSampledUtc")
.HasDatabaseName("IX_DriverResilience_LastSampled");
b.ToTable("DriverInstanceResilienceStatus", (string)null);
});
modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Equipment", b =>
{
b.Property<Guid>("EquipmentRowId")

View File

@@ -28,6 +28,7 @@ public sealed class OtOpcUaConfigDbContext(DbContextOptions<OtOpcUaConfigDbConte
public DbSet<ConfigAuditLog> ConfigAuditLogs => Set<ConfigAuditLog>();
public DbSet<ExternalIdReservation> ExternalIdReservations => Set<ExternalIdReservation>();
public DbSet<DriverHostStatus> DriverHostStatuses => Set<DriverHostStatus>();
public DbSet<DriverInstanceResilienceStatus> DriverInstanceResilienceStatuses => Set<DriverInstanceResilienceStatus>();
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
@@ -49,6 +50,7 @@ public sealed class OtOpcUaConfigDbContext(DbContextOptions<OtOpcUaConfigDbConte
ConfigureConfigAuditLog(modelBuilder);
ConfigureExternalIdReservation(modelBuilder);
ConfigureDriverHostStatus(modelBuilder);
ConfigureDriverInstanceResilienceStatus(modelBuilder);
}
private static void ConfigureServerCluster(ModelBuilder modelBuilder)
@@ -512,4 +514,21 @@ public sealed class OtOpcUaConfigDbContext(DbContextOptions<OtOpcUaConfigDbConte
e.HasIndex(x => x.LastSeenUtc).HasDatabaseName("IX_DriverHostStatus_LastSeen");
});
}
private static void ConfigureDriverInstanceResilienceStatus(ModelBuilder modelBuilder)
{
modelBuilder.Entity<DriverInstanceResilienceStatus>(e =>
{
e.ToTable("DriverInstanceResilienceStatus");
e.HasKey(x => new { x.DriverInstanceId, x.HostName });
e.Property(x => x.DriverInstanceId).HasMaxLength(64);
e.Property(x => x.HostName).HasMaxLength(256);
e.Property(x => x.LastCircuitBreakerOpenUtc).HasColumnType("datetime2(3)");
e.Property(x => x.LastRecycleUtc).HasColumnType("datetime2(3)");
e.Property(x => x.LastSampledUtc).HasColumnType("datetime2(3)");
// LastSampledUtc drives the Admin UI's stale-sample filter same way DriverHostStatus's
// LastSeenUtc index does for connectivity rows.
e.HasIndex(x => x.LastSampledUtc).HasDatabaseName("IX_DriverResilience_LastSampled");
});
}
}