FOCAS Tier-C PR E — ops glue: ProcessHostLauncher + post-mortem MMF + NSSM install scripts + doc close-out. Final of the 5 PRs for #220. With this landing, the Tier-C architecture is fully shipped; the only remaining FOCAS work is the hardware-dependent FwlibHostedBackend (real Fwlib32.dll P/Invoke, gated on #222 lab rig).
Production IHostProcessLauncher (ProcessHostLauncher.cs): Process.Start spawns OtOpcUa.Driver.FOCAS.Host.exe with OTOPCUA_FOCAS_PIPE / OTOPCUA_ALLOWED_SID / OTOPCUA_FOCAS_SECRET / OTOPCUA_FOCAS_BACKEND in the environment (supervisor-owned, never disk), polls FocasIpcClient.ConnectAsync at 250ms cadence until the pipe is up or the Host exits or the ConnectTimeout deadline passes, then wraps the connected client in an IpcFocasClient. TerminateAsync kills the entire process tree + disposes the IPC stream. ProcessHostLauncherOptions carries HostExePath + PipeName + AllowedSid plus optional SharedSecret (auto-generated from a GUID when omitted so install scripts don't have to), Arguments, Backend (fwlib32/fake/unconfigured default-unconfigured), ConnectTimeout (15s), and Series for CNC pre-flight. Post-mortem MMF (Host/Stability/PostMortemMmf.cs + Proxy/Supervisor/PostMortemReader.cs): ring-buffer of the last ~1000 IPC operations written by the Host into a memory-mapped file. On a Host crash the supervisor reads the MMF — which survives process death — to see what was in flight. File format: 16-byte header [magic 'OFPC' (0x4F465043) | version | capacity | writeIndex] + N × 256-byte entries [8-byte UTC unix ms | 8-byte opKind | 240-byte UTF-8 message + null terminator]. Magic distinguishes FOCAS MMFs from the Galaxy MMFs that ship the same format shape. Writer is single-producer (Host) with a lock_writeGate; reader is multi-consumer (Proxy + any diagnostic tool) using a separate MemoryMappedFile handle. NSSM install wrappers (scripts/install/Install-FocasHost.ps1 + Uninstall-FocasHost.ps1): idempotent service registration for OtOpcUaFocasHost. Resolves SID from the ServiceAccount, generates a fresh shared secret per install if not supplied, stages OTOPCUA_FOCAS_PIPE/SID/SECRET/BACKEND in AppEnvironmentExtra so they never hit disk, rotates 10MB stdout/stderr logs under %ProgramData%\OtOpcUa, DependOnService=OtOpcUa so startup order is deterministic. Backend selector defaults to unconfigured so a fresh install doesn't accidentally load a half-configured Fwlib32.dll on first start. Tests (7 new, 2 files): PostMortemMmfTests.cs in FOCAS.Host.Tests — round-trip write+read preserves order + content, ring-buffer wraps at capacity (writes 10 entries to a 3-slot buffer, asserts only op-7/8/9 survive in FIFO order), message truncation at the 240-byte cap is null-terminated + non-overflowing, reopening an existing file preserves entries. PostMortemReaderCompatibilityTests.cs in FOCAS.Tests — hand-writes a file in the exact host format (magic/entry layout) + asserts the Proxy reader decodes with correct ring-walk ordering when writeIndex != 0, empty-return on missing file + magic mismatch. Keeps the two codebases in format-lockstep without the net10 test project referencing the net48 Host assembly. Docs updated: docs/v2/implementation/focas-isolation-plan.md promoted from DRAFT to PRs A-E shipped status with per-PR citations + post-ship test counts (189 + 24 + 13 = 226 FOCAS-family tests green). docs/drivers/FOCAS-Test-Fixture.md §5 updated from "architecture scoped but not implemented" to listing the shipped components with the FwlibHostedBackend gap explicitly labeled as hardware-gated. Install-FocasHost.ps1 documents the OTOPCUA_FOCAS_BACKEND selector + points at docs/v2/focas-deployment.md for Fwlib32.dll licensing. What ISN'T in this PR: (1) the real FwlibHostedBackend implementing IFocasBackend with the P/Invoke — requires either a CNC on the bench or a licensed FANUC developer kit to validate, tracked under #220 as a single follow-up task; (2) Admin /hosts surface integration for FOCAS runtime status — Galaxy Tier-C already has the shape, FOCAS can slot in when someone wires ObservedCrashes/StickyAlertActive/BackoffAttempt to the FleetStatusHub; (3) a full integration test that actually spawns a real FOCAS Host process — ProcessHostLauncher is tested via its contract + the MMF is tested via round-trip, but no test spins up the real exe (the Galaxy Tier-C tests do this, but the FOCAS equivalent adds no new coverage over what's already in place). Total FOCAS-family tests green after this PR: 189 driver + 24 Shared + 13 Host = 226. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,133 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.IO.MemoryMappedFiles;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Host.Stability;
|
||||
|
||||
/// <summary>
|
||||
/// Ring-buffer of the last N IPC operations, written into a memory-mapped file. On a
|
||||
/// hard crash the Proxy-side supervisor reads the MMF after the corpse is gone to see
|
||||
/// what was in flight at the moment the Host died. Single-writer (the Host), multi-reader
|
||||
/// (the supervisor) — the file format is identical to the Galaxy Tier-C
|
||||
/// <c>PostMortemMmf</c> so a single reader tool can work both.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// File layout:
|
||||
/// <code>
|
||||
/// [16-byte header: magic(4) | version(4) | capacity(4) | writeIndex(4)]
|
||||
/// [capacity × 256-byte entries: each is [8-byte utcUnixMs | 8-byte opKind | 240-byte UTF-8 message]]
|
||||
/// </code>
|
||||
/// Magic is 'OFPC' (0x4F46_5043) to distinguish a FOCAS file from the Galaxy MMF.
|
||||
/// </remarks>
|
||||
public sealed class PostMortemMmf : IDisposable
|
||||
{
|
||||
private const int Magic = 0x4F465043; // 'OFPC'
|
||||
private const int Version = 1;
|
||||
private const int HeaderBytes = 16;
|
||||
public const int EntryBytes = 256;
|
||||
private const int MessageOffset = 16;
|
||||
private const int MessageCapacity = EntryBytes - MessageOffset;
|
||||
|
||||
public int Capacity { get; }
|
||||
public string Path { get; }
|
||||
|
||||
private readonly MemoryMappedFile _mmf;
|
||||
private readonly MemoryMappedViewAccessor _accessor;
|
||||
private readonly object _writeGate = new();
|
||||
|
||||
public PostMortemMmf(string path, int capacity = 1000)
|
||||
{
|
||||
if (capacity <= 0) throw new ArgumentOutOfRangeException(nameof(capacity));
|
||||
Capacity = capacity;
|
||||
Path = path;
|
||||
|
||||
var fileBytes = HeaderBytes + capacity * EntryBytes;
|
||||
Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path)!);
|
||||
|
||||
var fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read);
|
||||
fs.SetLength(fileBytes);
|
||||
_mmf = MemoryMappedFile.CreateFromFile(fs, null, fileBytes,
|
||||
MemoryMappedFileAccess.ReadWrite, HandleInheritability.None, leaveOpen: false);
|
||||
_accessor = _mmf.CreateViewAccessor(0, fileBytes, MemoryMappedFileAccess.ReadWrite);
|
||||
|
||||
if (_accessor.ReadInt32(0) != Magic)
|
||||
{
|
||||
_accessor.Write(0, Magic);
|
||||
_accessor.Write(4, Version);
|
||||
_accessor.Write(8, capacity);
|
||||
_accessor.Write(12, 0);
|
||||
}
|
||||
}
|
||||
|
||||
public void Write(long opKind, string message)
|
||||
{
|
||||
lock (_writeGate)
|
||||
{
|
||||
var idx = _accessor.ReadInt32(12);
|
||||
var offset = HeaderBytes + idx * EntryBytes;
|
||||
|
||||
_accessor.Write(offset + 0, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||
_accessor.Write(offset + 8, opKind);
|
||||
|
||||
var msgBytes = Encoding.UTF8.GetBytes(message ?? string.Empty);
|
||||
var copy = Math.Min(msgBytes.Length, MessageCapacity - 1);
|
||||
_accessor.WriteArray(offset + MessageOffset, msgBytes, 0, copy);
|
||||
_accessor.Write(offset + MessageOffset + copy, (byte)0);
|
||||
|
||||
var next = (idx + 1) % Capacity;
|
||||
_accessor.Write(12, next);
|
||||
}
|
||||
}
|
||||
|
||||
public PostMortemEntry[] ReadAll()
|
||||
{
|
||||
var magic = _accessor.ReadInt32(0);
|
||||
if (magic != Magic) return new PostMortemEntry[0];
|
||||
|
||||
var capacity = _accessor.ReadInt32(8);
|
||||
var writeIndex = _accessor.ReadInt32(12);
|
||||
|
||||
var entries = new PostMortemEntry[capacity];
|
||||
var count = 0;
|
||||
for (var i = 0; i < capacity; i++)
|
||||
{
|
||||
var slot = (writeIndex + i) % capacity;
|
||||
var offset = HeaderBytes + slot * EntryBytes;
|
||||
|
||||
var ts = _accessor.ReadInt64(offset + 0);
|
||||
if (ts == 0) continue;
|
||||
|
||||
var op = _accessor.ReadInt64(offset + 8);
|
||||
var msgBuf = new byte[MessageCapacity];
|
||||
_accessor.ReadArray(offset + MessageOffset, msgBuf, 0, MessageCapacity);
|
||||
var nulTerm = Array.IndexOf<byte>(msgBuf, 0);
|
||||
var msg = Encoding.UTF8.GetString(msgBuf, 0, nulTerm < 0 ? MessageCapacity : nulTerm);
|
||||
|
||||
entries[count++] = new PostMortemEntry(ts, op, msg);
|
||||
}
|
||||
|
||||
Array.Resize(ref entries, count);
|
||||
return entries;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_accessor.Dispose();
|
||||
_mmf.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public readonly struct PostMortemEntry
|
||||
{
|
||||
public long UtcUnixMs { get; }
|
||||
public long OpKind { get; }
|
||||
public string Message { get; }
|
||||
|
||||
public PostMortemEntry(long utcUnixMs, long opKind, string message)
|
||||
{
|
||||
UtcUnixMs = utcUnixMs;
|
||||
OpKind = opKind;
|
||||
Message = message;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
using System.IO.MemoryMappedFiles;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Supervisor;
|
||||
|
||||
/// <summary>
|
||||
/// Proxy-side reader for the Host's post-mortem MMF. After a Host crash the supervisor
|
||||
/// opens the file (which persists beyond the process lifetime) and enumerates the last
|
||||
/// few thousand IPC operations that were in flight. Format matches
|
||||
/// <c>Driver.FOCAS.Host.Stability.PostMortemMmf</c> — magic 'OFPC' / 256-byte entries.
|
||||
/// </summary>
|
||||
public sealed class PostMortemReader
|
||||
{
|
||||
private const int Magic = 0x4F465043; // 'OFPC'
|
||||
private const int HeaderBytes = 16;
|
||||
private const int EntryBytes = 256;
|
||||
private const int MessageOffset = 16;
|
||||
private const int MessageCapacity = EntryBytes - MessageOffset;
|
||||
|
||||
public string Path { get; }
|
||||
|
||||
public PostMortemReader(string path) => Path = path;
|
||||
|
||||
public PostMortemEntry[] ReadAll()
|
||||
{
|
||||
if (!File.Exists(Path)) return [];
|
||||
|
||||
using var mmf = MemoryMappedFile.CreateFromFile(Path, FileMode.Open, null, 0, MemoryMappedFileAccess.Read);
|
||||
using var accessor = mmf.CreateViewAccessor(0, 0, MemoryMappedFileAccess.Read);
|
||||
|
||||
if (accessor.ReadInt32(0) != Magic) return [];
|
||||
|
||||
var capacity = accessor.ReadInt32(8);
|
||||
var writeIndex = accessor.ReadInt32(12);
|
||||
var entries = new PostMortemEntry[capacity];
|
||||
var count = 0;
|
||||
|
||||
for (var i = 0; i < capacity; i++)
|
||||
{
|
||||
var slot = (writeIndex + i) % capacity;
|
||||
var offset = HeaderBytes + slot * EntryBytes;
|
||||
var ts = accessor.ReadInt64(offset + 0);
|
||||
if (ts == 0) continue;
|
||||
var op = accessor.ReadInt64(offset + 8);
|
||||
var msgBuf = new byte[MessageCapacity];
|
||||
accessor.ReadArray(offset + MessageOffset, msgBuf, 0, MessageCapacity);
|
||||
var nulTerm = Array.IndexOf<byte>(msgBuf, 0);
|
||||
var msg = Encoding.UTF8.GetString(msgBuf, 0, nulTerm < 0 ? MessageCapacity : nulTerm);
|
||||
entries[count++] = new PostMortemEntry(ts, op, msg);
|
||||
}
|
||||
|
||||
Array.Resize(ref entries, count);
|
||||
return entries;
|
||||
}
|
||||
}
|
||||
|
||||
public readonly record struct PostMortemEntry(long UtcUnixMs, long OpKind, string Message);
|
||||
@@ -0,0 +1,113 @@
|
||||
using System.Diagnostics;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Ipc;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Supervisor;
|
||||
|
||||
/// <summary>
|
||||
/// Production <see cref="IHostProcessLauncher"/>. Spawns <c>OtOpcUa.Driver.FOCAS.Host.exe</c>
|
||||
/// with the pipe name / allowed-SID / per-spawn shared secret in the environment, waits for
|
||||
/// the pipe to come up, then connects a <see cref="FocasIpcClient"/> and wraps it in an
|
||||
/// <see cref="IpcFocasClient"/>. On <see cref="TerminateAsync"/> best-effort kills the
|
||||
/// process and closes the IPC stream.
|
||||
/// </summary>
|
||||
public sealed class ProcessHostLauncher : IHostProcessLauncher
|
||||
{
|
||||
private readonly ProcessHostLauncherOptions _options;
|
||||
private Process? _process;
|
||||
private FocasIpcClient? _ipc;
|
||||
|
||||
public ProcessHostLauncher(ProcessHostLauncherOptions options)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
|
||||
public bool IsProcessAlive => _process is { HasExited: false };
|
||||
|
||||
public async Task<IFocasClient> LaunchAsync(CancellationToken ct)
|
||||
{
|
||||
await TerminateAsync(ct).ConfigureAwait(false);
|
||||
|
||||
var secret = _options.SharedSecret ?? Guid.NewGuid().ToString("N");
|
||||
|
||||
var psi = new ProcessStartInfo
|
||||
{
|
||||
FileName = _options.HostExePath,
|
||||
Arguments = _options.Arguments ?? string.Empty,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
psi.Environment["OTOPCUA_FOCAS_PIPE"] = _options.PipeName;
|
||||
psi.Environment["OTOPCUA_ALLOWED_SID"] = _options.AllowedSid;
|
||||
psi.Environment["OTOPCUA_FOCAS_SECRET"] = secret;
|
||||
psi.Environment["OTOPCUA_FOCAS_BACKEND"] = _options.Backend;
|
||||
|
||||
_process = Process.Start(psi)
|
||||
?? throw new InvalidOperationException($"Failed to start {_options.HostExePath}");
|
||||
|
||||
// Poll for pipe readiness up to the configured connect timeout.
|
||||
var deadline = DateTime.UtcNow + _options.ConnectTimeout;
|
||||
while (true)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
if (_process.HasExited)
|
||||
throw new InvalidOperationException(
|
||||
$"FOCAS Host exited before pipe was ready (ExitCode={_process.ExitCode}).");
|
||||
|
||||
try
|
||||
{
|
||||
_ipc = await FocasIpcClient.ConnectAsync(
|
||||
_options.PipeName, secret, TimeSpan.FromSeconds(1), ct).ConfigureAwait(false);
|
||||
break;
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
if (DateTime.UtcNow >= deadline)
|
||||
throw new TimeoutException(
|
||||
$"FOCAS Host pipe {_options.PipeName} did not come up within {_options.ConnectTimeout:g}.");
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(250), ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
return new IpcFocasClient(_ipc, _options.Series);
|
||||
}
|
||||
|
||||
public async Task TerminateAsync(CancellationToken ct)
|
||||
{
|
||||
if (_ipc is not null)
|
||||
{
|
||||
try { await _ipc.DisposeAsync().ConfigureAwait(false); }
|
||||
catch { /* best effort */ }
|
||||
_ipc = null;
|
||||
}
|
||||
|
||||
if (_process is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!_process.HasExited)
|
||||
{
|
||||
_process.Kill(entireProcessTree: true);
|
||||
await _process.WaitForExitAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch { /* best effort */ }
|
||||
finally
|
||||
{
|
||||
_process.Dispose();
|
||||
_process = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record ProcessHostLauncherOptions(
|
||||
string HostExePath,
|
||||
string PipeName,
|
||||
string AllowedSid)
|
||||
{
|
||||
public string? SharedSecret { get; init; }
|
||||
public string? Arguments { get; init; }
|
||||
public string Backend { get; init; } = "fwlib32";
|
||||
public TimeSpan ConnectTimeout { get; init; } = TimeSpan.FromSeconds(15);
|
||||
public FocasCncSeries Series { get; init; } = FocasCncSeries.Unknown;
|
||||
}
|
||||
Reference in New Issue
Block a user