feat(lmxproxy): phase 4 — host health monitoring, metrics, status web server

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
2026-03-22 00:14:40 -04:00
parent 16d1b95e9a
commit 9eb81180c0
12 changed files with 1546 additions and 12 deletions

View File

@@ -0,0 +1,57 @@
using System;
using System.Collections.Generic;
namespace ZB.MOM.WW.LmxProxy.Host.Status
{
public class StatusData
{
public DateTime Timestamp { get; set; }
public string ServiceName { get; set; } = "";
public string Version { get; set; } = "";
public ConnectionStatus Connection { get; set; } = new ConnectionStatus();
public SubscriptionStatus Subscriptions { get; set; } = new SubscriptionStatus();
public PerformanceStatus Performance { get; set; } = new PerformanceStatus();
public HealthInfo Health { get; set; } = new HealthInfo();
public HealthInfo? DetailedHealth { get; set; }
}
public class ConnectionStatus
{
public bool IsConnected { get; set; }
public string State { get; set; } = "";
public string NodeName { get; set; } = "";
public string GalaxyName { get; set; } = "";
}
public class SubscriptionStatus
{
public int TotalClients { get; set; }
public int TotalTags { get; set; }
public int ActiveSubscriptions { get; set; }
}
public class PerformanceStatus
{
public long TotalOperations { get; set; }
public double AverageSuccessRate { get; set; }
public Dictionary<string, OperationStatus> Operations { get; set; }
= new Dictionary<string, OperationStatus>();
}
public class OperationStatus
{
public long TotalCount { get; set; }
public double SuccessRate { get; set; }
public double AverageMilliseconds { get; set; }
public double MinMilliseconds { get; set; }
public double MaxMilliseconds { get; set; }
public double Percentile95Milliseconds { get; set; }
}
public class HealthInfo
{
public string Status { get; set; } = "";
public string Description { get; set; } = "";
public Dictionary<string, string> Data { get; set; } = new Dictionary<string, string>();
}
}

View File

@@ -0,0 +1,302 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
using Serilog;
using ZB.MOM.WW.LmxProxy.Host.Domain;
using ZB.MOM.WW.LmxProxy.Host.Health;
using HealthCheckService = ZB.MOM.WW.LmxProxy.Host.Health.HealthCheckService;
using ZB.MOM.WW.LmxProxy.Host.Metrics;
using ZB.MOM.WW.LmxProxy.Host.Subscriptions;
namespace ZB.MOM.WW.LmxProxy.Host.Status
{
/// <summary>
/// Aggregates health, metrics, and subscription data into status reports.
/// </summary>
public class StatusReportService
{
private static readonly ILogger Logger = Log.ForContext<StatusReportService>();
private readonly IScadaClient _scadaClient;
private readonly SubscriptionManager _subscriptionManager;
private readonly PerformanceMetrics _performanceMetrics;
private readonly HealthCheckService _healthCheckService;
private readonly DetailedHealthCheckService? _detailedHealthCheckService;
public StatusReportService(
IScadaClient scadaClient,
SubscriptionManager subscriptionManager,
PerformanceMetrics performanceMetrics,
HealthCheckService healthCheckService,
DetailedHealthCheckService? detailedHealthCheckService = null)
{
_scadaClient = scadaClient;
_subscriptionManager = subscriptionManager;
_performanceMetrics = performanceMetrics;
_healthCheckService = healthCheckService;
_detailedHealthCheckService = detailedHealthCheckService;
}
public async Task<string> GenerateHtmlReportAsync()
{
try
{
var statusData = await CollectStatusDataAsync();
return GenerateHtmlFromStatusData(statusData);
}
catch (Exception ex)
{
Logger.Error(ex, "Failed to generate HTML report");
return GenerateErrorHtml(ex);
}
}
public async Task<string> GenerateJsonReportAsync()
{
var statusData = await CollectStatusDataAsync();
var settings = new JsonSerializerSettings
{
Formatting = Formatting.Indented,
ContractResolver = new CamelCasePropertyNamesContractResolver()
};
return JsonConvert.SerializeObject(statusData, settings);
}
public async Task<bool> IsHealthyAsync()
{
var result = await _healthCheckService.CheckHealthAsync(new HealthCheckContext());
return result.Status == HealthStatus.Healthy;
}
private async Task<StatusData> CollectStatusDataAsync()
{
var statusData = new StatusData
{
Timestamp = DateTime.UtcNow,
ServiceName = "ZB.MOM.WW.LmxProxy.Host",
Version = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "0.0.0.0"
};
// Connection info
statusData.Connection = new ConnectionStatus
{
IsConnected = _scadaClient.IsConnected,
State = _scadaClient.ConnectionState.ToString()
};
// Subscription stats
var subStats = _subscriptionManager.GetStats();
statusData.Subscriptions = new SubscriptionStatus
{
TotalClients = subStats.TotalClients,
TotalTags = subStats.TotalTags,
ActiveSubscriptions = subStats.ActiveSubscriptions
};
// Performance stats
var allStats = _performanceMetrics.GetStatistics();
long totalOps = 0;
double totalSuccessRate = 0;
int opCount = 0;
foreach (var kvp in allStats)
{
totalOps += kvp.Value.TotalCount;
totalSuccessRate += kvp.Value.SuccessRate;
opCount++;
statusData.Performance.Operations[kvp.Key] = new OperationStatus
{
TotalCount = kvp.Value.TotalCount,
SuccessRate = kvp.Value.SuccessRate,
AverageMilliseconds = kvp.Value.AverageMilliseconds,
MinMilliseconds = kvp.Value.MinMilliseconds,
MaxMilliseconds = kvp.Value.MaxMilliseconds,
Percentile95Milliseconds = kvp.Value.Percentile95Milliseconds
};
}
statusData.Performance.TotalOperations = totalOps;
statusData.Performance.AverageSuccessRate = opCount > 0
? totalSuccessRate / opCount
: 1.0;
// Health check
var healthResult = await _healthCheckService.CheckHealthAsync(new HealthCheckContext());
statusData.Health = new HealthInfo
{
Status = healthResult.Status.ToString(),
Description = healthResult.Description ?? ""
};
if (healthResult.Data != null)
{
foreach (var kvp in healthResult.Data)
{
statusData.Health.Data[kvp.Key] = kvp.Value?.ToString() ?? "";
}
}
// Detailed health check (optional)
if (_detailedHealthCheckService != null)
{
var detailedResult = await _detailedHealthCheckService.CheckHealthAsync(new HealthCheckContext());
statusData.DetailedHealth = new HealthInfo
{
Status = detailedResult.Status.ToString(),
Description = detailedResult.Description ?? ""
};
if (detailedResult.Data != null)
{
foreach (var kvp in detailedResult.Data)
{
statusData.DetailedHealth.Data[kvp.Key] = kvp.Value?.ToString() ?? "";
}
}
}
return statusData;
}
private static string GenerateHtmlFromStatusData(StatusData statusData)
{
var sb = new StringBuilder();
sb.AppendLine("<!DOCTYPE html>");
sb.AppendLine("<html lang=\"en\">");
sb.AppendLine("<head>");
sb.AppendLine(" <meta charset=\"utf-8\">");
sb.AppendLine(" <meta http-equiv=\"refresh\" content=\"30\">");
sb.AppendLine(" <title>LmxProxy Status</title>");
sb.AppendLine(" <style>");
sb.AppendLine(" body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; margin: 20px; background: #f5f5f5; }");
sb.AppendLine(" h1 { color: #333; }");
sb.AppendLine(" .card { background: white; border-radius: 4px; padding: 16px; margin-bottom: 16px; box-shadow: 0 1px 3px rgba(0,0,0,0.12); }");
sb.AppendLine(" .card-green { border-left: 4px solid #28a745; }");
sb.AppendLine(" .card-yellow { border-left: 4px solid #ffc107; }");
sb.AppendLine(" .card-red { border-left: 4px solid #dc3545; }");
sb.AppendLine(" .grid { display: flex; flex-wrap: wrap; gap: 16px; }");
sb.AppendLine(" .grid-item { flex: 1; min-width: 300px; }");
sb.AppendLine(" table { width: 100%; border-collapse: collapse; }");
sb.AppendLine(" th, td { text-align: left; padding: 8px; border-bottom: 1px solid #eee; }");
sb.AppendLine(" th { background: #f8f9fa; font-weight: 600; }");
sb.AppendLine(" .status-healthy { color: #28a745; font-weight: bold; }");
sb.AppendLine(" .status-degraded { color: #ffc107; font-weight: bold; }");
sb.AppendLine(" .status-unhealthy { color: #dc3545; font-weight: bold; }");
sb.AppendLine(" .footer { color: #999; font-size: 0.85em; margin-top: 20px; }");
sb.AppendLine(" </style>");
sb.AppendLine("</head>");
sb.AppendLine("<body>");
sb.AppendLine(" <h1>LmxProxy Status Dashboard</h1>");
// Connection card
var connClass = statusData.Connection.IsConnected ? "card-green" : "card-red";
sb.AppendLine($" <div class=\"grid\">");
sb.AppendLine($" <div class=\"grid-item\"><div class=\"card {connClass}\">");
sb.AppendLine(" <h3>Connection</h3>");
sb.AppendLine($" <p><strong>Connected:</strong> {statusData.Connection.IsConnected}</p>");
sb.AppendLine($" <p><strong>State:</strong> {statusData.Connection.State}</p>");
if (!string.IsNullOrEmpty(statusData.Connection.NodeName))
sb.AppendLine($" <p><strong>Node:</strong> {statusData.Connection.NodeName}</p>");
if (!string.IsNullOrEmpty(statusData.Connection.GalaxyName))
sb.AppendLine($" <p><strong>Galaxy:</strong> {statusData.Connection.GalaxyName}</p>");
sb.AppendLine(" </div></div>");
// Health card
var healthClass = GetHealthCardClass(statusData.Health.Status);
var healthCss = GetHealthStatusCss(statusData.Health.Status);
sb.AppendLine($" <div class=\"grid-item\"><div class=\"card {healthClass}\">");
sb.AppendLine(" <h3>Health</h3>");
sb.AppendLine($" <p class=\"{healthCss}\">{statusData.Health.Status}</p>");
sb.AppendLine($" <p>{statusData.Health.Description}</p>");
sb.AppendLine(" </div></div>");
// Subscriptions card
sb.AppendLine(" <div class=\"grid-item\"><div class=\"card card-green\">");
sb.AppendLine(" <h3>Subscriptions</h3>");
sb.AppendLine($" <p><strong>Clients:</strong> {statusData.Subscriptions.TotalClients}</p>");
sb.AppendLine($" <p><strong>Tags:</strong> {statusData.Subscriptions.TotalTags}</p>");
sb.AppendLine($" <p><strong>Active:</strong> {statusData.Subscriptions.ActiveSubscriptions}</p>");
sb.AppendLine(" </div></div>");
sb.AppendLine(" </div>");
// Operations table
if (statusData.Performance.Operations.Count > 0)
{
sb.AppendLine(" <div class=\"card\">");
sb.AppendLine(" <h3>Operations</h3>");
sb.AppendLine(" <table>");
sb.AppendLine(" <tr><th>Operation</th><th>Count</th><th>Success Rate</th><th>Avg (ms)</th><th>Min (ms)</th><th>Max (ms)</th><th>P95 (ms)</th></tr>");
foreach (var op in statusData.Performance.Operations)
{
sb.AppendLine($" <tr>" +
$"<td>{op.Key}</td>" +
$"<td>{op.Value.TotalCount}</td>" +
$"<td>{op.Value.SuccessRate:P1}</td>" +
$"<td>{op.Value.AverageMilliseconds:F1}</td>" +
$"<td>{op.Value.MinMilliseconds:F1}</td>" +
$"<td>{op.Value.MaxMilliseconds:F1}</td>" +
$"<td>{op.Value.Percentile95Milliseconds:F1}</td>" +
$"</tr>");
}
sb.AppendLine(" </table>");
sb.AppendLine(" </div>");
}
// Detailed health (if available)
if (statusData.DetailedHealth != null)
{
var detailedClass = GetHealthCardClass(statusData.DetailedHealth.Status);
var detailedCss = GetHealthStatusCss(statusData.DetailedHealth.Status);
sb.AppendLine($" <div class=\"card {detailedClass}\">");
sb.AppendLine(" <h3>Detailed Health Check</h3>");
sb.AppendLine($" <p class=\"{detailedCss}\">{statusData.DetailedHealth.Status}</p>");
sb.AppendLine($" <p>{statusData.DetailedHealth.Description}</p>");
sb.AppendLine(" </div>");
}
sb.AppendLine($" <div class=\"footer\">Last updated: {statusData.Timestamp:yyyy-MM-dd HH:mm:ss} UTC | Service: {statusData.ServiceName} v{statusData.Version}</div>");
sb.AppendLine("</body>");
sb.AppendLine("</html>");
return sb.ToString();
}
private static string GetHealthCardClass(string status)
{
switch (status)
{
case "Healthy": return "card-green";
case "Degraded": return "card-yellow";
default: return "card-red";
}
}
private static string GetHealthStatusCss(string status)
{
switch (status)
{
case "Healthy": return "status-healthy";
case "Degraded": return "status-degraded";
default: return "status-unhealthy";
}
}
private static string GenerateErrorHtml(Exception ex)
{
var sb = new StringBuilder();
sb.AppendLine("<!DOCTYPE html>");
sb.AppendLine("<html><head><title>LmxProxy Status - Error</title></head>");
sb.AppendLine("<body>");
sb.AppendLine("<h1>Error generating status report</h1>");
sb.AppendLine($"<p>{ex.Message}</p>");
sb.AppendLine("</body></html>");
return sb.ToString();
}
}
}

View File

@@ -0,0 +1,215 @@
using System;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Serilog;
using ZB.MOM.WW.LmxProxy.Host.Configuration;
namespace ZB.MOM.WW.LmxProxy.Host.Status
{
/// <summary>
/// HTTP status server providing an HTML dashboard, JSON API, and health endpoint.
/// </summary>
public class StatusWebServer : IDisposable
{
private static readonly ILogger Logger = Log.ForContext<StatusWebServer>();
private readonly WebServerConfiguration _configuration;
private readonly StatusReportService _statusReportService;
private HttpListener? _httpListener;
private CancellationTokenSource? _cancellationTokenSource;
private Task? _listenerTask;
private bool _disposed;
public StatusWebServer(WebServerConfiguration configuration, StatusReportService statusReportService)
{
_configuration = configuration;
_statusReportService = statusReportService;
}
public bool Start()
{
if (!_configuration.Enabled)
{
Logger.Information("Status web server is disabled");
return true;
}
try
{
_httpListener = new HttpListener();
var prefix = _configuration.Prefix ?? $"http://+:{_configuration.Port}/";
if (!prefix.EndsWith("/"))
prefix += "/";
_httpListener.Prefixes.Add(prefix);
_httpListener.Start();
_cancellationTokenSource = new CancellationTokenSource();
_listenerTask = Task.Run(() => HandleRequestsAsync(_cancellationTokenSource.Token));
Logger.Information("Status web server started on {Prefix}", prefix);
return true;
}
catch (Exception ex)
{
Logger.Error(ex, "Failed to start status web server");
return false;
}
}
public bool Stop()
{
if (!_configuration.Enabled || _httpListener == null)
return true;
try
{
_cancellationTokenSource?.Cancel();
if (_listenerTask != null)
{
_listenerTask.Wait(TimeSpan.FromSeconds(5));
}
_httpListener.Stop();
_httpListener.Close();
Logger.Information("Status web server stopped");
return true;
}
catch (Exception ex)
{
Logger.Error(ex, "Error stopping status web server");
return false;
}
}
public void Dispose()
{
if (_disposed) return;
_disposed = true;
Stop();
_cancellationTokenSource?.Dispose();
if (_httpListener != null)
{
((IDisposable)_httpListener).Dispose();
}
}
private async Task HandleRequestsAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested && _httpListener != null && _httpListener.IsListening)
{
try
{
var context = await _httpListener.GetContextAsync();
_ = Task.Run(() => HandleRequestAsync(context));
}
catch (ObjectDisposedException)
{
// Expected during shutdown
break;
}
catch (HttpListenerException ex) when (ex.ErrorCode == 995)
{
// ERROR_OPERATION_ABORTED — expected during shutdown
break;
}
catch (Exception ex)
{
Logger.Error(ex, "Error accepting HTTP request");
await Task.Delay(1000, cancellationToken).ConfigureAwait(false);
}
}
}
private async Task HandleRequestAsync(HttpListenerContext context)
{
try
{
if (context.Request.HttpMethod != "GET")
{
context.Response.StatusCode = 405;
await WriteResponseAsync(context.Response, "Method Not Allowed", "text/plain");
return;
}
var path = context.Request.Url?.AbsolutePath?.ToLowerInvariant() ?? "/";
switch (path)
{
case "/":
await HandleStatusPageAsync(context.Response);
break;
case "/api/status":
await HandleStatusApiAsync(context.Response);
break;
case "/api/health":
await HandleHealthApiAsync(context.Response);
break;
default:
context.Response.StatusCode = 404;
await WriteResponseAsync(context.Response, "Not Found", "text/plain");
break;
}
}
catch (Exception ex)
{
Logger.Error(ex, "Error handling HTTP request");
try
{
context.Response.StatusCode = 500;
await WriteResponseAsync(context.Response, "Internal Server Error", "text/plain");
}
catch
{
// Ignore errors writing error response
}
}
}
private async Task HandleStatusPageAsync(HttpListenerResponse response)
{
var html = await _statusReportService.GenerateHtmlReportAsync();
await WriteResponseAsync(response, html, "text/html; charset=utf-8");
}
private async Task HandleStatusApiAsync(HttpListenerResponse response)
{
var json = await _statusReportService.GenerateJsonReportAsync();
await WriteResponseAsync(response, json, "application/json; charset=utf-8");
}
private async Task HandleHealthApiAsync(HttpListenerResponse response)
{
var isHealthy = await _statusReportService.IsHealthyAsync();
if (isHealthy)
{
response.StatusCode = 200;
await WriteResponseAsync(response, "OK", "text/plain");
}
else
{
response.StatusCode = 503;
await WriteResponseAsync(response, "UNHEALTHY", "text/plain");
}
}
private static async Task WriteResponseAsync(
HttpListenerResponse response, string content, string contentType)
{
response.ContentType = contentType;
response.Headers.Add("Cache-Control", "no-cache, no-store, must-revalidate");
response.Headers.Add("Pragma", "no-cache");
response.Headers.Add("Expires", "0");
var buffer = Encoding.UTF8.GetBytes(content);
response.ContentLength64 = buffer.Length;
await response.OutputStream.WriteAsync(buffer, 0, buffer.Length);
response.OutputStream.Close();
}
}
}