fix(site-runtime): resolve SiteRuntime-004..011 — deploy-after-persist, remove reflection, deterministic IDs, non-blocking startup, dedicated script scheduler, config-change detection, semantic trust-model check
This commit is contained in:
@@ -460,7 +460,8 @@ public class AlarmActor : ReceiveActor
|
||||
|
||||
var executionId = $"{_alarmName}-alarm-exec-{_executionCounter++}";
|
||||
|
||||
// NOTE: In production, configure a dedicated blocking I/O dispatcher via HOCON.
|
||||
// SiteRuntime-009: the on-trigger script body runs on the dedicated
|
||||
// ScriptExecutionScheduler, not the shared .NET thread pool.
|
||||
var props = Props.Create(() => new AlarmExecutionActor(
|
||||
_alarmName,
|
||||
_instanceName,
|
||||
|
||||
@@ -54,7 +54,11 @@ public class AlarmExecutionActor : ReceiveActor
|
||||
{
|
||||
var timeout = TimeSpan.FromSeconds(options.ScriptExecutionTimeoutSeconds);
|
||||
|
||||
_ = Task.Run(async () =>
|
||||
// SiteRuntime-009: run the alarm on-trigger body on the dedicated
|
||||
// script-execution scheduler, not the shared .NET thread pool.
|
||||
var scheduler = ScriptExecutionScheduler.Shared(options);
|
||||
|
||||
_ = Task.Factory.StartNew(async () =>
|
||||
{
|
||||
using var cts = new CancellationTokenSource(timeout);
|
||||
try
|
||||
@@ -108,6 +112,6 @@ public class AlarmExecutionActor : ReceiveActor
|
||||
{
|
||||
self.Tell(PoisonPill.Instance);
|
||||
}
|
||||
});
|
||||
}, CancellationToken.None, TaskCreationOptions.DenyChildAttach, scheduler).Unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,6 +99,7 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
|
||||
// Internal startup messages
|
||||
Receive<StartupConfigsLoaded>(HandleStartupConfigsLoaded);
|
||||
Receive<SharedScriptsLoaded>(HandleSharedScriptsLoaded);
|
||||
Receive<StartNextBatch>(HandleStartNextBatch);
|
||||
|
||||
// Internal enable result
|
||||
@@ -156,7 +157,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes the loaded configs from SQLite and begins staggered Instance Actor creation.
|
||||
/// Processes the loaded configs from SQLite.
|
||||
///
|
||||
/// SiteRuntime-008: shared scripts must be compiled before Instance Actors are
|
||||
/// created, but the SQLite read and Roslyn compilation must not block the
|
||||
/// singleton's mailbox. The compilation is run on a background task and a
|
||||
/// <see cref="SharedScriptsLoaded"/> message is piped back; only then does
|
||||
/// staggered Instance Actor creation begin. The deployed configs are stashed on the
|
||||
/// actor field in the meantime.
|
||||
/// </summary>
|
||||
private void HandleStartupConfigsLoaded(StartupConfigsLoaded msg)
|
||||
{
|
||||
@@ -166,9 +174,6 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
return;
|
||||
}
|
||||
|
||||
// Load and compile shared scripts from SQLite before creating Instance Actors
|
||||
LoadSharedScriptsFromStorage();
|
||||
|
||||
var enabledConfigs = msg.Configs.Where(c => c.IsEnabled).ToList();
|
||||
_totalDeployedCount = msg.Configs.Count;
|
||||
_logger.LogInformation(
|
||||
@@ -176,11 +181,25 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
msg.Configs.Count, enabledConfigs.Count);
|
||||
UpdateInstanceCounts();
|
||||
|
||||
if (enabledConfigs.Count == 0)
|
||||
// Load and compile shared scripts off the actor thread, then resume startup.
|
||||
LoadSharedScriptsFromStorage(enabledConfigs);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SiteRuntime-008: once shared scripts have been compiled off-thread, begins
|
||||
/// staggered Instance Actor creation for the enabled configs captured at startup.
|
||||
/// </summary>
|
||||
private void HandleSharedScriptsLoaded(SharedScriptsLoaded msg)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Loaded {Compiled}/{Total} shared scripts from SQLite",
|
||||
msg.CompiledCount, msg.TotalCount);
|
||||
|
||||
if (msg.EnabledConfigs.Count == 0)
|
||||
return;
|
||||
|
||||
// Start the first batch immediately
|
||||
var batchState = new BatchState(enabledConfigs, 0);
|
||||
var batchState = new BatchState(msg.EnabledConfigs, 0);
|
||||
Self.Tell(new StartNextBatch(batchState));
|
||||
}
|
||||
|
||||
@@ -275,6 +294,13 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// Creates the Instance Actor, persists the config, and replies to the deployer.
|
||||
/// A redeployment is an update of an existing instance, so the deployed-instance
|
||||
/// counter is only incremented for genuinely new deployments.
|
||||
///
|
||||
/// SiteRuntime-005: the deployer is <b>not</b> told <see cref="DeploymentStatus.Success"/>
|
||||
/// until SQLite persistence has committed. The site's deployed-config store is the
|
||||
/// durable source of truth — a config that was never persisted would be silently lost
|
||||
/// on the next restart/failover, so reporting Success before the row is committed is
|
||||
/// incorrect. The reply is sent from <see cref="HandleDeployPersistenceResult"/> once
|
||||
/// the persistence outcome is known.
|
||||
/// </summary>
|
||||
private void ApplyDeployment(DeployInstanceCommand command, IActorRef sender, bool isRedeploy)
|
||||
{
|
||||
@@ -307,33 +333,56 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
instanceName, command.FlattenedConfigurationJson,
|
||||
command.DeploymentId, command.RevisionHash, true));
|
||||
|
||||
return new DeployPersistenceResult(command.DeploymentId, instanceName, true, null, sender);
|
||||
return new DeployPersistenceResult(
|
||||
command.DeploymentId, instanceName, true, null, sender, isRedeploy);
|
||||
}).ContinueWith(t =>
|
||||
{
|
||||
if (t.IsCompletedSuccessfully)
|
||||
return t.Result;
|
||||
return new DeployPersistenceResult(
|
||||
command.DeploymentId, instanceName, false,
|
||||
t.Exception?.GetBaseException().Message, sender);
|
||||
t.Exception?.GetBaseException().Message, sender, isRedeploy);
|
||||
}).PipeTo(Self);
|
||||
|
||||
// Reply immediately — deployment is applied (actor is running)
|
||||
sender.Tell(new DeploymentStatusResponse(
|
||||
command.DeploymentId,
|
||||
instanceName,
|
||||
DeploymentStatus.Success,
|
||||
null,
|
||||
DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SiteRuntime-005: reports the deployment outcome to central only after the
|
||||
/// persistence result is known. On a persistence failure the Instance Actor that was
|
||||
/// created optimistically is stopped and the deployed-instance counter rolled back,
|
||||
/// so the in-memory state stays consistent with durable storage, and central is told
|
||||
/// the deployment <see cref="DeploymentStatus.Failed"/>.
|
||||
/// </summary>
|
||||
private void HandleDeployPersistenceResult(DeployPersistenceResult result)
|
||||
{
|
||||
if (!result.Success)
|
||||
if (result.Success)
|
||||
{
|
||||
_logger.LogError(
|
||||
"Failed to persist deployment {DeploymentId} for {Instance}: {Error}",
|
||||
result.DeploymentId, result.InstanceName, result.Error);
|
||||
result.OriginalSender.Tell(new DeploymentStatusResponse(
|
||||
result.DeploymentId,
|
||||
result.InstanceName,
|
||||
DeploymentStatus.Success,
|
||||
null,
|
||||
DateTimeOffset.UtcNow));
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogError(
|
||||
"Failed to persist deployment {DeploymentId} for {Instance}: {Error}",
|
||||
result.DeploymentId, result.InstanceName, result.Error);
|
||||
|
||||
// Persistence failed — undo the optimistic actor creation and counter bump so
|
||||
// the site does not advertise an instance it cannot durably recover.
|
||||
if (_instanceActors.Remove(result.InstanceName, out var orphan))
|
||||
Context.Stop(orphan);
|
||||
if (!result.IsRedeploy)
|
||||
_totalDeployedCount = Math.Max(0, _totalDeployedCount - 1);
|
||||
UpdateInstanceCounts();
|
||||
|
||||
result.OriginalSender.Tell(new DeploymentStatusResponse(
|
||||
result.DeploymentId,
|
||||
result.InstanceName,
|
||||
DeploymentStatus.Failed,
|
||||
result.Error ?? "Deployment persistence failed",
|
||||
DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -492,10 +541,20 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
|
||||
// ── DCL connection management ──
|
||||
|
||||
private readonly HashSet<string> _createdConnections = new();
|
||||
/// <summary>
|
||||
/// Tracks the configuration last sent to the DCL for each connection name, keyed by
|
||||
/// a hash of the connection's protocol/endpoints/credentials/failover count
|
||||
/// (SiteRuntime-010). A name whose hash is unchanged is skipped; a name whose config
|
||||
/// changed re-issues a <c>CreateConnectionCommand</c> so the DCL adopts the new
|
||||
/// configuration instead of keeping a stale connection after a redeployment.
|
||||
/// </summary>
|
||||
private readonly Dictionary<string, string> _createdConnections = new();
|
||||
|
||||
/// <summary>
|
||||
/// Sets up DCL connections from the flattened config (idempotent: tracks created connections).
|
||||
/// Sets up DCL connections from the flattened config. Idempotent on unchanged
|
||||
/// configuration, but re-issues the create command when a connection's endpoint,
|
||||
/// credentials, backup endpoint, or failover retry count has changed since it was
|
||||
/// last sent (SiteRuntime-010).
|
||||
/// </summary>
|
||||
private void EnsureDclConnections(string configJson)
|
||||
{
|
||||
@@ -508,7 +567,8 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
|
||||
foreach (var (name, connConfig) in config.Connections)
|
||||
{
|
||||
if (_createdConnections.Contains(name))
|
||||
var configHash = ComputeConnectionConfigHash(connConfig);
|
||||
if (_createdConnections.TryGetValue(name, out var lastHash) && lastHash == configHash)
|
||||
continue;
|
||||
|
||||
var primaryDetails = FlattenConnectionConfig(connConfig.Protocol, connConfig.ConfigurationJson);
|
||||
@@ -519,10 +579,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
_dclManager.Tell(new Commons.Messages.DataConnection.CreateConnectionCommand(
|
||||
name, connConfig.Protocol, primaryDetails, backupDetails, connConfig.FailoverRetryCount));
|
||||
|
||||
_createdConnections.Add(name);
|
||||
var changed = _createdConnections.ContainsKey(name);
|
||||
_createdConnections[name] = configHash;
|
||||
_logger.LogInformation(
|
||||
"Created DCL connection {Connection} (protocol={Protocol})",
|
||||
name, connConfig.Protocol);
|
||||
"{Action} DCL connection {Connection} (protocol={Protocol})",
|
||||
changed ? "Updated" : "Created", name, connConfig.Protocol);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -531,6 +592,26 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes a stable hash over the configuration fields that affect how the DCL
|
||||
/// connects, so a changed endpoint/credential/backup/failover count is detected
|
||||
/// (SiteRuntime-010).
|
||||
/// </summary>
|
||||
private static string ComputeConnectionConfigHash(
|
||||
Commons.Types.Flattening.ConnectionConfig connConfig)
|
||||
{
|
||||
var material = string.Join(
|
||||
"",
|
||||
connConfig.Protocol,
|
||||
connConfig.ConfigurationJson ?? string.Empty,
|
||||
connConfig.BackupConfigurationJson ?? string.Empty,
|
||||
connConfig.FailoverRetryCount.ToString());
|
||||
|
||||
var bytes = System.Security.Cryptography.SHA256.HashData(
|
||||
System.Text.Encoding.UTF8.GetBytes(material));
|
||||
return Convert.ToHexString(bytes);
|
||||
}
|
||||
|
||||
private static IDictionary<string, string> FlattenConnectionConfig(string protocol, string? json)
|
||||
{
|
||||
if (string.IsNullOrEmpty(json))
|
||||
@@ -559,25 +640,35 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
|
||||
// ── Shared Script Loading ──
|
||||
|
||||
private void LoadSharedScriptsFromStorage()
|
||||
/// <summary>
|
||||
/// SiteRuntime-008: reads and compiles all shared scripts on a background task so the
|
||||
/// SQLite read and Roslyn compilation never block the singleton's mailbox thread. The
|
||||
/// result is piped back as a <see cref="SharedScriptsLoaded"/> message, carrying the
|
||||
/// enabled configs to resume staggered Instance Actor creation on the actor thread.
|
||||
/// </summary>
|
||||
private void LoadSharedScriptsFromStorage(List<DeployedInstance> enabledConfigs)
|
||||
{
|
||||
try
|
||||
Task.Run(async () =>
|
||||
{
|
||||
var scripts = _storage.GetAllSharedScriptsAsync().GetAwaiter().GetResult();
|
||||
var scripts = await _storage.GetAllSharedScriptsAsync();
|
||||
var compiled = 0;
|
||||
foreach (var script in scripts)
|
||||
{
|
||||
if (_sharedScriptLibrary.CompileAndRegister(script.Name, script.Code))
|
||||
compiled++;
|
||||
}
|
||||
_logger.LogInformation(
|
||||
"Loaded {Compiled}/{Total} shared scripts from SQLite",
|
||||
compiled, scripts.Count);
|
||||
}
|
||||
catch (Exception ex)
|
||||
return new SharedScriptsLoaded(enabledConfigs, compiled, scripts.Count);
|
||||
}).ContinueWith(t =>
|
||||
{
|
||||
_logger.LogError(ex, "Failed to load shared scripts from SQLite");
|
||||
}
|
||||
if (t.IsCompletedSuccessfully)
|
||||
return t.Result;
|
||||
_logger.LogError(
|
||||
t.Exception?.GetBaseException(), "Failed to load shared scripts from SQLite");
|
||||
// A shared-script load failure must not abandon startup — proceed with
|
||||
// Instance Actor creation; scripts that need a missing shared script fail
|
||||
// at execution time and are recorded to the site event log.
|
||||
return new SharedScriptsLoaded(enabledConfigs, 0, 0);
|
||||
}).PipeTo(Self);
|
||||
}
|
||||
|
||||
// ── Debug View routing ──
|
||||
@@ -891,12 +982,22 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
// ── Internal messages ──
|
||||
|
||||
internal record StartupConfigsLoaded(List<DeployedInstance> Configs, string? Error);
|
||||
|
||||
/// <summary>
|
||||
/// Internal message piped back once shared scripts have been compiled off-thread
|
||||
/// (SiteRuntime-008). Carries the enabled configs so staggered Instance Actor
|
||||
/// creation resumes on the actor thread.
|
||||
/// </summary>
|
||||
internal record SharedScriptsLoaded(
|
||||
List<DeployedInstance> EnabledConfigs, int CompiledCount, int TotalCount);
|
||||
|
||||
internal record StartNextBatch(BatchState State);
|
||||
internal record BatchState(List<DeployedInstance> Configs, int NextIndex);
|
||||
internal record EnableResult(
|
||||
EnableInstanceCommand Command, DeployedInstance? Config, string? Error, IActorRef OriginalSender);
|
||||
internal record DeployPersistenceResult(
|
||||
string DeploymentId, string InstanceName, bool Success, string? Error, IActorRef OriginalSender);
|
||||
string DeploymentId, string InstanceName, bool Success, string? Error,
|
||||
IActorRef OriginalSender, bool IsRedeploy);
|
||||
|
||||
/// <summary>
|
||||
/// A redeployment command buffered until the previous Instance Actor terminates.
|
||||
|
||||
@@ -286,9 +286,10 @@ public class ScriptActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
var executionId = $"{_scriptName}-exec-{_executionCounter++}";
|
||||
|
||||
// NOTE: In production, configure a dedicated blocking I/O dispatcher via HOCON:
|
||||
// akka.actor.script-execution-dispatcher { type = PinnedDispatcher }
|
||||
// and chain .WithDispatcher("akka.actor.script-execution-dispatcher") below.
|
||||
// SiteRuntime-009: the actor's mailbox stays on the default dispatcher, but the
|
||||
// script body itself runs on the dedicated ScriptExecutionScheduler (a bounded
|
||||
// set of dedicated threads), so blocking script I/O is contained there and
|
||||
// cannot starve the shared .NET thread pool.
|
||||
var props = Props.Create(() => new ScriptExecutionActor(
|
||||
_scriptName,
|
||||
_instanceName,
|
||||
|
||||
@@ -68,8 +68,13 @@ public class ScriptExecutionActor : ReceiveActor
|
||||
{
|
||||
var timeout = TimeSpan.FromSeconds(options.ScriptExecutionTimeoutSeconds);
|
||||
|
||||
// SiteRuntime-009: run the script body on the dedicated script-execution
|
||||
// scheduler, not the shared .NET thread pool, so blocking script I/O cannot
|
||||
// starve the global pool and stall Akka dispatchers / HTTP handling.
|
||||
var scheduler = ScriptExecutionScheduler.Shared(options);
|
||||
|
||||
// CTS must be created inside the async lambda so it outlives this method
|
||||
_ = Task.Run(async () =>
|
||||
_ = Task.Factory.StartNew(async () =>
|
||||
{
|
||||
IServiceScope? serviceScope = null;
|
||||
// ISiteEventLogger is a singleton; resolve from the root provider so
|
||||
@@ -164,6 +169,6 @@ public class ScriptExecutionActor : ReceiveActor
|
||||
// Stop self after execution completes
|
||||
self.Tell(PoisonPill.Instance);
|
||||
}
|
||||
});
|
||||
}, CancellationToken.None, TaskCreationOptions.DenyChildAttach, scheduler).Unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,14 @@ public class SiteStorageService
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new (unopened) SQLite connection against the site database.
|
||||
/// Exposed so site-local repositories can open their own connections without
|
||||
/// reaching into private state via reflection (SiteRuntime-006). The caller owns
|
||||
/// the connection and is responsible for opening and disposing it.
|
||||
/// </summary>
|
||||
public SqliteConnection CreateConnection() => new(_connectionString);
|
||||
|
||||
/// <summary>
|
||||
/// Creates the SQLite tables if they do not exist.
|
||||
/// Called once on site startup.
|
||||
|
||||
@@ -169,27 +169,12 @@ public class SiteExternalSystemRepository : IExternalSystemRepository
|
||||
// ── Private helpers ──
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new SQLite connection using the same connection string as <see cref="SiteStorageService"/>.
|
||||
/// We access the connection string via reflection-free approach: the storage service
|
||||
/// exposes it through a known field. Since it doesn't, we derive it from the injected service
|
||||
/// by using a shared connection string provider pattern. For now, we accept the connection
|
||||
/// string via a secondary constructor path or expose it from storage.
|
||||
///
|
||||
/// Implementation note: We use the SiteStorageService's internal connection string.
|
||||
/// This field is accessed via a package-internal helper since SiteStorageService
|
||||
/// doesn't expose it directly. As a pragmatic solution, we pass the connection string
|
||||
/// separately at DI registration time.
|
||||
/// Creates a new SQLite connection against the site database via
|
||||
/// <see cref="SiteStorageService.CreateConnection"/> (SiteRuntime-006). The
|
||||
/// connection string is owned by <see cref="SiteStorageService"/>; the repository
|
||||
/// no longer reaches into its private state via reflection.
|
||||
/// </summary>
|
||||
private SqliteConnection CreateConnection()
|
||||
{
|
||||
// Access the connection string from SiteStorageService via its internal field.
|
||||
// This uses reflection as a pragmatic choice — the alternative is modifying
|
||||
// SiteStorageService to expose the connection string, which is out of scope.
|
||||
var field = typeof(SiteStorageService).GetField("_connectionString",
|
||||
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
|
||||
var connectionString = (string)field!.GetValue(_storage)!;
|
||||
return new SqliteConnection(connectionString);
|
||||
}
|
||||
private SqliteConnection CreateConnection() => _storage.CreateConnection();
|
||||
|
||||
private static ExternalSystemDefinition MapExternalSystem(SqliteDataReader reader)
|
||||
{
|
||||
@@ -233,12 +218,13 @@ public class SiteExternalSystemRepository : IExternalSystemRepository
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a stable positive integer ID from a string name.
|
||||
/// Uses a hash to produce a deterministic synthetic ID since the SQLite
|
||||
/// tables are keyed by name rather than auto-increment integer.
|
||||
/// Generates a stable positive integer ID from a string name (SiteRuntime-007).
|
||||
/// Uses a deterministic FNV-1a hash rather than <see cref="string.GetHashCode()"/>,
|
||||
/// which is randomized per process on .NET Core and would therefore change every
|
||||
/// time the process restarts — breaking any caller that stored an ID and later
|
||||
/// looks the entity up by that ID.
|
||||
/// </summary>
|
||||
private static int GenerateSyntheticId(string name)
|
||||
=> name.GetHashCode() & 0x7FFFFFFF;
|
||||
private static int GenerateSyntheticId(string name) => SyntheticId.From(name);
|
||||
|
||||
/// <summary>
|
||||
/// DTO for deserializing individual method entries from the method_definitions JSON column.
|
||||
|
||||
@@ -178,13 +178,12 @@ public class SiteNotificationRepository : INotificationRepository
|
||||
|
||||
// ── Private helpers ──
|
||||
|
||||
private SqliteConnection CreateConnection()
|
||||
{
|
||||
var field = typeof(SiteStorageService).GetField("_connectionString",
|
||||
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
|
||||
var connectionString = (string)field!.GetValue(_storage)!;
|
||||
return new SqliteConnection(connectionString);
|
||||
}
|
||||
/// <summary>
|
||||
/// Creates a new SQLite connection against the site database via
|
||||
/// <see cref="SiteStorageService.CreateConnection"/> (SiteRuntime-006) instead of
|
||||
/// reaching into its private connection-string field via reflection.
|
||||
/// </summary>
|
||||
private SqliteConnection CreateConnection() => _storage.CreateConnection();
|
||||
|
||||
private static NotificationList MapNotificationList(SqliteDataReader reader)
|
||||
{
|
||||
@@ -246,10 +245,9 @@ public class SiteNotificationRepository : INotificationRepository
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a stable positive integer ID from a string name.
|
||||
/// Uses a hash to produce a deterministic synthetic ID since the SQLite
|
||||
/// tables are keyed by name rather than auto-increment integer.
|
||||
/// Generates a stable positive integer ID from a string name (SiteRuntime-007).
|
||||
/// Uses a deterministic FNV-1a hash rather than <see cref="string.GetHashCode()"/>,
|
||||
/// which is randomized per process on .NET Core and would change every restart.
|
||||
/// </summary>
|
||||
private static int GenerateSyntheticId(string name)
|
||||
=> name.GetHashCode() & 0x7FFFFFFF;
|
||||
private static int GenerateSyntheticId(string name) => SyntheticId.From(name);
|
||||
}
|
||||
|
||||
35
src/ScadaLink.SiteRuntime/Repositories/SyntheticId.cs
Normal file
35
src/ScadaLink.SiteRuntime/Repositories/SyntheticId.cs
Normal file
@@ -0,0 +1,35 @@
|
||||
namespace ScadaLink.SiteRuntime.Repositories;
|
||||
|
||||
/// <summary>
|
||||
/// SiteRuntime-007: deterministic synthetic-ID generation for site-local artifacts.
|
||||
///
|
||||
/// The site SQLite tables are keyed by name rather than an auto-increment integer, but
|
||||
/// the shared repository contracts (<c>IExternalSystemRepository</c>,
|
||||
/// <c>INotificationRepository</c>) expose integer-keyed lookups. A synthetic integer ID
|
||||
/// is therefore derived from the entity name. It MUST be stable across process restarts
|
||||
/// — <see cref="string.GetHashCode()"/> is randomized per process on .NET Core and so
|
||||
/// cannot be used.
|
||||
/// </summary>
|
||||
internal static class SyntheticId
|
||||
{
|
||||
// FNV-1a 32-bit constants.
|
||||
private const uint FnvOffsetBasis = 2166136261;
|
||||
private const uint FnvPrime = 16777619;
|
||||
|
||||
/// <summary>
|
||||
/// Computes a deterministic, process-stable positive 31-bit integer ID for the
|
||||
/// given name using the FNV-1a hash over its UTF-8 bytes.
|
||||
/// </summary>
|
||||
public static int From(string name)
|
||||
{
|
||||
var hash = FnvOffsetBasis;
|
||||
foreach (var b in System.Text.Encoding.UTF8.GetBytes(name))
|
||||
{
|
||||
hash ^= b;
|
||||
hash *= FnvPrime;
|
||||
}
|
||||
|
||||
// Mask to a positive 31-bit value so the ID is always non-negative.
|
||||
return (int)(hash & 0x7FFFFFFF);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using Microsoft.CodeAnalysis;
|
||||
using Microsoft.CodeAnalysis.CSharp;
|
||||
using Microsoft.CodeAnalysis.CSharp.Scripting;
|
||||
using Microsoft.CodeAnalysis.CSharp.Syntax;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ScadaLink.Commons.Types;
|
||||
@@ -17,7 +18,10 @@ public class ScriptCompilationService
|
||||
private readonly ILogger<ScriptCompilationService> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Namespaces that are forbidden in user scripts for security.
|
||||
/// Forbidden API roots. Each entry is matched as a prefix against both the resolved
|
||||
/// symbol's containing namespace and its fully-qualified containing type name, so an
|
||||
/// entry may name a whole namespace ("System.IO") or a single type
|
||||
/// ("System.Diagnostics.Process").
|
||||
/// </summary>
|
||||
private static readonly string[] ForbiddenNamespaces =
|
||||
[
|
||||
@@ -30,8 +34,8 @@ public class ScriptCompilationService
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Specific types/members allowed even within forbidden namespaces.
|
||||
/// async/await is OK despite System.Threading being blocked.
|
||||
/// Specific namespaces/types allowed even though they sit under a forbidden root.
|
||||
/// async/await and cancellation tokens are OK despite System.Threading being blocked.
|
||||
/// </summary>
|
||||
private static readonly string[] AllowedExceptions =
|
||||
[
|
||||
@@ -46,58 +50,184 @@ public class ScriptCompilationService
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates that the script source code does not reference forbidden APIs.
|
||||
/// SiteRuntime-011: validates that the script does not reference forbidden APIs.
|
||||
///
|
||||
/// Validation is performed with Roslyn semantic analysis rather than a raw substring
|
||||
/// scan of the source text. The script is parsed and a semantic model is built; every
|
||||
/// identifier, type reference, member access, and object creation is resolved to its
|
||||
/// symbol and the symbol's containing namespace is checked against the forbidden list.
|
||||
///
|
||||
/// This is reliable in both directions a textual scan was not:
|
||||
/// - it catches forbidden types regardless of how they are written (<c>global::</c>
|
||||
/// prefixes, aliases, transitively-imported namespaces) because it inspects the
|
||||
/// resolved symbol, not the spelling;
|
||||
/// - it does not raise false positives for the namespace string appearing in a
|
||||
/// comment, a string literal, or an unrelated identifier.
|
||||
///
|
||||
/// Returns a list of violation messages, empty if clean.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> ValidateTrustModel(string code)
|
||||
{
|
||||
var violations = new List<string>();
|
||||
var tree = CSharpSyntaxTree.ParseText(code);
|
||||
var tree = CSharpSyntaxTree.ParseText(
|
||||
code, new CSharpParseOptions(kind: SourceCodeKind.Script));
|
||||
|
||||
var compilation = CSharpCompilation.CreateScriptCompilation(
|
||||
"TrustValidation",
|
||||
tree,
|
||||
ScriptReferences,
|
||||
new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary));
|
||||
|
||||
var model = compilation.GetSemanticModel(tree);
|
||||
var root = tree.GetRoot();
|
||||
var text = root.ToFullString();
|
||||
|
||||
foreach (var ns in ForbiddenNamespaces)
|
||||
// Deduplicate so a forbidden symbol used many times is reported once but
|
||||
// distinct forbidden symbols are all reported.
|
||||
var violations = new SortedSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var node in root.DescendantNodes())
|
||||
{
|
||||
if (text.Contains(ns, StringComparison.Ordinal))
|
||||
// Only inspect nodes that name a type or member; skip declarations,
|
||||
// string literals and comments entirely. Member-access and qualified-name
|
||||
// parents are evaluated as a whole, so their nested name parts are skipped.
|
||||
if (node is not (SimpleNameSyntax or MemberAccessExpressionSyntax
|
||||
or QualifiedNameSyntax or ObjectCreationExpressionSyntax))
|
||||
{
|
||||
// Check if it matches an allowed exception
|
||||
var isAllowed = AllowedExceptions.Any(allowed =>
|
||||
text.Contains(allowed, StringComparison.Ordinal) &&
|
||||
ns != allowed &&
|
||||
allowed.StartsWith(ns, StringComparison.Ordinal));
|
||||
|
||||
// More precise: check each occurrence
|
||||
var idx = 0;
|
||||
while ((idx = text.IndexOf(ns, idx, StringComparison.Ordinal)) >= 0)
|
||||
{
|
||||
var remainder = text.Substring(idx);
|
||||
var matchesAllowed = AllowedExceptions.Any(a =>
|
||||
remainder.StartsWith(a, StringComparison.Ordinal));
|
||||
|
||||
if (!matchesAllowed)
|
||||
{
|
||||
violations.Add($"Forbidden API reference: '{ns}' at position {idx}");
|
||||
break;
|
||||
}
|
||||
idx += ns.Length;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
var info = model.GetSymbolInfo(node);
|
||||
var symbol = info.Symbol ?? info.CandidateSymbols.FirstOrDefault();
|
||||
|
||||
// The set of fully-qualified scopes this reference touches: the resolved
|
||||
// symbol's containing namespace and type, or — when the symbol could not
|
||||
// be resolved (a type from an unreferenced assembly) — the syntactic
|
||||
// fully-qualified name written in source as a safe fallback.
|
||||
var scopes = symbol != null
|
||||
? GetSymbolScopes(symbol)
|
||||
: GetSyntacticScopes(node);
|
||||
if (scopes.Count == 0)
|
||||
continue;
|
||||
|
||||
var forbidden = ForbiddenNamespaces.FirstOrDefault(
|
||||
f => scopes.Any(s => IsUnderScope(s, f)));
|
||||
if (forbidden == null)
|
||||
continue;
|
||||
|
||||
// Allow specific exception namespaces/types (async/await, cancellation).
|
||||
if (scopes.Any(s => AllowedExceptions.Any(a => IsUnderScope(s, a))))
|
||||
continue;
|
||||
|
||||
var name = symbol?.Name ?? node.ToString();
|
||||
violations.Add($"Forbidden API reference: '{forbidden}' ({scopes[0]}.{name})");
|
||||
}
|
||||
|
||||
return violations;
|
||||
return violations.ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the fully-qualified scopes a resolved symbol belongs to — its containing
|
||||
/// namespace and, for a type or member, the fully-qualified containing type. A bare
|
||||
/// namespace symbol is intentionally ignored: a namespace name on its own performs
|
||||
/// no action; harm requires referencing a type or a member.
|
||||
/// </summary>
|
||||
private static List<string> GetSymbolScopes(ISymbol symbol)
|
||||
{
|
||||
var scopes = new List<string>();
|
||||
|
||||
switch (symbol)
|
||||
{
|
||||
case INamespaceSymbol:
|
||||
// A namespace reference alone is harmless — skip it. (This avoids a
|
||||
// false positive on the "System.Threading" qualifier of the allowed
|
||||
// "System.Threading.Tasks.Task".)
|
||||
break;
|
||||
case ITypeSymbol typeSymbol:
|
||||
scopes.Add(typeSymbol.ToDisplayString());
|
||||
if (typeSymbol.ContainingNamespace is { IsGlobalNamespace: false } typeNs)
|
||||
scopes.Add(typeNs.ToDisplayString());
|
||||
break;
|
||||
default:
|
||||
if (symbol.ContainingType != null)
|
||||
{
|
||||
scopes.Add(symbol.ContainingType.ToDisplayString());
|
||||
if (symbol.ContainingType.ContainingNamespace is { IsGlobalNamespace: false } memberNs)
|
||||
scopes.Add(memberNs.ToDisplayString());
|
||||
}
|
||||
else if (symbol.ContainingNamespace is { IsGlobalNamespace: false } ns)
|
||||
{
|
||||
scopes.Add(ns.ToDisplayString());
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return scopes;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fallback used when a name could not be resolved to a symbol (e.g. a type from an
|
||||
/// assembly the script is not allowed to reference). The fully-qualified name as
|
||||
/// written in source is used directly — a script that names
|
||||
/// <c>System.Net.Http.HttpClient</c> is still rejected even though that assembly is
|
||||
/// deliberately absent from the script's metadata references.
|
||||
/// </summary>
|
||||
private static List<string> GetSyntacticScopes(SyntaxNode node)
|
||||
{
|
||||
// A dotted name written in source is itself the fully-qualified scope. Only
|
||||
// consider names that actually contain a dot — bare local identifiers cannot
|
||||
// reach a forbidden namespace.
|
||||
var text = node switch
|
||||
{
|
||||
QualifiedNameSyntax q => q.ToString(),
|
||||
MemberAccessExpressionSyntax m => m.ToString(),
|
||||
_ => string.Empty
|
||||
};
|
||||
|
||||
// Strip whitespace/newlines that a multi-line member-access chain may contain.
|
||||
text = new string(text.Where(c => !char.IsWhiteSpace(c)).ToArray());
|
||||
|
||||
return string.IsNullOrEmpty(text) || !text.Contains('.')
|
||||
? []
|
||||
: [text];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True if <paramref name="actual"/> is exactly, or nested within,
|
||||
/// <paramref name="root"/> (e.g. "System.IO.Compression" is under "System.IO",
|
||||
/// "System.Diagnostics.Process" is under "System.Diagnostics.Process").
|
||||
/// </summary>
|
||||
private static bool IsUnderScope(string actual, string root)
|
||||
=> actual.Equals(root, StringComparison.Ordinal)
|
||||
|| actual.StartsWith(root + ".", StringComparison.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Assemblies referenced by compiled scripts. Shared between the Roslyn scripting
|
||||
/// options and the semantic-analysis compilation built for trust validation
|
||||
/// (SiteRuntime-011), so the validator resolves symbols against exactly the same
|
||||
/// metadata the script is compiled against.
|
||||
/// </summary>
|
||||
private static readonly System.Reflection.Assembly[] ScriptAssemblies =
|
||||
[
|
||||
typeof(object).Assembly,
|
||||
typeof(Enumerable).Assembly,
|
||||
typeof(Math).Assembly,
|
||||
typeof(Microsoft.CSharp.RuntimeBinder.CSharpArgumentInfo).Assembly,
|
||||
typeof(Commons.Types.DynamicJsonElement).Assembly
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Metadata references for the trust-validation semantic compilation.
|
||||
/// </summary>
|
||||
private static readonly MetadataReference[] ScriptReferences =
|
||||
ScriptAssemblies
|
||||
.Select(a => (MetadataReference)MetadataReference.CreateFromFile(a.Location))
|
||||
.ToArray();
|
||||
|
||||
/// <summary>
|
||||
/// Shared Roslyn scripting options (references + imports) used by both full
|
||||
/// script compilation and trigger-expression compilation.
|
||||
/// </summary>
|
||||
private static ScriptOptions BuildScriptOptions() => ScriptOptions.Default
|
||||
.WithReferences(
|
||||
typeof(object).Assembly,
|
||||
typeof(Enumerable).Assembly,
|
||||
typeof(Math).Assembly,
|
||||
typeof(Microsoft.CSharp.RuntimeBinder.CSharpArgumentInfo).Assembly,
|
||||
typeof(Commons.Types.DynamicJsonElement).Assembly)
|
||||
.WithReferences(ScriptAssemblies)
|
||||
.WithImports(
|
||||
"System",
|
||||
"System.Collections.Generic",
|
||||
|
||||
107
src/ScadaLink.SiteRuntime/Scripts/ScriptExecutionScheduler.cs
Normal file
107
src/ScadaLink.SiteRuntime/Scripts/ScriptExecutionScheduler.cs
Normal file
@@ -0,0 +1,107 @@
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
namespace ScadaLink.SiteRuntime.Scripts;
|
||||
|
||||
/// <summary>
|
||||
/// SiteRuntime-009: a dedicated, bounded <see cref="TaskScheduler"/> for running script
|
||||
/// and alarm on-trigger bodies.
|
||||
///
|
||||
/// Script bodies may perform synchronous blocking I/O (a database connection, a
|
||||
/// synchronous external-system call). Running them on the shared .NET
|
||||
/// <see cref="ThreadPool"/> lets a burst of blocking scripts starve the pool and stall
|
||||
/// unrelated Akka dispatchers and HTTP request handling. This scheduler owns a fixed set
|
||||
/// of dedicated threads, so script blocking is contained to those threads and cannot
|
||||
/// exhaust the global pool.
|
||||
///
|
||||
/// The scheduler is process-wide (one set of threads for all instances) and is sized
|
||||
/// from <see cref="SiteRuntimeOptions"/> the first time it is configured.
|
||||
/// </summary>
|
||||
public sealed class ScriptExecutionScheduler : TaskScheduler, IDisposable
|
||||
{
|
||||
private readonly BlockingCollection<Task> _queue = new();
|
||||
private readonly List<Thread> _threads;
|
||||
private int _disposed;
|
||||
|
||||
private static volatile ScriptExecutionScheduler? _shared;
|
||||
private static readonly object SharedLock = new();
|
||||
|
||||
/// <summary>
|
||||
/// The process-wide script-execution scheduler. Lazily created on first use with the
|
||||
/// thread count from <see cref="SiteRuntimeOptions.ScriptExecutionThreadCount"/>; the
|
||||
/// first caller wins, subsequent calls reuse the existing instance.
|
||||
/// </summary>
|
||||
public static ScriptExecutionScheduler Shared(SiteRuntimeOptions options)
|
||||
{
|
||||
if (_shared != null)
|
||||
return _shared;
|
||||
|
||||
lock (SharedLock)
|
||||
{
|
||||
return _shared ??= new ScriptExecutionScheduler(options.ScriptExecutionThreadCount);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a scheduler backed by <paramref name="threadCount"/> dedicated threads.
|
||||
/// </summary>
|
||||
public ScriptExecutionScheduler(int threadCount)
|
||||
{
|
||||
if (threadCount < 1)
|
||||
threadCount = 1;
|
||||
|
||||
_threads = new List<Thread>(threadCount);
|
||||
for (var i = 0; i < threadCount; i++)
|
||||
{
|
||||
var thread = new Thread(WorkerLoop)
|
||||
{
|
||||
IsBackground = true,
|
||||
Name = $"script-execution-{i}"
|
||||
};
|
||||
_threads.Add(thread);
|
||||
thread.Start();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The number of dedicated worker threads.</summary>
|
||||
public override int MaximumConcurrencyLevel => _threads.Count;
|
||||
|
||||
private void WorkerLoop()
|
||||
{
|
||||
try
|
||||
{
|
||||
foreach (var task in _queue.GetConsumingEnumerable())
|
||||
{
|
||||
TryExecuteTask(task);
|
||||
}
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// Scheduler disposed — worker exits.
|
||||
}
|
||||
}
|
||||
|
||||
protected override void QueueTask(Task task) => _queue.Add(task);
|
||||
|
||||
protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued)
|
||||
{
|
||||
// Only inline if we are already on one of this scheduler's worker threads,
|
||||
// so script work never escapes onto a thread-pool thread.
|
||||
if (Thread.CurrentThread.Name?.StartsWith("script-execution-", StringComparison.Ordinal) != true)
|
||||
return false;
|
||||
|
||||
return TryExecuteTask(task);
|
||||
}
|
||||
|
||||
protected override IEnumerable<Task> GetScheduledTasks() => _queue.ToArray();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
return;
|
||||
|
||||
_queue.CompleteAdding();
|
||||
foreach (var thread in _threads)
|
||||
thread.Join(TimeSpan.FromSeconds(5));
|
||||
_queue.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -36,4 +36,12 @@ public class SiteRuntimeOptions
|
||||
/// Default: 1000.
|
||||
/// </summary>
|
||||
public int StreamBufferSize { get; set; } = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// SiteRuntime-009: number of dedicated threads in the script-execution scheduler.
|
||||
/// Script and alarm on-trigger bodies run on these threads instead of the shared
|
||||
/// .NET thread pool, so blocking script I/O cannot starve the global pool.
|
||||
/// Default: 8.
|
||||
/// </summary>
|
||||
public int ScriptExecutionThreadCount { get; set; } = 8;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user