using System.IO.Compression; using System.Security.Cryptography; using System.Text.Json; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.Commons.Entities.ExternalSystems; using ZB.MOM.WW.ScadaBridge.Commons.Entities.InboundApi; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Scripts; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Scripting; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Scripting; using ZB.MOM.WW.ScadaBridge.Commons.Types.Transport; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase; using ZB.MOM.WW.ScadaBridge.ScriptAnalysis; using ZB.MOM.WW.ScadaBridge.TemplateEngine; using ZB.MOM.WW.ScadaBridge.TemplateEngine.Validation; using ZB.MOM.WW.ScadaBridge.Transport.Encryption; using ZB.MOM.WW.ScadaBridge.Transport.Serialization; namespace ZB.MOM.WW.ScadaBridge.Transport.Import; /// /// Three-phase bundle importer: validates the /// bundle envelope (manifest + content hash + decryption) and opens a /// session; diffs the bundle's DTOs against the /// current target database; writes the chosen /// resolutions through the audited repositories. All three phases are /// fully implemented. /// /// Audit-row responsibility: repository mutation methods in /// ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories are thin EF wrappers /// and do NOT emit audit rows. therefore writes /// each per-entity audit row explicitly via ; /// the scoped is /// automatically stamped on each row by the audit service. /// /// /// If repository methods are ever changed to emit audit rows themselves, /// the explicit LogAsync calls in this class must be removed to /// avoid double-logging. /// /// public sealed class BundleImporter : IBundleImporter { // All bundle content deserialization goes through BundleJsonOptions.Default. // IMPORTANT — unknown JSON properties must remain ALLOWED (the default // JsonUnmappedMemberHandling.Skip). Legacy bundles may carry a top-level // "apiKeys" array and/or "ApprovedApiKeyIds" inside "apiMethods[]" entries. // Setting JsonUnmappedMemberHandling.Disallow here would cause those bundles // to fail deserialization, breaking backward-compat. This tolerance is // load-bearing and must not be changed. (Fix I-2) private static readonly JsonSerializerOptions ContentJsonOptions = BundleJsonOptions.Default; private readonly BundleSerializer _bundleSerializer; private readonly ManifestValidator _manifestValidator; private readonly BundleSecretEncryptor _encryptor; private readonly ArtifactDiff _diff = new(); private readonly EntitySerializer _entitySerializer; private readonly IAuditService _auditService; private readonly IAuditCorrelationContext _correlationContext; private readonly ScadaBridgeDbContext _dbContext; private readonly ITemplateEngineRepository _templateRepo; private readonly IExternalSystemRepository _externalRepo; private readonly INotificationRepository _notificationRepo; private readonly IInboundApiRepository _inboundApiRepo; private readonly ISiteRepository _siteRepo; private readonly IBundleSessionStore _sessionStore; private readonly BundleUnlockRateLimiter _unlockRateLimiter; private readonly IOptions _options; private readonly TimeProvider _timeProvider; private readonly SemanticValidator _semanticValidator; // Optional. Implemented in DeploymentManager (where the flattening // pipeline lives) and surfaced via the Commons IStaleInstanceProbe seam. Null // in a host that registers Transport without DeploymentManager — staleness is // then best-effort empty (informational only, never gates the import). private readonly IStaleInstanceProbe? _staleInstanceProbe; private readonly IScriptArtifactChangeBus? _scriptArtifactChangeBus; private readonly ILogger? _logger; /// /// Initializes a new with all required dependencies. /// /// Serializer for reading bundle zip archives. /// Validates the bundle manifest on load. /// Handles AES-256-GCM decryption of encrypted bundles. /// Deserializes entity DTOs from bundle content. /// In-memory session store for loaded bundle state. /// Rate limiter for passphrase unlock attempts per client IP. /// Transport configuration options. /// Abstracted time provider for testability. /// Template engine repository for diff and apply. /// External system repository for diff and apply. /// Notification repository for diff and apply. /// Inbound API repository for diff and apply. /// Site repository — supplies the target sites and site-scoped data connections that the preview's site/connection auto-match resolves against. /// Audit service for writing per-entity import audit rows. /// Correlation context that carries the active BundleImportId. /// EF Core context used to commit the import transaction. /// Validates template references before applying. /// Optional: recomputes a deployed instance's current revision hash so the importer can enumerate instances stale-ed by a template overwrite. Null when no flattening pipeline is registered (Transport-without-DeploymentManager hosts) — staleness is then skipped. /// Optional: node-local bus notified AFTER a successful commit that script-bearing artifacts (ApiMethod / SharedScript / Template) were overwritten or renamed, so compiled-handler caches can invalidate. Null on hosts that don't register it (staleness/invalidation then relies on the consumer's own self-heal). /// Optional logger. public BundleImporter( BundleSerializer bundleSerializer, ManifestValidator manifestValidator, BundleSecretEncryptor encryptor, EntitySerializer entitySerializer, IBundleSessionStore sessionStore, BundleUnlockRateLimiter unlockRateLimiter, IOptions options, TimeProvider timeProvider, ITemplateEngineRepository templateRepo, IExternalSystemRepository externalRepo, INotificationRepository notificationRepo, IInboundApiRepository inboundApiRepo, ISiteRepository siteRepo, IAuditService auditService, IAuditCorrelationContext correlationContext, ScadaBridgeDbContext dbContext, SemanticValidator semanticValidator, IStaleInstanceProbe? staleInstanceProbe = null, IScriptArtifactChangeBus? scriptArtifactChangeBus = null, ILogger? logger = null) { _bundleSerializer = bundleSerializer ?? throw new ArgumentNullException(nameof(bundleSerializer)); _manifestValidator = manifestValidator ?? throw new ArgumentNullException(nameof(manifestValidator)); _encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor)); _entitySerializer = entitySerializer ?? throw new ArgumentNullException(nameof(entitySerializer)); _sessionStore = sessionStore ?? throw new ArgumentNullException(nameof(sessionStore)); _unlockRateLimiter = unlockRateLimiter ?? throw new ArgumentNullException(nameof(unlockRateLimiter)); _options = options ?? throw new ArgumentNullException(nameof(options)); _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); _templateRepo = templateRepo ?? throw new ArgumentNullException(nameof(templateRepo)); _externalRepo = externalRepo ?? throw new ArgumentNullException(nameof(externalRepo)); _notificationRepo = notificationRepo ?? throw new ArgumentNullException(nameof(notificationRepo)); _inboundApiRepo = inboundApiRepo ?? throw new ArgumentNullException(nameof(inboundApiRepo)); _siteRepo = siteRepo ?? throw new ArgumentNullException(nameof(siteRepo)); _auditService = auditService ?? throw new ArgumentNullException(nameof(auditService)); _correlationContext = correlationContext ?? throw new ArgumentNullException(nameof(correlationContext)); _dbContext = dbContext ?? throw new ArgumentNullException(nameof(dbContext)); _semanticValidator = semanticValidator ?? throw new ArgumentNullException(nameof(semanticValidator)); _staleInstanceProbe = staleInstanceProbe; _scriptArtifactChangeBus = scriptArtifactChangeBus; _logger = logger; } /// public async Task LoadAsync(Stream bundleStream, string? passphrase, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(bundleStream); // Copy to a seekable buffer — manifest + content readers each open a // fresh ZipArchive over the same bytes, so the upstream stream needs to // be seekable. A caller-supplied FileStream is seekable but a Kestrel // request stream is not, so we always normalise to MemoryStream. var ms = new MemoryStream(); await bundleStream.CopyToAsync(ms, ct).ConfigureAwait(false); ms.Position = 0; // Size cap is in MB; multiply in long arithmetic so the comparison // doesn't overflow at the int boundary for large MaxBundleSizeMb. var maxBytes = _options.Value.MaxBundleSizeMb * 1024L * 1024L; if (ms.Length > maxBytes) { throw new InvalidOperationException( $"Bundle exceeds maximum allowed size of {_options.Value.MaxBundleSizeMb} MB."); } // T-006: zip-bomb / decompression-bomb defences. We enforce three caps // BEFORE any entry is decompressed by ReadManifest / ReadContentBytes: // 1) total entry count // 2) per-entry decompressed length // 3) per-entry compression ratio (Length / CompressedLength) // Each cap is configurable via TransportOptions so an operator can tune // for an environment with legitimately large or unusually compressible // bundles. Using ZipArchiveEntry.Length / .CompressedLength avoids // reading the entry payload at all. ms.Position = 0; ValidateArchiveEnvelope(ms); BundleManifest manifest; try { ms.Position = 0; manifest = _bundleSerializer.ReadManifest(ms); } catch (InvalidDataException) { // Preserve the serializer's specific "manifest missing/null" message // — the caller wants to surface a precise diagnostic to the operator. throw; } catch (Exception ex) { throw new InvalidDataException("Bundle is missing or has a malformed manifest.json.", ex); } ms.Position = 0; var contentBytes = _bundleSerializer.ReadContentBytes(ms, manifest); // Validate format version + content-hash + manifest shape. Reject paths // surface as distinct exceptions so the UI can disambiguate the cause. var validation = _manifestValidator.Validate(manifest, contentBytes); switch (validation) { case ManifestValidationResult.UnsupportedFormatVersion: throw new NotSupportedException( $"Bundle format version {manifest.BundleFormatVersion} is not supported by this cluster."); case ManifestValidationResult.ContentHashMismatch: throw new InvalidDataException( "Bundle content hash does not match manifest — file may be corrupt."); case ManifestValidationResult.MalformedManifest: throw new InvalidDataException("Bundle manifest is malformed."); case ManifestValidationResult.Ok: break; default: throw new InvalidDataException($"Unrecognised manifest validation result: {validation}."); } // Decrypt when the manifest carries EncryptionMetadata. // // T-003: lockout enforcement is server-side and keyed by ContentHash so a // second tab / CLI caller re-uploading the same bundle bytes cannot // side-step the limit by skipping the Razor page. The counter is // consulted BEFORE attempting decrypt (rejects further attempts on an // already-locked bundle) and incremented on a CryptographicException // from _encryptor.Decrypt; a successful decrypt clears the counter so a // legitimate operator who eventually types the right passphrase is not // penalised for earlier typos. byte[] decryptedContent; if (manifest.Encryption is not null) { if (string.IsNullOrEmpty(passphrase)) { throw new ArgumentException( "Passphrase required for encrypted bundle.", nameof(passphrase)); } var maxAttempts = _options.Value.MaxUnlockAttemptsPerSession; var priorFailures = _sessionStore.GetUnlockFailureCount(manifest.ContentHash); if (priorFailures >= maxAttempts) { throw new BundleLockedException(manifest.ContentHash, priorFailures); } var aad = Encryption.BundleManifestAad.Compute(manifest); try { decryptedContent = _encryptor.Decrypt(contentBytes, manifest.Encryption, passphrase, aad); } catch (CryptographicException) { var newCount = _sessionStore.IncrementUnlockFailureCount(manifest.ContentHash); if (newCount >= maxAttempts) { // Surface the lockout as the typed exception so the caller can // distinguish "wrong passphrase, try again" from "no more attempts". throw new BundleLockedException(manifest.ContentHash, newCount); } // Otherwise rebubble the CryptographicException so the UI's // wrong-passphrase audit + retry path continues to work unchanged. throw; } _sessionStore.ClearUnlockFailures(manifest.ContentHash); } else { decryptedContent = contentBytes; } var ttl = TimeSpan.FromMinutes(_options.Value.BundleSessionTtlMinutes); var session = new BundleSession { SessionId = Guid.NewGuid(), Manifest = manifest, DecryptedContent = decryptedContent, ExpiresAt = _timeProvider.GetUtcNow() + ttl, }; return _sessionStore.Open(session); } /// /// T-006: validates the zip envelope against the configured caps BEFORE any /// entry payload is decompressed. Reads only the central-directory headers /// ( / ) /// so a hostile bundle can't OOM the central node through this method itself. /// /// Buffered bundle bytes. Position is preserved. private void ValidateArchiveEnvelope(MemoryStream bundleBytes) { var opts = _options.Value; var maxEntries = opts.MaxBundleEntryCount; var maxEntryDecompressed = opts.MaxBundleEntryDecompressedMb * 1024L * 1024L; var maxRatio = opts.MaxBundleEntryCompressionRatio; var savedPosition = bundleBytes.Position; try { bundleBytes.Position = 0; using var archive = new ZipArchive(bundleBytes, ZipArchiveMode.Read, leaveOpen: true); if (archive.Entries.Count > maxEntries) { throw new InvalidDataException( $"Bundle contains {archive.Entries.Count} zip entries; the configured maximum is {maxEntries}."); } foreach (var entry in archive.Entries) { if (entry.Length > maxEntryDecompressed) { throw new InvalidDataException( $"Bundle entry '{entry.FullName}' declares a decompressed size of {entry.Length} bytes; " + $"the configured maximum is {maxEntryDecompressed} bytes " + $"({opts.MaxBundleEntryDecompressedMb} MB)."); } // CompressedLength of 0 means store-only or empty — skip ratio check. if (entry.CompressedLength > 0 && entry.Length / entry.CompressedLength > maxRatio) { throw new InvalidDataException( $"Bundle entry '{entry.FullName}' has compression ratio " + $"{entry.Length / entry.CompressedLength}x; the configured maximum is {maxRatio}x."); } } } finally { bundleBytes.Position = savedPosition; } } /// public async Task PreviewAsync(Guid sessionId, CancellationToken ct = default) { var session = _sessionStore.Get(sessionId) ?? throw new InvalidOperationException($"Bundle session {sessionId} not found or expired."); if (session.Locked) { throw new InvalidOperationException($"Bundle session {sessionId} is locked."); } BundleContentDto content; try { content = JsonSerializer.Deserialize(session.DecryptedContent, ContentJsonOptions) ?? throw new InvalidDataException("Session content deserialized to null."); } catch (JsonException ex) { throw new InvalidDataException("Session content is not a valid BundleContentDto.", ex); } var items = new List(); // ---- TemplateFolders ---- var allFolders = await _templateRepo.GetAllFoldersAsync(ct).ConfigureAwait(false); var folderByName = allFolders.ToDictionary(f => f.Name, f => f, StringComparer.Ordinal); var folderNameById = allFolders.ToDictionary(f => f.Id, f => f.Name); foreach (var fDto in content.TemplateFolders) { folderByName.TryGetValue(fDto.Name, out var existing); items.Add(_diff.CompareTemplateFolder(fDto, existing, folderNameById)); } // ---- Templates ---- // Previously this loop iterated GetAllTemplatesAsync() // and called GetTemplateWithChildrenAsync(stub.Id) once per matching // name (classic N+1). The bulk variant fetches every matching template // with children eager-loaded in a single round-trip. var bundleTemplateNames = content.Templates.Select(t => t.Name); var hydratedTemplates = await _templateRepo .GetTemplatesWithChildrenAsync(bundleTemplateNames, ct) .ConfigureAwait(false); var hydratedByName = hydratedTemplates .ToDictionary(t => t.Name, t => t, StringComparer.Ordinal); var allTemplateStubs = await _templateRepo.GetAllTemplatesAsync(ct).ConfigureAwait(false); var templateNameById = allTemplateStubs.ToDictionary(t => t.Id, t => t.Name); foreach (var tDto in content.Templates) { hydratedByName.TryGetValue(tDto.Name, out var existing); items.Add(_diff.CompareTemplate(tDto, existing, folderNameById, templateNameById)); } // ---- SharedScripts ---- foreach (var s in content.SharedScripts) { var existing = await _templateRepo.GetSharedScriptByNameAsync(s.Name, ct).ConfigureAwait(false); items.Add(_diff.CompareSharedScript(s, existing)); } // ---- ExternalSystems (+ their methods) ---- foreach (var es in content.ExternalSystems) { var existing = await _externalRepo.GetExternalSystemByNameAsync(es.Name, ct).ConfigureAwait(false); IReadOnlyList? methods = null; if (existing is not null) { methods = await _externalRepo.GetMethodsByExternalSystemIdAsync(existing.Id, ct).ConfigureAwait(false); } items.Add(_diff.CompareExternalSystem(es, existing, methods)); } // ---- DatabaseConnections ---- foreach (var db in content.DatabaseConnections) { var existing = await _externalRepo.GetDatabaseConnectionByNameAsync(db.Name, ct).ConfigureAwait(false); items.Add(_diff.CompareDatabaseConnection(db, existing)); } // ---- NotificationLists ---- foreach (var nl in content.NotificationLists) { var existing = await _notificationRepo.GetListByNameAsync(nl.Name, ct).ConfigureAwait(false); items.Add(_diff.CompareNotificationList(nl, existing)); } // ---- SmtpConfigurations (no by-host lookup — scan GetAll) ---- var allSmtp = await _notificationRepo.GetAllSmtpConfigurationsAsync(ct).ConfigureAwait(false); var smtpByHost = allSmtp.ToDictionary(s => s.Host, s => s, StringComparer.Ordinal); foreach (var sm in content.SmtpConfigs) { smtpByHost.TryGetValue(sm.Host, out var existing); items.Add(_diff.CompareSmtpConfiguration(sm, existing)); } // ---- SmsConfigurations (no by-AccountSid lookup — scan GetAll) ---- var allSms = await _notificationRepo.GetAllSmsConfigurationsAsync(ct).ConfigureAwait(false); var smsBySid = allSms.ToDictionary(s => s.AccountSid, s => s, StringComparer.Ordinal); foreach (var sms in content.SmsConfigs) { smsBySid.TryGetValue(sms.AccountSid, out var existing); items.Add(_diff.CompareSmsConfiguration(sms, existing)); } // ---- ApiKeys ---- // Inbound API keys are not transported between environments. // New bundles never carry a keys section. A legacy bundle may still contain // one; we do NOT surface those keys as importable preview rows (they would // offer Add/Overwrite actions for keys that can't be meaningfully re-created // from a hash). They are counted, ignored, and reported at apply time. // ---- ApiMethods ---- foreach (var m in content.ApiMethods) { var existing = await _inboundApiRepo.GetMethodByNameAsync(m.Name, ct).ConfigureAwait(false); items.Add(_diff.CompareApiMethod(m, existing)); } // ---- Site/instance-scoped types ---- // Sites/DataConnections/Instances are referenced by stable string identity // (SiteIdentifier / connection Name / UniqueName) and resolved against the // TARGET environment's own surrogate keys. We auto-match the source site by // identifier; the result drives both the per-type diffs (a matched target // connection / instance feeds CompareDataConnection / CompareInstance) and // the operator-facing required-mapping list built below. // // Cache target-site lookups + their data connections so we don't re-query // the same site once per instance / connection / native-alarm override. var targetSiteByIdentifier = new Dictionary(StringComparer.Ordinal); var targetConnectionsBySiteIdentifier = new Dictionary>(StringComparer.Ordinal); async Task ResolveTargetSiteAsync(string siteIdentifier) { if (targetSiteByIdentifier.TryGetValue(siteIdentifier, out var cached)) return cached; var site = await _siteRepo.GetSiteByIdentifierAsync(siteIdentifier, ct).ConfigureAwait(false); targetSiteByIdentifier[siteIdentifier] = site; return site; } async Task> ResolveTargetConnectionsAsync(string siteIdentifier) { if (targetConnectionsBySiteIdentifier.TryGetValue(siteIdentifier, out var cached)) return cached; var site = await ResolveTargetSiteAsync(siteIdentifier).ConfigureAwait(false); IReadOnlyList conns = site is null ? Array.Empty() : await _siteRepo.GetDataConnectionsBySiteIdAsync(site.Id, ct).ConfigureAwait(false); targetConnectionsBySiteIdentifier[siteIdentifier] = conns; return conns; } // ---- Sites ---- foreach (var siteDto in content.Sites) { var existing = await ResolveTargetSiteAsync(siteDto.SiteIdentifier).ConfigureAwait(false); items.Add(_diff.CompareSite(siteDto, existing)); } // ---- DataConnections (site-scoped; matched by name within the auto-matched target site) ---- // Connection names are unique only WITHIN a site, so the preview item's // identity is SITE-QUALIFIED (`{SiteIdentifier}/{Name}`). The diff CONTENT is // unchanged — only the item Name is qualified (after CompareDataConnection // returns) so two sites' same-named connections resolve to distinct items and // the operator's per-item Skip/Overwrite applies to the right site's connection. // ApplyDataConnectionsAsync looks the resolution up by this same qualified key. foreach (var dcDto in content.DataConnections) { var targetConns = await ResolveTargetConnectionsAsync(dcDto.SiteIdentifier).ConfigureAwait(false); var existing = targetConns.FirstOrDefault(c => string.Equals(c.Name, dcDto.Name, StringComparison.Ordinal)); var item = _diff.CompareDataConnection(dcDto, existing); items.Add(item with { Name = QualifiedConnectionName(dcDto.SiteIdentifier, dcDto.Name) }); } // ---- Instances (hydrated target + resolved template/site/area names; review item I2) ---- foreach (var instDto in content.Instances) { // GetInstanceByUniqueNameAsync eagerly Includes all four child nav // collections (AttributeOverrides / AlarmOverrides / ConnectionBindings / // NativeAlarmSourceOverrides), so the entity handed to CompareInstance is // HYDRATED — its children diff correctly instead of every incoming child // reading as an addition (review item I2). var existing = await _templateRepo .GetInstanceByUniqueNameAsync(instDto.UniqueName, ct).ConfigureAwait(false); string? existingTemplateName = null; string? existingSiteIdentifier = null; string? existingAreaName = null; if (existing is not null) { // The entity stores template/site/area as numeric FKs that can't be // compared cross-environment, so resolve each to the same stable // string identity the incoming DTO carries. var tmpl = await _templateRepo.GetTemplateByIdAsync(existing.TemplateId, ct).ConfigureAwait(false); existingTemplateName = tmpl?.Name; var site = await _siteRepo.GetSiteByIdAsync(existing.SiteId, ct).ConfigureAwait(false); existingSiteIdentifier = site?.SiteIdentifier; if (existing.AreaId is int areaId) { var area = await _templateRepo.GetAreaByIdAsync(areaId, ct).ConfigureAwait(false); existingAreaName = area?.Name; } } items.Add(_diff.CompareInstance( instDto, existing, existingTemplateName, existingSiteIdentifier, existingAreaName)); } // ---- Required site/connection mappings ---- var (requiredSites, requiredConnections) = await BuildRequiredMappingsAsync( content, ResolveTargetSiteAsync, ResolveTargetConnectionsAsync).ConfigureAwait(false); // ---- Blocker detection ---- items.AddRange(await DetectBlockersAsync(content, ResolveTargetConnectionsAsync, ct).ConfigureAwait(false)); return new ImportPreview(sessionId, items, requiredSites, requiredConnections); } /// /// Collects the distinct set of source sites and (site, connection) pairs the /// bundle references, auto-matching each against the TARGET environment by /// identity, and returns the operator-facing required-mapping lists. /// /// Site references are drawn from every instance, every site, and every /// data-connection in the bundle. Connection references are drawn from every /// instance connection-binding, every non-null native-alarm-source /// ConnectionNameOverride, and every bundled data-connection. A source /// site auto-matches when the target has a site with the same identifier; a /// connection auto-matches when that target site additionally carries a /// connection of the same name. No match leaves AutoMatchTarget* null — /// the operator must supply an explicit mapping (or accept create-new) at /// apply time. The diff path above already populated the resolver caches, so /// these lookups are served from memory. /// /// private static async Task<(IReadOnlyList Sites, IReadOnlyList Connections)> BuildRequiredMappingsAsync( BundleContentDto content, Func> resolveTargetSite, Func>> resolveTargetConnections) { // Distinct source-site identifiers, with a best-effort display name. // SiteDto carries a Name; instances / data-connections only carry the // identifier, so default that to the identifier when no SiteDto is present. var siteNameByIdentifier = new Dictionary(StringComparer.Ordinal); void NoteSite(string identifier, string? name) { if (string.IsNullOrEmpty(identifier)) return; if (!siteNameByIdentifier.TryGetValue(identifier, out var existing) || (string.Equals(existing, identifier, StringComparison.Ordinal) && !string.IsNullOrEmpty(name))) { siteNameByIdentifier[identifier] = string.IsNullOrEmpty(name) ? identifier : name; } } foreach (var s in content.Sites) NoteSite(s.SiteIdentifier, s.Name); foreach (var i in content.Instances) NoteSite(i.SiteIdentifier, null); foreach (var dc in content.DataConnections) NoteSite(dc.SiteIdentifier, null); // Distinct (sourceSite, connectionName) pairs referenced anywhere. var connectionRefs = new HashSet<(string Site, string Name)>(); foreach (var i in content.Instances) { NoteSite(i.SiteIdentifier, null); foreach (var b in i.ConnectionBindings) { if (!string.IsNullOrEmpty(b.ConnectionName)) { connectionRefs.Add((i.SiteIdentifier, b.ConnectionName)); } } foreach (var n in i.NativeAlarmSourceOverrides) { if (!string.IsNullOrEmpty(n.ConnectionNameOverride)) { connectionRefs.Add((i.SiteIdentifier, n.ConnectionNameOverride)); } } } foreach (var dc in content.DataConnections) { if (!string.IsNullOrEmpty(dc.Name)) { connectionRefs.Add((dc.SiteIdentifier, dc.Name)); } } var siteMappings = new List(); foreach (var identifier in siteNameByIdentifier.Keys.OrderBy(k => k, StringComparer.Ordinal)) { var target = await resolveTargetSite(identifier).ConfigureAwait(false); siteMappings.Add(new RequiredSiteMapping( SourceSiteIdentifier: identifier, SourceSiteName: siteNameByIdentifier[identifier], AutoMatchTargetIdentifier: target?.SiteIdentifier)); } var connectionMappings = new List(); foreach (var (site, name) in connectionRefs .OrderBy(r => r.Site, StringComparer.Ordinal) .ThenBy(r => r.Name, StringComparer.Ordinal)) { // Auto-match only WITHIN the auto-matched target site: a connection of // the same name under a different site is not a valid match. var targetConns = await resolveTargetConnections(site).ConfigureAwait(false); var matched = targetConns.Any(c => string.Equals(c.Name, name, StringComparison.Ordinal)); connectionMappings.Add(new RequiredConnectionMapping( SourceSiteIdentifier: site, SourceConnectionName: name, AutoMatchTargetName: matched ? name : null)); } return (siteMappings, connectionMappings); } /// /// Surfaces unresolved cross-entity references: a TemplateScript or /// ApiMethod body that name-mentions a SharedScript or ExternalSystem that /// is in neither the bundle nor the target database. We reuse the same /// substring-with-word-boundary scan as DependencyResolver; the /// implementations are kept in lockstep but not factored out yet because /// the resolver's scan operates on entity Code while the importer's scan /// operates on DTO Code — same algorithm, different inputs. /// private async Task> DetectBlockersAsync( BundleContentDto content, Func>> resolveTargetConnections, CancellationToken ct) { var blockers = new List(); // ---- Instance template + referenced-connection blockers ---- // The set of template names the import can satisfy = (in-bundle templates) // ∪ (templates already in the target DB). An instance whose TemplateName is // in neither is unresolvable. var bundleTemplateNames = new HashSet(StringComparer.Ordinal); foreach (var t in content.Templates) bundleTemplateNames.Add(t.Name); // Honour a rename: a bundled template resolved as Rename is created under // its new name, but at preview time no resolution has been chosen yet, so // the in-bundle name is the original DTO name — which is what an instance // references. (Explicit rename remap is a D-wave apply-time concern.) var targetTemplates = await _templateRepo.GetAllTemplatesAsync(ct).ConfigureAwait(false); var targetTemplateNames = new HashSet(StringComparer.Ordinal); foreach (var t in targetTemplates) { targetTemplateNames.Add(t.Name); } foreach (var inst in content.Instances) { if (string.IsNullOrEmpty(inst.TemplateName)) continue; if (bundleTemplateNames.Contains(inst.TemplateName)) continue; if (targetTemplateNames.Contains(inst.TemplateName)) continue; blockers.Add(new ImportPreviewItem( EntityType: "Instance", Name: inst.UniqueName, ExistingVersion: null, IncomingVersion: null, Kind: ConflictKind.Blocker, FieldDiffJson: null, BlockerReason: $"Template '{inst.TemplateName}' not found in bundle or target.")); } // A referenced (sourceSite, connectionName) pair is resolvable when it is // either carried in the bundle's DataConnections OR auto-matches a connection // of the same name in the auto-matched target site. Genuinely-missing // references are blockers. (Explicit operator connection maps are applied in // a later wave; preview's auto-match is identity-based.) var bundleConnections = new HashSet<(string Site, string Name)>(); foreach (var dc in content.DataConnections) { if (!string.IsNullOrEmpty(dc.Name)) bundleConnections.Add((dc.SiteIdentifier, dc.Name)); } // Distinct referenced pairs from instance bindings + native-alarm overrides. var referencedConnections = new HashSet<(string Site, string Name)>(); foreach (var inst in content.Instances) { foreach (var b in inst.ConnectionBindings) { if (!string.IsNullOrEmpty(b.ConnectionName)) { referencedConnections.Add((inst.SiteIdentifier, b.ConnectionName)); } } foreach (var n in inst.NativeAlarmSourceOverrides) { if (!string.IsNullOrEmpty(n.ConnectionNameOverride)) { referencedConnections.Add((inst.SiteIdentifier, n.ConnectionNameOverride)); } } } foreach (var (site, name) in referencedConnections .OrderBy(r => r.Site, StringComparer.Ordinal) .ThenBy(r => r.Name, StringComparer.Ordinal)) { if (bundleConnections.Contains((site, name))) continue; var targetConns = await resolveTargetConnections(site).ConfigureAwait(false); if (targetConns.Any(c => string.Equals(c.Name, name, StringComparison.Ordinal))) continue; // Site-qualify the blocker Name so two sites' same-named-but-missing // connections surface as distinct blockers (a bare name would collide). blockers.Add(new ImportPreviewItem( EntityType: "Instance", Name: QualifiedConnectionName(site, name), ExistingVersion: null, IncomingVersion: null, Kind: ConflictKind.Blocker, FieldDiffJson: null, BlockerReason: $"Connection '{site}/{name}' unresolved — present in neither bundle nor target.")); } // Known-resolvable names = (in-bundle) ∪ (already-in-target). var allSharedScripts = await _templateRepo.GetAllSharedScriptsAsync(ct).ConfigureAwait(false); var allExternalSystems = await _externalRepo.GetAllExternalSystemsAsync(ct).ConfigureAwait(false); var sharedScriptNames = new HashSet(StringComparer.Ordinal); foreach (var s in content.SharedScripts) sharedScriptNames.Add(s.Name); foreach (var s in allSharedScripts) sharedScriptNames.Add(s.Name); var externalSystemNames = new HashSet(StringComparer.Ordinal); foreach (var e in content.ExternalSystems) externalSystemNames.Add(e.Name); foreach (var e in allExternalSystems) externalSystemNames.Add(e.Name); // Heuristic: collect a small candidate vocabulary of identifiers used // by the bundle's scripts that are NOT one of the known-good names, and // check whether each one was previously a SharedScript or ExternalSystem // (i.e. matches the naming-convention shape of an identifier reference). // For v1, we walk the SharedScripts the bundle *expects* — by scanning // bodies and reporting any identifier-shaped token that resolves to a // SharedScript by historical record... but that's overreach. // // Simpler + sufficient v1: scan every script body in the bundle's // templates + ApiMethods, and for each occurrence of "Name(" where // Name is a valid identifier, if Name appears in NEITHER set, surface // it as a Blocker. This catches the documented use-case // (HelperFn() / ErpSystem.Call()) without combinatorial blowup. var referencedFromTemplates = new HashSet(StringComparer.Ordinal); var referencedFromApiMethods = new HashSet(StringComparer.Ordinal); var locallyDeclaredInTemplates = new HashSet(StringComparer.Ordinal); var locallyDeclaredInApiMethods = new HashSet(StringComparer.Ordinal); foreach (var t in content.Templates) { foreach (var s in t.Scripts) { CollectCallIdentifiers(s.Code, referencedFromTemplates); CollectLocalDeclarations(s.Code, locallyDeclaredInTemplates); } foreach (var a in t.Attributes) { // Attribute.Value is a typed-literal default (int/double/string/list, // decoded by AttributeValueCodec — never compiled), so it is not a // trust surface; we still scan it for identifier-shaped tokens, but // any finding is only ever an advisory template warning. // DataSourceReference is an OPC UA node address path (e.g. // "ns=3;s=Tank.Level") owned by the device and must NOT be scanned, // or the dot delimiter trips the heuristic into flagging the address // segments as missing SharedScript/ExternalSystem references. CollectCallIdentifiers(a.Value, referencedFromTemplates); CollectLocalDeclarations(a.Value, locallyDeclaredInTemplates); } } foreach (var m in content.ApiMethods) { CollectCallIdentifiers(m.Script, referencedFromApiMethods); CollectLocalDeclarations(m.Script, locallyDeclaredInApiMethods); } // For each candidate, only report it if it looks like a resource // reference (PascalCase, length > 1), isn't a well-known language / // runtime / SQL token, AND isn't present anywhere we can satisfy it. A // candidate genuinely unresolved in an ApiMethod (referenced there and NOT // locally declared there) is a hard Blocker; a candidate genuinely // unresolved only in a template is a non-blocking Warning. var allCandidates = new HashSet(referencedFromTemplates, StringComparer.Ordinal); allCandidates.UnionWith(referencedFromApiMethods); foreach (var candidate in allCandidates.OrderBy(n => n, StringComparer.Ordinal)) { if (!LooksLikeResourceName(candidate)) continue; if (KnownNonReferenceNames.Contains(candidate)) continue; var isShared = sharedScriptNames.Contains(candidate); var isExternal = externalSystemNames.Contains(candidate); if (isShared || isExternal) continue; var apiUnresolved = referencedFromApiMethods.Contains(candidate) && !locallyDeclaredInApiMethods.Contains(candidate); var templateUnresolved = referencedFromTemplates.Contains(candidate) && !locallyDeclaredInTemplates.Contains(candidate); if (!apiUnresolved && !templateUnresolved) continue; // satisfied by a local decl in its own origin var fromApiMethod = apiUnresolved; blockers.Add(new ImportPreviewItem( EntityType: "Reference", Name: candidate, ExistingVersion: null, IncomingVersion: null, Kind: fromApiMethod ? ConflictKind.Blocker : ConflictKind.Warning, FieldDiffJson: null, BlockerReason: fromApiMethod ? $"References SharedScript or ExternalSystem '{candidate}' not present in bundle or target." : $"Script references SharedScript or ExternalSystem '{candidate}' not present in bundle or target — advisory; the deploy-time gate re-validates.")); } foreach (var (kind, entityName, scriptLabel, code) in EnumerateTrustGatedScripts(content, resolutionMap: null)) { IReadOnlyList violations; try { violations = ScriptTrustValidator.FindViolations(code); } catch (Exception ex) { // Fail closed (mirrors the apply-time gate): a validator throw on // one body surfaces as a Blocker for that script, not a preview crash. _logger?.LogWarning(ex, "Script trust analysis threw for {Kind} '{EntityName}' script '{ScriptLabel}' during preview; surfacing as a Blocker.", kind, entityName, scriptLabel); blockers.Add(new ImportPreviewItem( EntityType: kind, Name: $"{entityName}.{scriptLabel}", ExistingVersion: null, IncomingVersion: null, Kind: ConflictKind.Blocker, FieldDiffJson: null, BlockerReason: "Script trust analysis failed to complete — rejected.")); continue; } foreach (var violation in violations) { blockers.Add(new ImportPreviewItem( EntityType: kind, Name: $"{entityName}.{scriptLabel}", ExistingVersion: null, IncomingVersion: null, Kind: ConflictKind.Blocker, FieldDiffJson: null, BlockerReason: $"Script trust violation — {violation}")); } } // N4: advisory warning rows for instance overrides that target a LOCKED // template member — the flattener drops them, so surface the inert row in the // wizard (Kind: Warning, non-blocking). resolutionMap is null at preview time, // so every instance is scanned. foreach (var (instance, message) in CollectLockedOverrideWarnings(content, resolutionMap: null, targetTemplates)) { blockers.Add(new ImportPreviewItem( EntityType: "Instance", Name: instance, ExistingVersion: null, IncomingVersion: null, Kind: ConflictKind.Warning, FieldDiffJson: null, BlockerReason: message)); } return blockers; } private static void CollectCallIdentifiers(string? body, HashSet sink) { if (string.IsNullOrEmpty(body)) return; // Find every "Identifier(" or "Identifier." occurrence. The boundary // before the identifier must NOT be an identifier char so we don't // match the trailing portion of a longer token, AND must not be a // dot — otherwise `obj.Method()` would flag `Method` as a top-level // reference. Member-access trailing identifiers are skipped. for (var i = 0; i < body.Length; i++) { if (!IsIdentifierStart(body[i])) continue; if (i > 0 && (IsIdentifierChar(body[i - 1]) || body[i - 1] == '.')) continue; var start = i; while (i < body.Length && IsIdentifierChar(body[i])) i++; if (i >= body.Length) break; var trailing = body[i]; if (trailing == '(' || trailing == '.') { sink.Add(body[start..i]); } } } /// /// Collects the names of every local function / method DECLARED inside a /// script body, so the reference heuristic doesn't mistake a script's own /// helper (decimal ComputeRate(...) => ...; return ComputeRate(x);) /// for a missing SharedScript / ExternalSystem. Uses a Roslyn script-kind /// parse (partial trees are fine — a body with syntax errors still yields /// the declarations Roslyn could recover). Purely additive noise-suppression: /// on any parse failure the sink is simply left unchanged. /// private static void CollectLocalDeclarations(string? body, HashSet sink) { if (string.IsNullOrEmpty(body)) return; try { var root = CSharpSyntaxTree .ParseText(body, new CSharpParseOptions(kind: SourceCodeKind.Script)) .GetRoot(); foreach (var fn in root.DescendantNodes().OfType()) { sink.Add(fn.Identifier.ValueText); } foreach (var m in root.DescendantNodes().OfType()) { sink.Add(m.Identifier.ValueText); } } catch { // Best-effort: a parse failure just means no declarations are // excluded for this body — the finding (at worst) surfaces as an // advisory warning, never a hard error for template scripts. } } /// /// Enumerates every executable C# surface a bundle carries that the /// trust gate must vet: non-Skip template scripts + their Expression-trigger /// bodies, template alarm Expression-trigger bodies, shared scripts, /// ApiMethod scripts, AND instance alarm-override trigger expressions /// (InstanceAlarmOverrideDto.TriggerConfigurationOverride). Expression /// triggers — template alarms AND the instance overrides that replace them in /// the flattened config — compile and execute at the site /// (`TriggerExpressionGlobals`), so they are a genuine trust surface, not just /// script bodies. A null (preview time, before /// resolutions are chosen) gates everything. Yields (kind, entityName, /// scriptLabel, code) so callers can format either an error string or a /// preview row. /// private static IEnumerable<(string Kind, string EntityName, string ScriptLabel, string Code)> EnumerateTrustGatedScripts( BundleContentDto content, Dictionary<(string, string), ImportResolution>? resolutionMap) { foreach (var t in content.Templates) { if (IsSkipResolution(resolutionMap, "Template", t.Name)) continue; foreach (var s in t.Scripts) { if (!string.IsNullOrEmpty(s.Code)) { yield return ("Template", t.Name, s.Name, s.Code); } var scriptExpr = ExtractTriggerExpression(s.TriggerType, s.TriggerConfiguration); if (!string.IsNullOrEmpty(scriptExpr)) { yield return ("Template", t.Name, $"{s.Name} (trigger expression)", scriptExpr); } } foreach (var a in t.Alarms) { var alarmExpr = ExtractTriggerExpression(a.TriggerType.ToString(), a.TriggerConfiguration); if (!string.IsNullOrEmpty(alarmExpr)) { yield return ("Template", t.Name, $"{a.Name} (alarm trigger expression)", alarmExpr); } } } foreach (var s in content.SharedScripts) { if (IsSkipResolution(resolutionMap, "SharedScript", s.Name)) continue; if (!string.IsNullOrEmpty(s.Code)) { yield return ("SharedScript", s.Name, s.Name, s.Code); } } foreach (var m in content.ApiMethods) { if (IsSkipResolution(resolutionMap, "ApiMethod", m.Name)) continue; if (!string.IsNullOrEmpty(m.Script)) { yield return ("ApiMethod", m.Name, m.Name, m.Script); } } foreach (var i in content.Instances) { if (IsSkipResolution(resolutionMap, "Instance", i.UniqueName)) continue; foreach (var o in i.AlarmOverrides) { // The DTO carries no trigger type (the type lives on the template // alarm), so gate ANY override config carrying an {"expression":...} // string body: if the overridden alarm is not Expression-triggered // the body never executes, but vetting it anyway is fail-safe — the // structured (HiLo/threshold) configs have no "expression" key, so // there is no false-positive channel. var expr = ExtractExpressionBody(o.TriggerConfigurationOverride); if (!string.IsNullOrEmpty(expr)) { yield return ("Instance", i.UniqueName, $"{o.AlarmCanonicalName} (alarm trigger override expression)", expr); } } } } /// /// Extracts the C# boolean expression from an {"expression":"…"} trigger /// configuration when the trigger type is Expression — the same shape /// the site's TriggerExpressionGlobals compiles. Returns null for any /// non-Expression trigger, empty config, or malformed JSON (nothing to gate). /// private static string? ExtractTriggerExpression(string? triggerType, string? triggerConfigJson) { if (triggerType is null || !triggerType.Equals("Expression", StringComparison.OrdinalIgnoreCase)) { return null; } return ExtractExpressionBody(triggerConfigJson); } /// /// Extracts the C# boolean expression from an {"expression":"…"} trigger /// configuration, WITHOUT a trigger-type guard — used where the caller has no /// trigger type to check (an instance alarm override; the type lives on the /// template alarm). A config carrying no "expression" string key (the /// structured HiLo/threshold shapes) or malformed JSON yields null — nothing to /// gate. /// private static string? ExtractExpressionBody(string? triggerConfigJson) { if (string.IsNullOrWhiteSpace(triggerConfigJson)) { return null; } try { using var doc = JsonDocument.Parse(triggerConfigJson); if (doc.RootElement.ValueKind == JsonValueKind.Object && doc.RootElement.TryGetProperty("expression", out var expr) && expr.ValueKind == JsonValueKind.String) { return expr.GetString(); } } catch (JsonException) { // Malformed trigger config isn't executable as an expression — nothing // to gate here (a separate validation surfaces the malformed config). } return null; } private static bool IsSkipResolution( Dictionary<(string, string), ImportResolution>? resolutionMap, string entityType, string name) => resolutionMap != null && ResolveOrDefault(resolutionMap, entityType, name).Action == ResolutionAction.Skip; /// /// Best-effort scan for instance overrides that target a LOCKED template member. /// The flattener drops such overrides (an override on a locked attribute / alarm / /// native-alarm-source never takes effect), so the bundle carries an inert row — /// this surfaces an advisory warning so the operator learns the config won't apply, /// WITHOUT blocking the import or changing what gets written (bundle fidelity keeps /// the row; the flattener stays the behavioural authority). /// /// Matches locked members by DIRECT name only — own + inherited placeholder rows /// both carry the IsLocked flag, so a lock on either matches. Overrides /// addressing composed path-qualified members (y1.z.Val) or derived-shadow /// locks (LockedInDerived on a base the placeholder row doesn't reflect) may /// not match a direct row — an unmatched name emits NO warning (never a false /// positive; the ManagementActor and flattener remain the enforcement points). The /// instance's template is resolved from the bundle DTO (the version about to be /// written) first, else the pre-existing target template. /// /// private static IReadOnlyList<(string Instance, string Message)> CollectLockedOverrideWarnings( BundleContentDto content, Dictionary<(string, string), ImportResolution>? resolutionMap, IReadOnlyList