Files
ScadaBridge/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/BundleCommands.cs
T

576 lines
28 KiB
C#

using System.CommandLine;
using System.CommandLine.Parsing;
using System.Text.Json;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
/// <summary>
/// Transport (#24) bundle export / preview / import. The bundle bytes travel
/// through the management endpoint as base64 inside the standard JSON envelope
/// so no transport plumbing diverges from the other commands; the CLI handles
/// file I/O at the edges.
/// </summary>
public static class BundleCommands
{
private static readonly TimeSpan BundleCommandTimeout = TimeSpan.FromMinutes(5);
/// <summary>Builds the <c>bundle</c> command group with export, preview, and import sub-commands.</summary>
/// <param name="urlOption">Shared management URL option.</param>
/// <param name="formatOption">Shared output format option.</param>
/// <param name="usernameOption">Shared username option.</param>
/// <param name="passwordOption">Shared password option.</param>
/// <returns>The configured <see cref="Command"/> for the bundle group.</returns>
public static Command Build(
Option<string> urlOption, Option<string> formatOption,
Option<string> usernameOption, Option<string> passwordOption)
{
var command = new Command("bundle")
{
Description = "Export, preview, and import Transport bundles",
};
command.Add(BuildExport(urlOption, formatOption, usernameOption, passwordOption));
command.Add(BuildPreview(urlOption, formatOption, usernameOption, passwordOption));
command.Add(BuildImport(urlOption, formatOption, usernameOption, passwordOption));
return command;
}
// ====================================================================
// bundle export
// ====================================================================
private static Command BuildExport(
Option<string> urlOption, Option<string> formatOption,
Option<string> usernameOption, Option<string> passwordOption)
{
var outputOption = new Option<string>("--output")
{
Description = "Output file path (.scadabundle)",
Required = true,
};
var passphraseOption = new Option<string?>("--passphrase")
{
Description = "Encryption passphrase. Omit to produce an unencrypted bundle.",
};
var allOption = new Option<bool>("--all")
{
Description = "Export every entity of every supported type (ignores per-type name flags).",
};
var templatesOption = NameListOption("--templates", "Comma-separated template names");
var sharedScriptsOption = NameListOption("--shared-scripts", "Comma-separated shared-script names");
var externalSystemsOption = NameListOption("--external-systems", "Comma-separated external-system names");
var dbConnectionsOption = NameListOption("--db-connections", "Comma-separated database-connection names");
var notificationListsOption = NameListOption("--notification-lists", "Comma-separated notification-list names");
var smtpConfigsOption = NameListOption("--smtp-configs", "Comma-separated SMTP host names");
// Inbound API keys are not transported between environments (re-arch C4) — no
// --api-keys option. Re-create keys and re-grant their method scopes on the
// destination via the admin UI/CLI.
var apiMethodsOption = NameListOption("--api-methods", "Comma-separated API-method names");
// M8 (B4): site/instance-scoped export. Sites accept a SiteIdentifier
// (preferred) or friendly Name per token; instances accept a UniqueName.
var sitesOption = NameListOption("--sites", "Comma-separated site identifiers or names");
var instancesOption = NameListOption("--instances", "Comma-separated instance unique-names");
var includeDepsOption = new Option<bool>("--include-dependencies")
{
Description = "Pull transitive dependencies (referenced shared scripts, parents, composed members) into the bundle.",
};
var sourceEnvOption = new Option<string?>("--source-environment")
{
Description = "SourceEnvironment value stamped into the bundle manifest. Defaults to 'cli'.",
};
var cmd = new Command("export")
{
Description = "Export a bundle to a file",
};
cmd.Add(outputOption);
cmd.Add(passphraseOption);
cmd.Add(allOption);
cmd.Add(templatesOption);
cmd.Add(sharedScriptsOption);
cmd.Add(externalSystemsOption);
cmd.Add(dbConnectionsOption);
cmd.Add(notificationListsOption);
cmd.Add(smtpConfigsOption);
cmd.Add(apiMethodsOption);
cmd.Add(sitesOption);
cmd.Add(instancesOption);
cmd.Add(includeDepsOption);
cmd.Add(sourceEnvOption);
cmd.SetAction(async (ParseResult result) =>
{
var output = result.GetValue(outputOption)!;
var passphrase = result.GetValue(passphraseOption);
var all = result.GetValue(allOption);
var includeDeps = result.GetValue(includeDepsOption);
var sourceEnv = result.GetValue(sourceEnvOption) ?? "cli";
var payload = new ExportBundleCommand(
All: all,
TemplateNames: result.GetValue(templatesOption),
SharedScriptNames: result.GetValue(sharedScriptsOption),
ExternalSystemNames: result.GetValue(externalSystemsOption),
DatabaseConnectionNames: result.GetValue(dbConnectionsOption),
NotificationListNames: result.GetValue(notificationListsOption),
SmtpConfigurationNames: result.GetValue(smtpConfigsOption),
ApiMethodNames: result.GetValue(apiMethodsOption),
IncludeDependencies: includeDeps,
Passphrase: passphrase,
SourceEnvironment: sourceEnv,
SiteNames: result.GetValue(sitesOption),
InstanceNames: result.GetValue(instancesOption));
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
payload,
timeout: BundleCommandTimeout,
onSuccess: jsonOk =>
{
// CLI-020: previously the JSON envelope parse + property extraction +
// base64 decode all ran unguarded — a server-side bug that omits one of
// the two expected properties, returns a null base64 value, sends invalid
// base64, or returns a malformed JSON envelope would surface as one of
// KeyNotFoundException / InvalidOperationException / FormatException /
// JsonException, i.e. an unhandled stack trace rather than the
// documented "exit 1 with a clean INVALID_RESPONSE error". Wrap the
// envelope parse and the streamed write in a single try/catch matching
// the graceful-degradation theme established by CLI-002 / CLI-003 / CLI-005.
string base64;
int byteCount;
try
{
using var doc = JsonDocument.Parse(jsonOk);
base64 = doc.RootElement.GetProperty("base64Bundle").GetString()!;
byteCount = doc.RootElement.GetProperty("byteCount").GetInt32();
}
catch (Exception ex) when (ex is JsonException
or KeyNotFoundException
or InvalidOperationException)
{
OutputFormatter.WriteError(
$"Server returned a malformed bundle-export response: {ex.Message}",
"INVALID_RESPONSE");
return 1;
}
// CLI-019: stream the base64 → file write so a 100 MB bundle
// doesn't double-buffer through Convert.FromBase64String's
// ~100 MB byte[] on the LOH plus a synchronous File.WriteAllBytes.
// The management envelope's body is still buffered into the
// jsonOk string (wire-format limit), but the decode + write
// are now chunked, so peak working-set drops from
// ~base64+byte[]+envelope to ~base64+small-chunk.
long written;
try
{
written = StreamBase64ToFile(base64, output);
}
catch (FormatException ex)
{
OutputFormatter.WriteError(
$"Server returned invalid base64 in the bundle response: {ex.Message}",
"INVALID_RESPONSE");
return 1;
}
Console.WriteLine($"Wrote {written:N0} bytes to {output} (server reported {byteCount:N0}).");
return 0;
});
});
return cmd;
}
// ====================================================================
// bundle preview
// ====================================================================
private static Command BuildPreview(
Option<string> urlOption, Option<string> formatOption,
Option<string> usernameOption, Option<string> passwordOption)
{
var inputOption = new Option<string>("--input")
{
Description = "Bundle file path (.scadabundle)",
Required = true,
};
var passphraseOption = new Option<string?>("--passphrase")
{
Description = "Passphrase for encrypted bundles.",
};
var cmd = new Command("preview")
{
Description = "Load a bundle and print the diff preview without applying",
};
cmd.Add(inputOption);
cmd.Add(passphraseOption);
cmd.SetAction(async (ParseResult result) =>
{
var input = result.GetValue(inputOption)!;
if (!File.Exists(input))
{
OutputFormatter.WriteError($"Bundle file not found: {input}", "FILE_NOT_FOUND");
return 1;
}
var bytes = await File.ReadAllBytesAsync(input);
var payload = new PreviewBundleCommand(
Base64Bundle: Convert.ToBase64String(bytes),
Passphrase: result.GetValue(passphraseOption));
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
payload,
timeout: BundleCommandTimeout,
onSuccess: jsonOk =>
{
// Surface the required site/connection mappings so an operator
// sees which references need resolving on `bundle import`. The
// full PreviewBundleResult JSON (including the new fields) is
// still printed verbatim afterwards for tooling/jq.
PrintRequiredMappingSummary(jsonOk);
Console.WriteLine(jsonOk);
return 0;
});
});
return cmd;
}
/// <summary>
/// Parses the <see cref="PreviewBundleResult"/> JSON envelope and writes a
/// human-readable summary of the <c>requiredSiteMappings</c> /
/// <c>requiredConnectionMappings</c> arrays to stdout. Best-effort: any parse
/// hiccup is swallowed so the raw JSON (printed by the caller) remains the
/// authoritative output.
/// </summary>
/// <param name="jsonOk">The success-body JSON returned by the management endpoint.</param>
private static void PrintRequiredMappingSummary(string jsonOk)
{
try
{
using var doc = JsonDocument.Parse(jsonOk);
var root = doc.RootElement;
var sites = root.TryGetProperty("requiredSiteMappings", out var s)
&& s.ValueKind == JsonValueKind.Array ? s : default;
var conns = root.TryGetProperty("requiredConnectionMappings", out var c)
&& c.ValueKind == JsonValueKind.Array ? c : default;
var siteCount = sites.ValueKind == JsonValueKind.Array ? sites.GetArrayLength() : 0;
var connCount = conns.ValueKind == JsonValueKind.Array ? conns.GetArrayLength() : 0;
if (siteCount == 0 && connCount == 0) return;
Console.WriteLine("Mappings required before import:");
foreach (var site in sites.EnumerateArray())
{
var src = site.TryGetProperty("sourceSiteIdentifier", out var v) ? v.GetString() : null;
var name = site.TryGetProperty("sourceSiteName", out var n) ? n.GetString() : null;
var auto = site.TryGetProperty("autoMatchTargetIdentifier", out var a)
&& a.ValueKind == JsonValueKind.String ? a.GetString() : null;
var hint = auto is null
? "no auto-match (use --map-site or --create-missing-sites)"
: $"auto-matches '{auto}'";
Console.WriteLine($" site: {src} ({name}) — {hint}");
}
foreach (var conn in conns.EnumerateArray())
{
var src = conn.TryGetProperty("sourceSiteIdentifier", out var v) ? v.GetString() : null;
var cname = conn.TryGetProperty("sourceConnectionName", out var n) ? n.GetString() : null;
var auto = conn.TryGetProperty("autoMatchTargetName", out var a)
&& a.ValueKind == JsonValueKind.String ? a.GetString() : null;
var hint = auto is null
? "no auto-match (use --map-connection or --create-missing-connections)"
: $"auto-matches '{auto}'";
Console.WriteLine($" connection: {src}/{cname} — {hint}");
}
Console.WriteLine();
}
catch (JsonException)
{
// Best-effort: the raw JSON is still printed by the caller.
}
}
// ====================================================================
// bundle import
// ====================================================================
private static Command BuildImport(
Option<string> urlOption, Option<string> formatOption,
Option<string> usernameOption, Option<string> passwordOption)
{
var inputOption = new Option<string>("--input")
{
Description = "Bundle file path (.scadabundle)",
Required = true,
};
var passphraseOption = new Option<string?>("--passphrase")
{
Description = "Passphrase for encrypted bundles.",
};
var onConflictOption = new Option<string>("--on-conflict")
{
Description = "Resolution policy applied to every Modified row: skip, overwrite, or rename. Default: overwrite.",
DefaultValueFactory = _ => "overwrite",
};
// M8 (D3): site/connection name mapping. Repeatable. A token with no
// "=dst" part, or "=" with an empty right-hand side, means CreateNew (the
// destination is created from the bundle payload); otherwise the source
// is mapped to the named existing destination.
var mapSiteOption = new Option<string[]>("--map-site")
{
Description = "Map a source site to a destination: 'srcIdentifier=dstIdentifier'. " +
"'srcIdentifier' or 'srcIdentifier=' (no/empty dst) means create-new. Repeatable.",
AllowMultipleArgumentsPerToken = true,
};
var mapConnectionOption = new Option<string[]>("--map-connection")
{
Description = "Map a source connection to a destination: 'srcSiteIdentifier/srcName=dstName'. " +
"'srcSiteIdentifier/srcName' or '…=' (no/empty dst) means create-new. Repeatable.",
AllowMultipleArgumentsPerToken = true,
};
var createMissingSitesOption = new Option<bool>("--create-missing-sites")
{
Description = "Create any unmapped/unmatched source site from the bundle payload instead of failing.",
};
var createMissingConnectionsOption = new Option<bool>("--create-missing-connections")
{
Description = "Create any unmapped/unmatched source connection from the bundle payload instead of failing.",
};
var cmd = new Command("import")
{
Description = "Load + apply a bundle with a single global conflict policy",
};
cmd.Add(inputOption);
cmd.Add(passphraseOption);
cmd.Add(onConflictOption);
cmd.Add(mapSiteOption);
cmd.Add(mapConnectionOption);
cmd.Add(createMissingSitesOption);
cmd.Add(createMissingConnectionsOption);
cmd.SetAction(async (ParseResult result) =>
{
var input = result.GetValue(inputOption)!;
if (!File.Exists(input))
{
OutputFormatter.WriteError($"Bundle file not found: {input}", "FILE_NOT_FOUND");
return 1;
}
List<SiteMappingSpec> siteMappings;
List<ConnectionMappingSpec> connectionMappings;
try
{
siteMappings = ParseSiteMappings(result.GetValue(mapSiteOption));
connectionMappings = ParseConnectionMappings(result.GetValue(mapConnectionOption));
}
catch (FormatException ex)
{
OutputFormatter.WriteError(ex.Message, "INVALID_ARGUMENT");
return 1;
}
var bytes = await File.ReadAllBytesAsync(input);
var payload = new ImportBundleCommand(
Base64Bundle: Convert.ToBase64String(bytes),
Passphrase: result.GetValue(passphraseOption),
DefaultConflictPolicy: result.GetValue(onConflictOption)!,
SiteMappings: siteMappings.Count == 0 ? null : siteMappings,
ConnectionMappings: connectionMappings.Count == 0 ? null : connectionMappings,
CreateMissingSites: result.GetValue(createMissingSitesOption),
CreateMissingConnections: result.GetValue(createMissingConnectionsOption));
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
payload,
timeout: BundleCommandTimeout,
onSuccess: jsonOk =>
{
Console.WriteLine(jsonOk);
return 0;
});
});
return cmd;
}
// ====================================================================
// Shared HTTP plumbing
// ====================================================================
//
// CLI-017: bundle commands previously routed through a private
// RunBundleCommandAsync that re-implemented URL/credential resolution and
// skipped the IsAuthorizationFailure(...) check that ExecuteCommandAsync
// enforces — a server that signalled FORBIDDEN/UNAUTHORIZED via the error
// code on a non-403 status would exit 1 instead of the documented exit 2.
// The bundle path now delegates to CommandHelpers.ExecuteCommandAsync with
// the longer BundleCommandTimeout and a per-command success handler, so the
// exit-code contract is unified across every command group.
// CLI-019: chunked base64 → file streaming. The management envelope's
// success body is a single buffered JSON string (the wire format does not
// currently support response-body streaming), so we cannot remove the
// ~base64-string + ~envelope-string allocation. What we CAN — and do —
// remove is the intermediate ~bytecount-sized byte[] that
// Convert.FromBase64String allocates plus the synchronous File.WriteAllBytes:
// we slice the base64 string into 4-byte-multiple chunks (4 base64 chars
// decode into exactly 3 bytes, so any multiple of 4 is a clean boundary)
// and decode each chunk into a small rented buffer that we copy into the
// output FileStream. The chunk size is a tradeoff — large enough that the
// per-chunk loop overhead is negligible, small enough that we never put
// anything on the LOH (1 MB is below the 85 KB LOH threshold's larger
// cousin for buffers we don't keep). Returns the total decoded byte count
// for the post-write summary line.
internal const int Base64StreamChunkChars = 1024 * 1024; // 1 MB of base64 chars ≈ 768 KB decoded
/// <summary>
/// Decodes a base64 string into <paramref name="outputPath"/> in chunked fashion to avoid
/// large intermediate allocations. Returns the total number of decoded bytes written.
/// </summary>
/// <param name="base64">The base64-encoded content to decode and write.</param>
/// <param name="outputPath">Destination file path; created or overwritten.</param>
/// <returns>Total number of bytes written to the output file.</returns>
internal static long StreamBase64ToFile(string base64, string outputPath)
{
if (base64 is null) throw new ArgumentNullException(nameof(base64));
if (string.IsNullOrEmpty(outputPath)) throw new ArgumentException("Output path required.", nameof(outputPath));
// Skip any leading whitespace and trailing padding noise. Convert.TryFromBase64Chars
// tolerates internal whitespace, but slicing on arbitrary positions would split a
// run of base64 chars mid-quad — round the chunk to a multiple of 4 so each slice
// is independently decodable.
var chunkChars = Base64StreamChunkChars - (Base64StreamChunkChars % 4);
var totalChars = base64.Length;
var totalWritten = 0L;
using var fileStream = new FileStream(
outputPath, FileMode.Create, FileAccess.Write, FileShare.None,
bufferSize: 81920, useAsync: false);
// 4 base64 chars = 3 bytes, so the decoded buffer is sized accordingly.
var byteBuffer = new byte[(chunkChars / 4) * 3];
for (var offset = 0; offset < totalChars; offset += chunkChars)
{
var take = Math.Min(chunkChars, totalChars - offset);
var slice = base64.AsSpan(offset, take);
// The final slice may be shorter than chunkChars and may carry
// trailing '=' padding; TryFromBase64Chars handles that.
if (!Convert.TryFromBase64Chars(slice, byteBuffer, out var written))
{
throw new FormatException(
$"Bundle response contained invalid base64 at character offset {offset}.");
}
fileStream.Write(byteBuffer, 0, written);
totalWritten += written;
}
return totalWritten;
}
private static Option<IReadOnlyList<string>?> NameListOption(string name, string description)
{
var opt = new Option<IReadOnlyList<string>?>(name)
{
Description = description,
CustomParser = arg =>
ParseNameList(arg.Tokens.Count == 0 ? null : arg.Tokens[0].Value),
};
return opt;
}
/// <summary>
/// Splits a comma-separated CLI value into a trimmed, empty-entry-free name
/// list (the shared shape used by every <c>--templates</c>/<c>--sites</c>/…
/// option). Returns <c>null</c> for a null/blank token so the management
/// command sees "not specified" rather than an empty list. Exposed
/// <c>internal</c> so the flag-parse tests can assert the split semantics
/// without reaching into the per-command local options.
/// </summary>
/// <param name="token">The raw comma-separated option value, or <c>null</c>.</param>
/// <returns>The parsed name array, or <c>null</c> when the token is null/blank.</returns>
internal static IReadOnlyList<string>? ParseNameList(string? token)
{
if (string.IsNullOrWhiteSpace(token)) return null;
return token
.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.ToArray();
}
/// <summary>
/// Parses the repeatable <c>--map-site</c> tokens into
/// <see cref="SiteMappingSpec"/>s. Each token is
/// <c>srcIdentifier=dstIdentifier</c> (map to an existing destination) or
/// <c>srcIdentifier</c> / <c>srcIdentifier=</c> (no/empty right-hand side →
/// CreateNew, surfaced as a null target). Exposed <c>internal</c> so the
/// flag-parse tests can assert the split + CreateNew convention.
/// </summary>
/// <param name="tokens">Raw <c>--map-site</c> token values, or <c>null</c>.</param>
/// <returns>The parsed spec list (possibly empty).</returns>
/// <exception cref="FormatException">A token has an empty source identifier.</exception>
internal static List<SiteMappingSpec> ParseSiteMappings(string[]? tokens)
{
var result = new List<SiteMappingSpec>();
if (tokens is null) return result;
foreach (var token in tokens)
{
if (string.IsNullOrWhiteSpace(token)) continue;
var (left, right) = SplitOnFirst(token, '=');
var src = left.Trim();
if (src.Length == 0)
{
throw new FormatException(
$"Invalid --map-site '{token}': source identifier is required (use 'src=dst' or 'src' for create-new).");
}
// null target == CreateNew, per the documented convention.
result.Add(new SiteMappingSpec(src, string.IsNullOrWhiteSpace(right) ? null : right!.Trim()));
}
return result;
}
/// <summary>
/// Parses the repeatable <c>--map-connection</c> tokens into
/// <see cref="ConnectionMappingSpec"/>s. Each token is
/// <c>srcSiteIdentifier/srcName=dstName</c> (map to an existing destination)
/// or <c>srcSiteIdentifier/srcName</c> / <c>…=</c> (no/empty right-hand side →
/// CreateNew, surfaced as a null target). Exposed <c>internal</c> for the
/// flag-parse tests.
/// </summary>
/// <param name="tokens">Raw <c>--map-connection</c> token values, or <c>null</c>.</param>
/// <returns>The parsed spec list (possibly empty).</returns>
/// <exception cref="FormatException">A token is missing the <c>site/name</c> source shape.</exception>
internal static List<ConnectionMappingSpec> ParseConnectionMappings(string[]? tokens)
{
var result = new List<ConnectionMappingSpec>();
if (tokens is null) return result;
foreach (var token in tokens)
{
if (string.IsNullOrWhiteSpace(token)) continue;
var (lhs, right) = SplitOnFirst(token, '=');
var (site, name) = SplitOnFirst(lhs, '/');
var srcSite = site.Trim();
var srcName = (name ?? string.Empty).Trim();
if (srcSite.Length == 0 || name is null || srcName.Length == 0)
{
throw new FormatException(
$"Invalid --map-connection '{token}': expected 'srcSiteIdentifier/srcName[=dstName]'.");
}
// null target == CreateNew, per the documented convention.
result.Add(new ConnectionMappingSpec(srcSite, srcName, string.IsNullOrWhiteSpace(right) ? null : right!.Trim()));
}
return result;
}
/// <summary>
/// Splits <paramref name="value"/> on the first occurrence of
/// <paramref name="separator"/>. When the separator is absent the right side
/// is <c>null</c> (distinguishing "src" from "src=" — both mean CreateNew here,
/// but callers that need to detect the separator's presence can).
/// </summary>
private static (string Left, string? Right) SplitOnFirst(string value, char separator)
{
var idx = value.IndexOf(separator);
return idx < 0
? (value, null)
: (value[..idx], value[(idx + 1)..]);
}
}