diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs index 4c1f22fc..396e13bc 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs @@ -1,3 +1,4 @@ +using System.IO.Compression; using System.Security.Cryptography; using System.Text.Json; using Microsoft.EntityFrameworkCore; @@ -8,9 +9,10 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; /// /// Pure snapshot composer: reads the current live-edit state from -/// and serialises it into a deterministic byte[] artifact + SHA-256 hex revision hash. Determinism -/// comes from sorting every collection by its natural key before serialising, so two snapshots over -/// the same DB state always produce the same hash regardless of EF row ordering. +/// and serialises it into a deterministic, gzip-compressed byte[] artifact + SHA-256 hex revision hash. +/// Determinism comes from sorting every collection by its natural key before serialising, so two snapshots +/// over the same DB state always produce the same hash regardless of EF row ordering; compression wraps the +/// final bytes and reorders nothing. The hash is taken over the COMPRESSED bytes — see . /// public static class ConfigComposer { @@ -61,9 +63,49 @@ public static class ConfigComposer ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), }; - var blob = JsonSerializer.SerializeToUtf8Bytes(snapshot, JsonOptions); - var hash = Convert.ToHexStringLower(SHA256.HashData(blob)); - return new ConfigArtifact(blob, hash); + return Pack(JsonSerializer.SerializeToUtf8Bytes(snapshot, JsonOptions)); + } + + /// + /// Compress the serialised snapshot and hash the COMPRESSED bytes — the emitted artifact and its + /// revision hash, in one step, so the two can never be derived from different byte sequences. + /// + /// + /// + /// Compression happens before hashing on purpose. The invariant the whole deploy path + /// rests on is RevisionHash == SHA-256(stored/wire bytes): a FetchAndCache node + /// verifies the bytes it fetched against the hash its dispatch already carried, and the LocalDb + /// cache stores those same bytes. Hashing the uncompressed JSON instead would break every one of + /// those checks. Nothing between here and DeploymentArtifact.Open looks inside the blob, + /// which is what makes the change invisible to the cache and the gRPC serve path. + /// + /// + /// gzip with the standard magic 1F 8B and no framing of our own: valid JSON can never + /// start with 0x1F, so the read side sniffs the first two bytes and pre-upgrade + /// uncompressed rows keep parsing unchanged. because the + /// artifact is highly repetitive JSON — the marginal ratio from a slower level is small next to + /// the ~10–20x the first level already delivers. + /// + /// + /// gzip output is not guaranteed byte-stable across .NET releases. Harmless here: the hash is + /// computed from the bytes emitted at compose time and never re-derived from the JSON, so an + /// implementation change costs at most one spurious "changed" revision after a runtime upgrade. + /// Determinism of the snapshot itself is unaffected — compression wraps the final bytes and + /// reorders nothing. + /// + /// + /// The deterministic UTF-8 JSON snapshot. + /// The compressed artifact blob and the SHA-256 hex digest OF THAT BLOB. + internal static ConfigArtifact Pack(byte[] json) + { + using var output = new MemoryStream(json.Length / 8); + using (var gzip = new GZipStream(output, CompressionLevel.Fastest, leaveOpen: true)) + { + gzip.Write(json, 0, json.Length); + } + + var blob = output.ToArray(); + return new ConfigArtifact(blob, Convert.ToHexStringLower(SHA256.HashData(blob))); } /// Returns the SHA-256 hex digest of the supplied artifact bytes (lowercase, no prefix). diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DeploymentArtifact.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DeploymentArtifact.cs index c64a2c69..698dd077 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DeploymentArtifact.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DeploymentArtifact.cs @@ -1,3 +1,5 @@ +using System.Buffers; +using System.IO.Compression; using System.Text.Json; using System.Text.Json.Nodes; using ZB.MOM.WW.OtOpcUa.Commons.Types; @@ -93,13 +95,100 @@ public static class DeploymentArtifact /// every consumer of a single apply reads the same instead of rebuilding /// it per question. Dispose it when the apply completes. /// - /// The deployment artifact bytes. Not copied — the returned artifact reads directly - /// from this array, which must not be mutated for its lifetime (artifact blobs never are: they are - /// deployment-immutable bytes read from SQL, the LocalDb cache, or a config fetch). + /// + /// + /// This is the ONLY place an artifact blob is decompressed. ConfigComposer gzips the + /// serialised snapshot BEFORE hashing it, so the compressed bytes are the artifact everywhere: + /// Deployment.ArtifactBlob, the LocalDb cache chunks, the gRPC fetch stream, and the + /// RevisionHash the fetcher verifies against are all over these exact bytes. Nothing between + /// the composer and here looks inside the blob; every reader comes through this method, so a + /// gzipped artifact and a pre-upgrade uncompressed one are indistinguishable to every consumer. + /// + /// + /// The deployment artifact bytes, gzip-compressed or raw JSON (sniffed by the gzip + /// magic 1F 8B, which valid JSON can never start with). Raw-JSON bytes are not copied — the returned + /// artifact reads directly from this array, which must not be mutated for its lifetime (artifact blobs never + /// are: they are deployment-immutable bytes read from SQL, the LocalDb cache, or a config fetch). A gzipped + /// blob is decompressed into a fresh array the artifact owns. /// The parsed artifact. The caller owns it and must dispose it. - /// The blob is not valid JSON. Unlike the static wrappers — which own - /// the lenient "malformed ⇒ empty result" contract — opening is explicit about unusable bytes. - public static ParsedDeploymentArtifact Open(byte[] blob) => new(JsonDocument.Parse(blob)); + /// The blob is not valid JSON, is not decompressible, or inflates past + /// . Unlike the static wrappers — which own the lenient "malformed ⇒ + /// empty result" contract — opening is explicit about unusable bytes. + public static ParsedDeploymentArtifact Open(byte[] blob) => new(JsonDocument.Parse(Inflate(blob))); + + /// + /// Hard ceiling on the decompressed size of an artifact blob (512 MiB). A corrupt or hostile blob + /// must not be able to OOM the node: gzip over repetitive input reaches ~1000:1, so a few-hundred-KB + /// row could otherwise inflate to hundreds of megabytes. No honest artifact comes close — the real + /// ones are single-digit MB before compression. + /// + /// + /// Mutable and internal purely so tests can lower it: proving the guard fires with the production + /// value would mean crafting and inflating half a gigabyte per test run. Production never writes it. + /// + internal static int MaxDecompressedBytes = 512 * 1024 * 1024; + + /// + /// Return the artifact's JSON bytes, decompressing when the blob carries the gzip magic + /// 1F 8B at offset 0. Raw-JSON blobs (everything written before compression shipped) are + /// returned as-is — a JSON document can never begin with 0x1F, so the sniff is unambiguous + /// and no framing of our own is needed. + /// + /// + /// + /// gzip output is NOT guaranteed byte-stable across .NET releases. That is deliberately harmless + /// here: the revision hash is computed FROM the emitted bytes at compose time and is never + /// re-derived from the JSON, so the worst an implementation change can cost is one spurious + /// "changed" revision on the first compose after a runtime upgrade. + /// + /// + /// Every failure — corrupt stream, truncated stream, over-limit inflation — is reported as + /// so it lands in the exact catch the malformed-JSON path already + /// owns: the static wrappers degrade to empty results, and the apply paths fail the apply and + /// keep last-known-good (#485 "no bytes is no answer"). Unusable bytes are unusable bytes; the + /// reason they are unusable does not change any caller's decision. + /// + /// + /// The stored/wire artifact bytes. + /// The JSON bytes to parse. + /// The gzip stream is corrupt or inflates past . + internal static byte[] Inflate(byte[] blob) + { + if (blob.Length < 2 || blob[0] != 0x1F || blob[1] != 0x8B) return blob; + + var limit = MaxDecompressedBytes; + try + { + using var input = new MemoryStream(blob, writable: false); + using var gzip = new GZipStream(input, CompressionMode.Decompress); + using var output = new MemoryStream((int)Math.Min(blob.Length * 4L, 1 << 20)); + + var buffer = ArrayPool.Shared.Rent(64 * 1024); + try + { + int read; + while ((read = gzip.Read(buffer, 0, buffer.Length)) > 0) + { + if (output.Length + read > limit) + { + throw new JsonException( + $"Deployment artifact decompresses to more than the {limit}-byte limit; refusing to inflate it."); + } + output.Write(buffer, 0, read); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + + return output.ToArray(); + } + catch (Exception ex) when (ex is InvalidDataException or EndOfStreamException or IOException) + { + throw new JsonException("Deployment artifact is not a readable gzip stream.", ex); + } + } /// /// Parse a deployment artifact blob into the list of driver-instance specs to spawn. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ZB.MOM.WW.OtOpcUa.Runtime.csproj b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ZB.MOM.WW.OtOpcUa.Runtime.csproj index 094b2e67..b967ec9f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ZB.MOM.WW.OtOpcUa.Runtime.csproj +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ZB.MOM.WW.OtOpcUa.Runtime.csproj @@ -5,6 +5,12 @@ true + + + + + diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs index ac747ee3..4e431151 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs @@ -1,3 +1,6 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; @@ -86,6 +89,43 @@ public sealed class ConfigComposerTests : ControlPlaneActorTestBase artifact.RevisionHash.ShouldMatch("^[0-9a-f]{64}$"); } + /// + /// Task 8: the emitted blob is gzip AND the revision hash is the SHA-256 of THOSE bytes. The second + /// half is the load-bearing one — a FetchAndCache node verifies the bytes it fetched against + /// the RevisionHash its dispatch carried, so hashing the uncompressed JSON instead would fail + /// every fetch in the fleet while every hash-shape test above stayed green. + /// + [Fact] + public async Task Blob_is_gzip_and_the_revision_hash_is_over_the_compressed_bytes() + { + var f = NewInMemoryDbFactory(); + await using (var db = f.CreateDbContext()) + { + db.ServerClusters.Add(NewCluster("c1")); + await db.SaveChangesAsync(); + } + + var artifact = await ConfigComposer.SnapshotAndFlattenAsync(f.CreateDbContext()); + + artifact.Blob[0].ShouldBe((byte)0x1F); + artifact.Blob[1].ShouldBe((byte)0x8B); + artifact.RevisionHash.ShouldBe(Convert.ToHexStringLower(SHA256.HashData(artifact.Blob))); + // …which is exactly what ConfigComposer.HashOf (the fetcher's derivation) computes over the same bytes. + artifact.RevisionHash.ShouldBe(ConfigComposer.HashOf(artifact.Blob)); + + // Falsifiability control: the hash is NOT over the JSON. Decompress and hash that — it must differ, + // or "hash of the compressed bytes" would be an untested claim on a blob that happened to round-trip. + using var inflated = new MemoryStream(); + await using (var gz = new GZipStream(new MemoryStream(artifact.Blob), CompressionMode.Decompress)) + { + await gz.CopyToAsync(inflated); + } + var json = inflated.ToArray(); + Encoding.UTF8.GetString(json).ShouldStartWith("{"); + Convert.ToHexStringLower(SHA256.HashData(json)).ShouldNotBe(artifact.RevisionHash); + artifact.Blob.Length.ShouldBeLessThan(json.Length); + } + /// /// v3: verifies that serialises a /// 's endpoint into the artifact blob and that diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/ParsedDeploymentArtifactTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/ParsedDeploymentArtifactTests.cs index 11a101a7..8fda1c01 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/ParsedDeploymentArtifactTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/ParsedDeploymentArtifactTests.cs @@ -1,4 +1,6 @@ +using System.IO.Compression; using System.Text; +using System.Text.Json; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Authorization; @@ -340,6 +342,113 @@ public sealed class ParsedDeploymentArtifactTests DeploymentArtifact.ParseComposition(empty, NodeC1).EquipmentNodes.ShouldBeEmpty(); } + [Fact] + public void Open_reads_gzip_compressed_artifacts() + { + // Task 8: ConfigComposer now gzips the artifact BEFORE hashing it, so every blob that reaches a node + // (SQL row, LocalDb cache chunk, gRPC fetch) is gzip. Decompression lives in exactly one place — + // Open() — so this parity assertion covers every static wrapper and every consumer at once. + var plain = Blob; + var gzipped = Gzip(plain); + + gzipped[0].ShouldBe((byte)0x1F); + gzipped[1].ShouldBe((byte)0x8B); + gzipped.Length.ShouldBeLessThan(plain.Length); // the whole point + + using var parsedGzip = DeploymentArtifact.Open(gzipped); + using var parsedPlain = DeploymentArtifact.Open(plain); + + parsedGzip.ResolveClusterScope(NodeC1).ShouldBe(parsedPlain.ResolveClusterScope(NodeC1)); + parsedGzip.ParseDriverInstances().ShouldBe(parsedPlain.ParseDriverInstances()); + parsedGzip.ParseDriverInstances(NodeC1).ShouldBe(parsedPlain.ParseDriverInstances(NodeC1)); + AssertSameAuthority(parsedGzip.ParseDataPlaneAuthority(NodeC1), parsedPlain.ParseDataPlaneAuthority(NodeC1)); + AssertSameComposition(parsedGzip.ParseComposition(), parsedPlain.ParseComposition()); + AssertSameComposition(parsedGzip.ParseComposition(NodeC1), parsedPlain.ParseComposition(NodeC1)); + + // …and through the statics, which is what most call sites still use. + DeploymentArtifact.ParseDriverInstances(gzipped, NodeC1) + .ShouldBe(DeploymentArtifact.ParseDriverInstances(plain, NodeC1)); + AssertSameComposition( + DeploymentArtifact.ParseComposition(gzipped, NodeC1), + DeploymentArtifact.ParseComposition(plain, NodeC1)); + } + + [Fact] + public void Open_still_reads_uncompressed_artifacts() + { + // Backward compatibility: every pre-upgrade Deployment row (and every cached artifact chunk written + // before the upgrade) holds raw JSON. A JSON artifact can never start with 0x1F, so the magic sniff + // is unambiguous and these bytes take the identical path they always did. + using var parsed = DeploymentArtifact.Open(Blob); + + parsed.ParseDriverInstances().Select(s => s.DriverInstanceId).ShouldBe(new[] { "DI-modbus", "DI-s7" }); + parsed.ParseComposition(NodeC1).RawTags.ShouldNotBeEmpty(); + } + + [Fact] + public void Static_wrappers_stay_lenient_on_corrupt_gzip() + { + // Corrupt/truncated gzip must degrade exactly like malformed JSON does — the decompressor's + // InvalidDataException is translated to JsonException so the wrappers' existing lenient catch + // (and every apply path's failure handling) covers it unchanged. + var truncated = Gzip(Blob)[..24]; + + DeploymentArtifact.ParseDriverInstances(truncated).ShouldBeEmpty(); + DeploymentArtifact.ParseDriverInstances(truncated, NodeC1).ShouldBeEmpty(); + DeploymentArtifact.ResolveClusterScope(truncated, NodeC1).Mode.ShouldBe(ClusterFilterMode.None); + DeploymentArtifact.ParseDataPlaneAuthority(truncated, NodeC1).Keys.ShouldBeEmpty(); + DeploymentArtifact.ParseComposition(truncated).EquipmentNodes.ShouldBeEmpty(); + DeploymentArtifact.ParseComposition(truncated, NodeC1).EquipmentNodes.ShouldBeEmpty(); + + // Open() itself is explicit about unusable bytes, same as it is for malformed JSON. + Should.Throw(() => DeploymentArtifact.Open(truncated)); + } + + [Fact] + public void Decompression_size_guard_refuses_bombs() + { + // A hostile/corrupt blob must not be able to OOM the node: gzip of a highly repetitive payload + // reaches ~1000:1, so a few-hundred-KB artifact row could inflate to hundreds of MB. The production + // limit is 512 MB, which no honest artifact approaches — crafting a real 512 MB expansion in a unit + // test would burn seconds and gigabytes, so the limit is an internal seam this test lowers instead. + var original = DeploymentArtifact.MaxDecompressedBytes; + try + { + var payload = new byte[64 * 1024]; // all zeroes: compresses to a few dozen bytes + var bomb = Gzip(payload); + bomb.Length.ShouldBeLessThan(1024); // the small-in / huge-out shape the guard exists for + + DeploymentArtifact.MaxDecompressedBytes = 4096; + + var ex = Should.Throw(() => DeploymentArtifact.Open(bomb)); + ex.Message.ShouldContain("4096"); + + // And the lenient statics swallow it exactly like malformed JSON. + DeploymentArtifact.ParseComposition(bomb).EquipmentNodes.ShouldBeEmpty(); + DeploymentArtifact.ParseDriverInstances(bomb).ShouldBeEmpty(); + + // Falsifiability control: under the production limit these same bytes decompress fine (and then + // fail as JSON, not as a bomb) — so the assertion above is the guard firing, not gzip failing. + DeploymentArtifact.MaxDecompressedBytes = original; + Should.Throw(() => DeploymentArtifact.Open(bomb)) + .Message.ShouldNotContain("refusing to inflate"); + } + finally + { + DeploymentArtifact.MaxDecompressedBytes = original; + } + } + + private static byte[] Gzip(byte[] plain) + { + using var output = new MemoryStream(); + using (var gz = new GZipStream(output, CompressionLevel.Fastest, leaveOpen: true)) + { + gz.Write(plain, 0, plain.Length); + } + return output.ToArray(); + } + private static void AssertSameComposition(AddressSpaceComposition actual, AddressSpaceComposition expected) { actual.UnsAreas.ShouldBe(expected.UnsAreas);