perf(controlplane): gzip artifact blob inside the hashed bytes (~10-20x smaller)
ConfigComposer gzips the serialised snapshot (CompressionLevel.Fastest) immediately after SerializeToUtf8Bytes and BEFORE taking the SHA-256, so the compressed bytes ARE the artifact everywhere: Deployment.ArtifactBlob, the LocalDb cache chunks, the gRPC fetch stream, and the RevisionHash a FetchAndCache node verifies against. The invariant the deploy path rests on — RevisionHash == SHA-256(stored/wire bytes) — is preserved, which is what leaves the fetcher, the cache and the serve path untouched: nothing between the composer and DeploymentArtifact.Open ever looks inside the blob. Read side: Open() sniffs the gzip magic 1F 8B (valid JSON can never start with 0x1F, so no framing of our own is needed) and inflates through a size guard — MaxDecompressedBytes, 512 MiB, internal + mutable so the guard can be exercised without inflating half a gigabyte per test run. Pre-upgrade uncompressed blobs take the identical path they always did. Every decompression failure — corrupt stream, truncated stream, over-limit inflation — is reported as JsonException, the same exception family the malformed-JSON path already owns. So the lenient static wrappers still degrade to empty results and the apply paths still fail the apply and keep last-known-good (#485); no caller's decision changes because the reason the bytes are unusable changed. gzip output is not guaranteed byte-stable across .NET releases. Harmless here: the hash is computed FROM the emitted bytes at compose time and never re-derived from the JSON, so the worst an implementation change costs is one spurious "changed" revision after a runtime upgrade. Snapshot determinism is unaffected — compression wraps the final bytes and reorders nothing. Tests: gzip/uncompressed parity through both the instance parses and the statics, corrupt-gzip leniency through the wrappers, the size guard (with a falsifiability control proving the same bytes inflate fine under the production limit), and composer-side blob-starts-with-1F8B + RevisionHash == SHA256(blob) with a control proving the hash is NOT over the JSON.
This commit is contained in:
@@ -1,3 +1,5 @@
|
||||
using System.Buffers;
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
@@ -93,13 +95,100 @@ public static class DeploymentArtifact
|
||||
/// every consumer of a single apply reads the same <see cref="JsonDocument"/> instead of rebuilding
|
||||
/// it per question. Dispose it when the apply completes.
|
||||
/// </summary>
|
||||
/// <param name="blob">The deployment artifact bytes. Not copied — the returned artifact reads directly
|
||||
/// from this array, which must not be mutated for its lifetime (artifact blobs never are: they are
|
||||
/// deployment-immutable bytes read from SQL, the LocalDb cache, or a config fetch).</param>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>This is the ONLY place an artifact blob is decompressed.</b> <c>ConfigComposer</c> gzips the
|
||||
/// serialised snapshot BEFORE hashing it, so the compressed bytes are the artifact everywhere:
|
||||
/// <c>Deployment.ArtifactBlob</c>, the LocalDb cache chunks, the gRPC fetch stream, and the
|
||||
/// <c>RevisionHash</c> the fetcher verifies against are all over these exact bytes. Nothing between
|
||||
/// the composer and here looks inside the blob; every reader comes through this method, so a
|
||||
/// gzipped artifact and a pre-upgrade uncompressed one are indistinguishable to every consumer.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="blob">The deployment artifact bytes, gzip-compressed or raw JSON (sniffed by the gzip
|
||||
/// magic <c>1F 8B</c>, which valid JSON can never start with). Raw-JSON bytes are not copied — the returned
|
||||
/// artifact reads directly from this array, which must not be mutated for its lifetime (artifact blobs never
|
||||
/// are: they are deployment-immutable bytes read from SQL, the LocalDb cache, or a config fetch). A gzipped
|
||||
/// blob is decompressed into a fresh array the artifact owns.</param>
|
||||
/// <returns>The parsed artifact. The caller owns it and must dispose it.</returns>
|
||||
/// <exception cref="JsonException">The blob is not valid JSON. Unlike the static wrappers — which own
|
||||
/// the lenient "malformed ⇒ empty result" contract — opening is explicit about unusable bytes.</exception>
|
||||
public static ParsedDeploymentArtifact Open(byte[] blob) => new(JsonDocument.Parse(blob));
|
||||
/// <exception cref="JsonException">The blob is not valid JSON, is not decompressible, or inflates past
|
||||
/// <see cref="MaxDecompressedBytes"/>. Unlike the static wrappers — which own the lenient "malformed ⇒
|
||||
/// empty result" contract — opening is explicit about unusable bytes.</exception>
|
||||
public static ParsedDeploymentArtifact Open(byte[] blob) => new(JsonDocument.Parse(Inflate(blob)));
|
||||
|
||||
/// <summary>
|
||||
/// Hard ceiling on the decompressed size of an artifact blob (512 MiB). A corrupt or hostile blob
|
||||
/// must not be able to OOM the node: gzip over repetitive input reaches ~1000:1, so a few-hundred-KB
|
||||
/// row could otherwise inflate to hundreds of megabytes. No honest artifact comes close — the real
|
||||
/// ones are single-digit MB before compression.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Mutable and internal purely so tests can lower it: proving the guard fires with the production
|
||||
/// value would mean crafting and inflating half a gigabyte per test run. Production never writes it.
|
||||
/// </remarks>
|
||||
internal static int MaxDecompressedBytes = 512 * 1024 * 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Return the artifact's JSON bytes, decompressing when the blob carries the gzip magic
|
||||
/// <c>1F 8B</c> at offset 0. Raw-JSON blobs (everything written before compression shipped) are
|
||||
/// returned as-is — a JSON document can never begin with <c>0x1F</c>, so the sniff is unambiguous
|
||||
/// and no framing of our own is needed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// gzip output is NOT guaranteed byte-stable across .NET releases. That is deliberately harmless
|
||||
/// here: the revision hash is computed FROM the emitted bytes at compose time and is never
|
||||
/// re-derived from the JSON, so the worst an implementation change can cost is one spurious
|
||||
/// "changed" revision on the first compose after a runtime upgrade.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Every failure — corrupt stream, truncated stream, over-limit inflation — is reported as
|
||||
/// <see cref="JsonException"/> so it lands in the exact catch the malformed-JSON path already
|
||||
/// owns: the static wrappers degrade to empty results, and the apply paths fail the apply and
|
||||
/// keep last-known-good (#485 "no bytes is no answer"). Unusable bytes are unusable bytes; the
|
||||
/// reason they are unusable does not change any caller's decision.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="blob">The stored/wire artifact bytes.</param>
|
||||
/// <returns>The JSON bytes to parse.</returns>
|
||||
/// <exception cref="JsonException">The gzip stream is corrupt or inflates past <see cref="MaxDecompressedBytes"/>.</exception>
|
||||
internal static byte[] Inflate(byte[] blob)
|
||||
{
|
||||
if (blob.Length < 2 || blob[0] != 0x1F || blob[1] != 0x8B) return blob;
|
||||
|
||||
var limit = MaxDecompressedBytes;
|
||||
try
|
||||
{
|
||||
using var input = new MemoryStream(blob, writable: false);
|
||||
using var gzip = new GZipStream(input, CompressionMode.Decompress);
|
||||
using var output = new MemoryStream((int)Math.Min(blob.Length * 4L, 1 << 20));
|
||||
|
||||
var buffer = ArrayPool<byte>.Shared.Rent(64 * 1024);
|
||||
try
|
||||
{
|
||||
int read;
|
||||
while ((read = gzip.Read(buffer, 0, buffer.Length)) > 0)
|
||||
{
|
||||
if (output.Length + read > limit)
|
||||
{
|
||||
throw new JsonException(
|
||||
$"Deployment artifact decompresses to more than the {limit}-byte limit; refusing to inflate it.");
|
||||
}
|
||||
output.Write(buffer, 0, read);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
|
||||
return output.ToArray();
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or EndOfStreamException or IOException)
|
||||
{
|
||||
throw new JsonException("Deployment artifact is not a readable gzip stream.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parse a deployment artifact blob into the list of driver-instance specs to spawn.
|
||||
|
||||
@@ -5,6 +5,12 @@
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- DeploymentArtifact.MaxDecompressedBytes is internal + mutable so the artifact-decompression
|
||||
guard can be exercised without inflating half a gigabyte per test run. -->
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.OtOpcUa.Runtime.Tests"/>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Akka.Hosting"/>
|
||||
<PackageReference Include="Akka.Cluster.Tools"/>
|
||||
|
||||
Reference in New Issue
Block a user