feat(historian): outbox format v2 carries typed values + status; legacy entries still read
Serialize always writes v2 ("HV2\0" | ver 2 | id | tag | kind | payload | status u32 | ts).
TryDeserialize reads v2 by magic, falls back to the legacy exact-length layout (also when the
magic matches but v2 validation fails), else returns false - never throws.
FasterLogHistorizationOutbox: RecoverState and PeekBatchAsync both use TryDeserialize. An
unreadable record is indexed under a synthetic id (FIFO truncation passes it), skipped by peek,
counted once (UnreadableCount), and dropped by peek once it reaches the head so it can never
pin the outbox.
Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
+301
@@ -0,0 +1,301 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Text;
|
||||
using FASTER.core;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Recorder;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests.Recorder;
|
||||
|
||||
/// <summary>
|
||||
/// Outbox format v2: every <see cref="HistorianValueKind"/> + a real status code round-trips;
|
||||
/// pre-upgrade (legacy) entries still read; anything else is reported unreadable (never thrown),
|
||||
/// and the FasterLog outbox skips + counts such a record without letting it pin the head.
|
||||
/// </summary>
|
||||
public sealed class HistorizationOutboxEntrySerializerV2Tests : IDisposable
|
||||
{
|
||||
// Produced ONCE by the pre-v2 (Task-1 era) Serialize for
|
||||
// Id=00000000-0000-0000-0000-000000000001, Tag="t", Value=42.5, Quality=192, ts=2026-01-01T00:00:00Z.
|
||||
// This is what is sitting in a production outbox at upgrade time — never regenerate it.
|
||||
private const string PinnedLegacyHex =
|
||||
"0000000000000000000000000000000101000000740000000000404540C0000000F8B4C848DE48";
|
||||
|
||||
private static readonly DateTime Ts = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
private readonly List<string> _dirs = new();
|
||||
|
||||
public static TheoryData<string> Kinds() => new()
|
||||
{
|
||||
"empty", "double", "nan", "neginf", "int64min", "int64max", "true", "false",
|
||||
"string", "emptystring", "bigstring", "datetime",
|
||||
};
|
||||
|
||||
private static HistorianValue ValueFor(string kind) => kind switch
|
||||
{
|
||||
"empty" => HistorianValue.Empty,
|
||||
"double" => HistorianValue.FromDouble(123.456),
|
||||
"nan" => HistorianValue.FromDouble(double.NaN),
|
||||
"neginf" => HistorianValue.FromDouble(double.NegativeInfinity),
|
||||
"int64min" => HistorianValue.FromInt64(long.MinValue),
|
||||
"int64max" => HistorianValue.FromInt64(long.MaxValue),
|
||||
"true" => HistorianValue.FromBoolean(true),
|
||||
"false" => HistorianValue.FromBoolean(false),
|
||||
"string" => HistorianValue.FromString("héllo wörld 日本"),
|
||||
"emptystring" => HistorianValue.FromString(string.Empty),
|
||||
"bigstring" => HistorianValue.FromString(new string('x', 70 * 1024)),
|
||||
"datetime" => HistorianValue.FromDateTime(new DateTime(2025, 6, 7, 8, 9, 10, DateTimeKind.Utc)),
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(kind)),
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Kinds))]
|
||||
public void Every_kind_round_trips_with_a_bad_status(string kind)
|
||||
{
|
||||
var entry = new HistorizationOutboxEntry(Guid.NewGuid(), "Pumpe_Ω.Temp", ValueFor(kind), 0x80310000u, Ts);
|
||||
|
||||
byte[] bytes = HistorizationOutboxEntrySerializer.Serialize(entry);
|
||||
HistorizationOutboxEntrySerializer.TryDeserialize(bytes, out HistorizationOutboxEntry? round).ShouldBeTrue();
|
||||
|
||||
round.ShouldNotBeNull();
|
||||
round.Id.ShouldBe(entry.Id);
|
||||
round.Tag.ShouldBe(entry.Tag);
|
||||
round.Value.ShouldBe(entry.Value);
|
||||
round.StatusCode.ShouldBe(0x80310000u);
|
||||
round.TimestampUtc.ShouldBe(Ts);
|
||||
round.TimestampUtc.Kind.ShouldBe(DateTimeKind.Utc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void V2_layout_is_exactly_as_specified()
|
||||
{
|
||||
var id = Guid.Parse("00000000-0000-0000-0000-000000000001");
|
||||
byte[] bytes = HistorizationOutboxEntrySerializer.Serialize(
|
||||
new HistorizationOutboxEntry(id, "t", HistorianValue.FromString("ab"), 0x80310000u, Ts));
|
||||
|
||||
var expected = new List<byte>();
|
||||
expected.AddRange("HV2\0"u8.ToArray());
|
||||
expected.Add(2);
|
||||
expected.AddRange(id.ToByteArray());
|
||||
expected.AddRange(Le32(1));
|
||||
expected.AddRange("t"u8.ToArray());
|
||||
expected.Add((byte)HistorianValueKind.String);
|
||||
expected.AddRange(Le32(2));
|
||||
expected.AddRange("ab"u8.ToArray());
|
||||
expected.AddRange(Le32(unchecked((int)0x80310000u)));
|
||||
var ts = new byte[8];
|
||||
BinaryPrimitives.WriteInt64LittleEndian(ts, Ts.ToBinary());
|
||||
expected.AddRange(ts);
|
||||
|
||||
Convert.ToHexString(bytes).ShouldBe(Convert.ToHexString(expected.ToArray()));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Pinned_legacy_bytes_read_as_good_double()
|
||||
{
|
||||
byte[] legacy = Convert.FromHexString(PinnedLegacyHex);
|
||||
|
||||
HistorizationOutboxEntrySerializer.TryDeserialize(legacy, out HistorizationOutboxEntry? entry).ShouldBeTrue();
|
||||
|
||||
entry.ShouldNotBeNull();
|
||||
entry.Id.ShouldBe(Guid.Parse("00000000-0000-0000-0000-000000000001"));
|
||||
entry.Tag.ShouldBe("t");
|
||||
entry.Value.Kind.ShouldBe(HistorianValueKind.Double);
|
||||
entry.Value.Number.ShouldBe(42.5);
|
||||
entry.StatusCode.ShouldBe(0u);
|
||||
entry.TimestampUtc.ShouldBe(Ts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_legacy_record_whose_guid_starts_with_the_v2_magic_still_reads()
|
||||
{
|
||||
// Ambiguity guard (#18): a legacy id can begin with the bytes "HV2\0". The v2 structural
|
||||
// parse fails on it, and the legacy exact-length test must then run before giving up.
|
||||
byte[] legacy = Convert.FromHexString(PinnedLegacyHex);
|
||||
"HV2\0"u8.CopyTo(legacy);
|
||||
legacy[4] = 2;
|
||||
|
||||
HistorizationOutboxEntrySerializer.TryDeserialize(legacy, out HistorizationOutboxEntry? entry).ShouldBeTrue();
|
||||
entry!.Value.Number.ShouldBe(42.5);
|
||||
entry.Tag.ShouldBe("t");
|
||||
}
|
||||
|
||||
public static TheoryData<string> Garbage() => new() { "random3", "ver9", "truncated", "legacylength" };
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Garbage))]
|
||||
public void Unreadable_bytes_return_false_and_never_throw(string shape)
|
||||
{
|
||||
byte[] bytes = shape switch
|
||||
{
|
||||
"random3" => [0x13, 0x37, 0xFF],
|
||||
"ver9" => WithVersion(9),
|
||||
"truncated" => Serialize("tag", HistorianValue.FromString("payload"))[..^5],
|
||||
"legacylength" => [.. Convert.FromHexString(PinnedLegacyHex), 0x00], // 40 != 38 + 1
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(shape)),
|
||||
};
|
||||
|
||||
HistorizationOutboxEntrySerializer.TryDeserialize(bytes, out HistorizationOutboxEntry? entry).ShouldBeFalse();
|
||||
entry.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Corrupt_record_between_two_good_ones_is_skipped_counted_and_does_not_block()
|
||||
{
|
||||
string dir = NewTempDir();
|
||||
var a = Entry("A");
|
||||
var c = Entry("C");
|
||||
{
|
||||
using var o = new FasterLogHistorizationOutbox(dir);
|
||||
await o.AppendAsync(a, TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
AppendRaw(dir, [0xDE, 0xAD, 0xBE]);
|
||||
|
||||
{
|
||||
using var o = new FasterLogHistorizationOutbox(dir);
|
||||
await o.AppendAsync(c, TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
using var outbox = new FasterLogHistorizationOutbox(dir);
|
||||
outbox.UnreadableCount.ShouldBe(1);
|
||||
|
||||
var batch = await outbox.PeekBatchAsync(10, TestContext.Current.CancellationToken);
|
||||
batch.Select(e => e.Id).ShouldBe([a.Id, c.Id]);
|
||||
outbox.UnreadableCount.ShouldBe(1); // re-peeking a counted record does not count it again
|
||||
|
||||
await outbox.RemoveAsync(c.Id, TestContext.Current.CancellationToken); // FIFO ack passes the garbage
|
||||
(await outbox.CountAsync(TestContext.Current.CancellationToken)).ShouldBe(0);
|
||||
(await outbox.PeekBatchAsync(10, TestContext.Current.CancellationToken)).ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Garbage_at_the_head_survives_a_restart_and_does_not_pin_the_head()
|
||||
{
|
||||
// Review amendment #1: RecoverState (ctor) must not throw on the garbage, the good entry behind
|
||||
// it must drain, and the head must advance past the garbage durably.
|
||||
string dir = NewTempDir();
|
||||
AppendRaw(dir, [0x01, 0x02, 0x03, 0x04, 0x05]);
|
||||
var good = Entry("G");
|
||||
{
|
||||
using var o = new FasterLogHistorizationOutbox(dir);
|
||||
o.UnreadableCount.ShouldBe(1);
|
||||
await o.AppendAsync(good, TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
using (var o = new FasterLogHistorizationOutbox(dir)) // restart: ctor scans the garbage
|
||||
{
|
||||
o.UnreadableCount.ShouldBe(1);
|
||||
var batch = await o.PeekBatchAsync(10, TestContext.Current.CancellationToken);
|
||||
batch.Select(e => e.Id).ShouldBe([good.Id]);
|
||||
await o.RemoveAsync(good.Id, TestContext.Current.CancellationToken);
|
||||
(await o.CountAsync(TestContext.Current.CancellationToken)).ShouldBe(0);
|
||||
}
|
||||
|
||||
using var reopened = new FasterLogHistorizationOutbox(dir);
|
||||
reopened.UnreadableCount.ShouldBe(0); // the garbage was truncated away, not re-found
|
||||
(await reopened.CountAsync(TestContext.Current.CancellationToken)).ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Trailing_garbage_with_no_good_entry_behind_it_is_removed_by_peek()
|
||||
{
|
||||
// The pin case: the last good entry is acked (truncating TO the garbage), and nothing readable
|
||||
// follows, so no RemoveAsync will ever pass it. Peek must drop leading unreadable records.
|
||||
string dir = NewTempDir();
|
||||
var a = Entry("A");
|
||||
{
|
||||
using var o = new FasterLogHistorizationOutbox(dir);
|
||||
await o.AppendAsync(a, TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
AppendRaw(dir, [0xAA, 0xBB, 0xCC]);
|
||||
|
||||
using (var o = new FasterLogHistorizationOutbox(dir))
|
||||
{
|
||||
(await o.CountAsync(TestContext.Current.CancellationToken)).ShouldBe(2);
|
||||
(await o.PeekBatchAsync(10, TestContext.Current.CancellationToken)).Select(e => e.Id).ShouldBe([a.Id]);
|
||||
await o.RemoveAsync(a.Id, TestContext.Current.CancellationToken);
|
||||
(await o.CountAsync(TestContext.Current.CancellationToken)).ShouldBe(1); // only the garbage
|
||||
|
||||
(await o.PeekBatchAsync(10, TestContext.Current.CancellationToken)).ShouldBeEmpty();
|
||||
(await o.CountAsync(TestContext.Current.CancellationToken)).ShouldBe(0); // head advanced
|
||||
o.UnreadableCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
using var reopened = new FasterLogHistorizationOutbox(dir);
|
||||
(await reopened.CountAsync(TestContext.Current.CancellationToken)).ShouldBe(0);
|
||||
reopened.UnreadableCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_pinned_legacy_record_in_the_log_drains_after_upgrade()
|
||||
{
|
||||
string dir = NewTempDir();
|
||||
AppendRaw(dir, Convert.FromHexString(PinnedLegacyHex));
|
||||
|
||||
using var o = new FasterLogHistorizationOutbox(dir);
|
||||
o.UnreadableCount.ShouldBe(0);
|
||||
var batch = await o.PeekBatchAsync(10, TestContext.Current.CancellationToken);
|
||||
batch.Count.ShouldBe(1);
|
||||
batch[0].Value.ShouldBe(HistorianValue.FromDouble(42.5));
|
||||
await o.RemoveAsync(batch[0].Id, TestContext.Current.CancellationToken);
|
||||
(await o.CountAsync(TestContext.Current.CancellationToken)).ShouldBe(0);
|
||||
}
|
||||
|
||||
private static HistorizationOutboxEntry Entry(string tag) =>
|
||||
new(Guid.NewGuid(), tag, HistorianValue.FromDouble(1), 0u, Ts);
|
||||
|
||||
private static byte[] Serialize(string tag, HistorianValue v) =>
|
||||
HistorizationOutboxEntrySerializer.Serialize(new HistorizationOutboxEntry(Guid.NewGuid(), tag, v, 0u, Ts));
|
||||
|
||||
private static byte[] WithVersion(byte ver)
|
||||
{
|
||||
byte[] b = Serialize("tag", HistorianValue.FromDouble(1));
|
||||
b[4] = ver;
|
||||
return b;
|
||||
}
|
||||
|
||||
private static byte[] Le32(int v)
|
||||
{
|
||||
var b = new byte[4];
|
||||
BinaryPrimitives.WriteInt32LittleEndian(b, v);
|
||||
return b;
|
||||
}
|
||||
|
||||
// Opens the outbox's FasterLog directly (no outbox) and commits a raw record, as a corrupted or
|
||||
// pre-upgrade record would sit on disk.
|
||||
private static void AppendRaw(string dir, byte[] record)
|
||||
{
|
||||
using var device = new ManagedLocalStorageDevice(Path.Combine(dir, "hlog.log"));
|
||||
using var log = new FasterLog(new FasterLogSettings { LogDevice = device });
|
||||
log.Enqueue(record);
|
||||
log.Commit(spinWait: true);
|
||||
}
|
||||
|
||||
private string NewTempDir()
|
||||
{
|
||||
string dir = Path.Combine(Path.GetTempPath(), "histgw-outbox-v2-" + Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(dir);
|
||||
_dirs.Add(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (string dir in _dirs)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(dir, recursive: true);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user