From dce6f834882746efa66bc2c722d72a2fad4563af Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 6 Jun 2026 16:19:53 -0400 Subject: [PATCH] loader: add populate-equipment (company-shape Equipment overlay) + scope verify-equipment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit populate-equipment loads the Northwind Enterprise/Site/Area/Line/Equipment/Signal shape from company-uns.json as a second Equipment-kind namespace (nw-uns) alongside the galaxy mirror — 3 areas / 8 lines / 40 equipment / 1036 signals. Friendly DisplayName, stable logical-Id NodeId. verify-equipment now scopes to the nw-area-* overlay by default (--all for the whole tree). Verified live on :4840 against OtOpcUa master's Equipment-namespace materialization (structure-only; leaves are BadWaitingForInitialData). clean now drops the overlay too. --- otopcua-uns-loader/README.md | 31 +++++-- otopcua-uns-loader/otopcua_uns.py | 137 ++++++++++++++++++++++++++++-- 2 files changed, 151 insertions(+), 17 deletions(-) diff --git a/otopcua-uns-loader/README.md b/otopcua-uns-loader/README.md index 2d6800c..8f54321 100644 --- a/otopcua-uns-loader/README.md +++ b/otopcua-uns-loader/README.md @@ -109,13 +109,26 @@ Defaults target docker-dev; override via flags or env: - `../galaxy-hierarchy.json` — the source of truth, pulled live from the gateway - `requirements.txt`, `.venv/` -## Scope note — company-UNS shape +## Company-shape overlay (`populate-equipment`) -This tool loads the galaxy in its **native hierarchy** -(`OtOpcUa/TestMachine_NNN/`), which is the only shape that can carry live -Galaxy values: OtOpcUa forbids the `GalaxyMxGateway` driver in an `Equipment` -namespace, so a custom `Enterprise/Site/Area/Line/Equipment` UNS (e.g. the -Northwind model in `../company-uns.json`) must be a separate **Equipment** -namespace fed by an `OpcUaClient` driver + `UnsMappingTable` that remaps this -mirror. That overlay is the designed next layer; `../company-uns.json` already -carries the area/line/equipment → galaxy-ref mapping it needs. +Besides the galaxy-native mirror, the tool can load the **Northwind company +shape** (`filling / line-1 / rinser-01 / speed-rpm`) as a second, **Equipment**-kind +namespace (`nw-uns`, in cluster `MAIN`) from `../company-uns.json`. This needs +OtOpcUa `master` ≥ the Equipment-namespace structure milestone +(`febe462…9a67ebc`), which materialises Equipment `Tag`/`VirtualTag` rows on +deploy and added a **headless deploy** endpoint. + +```bash +./.venv/bin/python otopcua_uns.py populate-equipment # 3 areas / 8 lines / 40 equipment / 1036 signals +curl -s -X POST http://localhost:9200/api/deployments -H 'X-Api-Key: docker-dev-deploy-key' # headless deploy +./.venv/bin/python otopcua_uns.py verify-equipment --expect 1036 # browse the company tree (nw-area-* scope) +``` + +UNS folders carry the friendly **DisplayName** (`filling`); the BrowseName/NodeId +stay the stable logical Id (`nw-area-filling`) — standard OPC UA. **Structure-only:** +the company leaves materialise as `BadWaitingForInitialData` — live **values** in +the company shape are the next OtOpcUa milestone (driver/VirtualTag source), tracked +in `OtOpcUa/docs/plans/2026-06-06-equipment-namespace-materialization-scope.md` (WS-3). +The galaxy-native mirror (`populate`) still carries live values. + +`clean` removes both the mirror tags and the company overlay. diff --git a/otopcua-uns-loader/otopcua_uns.py b/otopcua-uns-loader/otopcua_uns.py index c0d1404..e18ea3e 100644 --- a/otopcua-uns-loader/otopcua_uns.py +++ b/otopcua-uns-loader/otopcua_uns.py @@ -38,11 +38,13 @@ and clean print the reminder and `verify --wait` polls until it lands. Deps: pymssql, asyncua (see requirements.txt; use the bundled .venv). """ import argparse +import hashlib import json import os import re import sys import time +import uuid # ── config (overridable via env / flags) ─────────────────────────────────── DEF_MSSQL = dict( @@ -58,9 +60,25 @@ DEF_GALAXY_JSON = os.environ.get( "OTOPCUA_GALAXY_JSON", os.path.join(os.path.dirname(__file__), "..", "galaxy-hierarchy.json"), ) -ID_PREFIX = "nw-mirror-" # all rows we own carry this TagId prefix +DEF_COMPANY_JSON = os.environ.get( + "OTOPCUA_COMPANY_JSON", + os.path.join(os.path.dirname(__file__), "..", "company-uns.json"), +) +ID_PREFIX = "nw-mirror-" # SystemPlatform galaxy-mirror TagId prefix LOAD_PLAN = os.path.join(os.path.dirname(__file__), "load-plan.json") +# Equipment-overlay (company-shape) object ids — all carry the nw- prefix so +# `clean` can remove them. The Equipment namespace is a SECOND namespace loaded +# alongside the galaxy mirror; its leaves stay BadWaitingForInitialData until the +# value milestone (scope doc WS-3) wires a driver/VirtualTag source. +EQ_CLUSTER = os.environ.get("OTOPCUA_EQ_CLUSTER", "MAIN") +EQ_NS = "nw-uns" +EQ_DRIVER = "nw-uns-modbus" # non-Galaxy FK driver (structure-only; doesn't stream) + +# galaxy dataTypeName / gen_uns dtype → valid OtOpcUa DriverDataType +_DTYPE_FIX = {"Double": "Float64", "Float": "Float32"} +_ACCESS = {"ReadOnly": "0", "Read": "0", "ReadWrite": "1"} + # ── the value signals we mirror, per $TestMachine instance ────────────────── # (galaxy attribute name, OtOpcUa DriverDataType, access '0'=Read/'1'=ReadWrite) SIGNALS = [ @@ -194,13 +212,95 @@ def cmd_populate(args): return 0 +def cmd_populate_equipment(args): + """Load the company-shape Equipment namespace from company-uns.json: a second + (Equipment-kind) namespace alongside the galaxy mirror, with the Northwind + Area/Line/Equipment/Signal tree. Structure-only — leaves materialise as + BadWaitingForInitialData (the value milestone is separate). Idempotent: + drop-and-recreate of the nw- overlay rows.""" + with open(args.company_json) as f: + doc = json.load(f) + u = doc["uns"] + conn, cur = connect(args.mssql) + + # Drop any prior overlay (child rows first), then recreate. + cur.execute("DELETE FROM dbo.Tag WHERE DriverInstanceId=%s", (EQ_DRIVER,)) + cur.execute("DELETE FROM dbo.Equipment WHERE DriverInstanceId=%s", (EQ_DRIVER,)) + cur.execute("DELETE FROM dbo.UnsLine WHERE UnsLineId LIKE 'nw-line-%'") + cur.execute("DELETE FROM dbo.UnsArea WHERE UnsAreaId LIKE 'nw-area-%'") + cur.execute("DELETE FROM dbo.DriverInstance WHERE DriverInstanceId=%s", (EQ_DRIVER,)) + cur.execute("DELETE FROM dbo.Namespace WHERE NamespaceId=%s", (EQ_NS,)) + + cur.execute( + "INSERT INTO dbo.Namespace (NamespaceRowId, NamespaceId, ClusterId, Kind, NamespaceUri, Enabled) " + "VALUES (NEWID(), %s, %s, 'Equipment', %s, 1)", + (EQ_NS, EQ_CLUSTER, doc.get("namespace", {}).get("namespaceUri", "urn:northwind:birmingham:uns"))) + cur.execute( + "INSERT INTO dbo.DriverInstance (DriverInstanceRowId, DriverInstanceId, ClusterId, NamespaceId, " + "Name, DriverType, Enabled, DriverConfig) VALUES (NEWID(), %s, %s, %s, 'Northwind UNS placeholder', 'Modbus', 1, '{}')", + (EQ_DRIVER, EQ_CLUSTER, EQ_NS)) + + for a in u["unsAreas"]: + cur.execute("INSERT INTO dbo.UnsArea (UnsAreaRowId, UnsAreaId, ClusterId, Name) VALUES (NEWID(), %s, %s, %s)", + ("nw-" + a["unsAreaId"], EQ_CLUSTER, a["name"])) + for l in u["unsLines"]: + cur.execute("INSERT INTO dbo.UnsLine (UnsLineRowId, UnsLineId, UnsAreaId, Name) VALUES (NEWID(), %s, %s, %s)", + ("nw-" + l["unsLineId"], "nw-" + l["unsAreaId"], l["name"])) + + eq_n = tag_n = 0 + for e in u["equipment"]: + eq_id = "nw-" + e["equipmentId"] + eq_uuid = str(uuid.uuid5(uuid.NAMESPACE_URL, "otopcua-nw-eq/" + e["equipmentId"])) + cur.execute( + "INSERT INTO dbo.Equipment (EquipmentRowId, EquipmentId, EquipmentUuid, DriverInstanceId, UnsLineId, " + "Name, MachineCode, Manufacturer, Model, Enabled) VALUES (NEWID(), %s, %s, %s, %s, %s, %s, %s, %s, 1)", + (eq_id, eq_uuid, EQ_DRIVER, "nw-" + e["unsLineId"], e["name"], e["machineCode"], + e.get("manufacturer"), e.get("model"))) + eq_n += 1 + for t in e["tags"]: + dtype = _DTYPE_FIX.get(t["dataType"], t["dataType"]) + access = _ACCESS.get(t["accessLevel"], "0") + folder = t.get("folderPath") or None + # Local NodeId == TagConfig.FullName; prefix with nw: so it never collides with the + # galaxy-mirror SystemPlatform NodeIds (which use the bare MXAccess ref). + full = "nw:" + t["source"]["fullTagReference"] + # TagId is capped at 64 chars; a short stable hash keeps it unique. Cleanup is by + # DriverInstanceId (not TagId), so no prefix scan is needed. + tag_id = "nweq-" + hashlib.sha1( + f"{e['equipmentId']}|{folder}|{t['name']}".encode()).hexdigest()[:20] + cfg = json.dumps({"FullName": full, "DataType": dtype}) + cur.execute( + "INSERT INTO dbo.Tag (TagRowId, TagId, DriverInstanceId, EquipmentId, Name, FolderPath, " + "DataType, AccessLevel, WriteIdempotent, TagConfig) VALUES (NEWID(), %s, %s, %s, %s, %s, %s, %s, 0, %s)", + (tag_id, EQ_DRIVER, eq_id, t["name"], folder, dtype, access, cfg)) + tag_n += 1 + + conn.commit() + conn.close() + print(f"populated equipment overlay: namespace {EQ_NS} ({EQ_CLUSTER}), " + f"{len(u['unsAreas'])} areas, {len(u['unsLines'])} lines, {eq_n} equipment, {tag_n} signals") + print() + print(f">>> NEXT: deploy (headless) — curl -s -X POST {args.deploy_url.replace('/deployments','')}/api/deployments " + f"-H 'X-Api-Key: {args.deploy_key}'") + print(">>> then run: otopcua_uns.py verify-equipment") + return 0 + + def cmd_clean(args): conn, cur = connect(args.mssql) cur.execute("DELETE FROM dbo.Tag WHERE TagId LIKE %s", (ID_PREFIX + "%",)) n = cur.rowcount + # Also drop the company-shape Equipment overlay (child rows first). + cur.execute("DELETE FROM dbo.Tag WHERE DriverInstanceId=%s", (EQ_DRIVER,)) + cur.execute("DELETE FROM dbo.Equipment WHERE DriverInstanceId=%s", (EQ_DRIVER,)) + cur.execute("DELETE FROM dbo.UnsLine WHERE UnsLineId LIKE 'nw-line-%'") + cur.execute("DELETE FROM dbo.UnsArea WHERE UnsAreaId LIKE 'nw-area-%'") + cur.execute("DELETE FROM dbo.DriverInstance WHERE DriverInstanceId=%s", (EQ_DRIVER,)) + cur.execute("DELETE FROM dbo.Namespace WHERE NamespaceId=%s", (EQ_NS,)) conn.commit() conn.close() - print(f"removed {n} nw-* mirror tag(s). Deploy again at {args.deploy_url} to drop them from the address space.") + print(f"removed {n} nw-* mirror tag(s) + the {EQ_NS} equipment overlay. " + f"Deploy again at {args.deploy_url} to drop them from the address space.") return 0 @@ -298,11 +398,13 @@ def sample_values(endpoint, n): return [("", str(e), "?")] -def browse_tree(endpoint, max_depth=8): +def browse_tree(endpoint, max_depth=8, top_prefix=None): """Recursively descend the OtOpcUa address space and count leaf variables, returning (folder_count, leaf_count, leaf_paths). A node with no children is a leaf signal — this correctly handles the DEEP Equipment UNS tree (Area/Line/Equipment/[FolderPath]/Signal), - unlike browse_summary which assumes the flat 2-level Galaxy hierarchy.""" + unlike browse_summary which assumes the flat 2-level Galaxy hierarchy. When top_prefix is + set, only top-level OtOpcUa folders whose browse name starts with it are counted (e.g. + 'nw-area-' scopes to the company Equipment overlay, excluding the Galaxy mirror folders).""" import asyncio from asyncua import Client @@ -329,7 +431,16 @@ def browse_tree(endpoint, max_depth=8): for k in await c.nodes.objects.get_children(): if (await k.read_browse_name()).Name != "OtOpcUa": continue - await walk(k, "OtOpcUa", 0, acc) + for top in await k.get_children(): + tn = (await top.read_browse_name()).Name + if top_prefix and not tn.startswith(top_prefix): + continue + if await top.get_children(): + acc["folders"] += 1 + await walk(top, "OtOpcUa/" + tn, 1, acc) + else: + acc["leaves"] += 1 + acc["paths"].append("OtOpcUa/" + tn) return acc["folders"], acc["leaves"], acc["paths"] try: return asyncio.run(run()) @@ -341,8 +452,10 @@ def cmd_verify_equipment(args): """Browse the full UNS tree by friendly Area/Line/Equipment/Signal names and report the leaf signal count. With --expect N, exit non-zero unless exactly N leaf signals are present (the equipment-namespace structure-materialisation check).""" - folders, leaves, paths = browse_tree(args.opcua_endpoint) - print(f"equipment tree : {folders} folder(s), {leaves} leaf signal(s) on {args.opcua_endpoint}") + top_prefix = None if args.all else "nw-area-" + folders, leaves, paths = browse_tree(args.opcua_endpoint, top_prefix=top_prefix) + scope = "whole address space" if args.all else "company overlay (nw-area-*)" + print(f"equipment tree : {folders} folder(s), {leaves} leaf signal(s) on {args.opcua_endpoint} [{scope}]") for p in sorted(paths)[:args.show]: print(f" {p}") if len(paths) > args.show: @@ -363,6 +476,9 @@ def main(argv): p.add_argument("--driver", default=DEF_DRIVER, help="SystemPlatform GalaxyMxGateway driver instance id") p.add_argument("--opcua-endpoint", default=DEF_OPCUA) p.add_argument("--deploy-url", default="http://localhost:9200/deployments") + p.add_argument("--deploy-key", default=os.environ.get("OTOPCUA_DEPLOY_KEY", "docker-dev-deploy-key"), + help="X-Api-Key for the headless POST /api/deployments endpoint") + p.add_argument("--company-json", default=DEF_COMPANY_JSON) p.add_argument("--sql-host", default=DEF_MSSQL["host"]) p.add_argument("--sql-port", type=int, default=DEF_MSSQL["port"]) p.add_argument("--sql-user", default=DEF_MSSQL["user"]) @@ -371,6 +487,8 @@ def main(argv): sub = p.add_subparsers(dest="cmd", required=True) sub.add_parser("generate") sub.add_parser("populate") + sub.add_parser("populate-equipment", + help="load the company-shape Equipment namespace from company-uns.json (structure-only)") sub.add_parser("clean") sub.add_parser("status") vp = sub.add_parser("verify") @@ -380,12 +498,15 @@ def main(argv): help="recursively browse the Equipment UNS tree + count leaf signals") ep.add_argument("--expect", type=int, default=None, help="assert exactly N leaf signals") ep.add_argument("--show", type=int, default=20, help="how many leaf paths to print") + ep.add_argument("--all", action="store_true", + help="count the whole address space (default: only the nw-area-* company overlay)") a = p.parse_args(argv) a.mssql = dict(host=a.sql_host, port=a.sql_port, user=a.sql_user, password=a.sql_password, database=a.sql_db) return { - "generate": cmd_generate, "populate": cmd_populate, "clean": cmd_clean, + "generate": cmd_generate, "populate": cmd_populate, + "populate-equipment": cmd_populate_equipment, "clean": cmd_clean, "status": cmd_status, "verify": cmd_verify, "verify-equipment": cmd_verify_equipment, }[a.cmd](a)