#!/bin/bash
set -euo pipefail

# End-to-end DelmiaNotifier verification against the docker rig, including an ACTIVE central
# crash (docker kill = SIGKILL, so auto-down failover is exercised, not CoordinatedShutdown's
# graceful handover) + rejoin. Requires: seed-delmia-sim.sh + seed-delmia-api.sh already run, and
# DELMIA_API_KEY exported (the token printed by seed-delmia-api.sh). FLOW_ONLY=1 skips A-G (no
# docker kill) and runs the Flow Debugger steps H, H2, I and J alone.
#
# HISTORY (2026-09-02): the first run of this script found that inbound API keys were a PER-NODE
# SQLite store, so the key minted through the active node did not exist on the standby and every
# call after a takeover answered 401 KeyNotFound (deferred-work register row 108; the rig carried
# a sync-inbound-api-keys.sh file-copy workaround). Both rig centrals now run
# ScadaBridge__InboundApi__ApiKeyStore__Provider=SqlServer (one shared zbauth schema in the config
# DB), so the preflight below — which lists the key via EACH node's own management API — is no
# longer a gate that needs a script to satisfy: it is the proof that a key minted on one node is
# visible on the other without any file copy.
#
# The notifier is run as the managed build (dotnet WWNotifier.dll) from a scratch copy so the
# API key is written to THAT copy's appsettings.json (ScadaBridge:ApiKey) and never into the
# repo. BaseUrls lists the two central nodes DIRECTLY (not Traefik) so the notifier's own
# connect-failure failover is what gets exercised.
#
# Expected outcomes:
#   A  baseline (active listed first)            -> YES
#   B  standby listed first (standby relays)     -> YES   (StandbyForwardPeerEndpoints)
#   C  bad key                                   -> NO    (HTTP 401, no failover: 401 is an answer)
#   D  active KILLED, before takeover completes  -> NO or YES — informational only. A NO here is the
#      documented connect-failure-only design (standby answers 503 ACTIVE_NODE_UNREACHABLE).
#   E  active KILLED, after survivor /health/active=200 -> YES (first URL refused -> failover)
#   F  victim restarted and rejoined (memberCount=2) -> YES
#   G  reject path (DownloadPath *.bad)           -> NO + "Simulated reject"
#   H  Flow Debugger live run + replay            -> YES  (flow-debug run-method returns the traced
#      site steps inline; flow-debug replay rebuilds the same chain from the central audit store)
#   H2 Flow Debugger SCRIPT entry (2026-09-08)    -> YES  (flow-debug run-script calls $MACHINE's
#      OutboundDemo Call script directly — no inbound method in front of it — and the reply leads with
#      the operator-attributed RouteCall root, then the site's inline ScriptRun. Skipped with a note
#      when the instance is not seeded.)
#   I  Flow Debugger OUTBOUND calls (2026-09-06)   -> YES  (FlowOutboundDemo — seed-flow-outbound-demo.sh —
#      returns ApiOutbound + DbOutbound + Notification rows inline: the sync ApiCall Delivered, the cached
#      DB write Submitted or — DB reachable, immediate-completion path — already Delivered, the
#      notification Submitted; replay is retried <= 90 s until the write is Delivered and the
#      notification has a terminal NotifyDeliver row). Skipped with a note when the demo is not seeded.
#   J  Flow Debugger STEP LOGS (2026-09-09)      -> YES  (FlowLogDemo — seed-flow-log-demo.sh — logs
#      info/debug/warn, writes a static attribute, then throws: the ScriptRun step comes back Failed with
#      errorDetail naming InvalidOperationException, and >= 3 ScriptTrace/ScriptLog rows carry levels
#      {info,debug,warn}. Replay asserts the same from the central audit store. Skipped with a note when
#      the demo is not seeded.)

: "${DELMIA_API_KEY:?export DELMIA_API_KEY=sbk_... first}"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"; PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
RUN_DIR="${RUN_DIR:-${TMPDIR:-/tmp}/delmia-notifier-run}"
MACHINE="${MACHINE:-DelmiaSim01}"
A_URL=http://localhost:9001; B_URL=http://localhost:9002

echo "== build notifier + stage a runnable copy in $RUN_DIR"
dotnet build "$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.DelmiaNotifier" -c Debug -nologo -v q
rm -rf "$RUN_DIR"; mkdir -p "$RUN_DIR"; cp -R "$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.DelmiaNotifier/bin/Debug/net10.0/." "$RUN_DIR/"

write_cfg() { # $1 = BaseUrls, $2 = ApiKey
  printf '{"ScadaBridge":{"BaseUrls":"%s","TimeoutSeconds":30,"LogPath":"delmia-notifier.log","ApiKey":"%s"}}\n' "$1" "$2" > "$RUN_DIR/appsettings.json"
}
notify() { # $1 = label, $2 = expected YES|NO, rest = extra args; prints stdout + exit code
  local label=$1 expect=$2; shift 2
  local out rc; set +e; out=$(env -u SCADABRIDGE_API_KEY dotnet "$RUN_DIR/WWNotifier.dll" -m "$MACHINE" -d 'C:\recipes\wo1.nc' -w "WO-$label" -p PN-1 -s 0100 -u drill "$@" 2>/dev/null); rc=$?; set -e
  local first; first=$(echo "$out" | head -1)
  printf '%-3s stdout=%-4s exit=%-3s | %s\n' "$label" "$first" "$rc" "$(echo "$out" | sed -n 2p)"
  [ "$first" = "$expect" ] || { echo "   ^^ EXPECTED $expect"; return 1; }
}
active() { curl -sf -o /dev/null "$A_URL/health/active" && echo a || { curl -sf -o /dev/null "$B_URL/health/active" && echo b; }; }
url_of() { [ "$1" = a ] && echo "$A_URL" || echo "$B_URL"; }
container_of() { [ "$1" = a ] && echo scadabridge-central-a || echo scadabridge-central-b; }
other() { [ "$1" = a ] && echo b || echo a; }
wait_for() { # $1 url $2 path $3 seconds
  local i; for i in $(seq 1 "$3"); do curl -sf -o /dev/null "$1$2" && return 0; sleep 1; done; return 1; }

ACT=$(active); STB=$(other "$ACT")
echo "== active=central-$ACT standby=central-$STB"

echo "== preflight/proof: the key minted on the active node must be visible from BOTH nodes (shared store, no file copy)"
CLI="$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI/bin/Debug/net10.0/scadabridge"
KEY_ID="${DELMIA_API_KEY#sbk_}"; KEY_ID="${KEY_ID%%_*}"
for n in a b; do
  if "$CLI" security api-key list --url "$(url_of "$n")" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}"       | grep -q "\"keyId\":\"$KEY_ID\""; then echo "   central-$n: key $KEY_ID present"
  else echo "FAIL: central-$n does not see key $KEY_ID — is ScadaBridge__InboundApi__ApiKeyStore__Provider=SqlServer set on both centrals (docker-compose.yml)?" >&2; exit 1; fi
done

if [ "${FLOW_ONLY:-}" = 1 ]; then
  echo "== FLOW_ONLY=1: skipping A-G (notifier + failover drill); running the Flow Debugger steps H, H2, I and J only"
else
echo "== A baseline (active first)";        write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"; notify A YES
echo "== B standby first (relay)";           write_cfg "$(url_of "$STB"),$(url_of "$ACT")" "$DELMIA_API_KEY"; notify B YES
echo "== C bad key";                         write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "sbk_bogus_bogus"; notify C NO || true
write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"

echo "== D/E KILL the ACTIVE central ($(container_of "$ACT"))"; docker kill "$(container_of "$ACT")" > /dev/null
T0=$(date +%s)
echo "-- D immediate attempt (informational):"; notify D NO || notify D YES || true
echo "-- waiting for survivor $(url_of "$STB")/health/active (auto-down ~25-30 s)..."
wait_for "$(url_of "$STB")" /health/active 90 || { echo "FAIL: survivor never became active"; exit 1; }
echo "   takeover after $(( $(date +%s) - T0 )) s"
echo "== E after takeover (first URL refused -> failover)"; notify E YES

echo "== F restart victim + wait for rejoin"; docker start "$(container_of "$ACT")" > /dev/null
T1=$(date +%s)
wait_for "$(url_of "$ACT")" /health/ready 120 || { echo "FAIL: victim never ready"; exit 1; }
for i in $(seq 1 60); do
  curl -s "$(url_of "$STB")/health/ready" | grep -q '"memberCount":2,"unreachableCount":0' && break; sleep 2
done
echo "   victim ready + memberCount=2 on survivor after $(( $(date +%s) - T1 )) s; victim /health/active=$(curl -s -o /dev/null -w '%{http_code}' "$(url_of "$ACT")/health/active") (503 = rejoined as standby)"
notify F YES

echo "== G reject path"; notify G NO -d 'C:\recipes\x.bad' || true
fi

# H — Flow Debugger (design 2026-09-05). A Live run executes the real method on the active node
# with tracing on, so the site's ScriptRun / AttributeWrite / AttributeWait rows come back inline
# in the same reply; Replay reads the same chain back from the central AuditLog, which the site
# feeds through the 5 s telemetry push — hence the bounded retry. Both must show the full kind set.
echo "== H Flow Debugger live run + replay"
ACT=$(active)
H_ARGS=(--url "$(url_of "$ACT")" --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}" --format json)
h_check() { # stdin = FlowDebugRunResult JSON; prints a one-line verdict, exits 1 when incomplete or kinds missing
  python3 -c '
import json, sys
r = json.load(sys.stdin)
kinds = sorted(json.loads(e["detailsJson"]).get("kind", "?") for e in r.get("events", []))
need = ["AttributeWait", "AttributeWrite", "InboundRequest", "RouteCall", "ScriptRun"]
missing = [k for k in need if k not in kinds]
ok = r.get("completed") is True and not missing
print(("YES" if ok else "NO") + " root=" + str(r.get("rootExecutionId")) + " events=" + str(len(kinds)) + " kinds=" + ",".join(kinds)
      + (" missing=" + ",".join(missing) if missing else "") + (" error=" + str(r.get("errorCode")) if r.get("errorCode") else ""))
sys.exit(0 if ok else 1)'
}
METHOD_ID=$("$CLI" api-method list "${H_ARGS[@]}" | python3 -c 'import json,sys; ms=json.load(sys.stdin); print(next(m["id"] for m in ms if m["name"]=="DelmiaRecipeDownload"))')
H_PARAMS=$(python3 -c 'import json,sys; print(json.dumps({"MachineCode":sys.argv[1],"DownloadPath":"C:\\recipes\\wo1.nc","WorkOrderNumber":"WO-H","PartNumber":"PN-1","JobStepNumber":"0100","Username":"drill"}))' "$MACHINE")
set +e
H_RUN=$("$CLI" flow-debug run-method --method-id "$METHOD_ID" --params "$H_PARAMS" "${H_ARGS[@]}")
H_RC=$?; set -e
[ "$H_RC" -eq 0 ] || { echo "FAIL: flow-debug run-method exited $H_RC: $H_RUN"; exit 1; }
H_LIVE=$(echo "$H_RUN" | h_check) || { echo "FAIL: live run incomplete: $H_LIVE"; exit 1; }
echo "   live:   $H_LIVE"
H_ROOT=$(echo "$H_RUN" | python3 -c 'import json,sys; print(json.load(sys.stdin)["rootExecutionId"])')
H_REPLAY=""
for i in $(seq 1 20); do
  set +e; H_REPLAY=$("$CLI" flow-debug replay --execution-id "$H_ROOT" "${H_ARGS[@]}" | h_check); H_RRC=$?; set -e
  [ "$H_RRC" -eq 0 ] && break
  sleep 2
done
[ "${H_RRC:-1}" -eq 0 ] || { echo "FAIL: replay never showed the full chain after 40 s: $H_REPLAY"; exit 1; }
echo "   replay: $H_REPLAY"
echo "H   live+replay -> YES"

# H2 — Flow Debugger SCRIPT entry (design 2026-09-08): flow-debug run-script invokes $MACHINE's
# OutboundDemo Call script directly through the traced RouteToCall path — no inbound method in front
# of it. The reply's first row is the operator-attributed RouteCall root written centrally, followed
# by the site's inline ScriptRun (and this script's outbound rows). OutboundDemo declares no
# ParameterDefinitions (seed-flow-outbound-demo.sh passes no --parameters; it reads an optional
# WorkOrder and defaults it), so no --params is needed.
echo "== H2 Flow Debugger script entry (direct call)"
H2_INSTANCE_ID=$("$CLI" instance list "${H_ARGS[@]}" | python3 -c 'import json,sys; xs=json.load(sys.stdin); print(next((i["id"] for i in xs if i.get("uniqueName")==sys.argv[1]), ""))' "$MACHINE")
if [ -z "$H2_INSTANCE_ID" ]; then
  echo "H2  script entry -> SKIPPED ($MACHINE not seeded — run docker/seed-delmia-sim.sh)"
else
  set +e; H2_RUN=$("$CLI" flow-debug run-script --instance-id "$H2_INSTANCE_ID" --script OutboundDemo "${H_ARGS[@]}"); H2_RC=$?; set -e
  [ "$H2_RC" -eq 0 ] || { echo "FAIL: flow-debug run-script exited $H2_RC: $H2_RUN"; exit 1; }
  H2_OK=$(echo "$H2_RUN" | python3 -c '
import json, sys
r = json.load(sys.stdin)
kinds = [json.loads(e["detailsJson"]).get("kind") or "?" for e in r.get("events", [])]
ok = r.get("completed") is True and kinds[:1] == ["RouteCall"] and "ScriptRun" in kinds
print(("YES" if ok else "NO") + " root=" + str(r.get("rootExecutionId")) + " kinds=" + ",".join(kinds)
      + (" error=" + str(r.get("errorCode")) if r.get("errorCode") else ""))
sys.exit(0 if ok else 1)') || { echo "FAIL: run-script incomplete: $H2_OK"; exit 1; }
  echo "   live:   $H2_OK"
  echo "H2  script entry (direct call) -> YES"
fi

# I — Flow Debugger outbound calls (plan 2026-09-06). The demo script makes a sync ExternalSystem.Call,
# a Database.CachedWrite and a Notify.Send. Live: the site returns the rows it emitted inline — the
# sync ApiCall is already Delivered, the cached write and the notification are Submitted (pending).
# Replay: the S&F forwarder (site) and the Notification Outbox (central) write the later lifecycle
# rows minutes apart, so replay is retried until both operations have a terminal row.
echo "== I Flow Debugger outbound calls (live + replay)"
I_METHOD_ID=$("$CLI" api-method list "${H_ARGS[@]}" | python3 -c 'import json,sys; ms=json.load(sys.stdin); print(next((m["id"] for m in ms if m["name"]=="FlowOutboundDemo"), ""))')
if [ -z "$I_METHOD_ID" ]; then
  echo "I   outbound live+replay -> SKIPPED (FlowOutboundDemo not seeded — run docker/seed-flow-outbound-demo.sh)"
else
  i_check() { # $1 = live|replay; stdin = FlowDebugRunResult JSON
    python3 -c '
import json, sys
mode = sys.argv[1]
r = json.load(sys.stdin)
rows = [json.loads(e["detailsJson"]) for e in r.get("events", [])]
by = lambda ch: [d for d in rows if d.get("channel") == ch]
api, db, nt = by("ApiOutbound"), by("DbOutbound"), by("Notification")
terminal = {"Delivered", "Failed", "Parked", "Discarded"}
rank = {"Submitted": 0, "Forwarded": 1, "Attempted": 2}
# The operation status is the furthest-along lifecycle row, not the last in list order: a same-tick
# Attempted + Resolve pair (the immediate-completion path) can arrive in either order.
last = lambda ds: (max((d.get("status") for d in ds), key=lambda st: 3 if st in terminal else rank.get(st, -1)) if ds else None)
problems = []
if r.get("completed") is not True: problems.append("incomplete")
if not any(d.get("kind") == "ApiCall" and d.get("status") == "Delivered" for d in api): problems.append("no Delivered ApiCall")
if not db: problems.append("no DbOutbound rows")
if not nt: problems.append("no Notification rows")
if mode == "live":
    # The cached write is Submitted inline when the S&F engine buffers it, but with the DB reachable the
    # site resolves it INSIDE the run on the immediate-completion path (Submit, Attempted, Delivered all
    # inline). Either is a truthful live picture; only a Failed/Parked write is wrong here.
    if last(db) not in ("Submitted", "Delivered"): problems.append("db inline ended " + str(last(db)))
    # The notification is delivered by the central Notification Outbox after the run: always pending inline.
    if last(nt) != "Submitted": problems.append("notify not pending inline")
else:
    if last(db) not in terminal: problems.append("db still " + str(last(db)))
    if last(nt) not in terminal: problems.append("notify still " + str(last(nt)))
    if last(db) != "Delivered": problems.append("db ended " + str(last(db)))
ok = not problems
print(("YES" if ok else "NO") + " root=" + str(r.get("rootExecutionId")) + " events=" + str(len(rows))
      + " api=" + str(last(api)) + " db=" + str(last(db)) + " notify=" + str(last(nt))
      + (" problems=" + ";".join(problems) if problems else "") + (" error=" + str(r.get("errorCode")) if r.get("errorCode") else ""))
sys.exit(0 if ok else 1)' "$1"
  }
  I_PARAMS=$(python3 -c 'import json,sys; print(json.dumps({"MachineCode":sys.argv[1],"WorkOrder":"WO-I"}))' "$MACHINE")
  set +e; I_RUN=$("$CLI" flow-debug run-method --method-id "$I_METHOD_ID" --params "$I_PARAMS" "${H_ARGS[@]}"); I_RC=$?; set -e
  [ "$I_RC" -eq 0 ] || { echo "FAIL: flow-debug run-method exited $I_RC: $I_RUN"; exit 1; }
  I_LIVE=$(echo "$I_RUN" | i_check live) || { echo "FAIL: live run: $I_LIVE"; exit 1; }
  echo "   live:   $I_LIVE"
  I_ROOT=$(echo "$I_RUN" | python3 -c 'import json,sys; print(json.load(sys.stdin)["rootExecutionId"])')
  I_REPLAY=""
  for i in $(seq 1 45); do
    set +e; I_REPLAY=$("$CLI" flow-debug replay --execution-id "$I_ROOT" "${H_ARGS[@]}" | i_check replay); I_RRC=$?; set -e
    [ "$I_RRC" -eq 0 ] && break
    sleep 2
  done
  [ "${I_RRC:-1}" -eq 0 ] || { echo "FAIL: replay never showed both outbound operations terminal after 90 s: $I_REPLAY"; exit 1; }
  echo "   replay: $I_REPLAY"
  echo "I   outbound live+replay -> YES"
fi
# J — Flow Debugger step logs + exception detail (design 2026-09-09). FlowLogDemo logs at three
# levels, writes a static attribute, then throws. Live: the site returns its ScriptRun row (Failed,
# errorDetail = the exception's ToString()) and the ScriptLog rows inline on the same reply. Replay:
# the same rows are read back from the central AuditLog once the 5 s telemetry push has landed.
echo "== J Flow Debugger step logs (live + replay)"
J_INSTANCE_ID=$("$CLI" instance list "${H_ARGS[@]}" | python3 -c 'import json,sys; xs=json.load(sys.stdin); print(next((i["id"] for i in xs if i.get("uniqueName")==sys.argv[1]), ""))' "$MACHINE")
if [ -z "$J_INSTANCE_ID" ]; then
  echo "J   step logs live+replay -> SKIPPED ($MACHINE not seeded — run docker/seed-delmia-sim.sh)"
else
  j_check() { # $1 = live|replay; stdin = FlowDebugRunResult JSON
    python3 -c '
import json, sys
mode = sys.argv[1]
r = json.load(sys.stdin)
rows = []
for e in r.get("events", []):
    d = json.loads(e["detailsJson"])
    d["_extra"] = json.loads(d["extra"]) if d.get("extra") else {}
    rows.append(d)
runs = [d for d in rows if d.get("kind") == "ScriptRun"]
logs = [d for d in rows if d.get("kind") == "ScriptLog"]
levels = sorted({(d["_extra"].get("logLevel") or "?") for d in logs})
problems = []
if not runs:
    problems.append("no ScriptRun row")
else:
    failed = [d for d in runs if d.get("status") == "Failed"]
    if not failed:
        problems.append("ScriptRun not Failed (" + ",".join(str(d.get("status")) for d in runs) + ")")
    elif not any("InvalidOperationException" in (d.get("errorDetail") or "") for d in failed):
        problems.append("no InvalidOperationException in errorDetail")
if len(logs) < 3:
    problems.append("only " + str(len(logs)) + " ScriptLog row(s)")
missing = [lv for lv in ("info", "debug", "warn") if lv not in levels]
if missing:
    problems.append("levels missing " + ",".join(missing))
ok = not problems
print(("YES" if ok else "NO") + " root=" + str(r.get("rootExecutionId")) + " logs=" + str(len(logs))
      + " levels=" + ",".join(levels) + " scriptRun=" + ",".join(str(d.get("status")) for d in runs)
      + (" problems=" + ";".join(problems) if problems else "") + (" error=" + str(r.get("errorCode")) if r.get("errorCode") else ""))
sys.exit(0 if ok else 1)' "$1"
  }
  # The script throws by design, so the run reports Completed=false with CALL_FAILED — that is the
  # expected outcome here, and the CLI still prints the full result. Do not gate on the exit code.
  set +e; J_RUN=$("$CLI" flow-debug run-script --instance-id "$J_INSTANCE_ID" --script FlowLogDemo "${H_ARGS[@]}"); set -e
  case "$J_RUN" in
    *rootExecutionId*) ;;
    *) echo "J   step logs live+replay -> SKIPPED (FlowLogDemo not seeded — run docker/seed-flow-log-demo.sh): $(echo "$J_RUN" | head -2)"; J_RUN=""; ;;
  esac
  if [ -n "$J_RUN" ]; then
    J_LIVE=$(echo "$J_RUN" | j_check live) || { echo "FAIL: live run: $J_LIVE"; exit 1; }
    echo "   live:   $J_LIVE"
    J_ROOT=$(echo "$J_RUN" | python3 -c 'import json,sys; print(json.load(sys.stdin)["rootExecutionId"])')
    J_REPLAY=""
    for i in $(seq 1 20); do
      set +e; J_REPLAY=$("$CLI" flow-debug replay --execution-id "$J_ROOT" "${H_ARGS[@]}" | j_check replay); J_RRC=$?; set -e
      [ "$J_RRC" -eq 0 ] && break
      sleep 2
    done
    [ "${J_RRC:-1}" -eq 0 ] || { echo "FAIL: replay never showed the logs + exception after 40 s: $J_REPLAY"; exit 1; }
    echo "   replay: $J_REPLAY"
    echo "   root:   $J_ROOT  (open /deployment/flow-debugger?executionId=$J_ROOT)"
    echo "J   step logs live+replay -> YES"
  fi
fi
echo "== done. Notifier log: $RUN_DIR/delmia-notifier.log"