fix(install): swap waits for the host process to exit before moving the folder (#908)

On QA (76e73e7d deploy) Invoke-OtOpcUaInstallSwap.ps1's first Move-Item of app\ failed "file in use" right after the service reported Stopped: SCM reports Stopped while the host is still exiting (e.g. a log file under InstallRoot still open). - After the stop, wait (bounded, -ProcessExitTimeoutSeconds, default 60) until the service's host process (PID + start time captured before the stop) and every process imaged under InstallRoot have exited; otherwise throw naming them, with nothing moved. - The renames use [IO.Directory]::Move instead of Move-Item. On windev, PowerShell 7's Move-Item falls back to an item-by-item move when the rename fails because a file under the folder is open, and leaves a partial PreviousInstall tree behind (a retry then nests into it). - The first rename is retried for at most -MoveRetrySeconds (default 10) while it fails because something under InstallRoot is open (a backstop for a holder whose image is outside InstallRoot). - -Verify/-Reapply/-Scope semantics unchanged. Tests: dry-run harness gains fake-process-list tests for the wait and the retry plus AST order checks (145 passed); the Windows harness gains three real cases (holder past the budget, holder released 2 s into the wait, holder outside InstallRoot) - 77/0 on pwsh 7 and Windows PowerShell 5.1 on windev; with the wait removed the first two fail. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 08:46:26 -05:00
parent f416b88324
commit a157b27628
6 changed files with 332 additions and 13 deletions
@@ -692,8 +692,125 @@ function Get-OtOpcUaCoverageWalkRoots {
    [pscustomobject]@{ Roots = $outer.ToArray(); Skip = $(if ($Scope -eq 'Outside') { @($root) } else { @() }) }
 }

+function Get-OtOpcUaProcessSnapshot {
+    <#
+    .SYNOPSIS
+        Every process as Id, Name, Path (image path; '' when unreadable), StartTime (UTC or $null). Windows: Win32_Process
+        (sees service processes in session 0 when elevated); elsewhere Get-Process (the dry-run harness injects its own).
+    #>
+    if ($PSVersionTable.PSEdition -eq 'Desktop' -or $IsWindows) {
+        foreach ($p in @(Get-CimInstance -ClassName Win32_Process -ErrorAction Stop)) {
+            [pscustomobject]@{
+                Id = [int]$p.ProcessId; Name = [string]$p.Name; Path = [string]$p.ExecutablePath
+                StartTime = $(if ($p.CreationDate) { ([datetime]$p.CreationDate).ToUniversalTime() } else { $null })
+            }
+        }
+    }
+    else {
+        foreach ($p in @(Get-Process)) {
+            $path = ''; $start = $null
+            try { $path = [string]$p.Path } catch { }
+            try { $start = $p.StartTime.ToUniversalTime() } catch { }
+            [pscustomobject]@{ Id = $p.Id; Name = $p.ProcessName; Path = $path; StartTime = $start }
+        }
+    }
+}
+
+function Find-OtOpcUaInstallHolder {
+    <#
+    .SYNOPSIS
+        The processes that still pin InstallRoot (lmxopcua#908): the service's host process (by the PID captured before the
+        stop; when its start time was captured too, a different process reusing the PID does not count) and any process
+        whose image is under InstallRoot (OtOpcUa.Host, ZB.MOM.WW.OtOpcUa.ScriptWorker in script-worker\). Pure: the
+        snapshot is passed in.
+    #>
+    param(
+        [Parameter(Mandatory)] [AllowEmptyCollection()] [object[]]$Snapshot,
+        [Parameter(Mandatory)] [string]$InstallRoot,
+        [int]$ProcessId = 0,
+        [Nullable[datetime]]$ProcessStartTime = $null
+    )
+    @($Snapshot | Where-Object {
+        $isHost = $ProcessId -gt 0 -and $_.Id -eq $ProcessId -and
+            ($null -eq $ProcessStartTime -or $null -eq $_.StartTime -or [math]::Abs(([datetime]$_.StartTime - [datetime]$ProcessStartTime).TotalSeconds) -lt 1)
+        $isUnder = $_.Path -and (Test-OtOpcUaPathWithin -Path ([string]$_.Path) -Ancestor $InstallRoot)
+        $isHost -or $isUnder
+    })
+}
+
+function Wait-OtOpcUaInstallReleased {
+    <#
+    .SYNOPSIS
+        After the service reports Stopped, waits until no process pins InstallRoot (Find-OtOpcUaInstallHolder), polling
+        every -PollMilliseconds for at most -TimeoutSeconds; throws with the holders' names, PIDs and paths when they are
+        still there (lmxopcua#908: SCM reports Stopped while the host is still exiting, and the folder move fails "in use").
+        Returns the seconds waited. -GetSnapshot replaces the process list (tests).
+    #>
+    param(
+        [Parameter(Mandatory)] [string]$InstallRoot,
+        [int]$ProcessId = 0,
+        [Nullable[datetime]]$ProcessStartTime = $null,
+        [int]$TimeoutSeconds = 60,
+        [int]$PollMilliseconds = 250,
+        [scriptblock]$GetSnapshot = { Get-OtOpcUaProcessSnapshot }
+    )
+    $sw = [System.Diagnostics.Stopwatch]::StartNew()
+    while ($true) {
+        $holders = @(Find-OtOpcUaInstallHolder -Snapshot @(& $GetSnapshot) -InstallRoot $InstallRoot -ProcessId $ProcessId -ProcessStartTime $ProcessStartTime)
+        if ($holders.Count -eq 0) { return $sw.Elapsed.TotalSeconds }
+        if ($sw.Elapsed.TotalSeconds -ge $TimeoutSeconds) {
+            $list = ($holders | ForEach-Object { '{0} (PID {1}, {2})' -f $_.Name, $_.Id, $(if ($_.Path) { $_.Path } else { 'path unreadable' }) }) -join '; '
+            throw ("$InstallRoot is still in use $TimeoutSeconds s after the service stopped (lmxopcua#908): $list. Nothing was moved. " +
+                'Wait for or end those processes, then re-run the swap (or raise -ProcessExitTimeoutSeconds).')
+        }
+        Start-Sleep -Milliseconds $PollMilliseconds
+    }
+}
+
+function Test-OtOpcUaSharingViolation {
+    # True when a folder rename failed because something under it is open (lmxopcua#908, all seen on windev):
+    # [IO.Directory]::Move reports an IOException 0x80070005 "Access to the path '...' is denied." on both PowerShell hosts;
+    # Move-Item reports "The process cannot access the file because it is being used by another process" (IOException,
+    # ERROR_SHARING_VIOLATION 32 / ERROR_LOCK_VIOLATION 33) on PowerShell 7 and IOException COR_E_IO "Access to the path
+    # ... is denied." on 5.1. An ACL denial is an UnauthorizedAccessException and is not matched; an IOException of the
+    # access-denied shape that is a real denial only costs the bounded retry budget. Checks the exception and every inner one.
+    param([Exception]$Exception)
+    for ($e = $Exception; $e; $e = $e.InnerException) {
+        if ($e -is [System.IO.IOException] -and (($e.HResult -band 0xFFFF) -in 32, 33)) { return $true }
+        if ($e.Message -like '*being used by another process*') { return $true }
+        if ($e -is [System.IO.IOException] -and (($e.HResult -band 0xFFFF) -eq 5 -or $e.Message -like 'Access to the path*is denied*')) { return $true }
+    }
+    $false
+}
+
+function Invoke-OtOpcUaMoveWithRetry {
+    <#
+    .SYNOPSIS
+        Runs -Move (the first folder rename of the swap, before anything has moved) and retries it while it fails with a
+        sharing violation, for at most -RetrySeconds (lmxopcua#908 backstop; the process wait is the fix). Any other error,
+        or a sharing violation past the budget, is rethrown. Returns the number of attempts.
+    #>
+    param(
+        [Parameter(Mandatory)] [scriptblock]$Move,
+        [int]$RetrySeconds = 10,
+        [int]$DelayMilliseconds = 500
+    )
+    $sw = [System.Diagnostics.Stopwatch]::StartNew()
+    $attempt = 0
+    while ($true) {
+        $attempt++
+        try { & $Move; return $attempt }
+        catch {
+            if (-not (Test-OtOpcUaSharingViolation -Exception $_.Exception) -or $sw.Elapsed.TotalSeconds -ge $RetrySeconds) { throw }
+            Write-Warning ("move attempt {0} failed (in use), retrying in {1} ms: {2}" -f $attempt, $DelayMilliseconds, $_.Exception.Message)
+            Start-Sleep -Milliseconds $DelayMilliseconds
+        }
+    }
+}
+
 Export-ModuleMember -Function Get-OtOpcUaServiceSid, Get-OtOpcUaPeCacheSddl, Get-OtOpcUaPeEntriesAce,
    Get-OtOpcUaScriptWorkerFolderAce, Get-OtOpcUaInstallRootDenyAce, Resolve-OtOpcUaDenyPaths, Get-ScriptWorkerIsolationPlan,
    Invoke-ScriptWorkerIsolationStep, Get-OtOpcUaRecordedDenyPaths, Find-OtOpcUaDenyCoverageGap, Get-OtOpcUaIsolationSettings, Test-OtOpcUaPathWithin,
    Test-ScriptWorkerIsolationStep, Test-OtOpcUaRecordKeyAcl, Test-OtOpcUaStepInInstallRoot, Select-ScriptWorkerIsolationPlanScope,
-    Get-OtOpcUaCoverageWalkRoots
+    Get-OtOpcUaCoverageWalkRoots, Get-OtOpcUaProcessSnapshot, Find-OtOpcUaInstallHolder, Wait-OtOpcUaInstallReleased,
+    Test-OtOpcUaSharingViolation, Invoke-OtOpcUaMoveWithRetry