Prevent and recover from wedged Wine-hosted processes

Prompted by a real incident: an MSBuild node-reuse worker (its own
/nodeReuse:true default) survived a build getting interrupted, came
back deadlocked, and got reused by the next `msbuild` invocation -
which then failed with a confusing, unrelated-looking
`System.TypeLoadException` on Microsoft.VisualStudio.Telemetry on
every call for hours, until the stale process was killed by hand.
That's exactly the "unrelated blocker" noted in this repo's own
earlier session notes (CLAUDE.md) while debugging a real project's
build - it wasn't a missing dependency, it was a corrupted reused
process.

Two changes:

- vintner now forces /nodeReuse:false on every msbuild invocation
  (unless the caller already passed their own /nodeReuse or /nr
  switch), so a wedged worker can never poison a later, unrelated
  build in the first place. Costs each invocation the couple-hundred-
  ms/node startup time node reuse exists to save.

- VINTNER_TIMEOUT (a duration string, e.g. "30m") bounds how long any
  single tool invocation is allowed to run, for the case something
  wedges that isn't MSBuild-specific. Every exec.Command site in
  internal/wrapper now goes through a shared newToolCommand
  constructor that, when the timeout is set, kills the *whole*
  process group (not just the immediate `wine` process - a wedged
  child surviving under it is exactly the scenario this needs to
  reach) via a context deadline, and reports a clear "timed out after
  Xm" message (exit 124, matching the timeout(1) convention) instead
  of a bare "signal: killed". Unset by default - every real build
  observed stays unbounded, matching Windows' own behavior.

Verified end-to-end, not just at the unit level: a real `sleep 30`
through the `cmd` native wrapper with VINTNER_TIMEOUT=1s was killed
within the deadline and reported the timeout clearly (exit 124); a
real `cl` invocation with the same 1s timeout finished normally
(0.26s) without being mistaken for a hang.
This commit is contained in:
Cheviiot
2026-07-25 15:58:22 +10:00
parent bdea270d71
commit f6b9a0811a
7 changed files with 313 additions and 39 deletions
+45 -31
View File
@@ -84,27 +84,29 @@ func Run(tool string, args []string) int {
// read as-is), and add the extra environment MSBuild's own
// toolset/SDK-detection props need on top of the generic
// INCLUDE/LIB/WINEPATH, plus any global properties a project file
// itself could otherwise override (see msbuildGlobalArgs).
// itself could otherwise override (see msbuildGlobalArgs) and a
// forced /nodeReuse:false (see msbuildNodeReuseArgs).
msArgs := append(msbuildGlobalArgs(cfg, rewritten), rewritten...)
cmd := exec.Command(wineBin, append([]string{toolExePath}, msArgs...)...)
msArgs = append(msbuildNodeReuseArgs(rewritten), msArgs...)
tc, cleanup := newToolCommand(wineBin, append([]string{toolExePath}, msArgs...)...)
defer cleanup()
env := buildEnv(paths)
for k, v := range msbuildEnv(cfg, paths) {
env = append(env, k+"="+v)
}
cmd.Env = env
cmd.Stdin = os.Stdin
setNewProcessGroup(cmd)
exitCode = runRawStdout(cmd)
tc.Env = env
tc.Stdin = os.Stdin
exitCode = runRawStdout(tc)
default:
relay := filepath.Join(paths.BaseUnix, "bin", toolRelayName)
if fi, err := os.Stat(relay); err == nil && !fi.IsDir() {
exitCode = runViaToolRelay(wineBin, relay, toolExePath, rewritten, paths, s.stdoutFilter, s.stderrFilter)
} else {
cmd := exec.Command(wineBin, append([]string{toolExePath}, rewritten...)...)
cmd.Env = buildEnv(paths)
cmd.Stdin = os.Stdin
setNewProcessGroup(cmd)
exitCode = runFiltered(cmd, s.stdoutFilter, s.stderrFilter)
tc, cleanup := newToolCommand(wineBin, append([]string{toolExePath}, rewritten...)...)
defer cleanup()
tc.Env = buildEnv(paths)
tc.Stdin = os.Stdin
exitCode = runFiltered(tc, s.stdoutFilter, s.stderrFilter)
}
}
@@ -139,20 +141,20 @@ func runViaToolRelay(wineBin, relayExe, exePath string, args []string, paths *wi
defer os.Remove(stderrFifo)
cmdArgs := append([]string{relayExe, exePath}, args...)
cmd := exec.Command(wineBin, cmdArgs...)
cmd.Env = append(buildEnv(paths), "MSVCGOWINE_STDOUT="+stdoutFifo, "MSVCGOWINE_STDERR="+stderrFifo)
setNewProcessGroup(cmd)
tc, cleanup := newToolCommand(wineBin, cmdArgs...)
defer cleanup()
tc.Env = append(buildEnv(paths), "MSVCGOWINE_STDOUT="+stdoutFifo, "MSVCGOWINE_STDERR="+stderrFifo)
if devNull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0); err == nil {
defer devNull.Close()
cmd.Stdout = devNull
cmd.Stderr = devNull
tc.Stdout = devNull
tc.Stderr = devNull
}
if err := cmd.Start(); err != nil {
if err := tc.Start(); err != nil {
fmt.Fprintln(os.Stderr, "vintner:", err)
return 1
}
stopSignals := forwardSignals(cmd.Process)
stopSignals := forwardSignals(tc.Process)
defer stopSignals()
var wg sync.WaitGroup
@@ -176,10 +178,14 @@ func runViaToolRelay(wineBin, relayExe, exePath string, args []string, paths *wi
pumpLines(f, os.Stderr, stderrF)
}()
err := cmd.Wait()
err := tc.Wait()
wg.Wait()
if err != nil {
if tc.timedOut() {
fmt.Fprintln(os.Stderr, tc.timeoutMessage())
return 124
}
if exitErr, ok := err.(*exec.ExitError); ok {
return exitErr.ExitCode()
}
@@ -195,23 +201,23 @@ func runViaToolRelay(wineBin, relayExe, exePath string, args []string, paths *wi
// own copy goroutines - not the caller's terminal or pipe - are exposed to
// Wine's background processes holding those descriptors open; see
// pipeDrainGrace.
func runRawStdout(cmd *exec.Cmd) int {
stdout, err := cmd.StdoutPipe()
func runRawStdout(tc *toolCommand) int {
stdout, err := tc.StdoutPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "vintner:", err)
return 1
}
stderr, err := cmd.StderrPipe()
stderr, err := tc.StderrPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "vintner:", err)
return 1
}
if err := cmd.Start(); err != nil {
if err := tc.Start(); err != nil {
fmt.Fprintln(os.Stderr, "vintner:", err)
return 1
}
stopSignals := forwardSignals(cmd.Process)
stopSignals := forwardSignals(tc.Process)
defer stopSignals()
doneOut := make(chan struct{})
@@ -219,11 +225,15 @@ func runRawStdout(cmd *exec.Cmd) int {
go func() { io.Copy(os.Stdout, stdout); close(doneOut) }()
go func() { io.Copy(os.Stderr, stderr); close(doneErr) }()
err = cmd.Wait()
err = tc.Wait()
drain(doneOut)
drain(doneErr)
if err != nil {
if tc.timedOut() {
fmt.Fprintln(os.Stderr, tc.timeoutMessage())
return 124
}
if exitErr, ok := err.(*exec.ExitError); ok {
return exitErr.ExitCode()
}
@@ -271,23 +281,23 @@ func buildEnv(p *wineenv.Paths) []string {
// runFiltered streams stdout/stderr line by line through the tool's
// filters (CR-stripping always applied first), then waits for completion.
func runFiltered(cmd *exec.Cmd, stdoutF, stderrF lineFilter) int {
stdout, err := cmd.StdoutPipe()
func runFiltered(tc *toolCommand, stdoutF, stderrF lineFilter) int {
stdout, err := tc.StdoutPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "vintner:", err)
return 1
}
stderr, err := cmd.StderrPipe()
stderr, err := tc.StderrPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "vintner:", err)
return 1
}
if err := cmd.Start(); err != nil {
if err := tc.Start(); err != nil {
fmt.Fprintln(os.Stderr, "vintner:", err)
return 1
}
stopSignals := forwardSignals(cmd.Process)
stopSignals := forwardSignals(tc.Process)
defer stopSignals()
doneOut := make(chan struct{})
@@ -295,11 +305,15 @@ func runFiltered(cmd *exec.Cmd, stdoutF, stderrF lineFilter) int {
go func() { pumpLines(stdout, os.Stdout, stdoutF); close(doneOut) }()
go func() { pumpLines(stderr, os.Stderr, stderrF); close(doneErr) }()
err = cmd.Wait()
err = tc.Wait()
drain(doneOut)
drain(doneErr)
if err != nil {
if tc.timedOut() {
fmt.Fprintln(os.Stderr, tc.timeoutMessage())
return 124
}
if exitErr, ok := err.(*exec.ExitError); ok {
return exitErr.ExitCode()
}