mirror of
https://github.com/Cheviiot/Vintner.git
synced 2026-08-04 00:07:24 +00:00
Prevent and recover from wedged Wine-hosted processes
Prompted by a real incident: an MSBuild node-reuse worker (its own /nodeReuse:true default) survived a build getting interrupted, came back deadlocked, and got reused by the next `msbuild` invocation - which then failed with a confusing, unrelated-looking `System.TypeLoadException` on Microsoft.VisualStudio.Telemetry on every call for hours, until the stale process was killed by hand. That's exactly the "unrelated blocker" noted in this repo's own earlier session notes (CLAUDE.md) while debugging a real project's build - it wasn't a missing dependency, it was a corrupted reused process. Two changes: - vintner now forces /nodeReuse:false on every msbuild invocation (unless the caller already passed their own /nodeReuse or /nr switch), so a wedged worker can never poison a later, unrelated build in the first place. Costs each invocation the couple-hundred- ms/node startup time node reuse exists to save. - VINTNER_TIMEOUT (a duration string, e.g. "30m") bounds how long any single tool invocation is allowed to run, for the case something wedges that isn't MSBuild-specific. Every exec.Command site in internal/wrapper now goes through a shared newToolCommand constructor that, when the timeout is set, kills the *whole* process group (not just the immediate `wine` process - a wedged child surviving under it is exactly the scenario this needs to reach) via a context deadline, and reports a clear "timed out after Xm" message (exit 124, matching the timeout(1) convention) instead of a bare "signal: killed". Unset by default - every real build observed stays unbounded, matching Windows' own behavior. Verified end-to-end, not just at the unit level: a real `sleep 30` through the `cmd` native wrapper with VINTNER_TIMEOUT=1s was killed within the deadline and reported the timeout clearly (exit 124); a real `cl` invocation with the same 1s timeout finished normally (0.26s) without being mistaken for a hang.
This commit is contained in:
+45
-31
@@ -84,27 +84,29 @@ func Run(tool string, args []string) int {
|
||||
// read as-is), and add the extra environment MSBuild's own
|
||||
// toolset/SDK-detection props need on top of the generic
|
||||
// INCLUDE/LIB/WINEPATH, plus any global properties a project file
|
||||
// itself could otherwise override (see msbuildGlobalArgs).
|
||||
// itself could otherwise override (see msbuildGlobalArgs) and a
|
||||
// forced /nodeReuse:false (see msbuildNodeReuseArgs).
|
||||
msArgs := append(msbuildGlobalArgs(cfg, rewritten), rewritten...)
|
||||
cmd := exec.Command(wineBin, append([]string{toolExePath}, msArgs...)...)
|
||||
msArgs = append(msbuildNodeReuseArgs(rewritten), msArgs...)
|
||||
tc, cleanup := newToolCommand(wineBin, append([]string{toolExePath}, msArgs...)...)
|
||||
defer cleanup()
|
||||
env := buildEnv(paths)
|
||||
for k, v := range msbuildEnv(cfg, paths) {
|
||||
env = append(env, k+"="+v)
|
||||
}
|
||||
cmd.Env = env
|
||||
cmd.Stdin = os.Stdin
|
||||
setNewProcessGroup(cmd)
|
||||
exitCode = runRawStdout(cmd)
|
||||
tc.Env = env
|
||||
tc.Stdin = os.Stdin
|
||||
exitCode = runRawStdout(tc)
|
||||
default:
|
||||
relay := filepath.Join(paths.BaseUnix, "bin", toolRelayName)
|
||||
if fi, err := os.Stat(relay); err == nil && !fi.IsDir() {
|
||||
exitCode = runViaToolRelay(wineBin, relay, toolExePath, rewritten, paths, s.stdoutFilter, s.stderrFilter)
|
||||
} else {
|
||||
cmd := exec.Command(wineBin, append([]string{toolExePath}, rewritten...)...)
|
||||
cmd.Env = buildEnv(paths)
|
||||
cmd.Stdin = os.Stdin
|
||||
setNewProcessGroup(cmd)
|
||||
exitCode = runFiltered(cmd, s.stdoutFilter, s.stderrFilter)
|
||||
tc, cleanup := newToolCommand(wineBin, append([]string{toolExePath}, rewritten...)...)
|
||||
defer cleanup()
|
||||
tc.Env = buildEnv(paths)
|
||||
tc.Stdin = os.Stdin
|
||||
exitCode = runFiltered(tc, s.stdoutFilter, s.stderrFilter)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,20 +141,20 @@ func runViaToolRelay(wineBin, relayExe, exePath string, args []string, paths *wi
|
||||
defer os.Remove(stderrFifo)
|
||||
|
||||
cmdArgs := append([]string{relayExe, exePath}, args...)
|
||||
cmd := exec.Command(wineBin, cmdArgs...)
|
||||
cmd.Env = append(buildEnv(paths), "MSVCGOWINE_STDOUT="+stdoutFifo, "MSVCGOWINE_STDERR="+stderrFifo)
|
||||
setNewProcessGroup(cmd)
|
||||
tc, cleanup := newToolCommand(wineBin, cmdArgs...)
|
||||
defer cleanup()
|
||||
tc.Env = append(buildEnv(paths), "MSVCGOWINE_STDOUT="+stdoutFifo, "MSVCGOWINE_STDERR="+stderrFifo)
|
||||
if devNull, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0); err == nil {
|
||||
defer devNull.Close()
|
||||
cmd.Stdout = devNull
|
||||
cmd.Stderr = devNull
|
||||
tc.Stdout = devNull
|
||||
tc.Stderr = devNull
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
if err := tc.Start(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "vintner:", err)
|
||||
return 1
|
||||
}
|
||||
stopSignals := forwardSignals(cmd.Process)
|
||||
stopSignals := forwardSignals(tc.Process)
|
||||
defer stopSignals()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
@@ -176,10 +178,14 @@ func runViaToolRelay(wineBin, relayExe, exePath string, args []string, paths *wi
|
||||
pumpLines(f, os.Stderr, stderrF)
|
||||
}()
|
||||
|
||||
err := cmd.Wait()
|
||||
err := tc.Wait()
|
||||
wg.Wait()
|
||||
|
||||
if err != nil {
|
||||
if tc.timedOut() {
|
||||
fmt.Fprintln(os.Stderr, tc.timeoutMessage())
|
||||
return 124
|
||||
}
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
return exitErr.ExitCode()
|
||||
}
|
||||
@@ -195,23 +201,23 @@ func runViaToolRelay(wineBin, relayExe, exePath string, args []string, paths *wi
|
||||
// own copy goroutines - not the caller's terminal or pipe - are exposed to
|
||||
// Wine's background processes holding those descriptors open; see
|
||||
// pipeDrainGrace.
|
||||
func runRawStdout(cmd *exec.Cmd) int {
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
func runRawStdout(tc *toolCommand) int {
|
||||
stdout, err := tc.StdoutPipe()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "vintner:", err)
|
||||
return 1
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
stderr, err := tc.StderrPipe()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "vintner:", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
if err := tc.Start(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "vintner:", err)
|
||||
return 1
|
||||
}
|
||||
stopSignals := forwardSignals(cmd.Process)
|
||||
stopSignals := forwardSignals(tc.Process)
|
||||
defer stopSignals()
|
||||
|
||||
doneOut := make(chan struct{})
|
||||
@@ -219,11 +225,15 @@ func runRawStdout(cmd *exec.Cmd) int {
|
||||
go func() { io.Copy(os.Stdout, stdout); close(doneOut) }()
|
||||
go func() { io.Copy(os.Stderr, stderr); close(doneErr) }()
|
||||
|
||||
err = cmd.Wait()
|
||||
err = tc.Wait()
|
||||
drain(doneOut)
|
||||
drain(doneErr)
|
||||
|
||||
if err != nil {
|
||||
if tc.timedOut() {
|
||||
fmt.Fprintln(os.Stderr, tc.timeoutMessage())
|
||||
return 124
|
||||
}
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
return exitErr.ExitCode()
|
||||
}
|
||||
@@ -271,23 +281,23 @@ func buildEnv(p *wineenv.Paths) []string {
|
||||
|
||||
// runFiltered streams stdout/stderr line by line through the tool's
|
||||
// filters (CR-stripping always applied first), then waits for completion.
|
||||
func runFiltered(cmd *exec.Cmd, stdoutF, stderrF lineFilter) int {
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
func runFiltered(tc *toolCommand, stdoutF, stderrF lineFilter) int {
|
||||
stdout, err := tc.StdoutPipe()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "vintner:", err)
|
||||
return 1
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
stderr, err := tc.StderrPipe()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "vintner:", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
if err := tc.Start(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "vintner:", err)
|
||||
return 1
|
||||
}
|
||||
stopSignals := forwardSignals(cmd.Process)
|
||||
stopSignals := forwardSignals(tc.Process)
|
||||
defer stopSignals()
|
||||
|
||||
doneOut := make(chan struct{})
|
||||
@@ -295,11 +305,15 @@ func runFiltered(cmd *exec.Cmd, stdoutF, stderrF lineFilter) int {
|
||||
go func() { pumpLines(stdout, os.Stdout, stdoutF); close(doneOut) }()
|
||||
go func() { pumpLines(stderr, os.Stderr, stderrF); close(doneErr) }()
|
||||
|
||||
err = cmd.Wait()
|
||||
err = tc.Wait()
|
||||
drain(doneOut)
|
||||
drain(doneErr)
|
||||
|
||||
if err != nil {
|
||||
if tc.timedOut() {
|
||||
fmt.Fprintln(os.Stderr, tc.timeoutMessage())
|
||||
return 124
|
||||
}
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
return exitErr.ExitCode()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user