llama-swap should never need to be restarted due to upstream issues

misc/process-cmd-test/main.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/signal"
+	"syscall"
+	"time"
+)
+
+/*
+**
+Test how exec.Cmd.CommandContext behaves under certain conditions:*
+
+  - process is killed externally, what happens with cmd.Wait() *
+    ✔︎ it returns. catches crashes.*
+
+  - process ignores SIGTERM*
+    ✔︎ `kill()` is called after cmd.WaitDelay*
+
+  - this process exits, what happens with children (kill -9 <this process' pid>)*
+    x they stick around. have to be manually killed.*
+
+  - .WithTimeout()'s cancel is called *
+    ✔︎ process is killed after it ignores sigterm, cmd.Wait() catches it.*
+
+  - parent receives SIGINT/SIGTERM, what happens
+    ✔︎ waits for child process to exit, then exits gracefully.
+*/
+func main() {
+
+	// swap between these to use kill -9 <pid> on the cli to sim external crash
+	ctx, cancel := context.WithCancel(context.Background())
+	//ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
+	defer cancel()
+
+	//cmd := exec.CommandContext(ctx, "sleep", "1")
+	cmd := exec.CommandContext(ctx,
+		"../../build/simple-responder_darwin_arm64",
+		"-ignore-sig-term", /* so it doesn't exit on receiving SIGTERM, test cmd.WaitTimeout */
+	)
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	// set a wait delay before signing sig kill
+	cmd.WaitDelay = 500 * time.Millisecond
+	cmd.Cancel = func() error {
+		fmt.Println("✔︎ Cancel() called, sending SIGTERM")
+		cmd.Process.Signal(syscall.SIGTERM)
+		return nil
+	}
+
+	if err := cmd.Start(); err != nil {
+		fmt.Println("Error starting process:", err)
+		return
+	}
+
+	// catch signals. Calls cancel() which will cause cmd.Wait() to return and
+	// this program to eventually exit gracefully.
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		signal := <-sigChan
+		fmt.Printf("✔︎ Received signal: %d, Killing process... with cancel before exiting\n", signal)
+		cancel()
+	}()
+
+	fmt.Printf("✔︎ Parent Pid: %d, Process Pid: %d\n", os.Getpid(), cmd.Process.Pid)
+	fmt.Println("✔︎ Process started, cmd.Wait() ... ")
+	if err := cmd.Wait(); err != nil {
+		fmt.Println("✔︎ cmd.Wait returned, Error:", err)
+	} else {
+		fmt.Println("✔︎ cmd.Wait returned, Process exited on its own")
+	}
+	fmt.Println("✔︎ Child process exited, Done.")
+}

proxy/process.go

@@ -62,10 +62,6 @@ type Process struct {
 	// used to block on multiple start() calls
 	waitStarting sync.WaitGroup
 
-	// for managing shutdown state
-	shutdownCtx    context.Context
-	shutdownCancel context.CancelFunc
-
 	// for managing concurrency limits
 	concurrencyLimitSemaphore chan struct{}
 
@@ -77,7 +73,6 @@ type Process struct {
 }
 
 func NewProcess(ID string, healthCheckTimeout int, config ModelConfig, processLogger *LogMonitor, proxyLogger *LogMonitor) *Process {
-	ctx, cancel := context.WithCancel(context.Background())
 	concurrentLimit := 10
 	if config.ConcurrencyLimit > 0 {
 		concurrentLimit = config.ConcurrencyLimit
@@ -93,8 +88,6 @@ func NewProcess(ID string, healthCheckTimeout int, config ModelConfig, processLo
 		healthCheckTimeout:      healthCheckTimeout,
 		healthCheckLoopInterval: 5 * time.Second, /* default, can not be set by user - used for testing */
 		state:                   StateStopped,
-		shutdownCtx:             ctx,
-		shutdownCancel:          cancel,
 
 		// concurrency limit
 		concurrencyLimitSemaphore: make(chan struct{}, concurrentLimit),
@@ -266,15 +259,18 @@ func (p *Process) start() error {
 	loop:
 		// Ready Check loop
 		for {
+			currentState := p.CurrentState()
+			if currentState != StateStarting {
+				return errors.New("health check interrupted due to shutdown")
+			}
+
 			select {
 			case <-checkDeadline.Done():
 				if curState, err := p.swapState(StateStarting, StateFailed); err != nil {
 					return fmt.Errorf("health check timed out after %vs AND state swap failed: %v, current state: %v", maxDuration.Seconds(), err, curState)
 				} else {
 					return fmt.Errorf("health check timed out after %vs", maxDuration.Seconds())
 				}
-			case <-p.shutdownCtx.Done():
-				return errors.New("health check interrupted due to shutdown")
 			case exitErr := <-p.cmdWaitChan:
 				if exitErr != nil {
 					p.proxyLogger.Warnf("<%s> upstream command exited prematurely with error: %v", p.ID, exitErr)
@@ -392,7 +388,6 @@ func (p *Process) Shutdown() {
 		return
 	}
 
-	p.shutdownCancel()
 	p.stopCommand(p.gracefulStopTimeout)
 
 	// just force it to this state since there is no recovery from shutdown