Fix the StateFailed issue when calling /unload and resulting SIGTERM

proxy/process.go

@@ -30,11 +30,15 @@ const (
 	StateShutdown ProcessState = ProcessState("shutdown")
 )
 
+// Strategies used for controlling the desired stopping behaviour
 type StopStrategy int
 
 const (
 	StopImmediately StopStrategy = iota
 	StopWaitForInflightRequest
+
+	// stop the process and reset its state to pristine
+	StopAndReset
 )
 
 type Process struct {
@@ -72,6 +76,7 @@ type Process struct {
 	gracefulStopTimeout time.Duration
 }
 
+// Create a new Process
 func NewProcess(ID string, healthCheckTimeout int, config ModelConfig, processLogger *LogMonitor, proxyLogger *LogMonitor) *Process {
 	ctx, cancel := context.WithCancel(context.Background())
 	concurrentLimit := 10
@@ -201,6 +206,7 @@ func (p *Process) start() error {
 
 	// Set process state to failed
 	if err != nil {
+		p.proxyLogger.Errorf("<%s> cmd.Start() failed: %s", p.ID, err.Error())
 		if curState, swapErr := p.swapState(StateStarting, StateFailed); swapErr != nil {
 			return fmt.Errorf(
 				"failed to start command and state swap failed. command error: %v, current state: %v, state swap error: %v",

proxy/processgroup.go

@@ -21,6 +21,7 @@ type ProcessGroup struct {
 
 	// map of current processes
 	processes       map[string]*Process
+	processMutex    sync.Mutex
 	lastUsedProcess string
 }
 
@@ -86,17 +87,26 @@ func (pg *ProcessGroup) StopProcesses(strategy StopStrategy) {
 
 	// stop Processes in parallel
 	var wg sync.WaitGroup
-	for _, process := range pg.processes {
+	for modelID, process := range pg.processes {
 		wg.Add(1)
-		go func(process *Process) {
+		go func(modelID string, process *Process) {
 			defer wg.Done()
 			switch strategy {
 			case StopImmediately:
 				process.StopImmediately()
+
+			// stop all processes and replace with a new prestine Process
+			case StopAndReset:
+				process.StopImmediately()
+				modelConfig, actualModelID, _ := pg.config.FindConfig(modelID)
+				pg.processMutex.Lock()
+				pg.proxyLogger.Debugf("<%s> Stopped and replaced with new Process", modelID)
+				pg.processes[actualModelID] = NewProcess(modelID, pg.config.HealthCheckTimeout, modelConfig, pg.upstreamLogger, pg.proxyLogger)
+				pg.processMutex.Unlock()
 			default:
 				process.Stop()
 			}
-		}(process)
+		}(modelID, process)
 	}
 	wg.Wait()
 }

proxy/proxymanager.go

@@ -504,7 +504,7 @@ func (pm *ProxyManager) sendErrorResponse(c *gin.Context, statusCode int, messag
 }
 
 func (pm *ProxyManager) unloadAllModelsHandler(c *gin.Context) {
-	pm.StopProcesses(StopImmediately)
+	pm.StopProcesses(StopAndReset)
 	c.String(http.StatusOK, "OK")
 }