feat(langchain/createAgent): implement retry model request hook in middleware

libs/langchain/src/agents/middlewareAgent/ReactAgent.ts

@@ -173,6 +173,13 @@ export class ReactAgent<
        */
       () => any
     ][] = [];
+    const retryModelRequestHookMiddleware: [
+      AgentMiddleware,
+      /**
+       * ToDo: better type to get the state of middleware
+       */
+      () => any
+    ][] = [];
 
     this.#agentNode = new AgentNode({
       model: this.options.model,
@@ -185,6 +192,7 @@ export class ReactAgent<
       shouldReturnDirect,
       signal: this.options.signal,
       modifyModelRequestHookMiddleware,
+      retryModelRequestHookMiddleware,
     });
 
     const middlewareNames = new Set<string>();
@@ -240,6 +248,16 @@ export class ReactAgent<
           }),
         ]);
       }
+
+      if (m.retryModelRequest) {
+        retryModelRequestHookMiddleware.push([
+          m,
+          () => ({
+            ...beforeModelNode?.getState(),
+            ...afterModelNode?.getState(),
+          }),
+        ]);
+      }
     }
 
     /**

libs/langchain/src/agents/middlewareAgent/middleware.ts

@@ -116,6 +116,34 @@ export function createMiddleware<
         : never
     >
   ) => Promise<ModelRequest | void> | ModelRequest | void;
+  /**
+   * The function to handle model invocation errors and optionally retry.
+   *
+   * @param error - The exception that occurred during model invocation
+   * @param request - The original model request that failed
+   * @param state - The current agent state
+   * @param runtime - The runtime context
+   * @param attempt - The current attempt number (1-indexed)
+   * @returns Modified request to retry with, or undefined/null to propagate the error (re-raise)
+   */
+  retryModelRequest?: (
+    error: Error,
+    request: ModelRequest,
+    state: (TSchema extends InteropZodObject
+      ? InferInteropZodInput<TSchema>
+      : {}) &
+      AgentBuiltInState,
+    runtime: Runtime<
+      TContextSchema extends InteropZodObject
+        ? InferInteropZodOutput<TContextSchema>
+        : TContextSchema extends InteropZodDefault<any>
+        ? InferInteropZodOutput<TContextSchema>
+        : TContextSchema extends InteropZodOptional<any>
+        ? Partial<InferInteropZodOutput<TContextSchema>>
+        : never
+    >,
+    attempt: number
+  ) => Promise<ModelRequest | void> | ModelRequest | void;
   /**
    * The function to run before the model call. This function is called before the model is invoked and before the `modifyModelRequest` hook.
    * It allows to modify the state of the agent.
@@ -219,6 +247,33 @@ export function createMiddleware<
       );
   }
 
+  if (config.retryModelRequest) {
+    middleware.retryModelRequest = async (
+      error,
+      request,
+      state,
+      runtime,
+      attempt
+    ) =>
+      Promise.resolve(
+        config.retryModelRequest!(
+          error,
+          request,
+          state,
+          runtime as Runtime<
+            TContextSchema extends InteropZodObject
+              ? InferInteropZodOutput<TContextSchema>
+              : TContextSchema extends InteropZodDefault<any>
+              ? InferInteropZodOutput<TContextSchema>
+              : TContextSchema extends InteropZodOptional<any>
+              ? Partial<InferInteropZodOutput<TContextSchema>>
+              : never
+          >,
+          attempt
+        )
+      );
+  }
+
   if (config.beforeModel) {
     middleware.beforeModel = async (state, runtime) =>
       Promise.resolve(

libs/langchain/src/agents/middlewareAgent/middleware/index.ts

@@ -30,4 +30,5 @@ export {
   modelCallLimitMiddleware,
   type ModelCallLimitMiddlewareConfig,
 } from "./callLimit.js";
+export { modelFallbackMiddleware } from "./modelFallback.js";
 export { type AgentMiddleware } from "../types.js";

libs/langchain/src/agents/middlewareAgent/middleware/modelFallback.ts

@@ -0,0 +1,83 @@
+import type { LanguageModelLike } from "@langchain/core/language_models/base";
+import { initChatModel } from "../../../chat_models/universal.js";
+import type { ModelRequest, AgentMiddleware } from "../types.js";
+import { createMiddleware } from "../middleware.js";
+
+/**
+ * Middleware that provides automatic model fallback on errors.
+ *
+ * This middleware attempts to retry failed model calls with alternative models
+ * in sequence. When a model call fails, it tries the next model in the fallback
+ * list until either a call succeeds or all models have been exhausted.
+ *
+ * @example
+ * ```ts
+ * import { createAgent, modelFallbackMiddleware } from "langchain";
+ *
+ * // Create middleware with fallback models (not including primary)
+ * const fallback = modelFallbackMiddleware({
+ *   "openai:gpt-4o-mini",  // First fallback
+ *   "anthropic:claude-3-5-sonnet-20241022",  // Second fallback
+ * });
+ *
+ * const agent = createAgent({
+ *   model: "openai:gpt-4o",  // Primary model
+ *   middleware: [fallback],
+ *   tools: [],
+ * });
+ *
+ * // If gpt-4o fails, automatically tries gpt-4o-mini, then claude
+ * const result = await agent.invoke({
+ *   messages: [{ role: "user", content: "Hello" }]
+ * });
+ * ```
+ *
+ * @param fallbackModels - The fallback models to try, in order.
+ * @returns A middleware instance that handles model failures with fallbacks
+ */
+export function modelFallbackMiddleware(
+  /**
+   * The fallback models to try, in order.
+   */
+  ...fallbackModels: (string | LanguageModelLike)[]
+): AgentMiddleware {
+  return createMiddleware({
+    name: "modelFallbackMiddleware",
+    retryModelRequest: async (
+      _error,
+      request,
+      _state,
+      _runtime,
+      attempt
+    ): Promise<ModelRequest | undefined> => {
+      /**
+       * attempt 1 = primary model failed, try models[0] (first fallback)
+       */
+      const fallbackIndex = attempt - 1;
+
+      /**
+       * All fallback models exhausted
+       */
+      if (fallbackIndex >= fallbackModels.length) {
+        return undefined;
+      }
+
+      /**
+       * Get or initialize the fallback model
+       */
+      const fallbackModel = fallbackModels[fallbackIndex];
+      const model =
+        typeof fallbackModel === "string"
+          ? await initChatModel(fallbackModel)
+          : fallbackModel;
+
+      /**
+       * Try next fallback model
+       */
+      return {
+        ...request,
+        model,
+      };
+    },
+  });
+}

libs/langchain/src/agents/middlewareAgent/middleware/tests/modelFallback.test.ts

@@ -0,0 +1,97 @@
+import { expect, describe, it, vi } from "vitest";
+import { HumanMessage, AIMessage } from "@langchain/core/messages";
+import { LanguageModelLike } from "@langchain/core/language_models/base";
+
+import { createAgent } from "../../index.js";
+import { modelFallbackMiddleware } from "../modelFallback.js";
+
+function createMockModel(name = "ChatAnthropic", model = "anthropic") {
+  // Mock Anthropic model
+  const invokeCallback = vi
+    .fn()
+    .mockResolvedValue(new AIMessage("Response from model"));
+  return {
+    getName: () => name,
+    bindTools: vi.fn().mockReturnThis(),
+    _streamResponseChunks: vi.fn().mockReturnThis(),
+    bind: vi.fn().mockReturnThis(),
+    invoke: invokeCallback,
+    lc_runnable: true,
+    _modelType: model,
+    _generate: vi.fn(),
+    _llmType: () => model,
+  } as unknown as LanguageModelLike;
+}
+
+describe("modelFallbackMiddleware", () => {
+  it("should retry the model request with the new model", async () => {
+    const model = createMockModel();
+    model.invoke = vi.fn().mockRejectedValue(new Error("Model error"));
+    const retryModel = createMockModel("ChatAnthropic", "anthropic");
+    const agent = createAgent({
+      model,
+      tools: [],
+      middleware: [modelFallbackMiddleware(retryModel)] as const,
+    });
+    await agent.invoke({ messages: [new HumanMessage("Hello, world!")] });
+    expect(model.invoke).toHaveBeenCalledTimes(1);
+    expect(retryModel.invoke).toHaveBeenCalledTimes(1);
+  });
+
+  it("should allow to configure additional models", async () => {
+    const model = createMockModel();
+    model.invoke = vi
+      .fn()
+      .mockRejectedValueOnce(new Error("Model error"))
+      .mockResolvedValueOnce(new AIMessage("Response from model"));
+    const anotherFailingModel = createMockModel();
+    anotherFailingModel.invoke = vi
+      .fn()
+      .mockRejectedValue(new Error("Model error"));
+    const retryModel = createMockModel("ChatAnthropic", "anthropic");
+    const agent = createAgent({
+      model,
+      tools: [],
+      middleware: [
+        modelFallbackMiddleware(
+          anotherFailingModel,
+          anotherFailingModel,
+          anotherFailingModel,
+          retryModel
+        ),
+      ] as const,
+    });
+
+    await agent.invoke({ messages: [new HumanMessage("Hello, world!")] });
+    expect(model.invoke).toHaveBeenCalledTimes(1);
+    expect(anotherFailingModel.invoke).toHaveBeenCalledTimes(3);
+    expect(retryModel.invoke).toHaveBeenCalledTimes(1);
+  });
+
+  it("should throw if list is exhausted", async () => {
+    const model = createMockModel();
+    model.invoke = vi
+      .fn()
+      .mockRejectedValueOnce(new Error("Model error"))
+      .mockResolvedValueOnce(new AIMessage("Response from model"));
+    const anotherFailingModel = createMockModel();
+    anotherFailingModel.invoke = vi
+      .fn()
+      .mockRejectedValue(new Error("Model error"));
+    const agent = createAgent({
+      model,
+      tools: [],
+      middleware: [
+        modelFallbackMiddleware(
+          anotherFailingModel,
+          anotherFailingModel,
+          anotherFailingModel
+        ),
+      ] as const,
+    });
+
+    await expect(
+      agent.invoke({ messages: [new HumanMessage("Hello, world!")] })
+    ).rejects.toThrow("Model error");
+  });
+});