Dev into main

config/config.yaml

@@ -23,7 +23,7 @@ RPM: 10
 #SPARK_URL : "ws://spark-api.xf-yun.com/v2.1/chat"
 
 #### if Anthropic
-#Anthropic_API_KEY: "YOUR_API_KEY"
+#ANTHROPIC_API_KEY: "YOUR_API_KEY"
 
 #### if AZURE, check https://github.com/openai/openai-cookbook/blob/main/examples/azure/chat.ipynb
 #### You can use ENGINE or DEPLOYMENT mode

docs/ROADMAP.md

@@ -30,10 +30,10 @@ To reach version v0.5, approximately 70% of the following tasks need to be compl
    4. Complete the design and implementation of module breakdown
    5. Support various modes of memory: clearly distinguish between long-term and short-term memory
    6. Perfect the test role, and carry out necessary interactions with humans
-   7. Allowing natural communication between roles (expected v0.5.0)
+   7. ~~Allowing natural communication between roles~~ (v0.5.0)
    8. Implement SkillManager and the process of incremental Skill learning (experimentation done with game agents)
    9. Automatically get RPM and configure it by calling the corresponding openai page, so that each key does not need to be manually configured
-   10. IMPORTANT: Support incremental development (expected v0.5.0)
+   10. ~~IMPORTANT: Support incremental development~~ (v0.5.0)
 3. Strategies
    1. Support ReAct strategy (experimentation done with game agents)
    2. Support CoT strategy (experimentation done with game agents)
@@ -45,8 +45,8 @@ To reach version v0.5, approximately 70% of the following tasks need to be compl
    2. Implementation: Knowledge search, supporting 10+ data formats
    3. Implementation: Data EDA (expected v0.6.0)
    4. Implementation: Review
-   5. Implementation: Add Document (expected v0.5.0)
-   6. Implementation: Delete Document (expected v0.5.0)
+   5. ~~Implementation~~: Add Document (v0.5.0)
+   6. ~~Implementation~~: Delete Document (v0.5.0)
    7. Implementation: Self-training
    8. ~~Implementation: DebugError~~ (v0.2.1)
    9. Implementation: Generate reliable unit tests based on YAPI

examples/agent_creator.py

@@ -12,9 +12,8 @@
 from metagpt.roles import Role
 from metagpt.schema import Message
 
-with open(METAGPT_ROOT / "examples/build_customized_agent.py", "r") as f:
-    # use official example script to guide AgentCreator
-    MULTI_ACTION_AGENT_CODE_EXAMPLE = f.read()
+EXAMPLE_CODE_FILE = METAGPT_ROOT / "examples/build_customized_agent.py"
+MULTI_ACTION_AGENT_CODE_EXAMPLE = EXAMPLE_CODE_FILE.read_text()
 
 
 class CreateAgent(Action):
@@ -50,8 +49,8 @@ def parse_code(rsp):
         match = re.search(pattern, rsp, re.DOTALL)
         code_text = match.group(1) if match else ""
         CONFIG.workspace_path.mkdir(parents=True, exist_ok=True)
-        with open(CONFIG.workspace_path / "agent_created_agent.py", "w") as f:
-            f.write(code_text)
+        new_file = CONFIG.workspace_path / "agent_created_agent.py"
+        new_file.write_text(code_text)
         return code_text
 
 

metagpt/actions/action.py

@@ -6,43 +6,39 @@
 @File    : action.py
 """
 
+from __future__ import annotations
+
 from abc import ABC
 from typing import Optional
 
-from tenacity import retry, stop_after_attempt, wait_random_exponential
-
-from metagpt.actions.action_output import ActionOutput
+from metagpt.actions.action_node import ActionNode
 from metagpt.llm import LLM
-from metagpt.logs import logger
-from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess
-from metagpt.utils.common import OutputParser
-from metagpt.utils.utils import general_after_log
+from metagpt.schema import BaseContext
 
 
 class Action(ABC):
+    """Action abstract class, requiring all inheritors to provide a series of standard capabilities"""
+
+    name: str
+    llm: LLM
+    context: dict | BaseContext | str | None
+    prefix: str
+    desc: str
+    node: ActionNode | None
+
     def __init__(self, name: str = "", context=None, llm: LLM = None):
         self.name: str = name
         if llm is None:
             llm = LLM()
         self.llm = llm
         self.context = context
         self.prefix = ""  # aask*时会加上prefix，作为system_message
-        self.profile = ""  # FIXME: USELESS
         self.desc = ""  # for skill manager
-        self.nodes = ...
-
-        # Output, useless
-        # self.content = ""
-        # self.instruct_content = None
-        # self.env = None
+        self.node = None
 
-    # def set_env(self, env):
-    #     self.env = env
-
-    def set_prefix(self, prefix, profile):
+    def set_prefix(self, prefix):
         """Set prefix for later usage"""
         self.prefix = prefix
-        self.profile = profile
         return self
 
     def __str__(self):
@@ -58,33 +54,6 @@ async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> s
         system_msgs.append(self.prefix)
         return await self.llm.aask(prompt, system_msgs)
 
-    @retry(
-        wait=wait_random_exponential(min=1, max=60),
-        stop=stop_after_attempt(6),
-        after=general_after_log(logger),
-    )
-    async def _aask_v1(
-        self,
-        prompt: str,
-        output_class_name: str,
-        output_data_mapping: dict,
-        system_msgs: Optional[list[str]] = None,
-        format="markdown",  # compatible to original format
-    ) -> ActionOutput:
-        content = await self.llm.aask(prompt, system_msgs)
-        logger.debug(f"llm raw output:\n{content}")
-        output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping)
-
-        if format == "json":
-            parsed_data = llm_output_postprecess(output=content, schema=output_class.schema(), req_key="[/CONTENT]")
-
-        else:  # using markdown parser
-            parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping)
-
-        logger.debug(f"parsed_data:\n{parsed_data}")
-        instruct_content = output_class(**parsed_data)
-        return ActionOutput(content, instruct_content)
-
     async def run(self, *args, **kwargs):
         """Run action"""
         raise NotImplementedError("The run method should be implemented in a subclass.")

metagpt/actions/action_node.py

@@ -6,17 +6,15 @@
 @File    : action_node.py
 """
 import json
-import re
-from typing import Any, Dict, List, Optional, Type
+from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar
 
 from pydantic import BaseModel, create_model, root_validator, validator
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
-from metagpt.actions import ActionOutput
 from metagpt.llm import BaseGPTAPI
 from metagpt.logs import logger
-from metagpt.utils.common import OutputParser
-from metagpt.utils.custom_decoder import CustomDecoder
+from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess
+from metagpt.utils.common import OutputParser, general_after_log
 
 CONSTRAINT = """
 - Language: Please use the same language as the user input.
@@ -50,8 +48,12 @@ def dict_to_markdown(d, prefix="-", postfix="\n"):
     return markdown_str
 
 
-class ActionNode:
+T = TypeVar("T")
+
+
+class ActionNode(Generic[T]):
     """ActionNode is a tree of nodes."""
+
     mode: str
 
     # Action Context
@@ -64,14 +66,21 @@ class ActionNode:
     expected_type: Type  # such as str / int / float etc.
     # context: str  # everything in the history.
     instruction: str  # the instructions should be followed.
-    example: Any  # example for In Context-Learning.
+    example: T  # example for In Context-Learning.
 
     # Action Output
     content: str
     instruct_content: BaseModel
 
-    def __init__(self, key: str, expected_type: Type, instruction: str, example: str, content: str = "",
-                 children: dict[str, "ActionNode"] = None):
+    def __init__(
+        self,
+        key: str,
+        expected_type: Type,
+        instruction: str,
+        example: T,
+        content: str = "",
+        children: dict[str, "ActionNode"] = None,
+    ):
         self.key = key
         self.expected_type = expected_type
         self.instruction = instruction
@@ -118,7 +127,7 @@ def get_mapping(self, mode="children") -> Dict[str, Type]:
         return self.get_self_mapping()
 
     @classmethod
-    def create_model_class(cls, class_name: str, mapping: Dict[str, Type]):
+    def create_model_class(cls, class_name: str, mapping: Dict[str, Tuple[Type, Any]]):
         """基于pydantic v1的模型动态生成，用来检验结果类型正确性"""
         new_class = create_model(class_name, **mapping)
 
@@ -140,29 +149,6 @@ def check_missing_fields(values):
         new_class.__root_validator_check_missing_fields = classmethod(check_missing_fields)
         return new_class
 
-    @classmethod
-    def create_model_class_v2(cls, class_name: str, mapping: Dict[str, Type]):
-        """基于pydantic v2的模型动态生成，用来检验结果类型正确性，待验证"""
-        new_class = create_model(class_name, **mapping)
-
-        @model_validator(mode="before")
-        def check_missing_fields(data):
-            required_fields = set(mapping.keys())
-            missing_fields = required_fields - set(data.keys())
-            if missing_fields:
-                raise ValueError(f"Missing fields: {missing_fields}")
-            return data
-
-        @field_validator("*")
-        def check_name(v: Any, field: str) -> Any:
-            if field not in mapping.keys():
-                raise ValueError(f"Unrecognized block: {field}")
-            return v
-
-        new_class.__model_validator_check_missing_fields = classmethod(check_missing_fields)
-        new_class.__field_validator_check_name = classmethod(check_name)
-        return new_class
-
     def create_children_class(self):
         """使用object内有的字段直接生成model_class"""
         class_name = f"{self.key}_AN"
@@ -237,43 +223,40 @@ def compile(self, context, to="json", mode="children", template=SIMPLE_TEMPLATE)
         """
 
         # FIXME: json instruction会带来格式问题，如："Project name": "web_2048  # 项目名称使用下划线",
+        # compile example暂时不支持markdown
         self.instruction = self.compile_instruction(to="markdown", mode=mode)
         self.example = self.compile_example(to=to, tag="CONTENT", mode=mode)
         prompt = template.format(
             context=context, example=self.example, instruction=self.instruction, constraint=CONSTRAINT
         )
         return prompt
 
-    @retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(6))
+    @retry(
+        wait=wait_random_exponential(min=1, max=20),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
     async def _aask_v1(
         self,
         prompt: str,
         output_class_name: str,
         output_data_mapping: dict,
         system_msgs: Optional[list[str]] = None,
-        format="markdown",  # compatible to original format
-    ) -> ActionOutput:
+        schema="markdown",  # compatible to original format
+    ) -> (str, BaseModel):
+        """Use ActionOutput to wrap the output of aask"""
         content = await self.llm.aask(prompt, system_msgs)
-        logger.debug(content)
-        output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping)
-
-        if format == "json":
-            pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]"
-            matches = re.findall(pattern, content, re.DOTALL)
-
-            for match in matches:
-                if match:
-                    content = match
-                    break
-
-            parsed_data = CustomDecoder(strict=False).decode(content)
+        logger.debug(f"llm raw output:\n{content}")
+        output_class = self.create_model_class(output_class_name, output_data_mapping)
 
+        if schema == "json":
+            parsed_data = llm_output_postprecess(output=content, schema=output_class.schema(), req_key="[/CONTENT]")
         else:  # using markdown parser
             parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping)
 
-        logger.debug(parsed_data)
+        logger.debug(f"parsed_data:\n{parsed_data}")
         instruct_content = output_class(**parsed_data)
-        return ActionOutput(content, instruct_content)
+        return content, instruct_content
 
     def get(self, key):
         return self.instruct_content.dict()[key]
@@ -294,9 +277,9 @@ async def simple_fill(self, to, mode):
         mapping = self.get_mapping(mode)
 
         class_name = f"{self.key}_AN"
-        output = await self._aask_v1(prompt, class_name, mapping, format=to)
-        self.content = output.content
-        self.instruct_content = output.instruct_content
+        content, scontent = await self._aask_v1(prompt, class_name, mapping, schema=to)
+        self.content = content
+        self.instruct_content = scontent
         return self
 
     async def fill(self, context, llm, to="json", mode="auto", strgy="simple"):

metagpt/actions/action_output.py

@@ -6,9 +6,7 @@
 @File    : action_output
 """
 
-from typing import Dict, Type
-
-from pydantic import BaseModel, create_model, root_validator, validator
+from pydantic import BaseModel
 
 
 class ActionOutput:
@@ -18,25 +16,3 @@ class ActionOutput:
     def __init__(self, content: str, instruct_content: BaseModel):
         self.content = content
         self.instruct_content = instruct_content
-
-    @classmethod
-    def create_model_class(cls, class_name: str, mapping: Dict[str, Type]):
-        new_class = create_model(class_name, **mapping)
-
-        @validator("*", allow_reuse=True)
-        def check_name(v, field):
-            if field.name not in mapping.keys():
-                raise ValueError(f"Unrecognized block: {field.name}")
-            return v
-
-        @root_validator(pre=True, allow_reuse=True)
-        def check_missing_fields(values):
-            required_fields = set(mapping.keys())
-            missing_fields = required_fields - set(values.keys())
-            if missing_fields:
-                raise ValueError(f"Missing fields: {missing_fields}")
-            return values
-
-        new_class.__validator_check_name = classmethod(check_name)
-        new_class.__root_validator_check_missing_fields = classmethod(check_missing_fields)
-        return new_class

metagpt/actions/design_api.py

@@ -50,7 +50,7 @@ def __init__(self, name, context=None, llm=None):
             "clearly and in detail."
         )
 
-    async def run(self, with_messages, format=CONFIG.prompt_format):
+    async def run(self, with_messages, schema=CONFIG.prompt_schema):
         # Use `git diff` to identify which PRD documents have been modified in the `docs/prds` directory.
         prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO)
         changed_prds = prds_file_repo.changed_files
@@ -80,13 +80,13 @@ async def run(self, with_messages, format=CONFIG.prompt_format):
         # leaving room for global optimization in subsequent steps.
         return ActionOutput(content=changed_files.json(), instruct_content=changed_files)
 
-    async def _new_system_design(self, context, format=CONFIG.prompt_format):
-        node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=format)
+    async def _new_system_design(self, context, schema=CONFIG.prompt_schema):
+        node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=schema)
         return node
 
-    async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format):
+    async def _merge(self, prd_doc, system_design_doc, schema=CONFIG.prompt_schema):
         context = NEW_REQ_TEMPLATE.format(old_design=system_design_doc.content, context=prd_doc.content)
-        node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=format)
+        node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=schema)
         system_design_doc.content = node.instruct_content.json(ensure_ascii=False)
         return system_design_doc