Merge branch 'fix/chore-fix' into dev/plugin-deploy

api/core/agent/cot_agent_runner.py

@@ -309,13 +309,13 @@ def _handle_invoke_action(
         )
 
         # publish files
-        for message_file_id, save_as in message_files:
+        for message_file_id in message_files:
             # publish message file
             self.queue_manager.publish(
-                QueueMessageFileEvent(message_file_id=message_file_id.id), PublishFrom.APPLICATION_MANAGER
+                QueueMessageFileEvent(message_file_id=message_file_id), PublishFrom.APPLICATION_MANAGER
             )
             # add message file ids
-            message_file_ids.append(message_file_id.id)
+            message_file_ids.append(message_file_id)
 
         return tool_invoke_response, tool_invoke_meta
 

api/core/agent/fc_agent_runner.py

@@ -246,13 +246,13 @@ def increase_usage(final_llm_usage_dict: dict[str, Optional[LLMUsage]], usage: L
                         conversation_id=self.conversation.id,
                     )
                     # publish files
-                    for message_file_id, save_as in message_files:
+                    for message_file_id in message_files:
                         # publish message file
                         self.queue_manager.publish(
-                            QueueMessageFileEvent(message_file_id=message_file_id.id), PublishFrom.APPLICATION_MANAGER
+                            QueueMessageFileEvent(message_file_id=message_file_id), PublishFrom.APPLICATION_MANAGER
                         )
                         # add message file ids
-                        message_file_ids.append(message_file_id.id)
+                        message_file_ids.append(message_file_id)
 
                     tool_response = {
                         "tool_call_id": tool_call_id,

api/core/app/apps/agent_chat/app_generator.py

@@ -172,7 +172,7 @@ def generate(
             target=self._generate_worker,
             kwargs={
                 "flask_app": current_app._get_current_object(),  # type: ignore
-                "contexts": contextvars.copy_context(),
+                "context": contextvars.copy_context(),
                 "application_generate_entity": application_generate_entity,
                 "queue_manager": queue_manager,
                 "conversation_id": conversation.id,

api/core/tools/__base/tool.py

@@ -157,48 +157,51 @@ def get_merged_runtime_parameters(
 
         return parameters
 
-    def create_image_message(self, image: str, save_as: str = "") -> ToolInvokeMessage:
+    def create_image_message(
+        self,
+        image: str,
+    ) -> ToolInvokeMessage:
         """
         create an image message
 
         :param image: the url of the image
         :return: the image message
         """
         return ToolInvokeMessage(
-            type=ToolInvokeMessage.MessageType.IMAGE, message=ToolInvokeMessage.TextMessage(text=image), save_as=save_as
+            type=ToolInvokeMessage.MessageType.IMAGE, message=ToolInvokeMessage.TextMessage(text=image)
         )
 
     def create_file_message(self, file: "File") -> ToolInvokeMessage:
         return ToolInvokeMessage(
             type=ToolInvokeMessage.MessageType.FILE,
             message=ToolInvokeMessage.FileMessage(),
             meta={"file": file},
-            save_as="",
         )
 
-    def create_link_message(self, link: str, save_as: str = "") -> ToolInvokeMessage:
+    def create_link_message(self, link: str) -> ToolInvokeMessage:
         """
         create a link message
 
         :param link: the url of the link
         :return: the link message
         """
         return ToolInvokeMessage(
-            type=ToolInvokeMessage.MessageType.LINK, message=ToolInvokeMessage.TextMessage(text=link), save_as=save_as
+            type=ToolInvokeMessage.MessageType.LINK, message=ToolInvokeMessage.TextMessage(text=link)
         )
 
-    def create_text_message(self, text: str, save_as: str = "") -> ToolInvokeMessage:
+    def create_text_message(self, text: str) -> ToolInvokeMessage:
         """
         create a text message
 
         :param text: the text
         :return: the text message
         """
         return ToolInvokeMessage(
-            type=ToolInvokeMessage.MessageType.TEXT, message=ToolInvokeMessage.TextMessage(text=text), save_as=save_as
+            type=ToolInvokeMessage.MessageType.TEXT,
+            message=ToolInvokeMessage.TextMessage(text=text),
         )
 
-    def create_blob_message(self, blob: bytes, meta: Optional[dict] = None, save_as: str = "") -> ToolInvokeMessage:
+    def create_blob_message(self, blob: bytes, meta: Optional[dict] = None) -> ToolInvokeMessage:
         """
         create a blob message
 
@@ -209,7 +212,6 @@ def create_blob_message(self, blob: bytes, meta: Optional[dict] = None, save_as:
             type=ToolInvokeMessage.MessageType.BLOB,
             message=ToolInvokeMessage.BlobMessage(blob=blob),
             meta=meta,
-            save_as=save_as,
         )
 
     def create_json_message(self, object: dict) -> ToolInvokeMessage:

api/core/tools/builtin_tool/providers/audio/audio.py

@@ -0,0 +1,6 @@
+from core.tools.builtin_tool.provider import BuiltinToolProviderController
+
+
+class AudioToolProvider(BuiltinToolProviderController):
+    def _validate_credentials(self, credentials: dict) -> None:
+        pass

api/core/tools/builtin_tool/providers/audio/audio.yaml

@@ -0,0 +1,11 @@
+identity:
+  author: hjlarry
+  name: audio
+  label:
+    en_US: Audio
+  description:
+    en_US: A tool for tts and asr.
+    zh_Hans: 一个用于文本转语音和语音转文本的工具。
+  icon: icon.svg
+  tags:
+    - utilities

api/core/tools/builtin_tool/providers/audio/tools/asr.py

@@ -0,0 +1,71 @@
+import io
+from collections.abc import Generator
+from typing import Any
+
+from core.file.enums import FileType
+from core.file.file_manager import download
+from core.model_manager import ModelManager
+from core.model_runtime.entities.model_entities import ModelType
+from core.tools.builtin_tool.tool import BuiltinTool
+from core.tools.entities.common_entities import I18nObject
+from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolParameterOption
+from services.model_provider_service import ModelProviderService
+
+
+class ASRTool(BuiltinTool):
+    def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Generator[ToolInvokeMessage, None, None]:
+        file = tool_parameters.get("audio_file")
+        if file.type != FileType.AUDIO:  # type: ignore
+            yield self.create_text_message("not a valid audio file")
+            return
+        audio_binary = io.BytesIO(download(file))  # type: ignore
+        audio_binary.name = "temp.mp3"
+        provider, model = tool_parameters.get("model").split("#")  # type: ignore
+        model_manager = ModelManager()
+        model_instance = model_manager.get_model_instance(
+            tenant_id=self.runtime.tenant_id,
+            provider=provider,
+            model_type=ModelType.SPEECH2TEXT,
+            model=model,
+        )
+        text = model_instance.invoke_speech2text(
+            file=audio_binary,
+            user=user_id,
+        )
+        yield self.create_text_message(text)
+
+    def get_available_models(self) -> list[tuple[str, str]]:
+        model_provider_service = ModelProviderService()
+        models = model_provider_service.get_models_by_model_type(
+            tenant_id=self.runtime.tenant_id, model_type="speech2text"
+        )
+        items = []
+        for provider_model in models:
+            provider = provider_model.provider
+            for model in provider_model.models:
+                items.append((provider, model.model))
+        return items
+
+    def get_runtime_parameters(self) -> list[ToolParameter]:
+        parameters = []
+
+        options = []
+        for provider, model in self.get_available_models():
+            option = ToolParameterOption(value=f"{provider}#{model}", label=I18nObject(en_US=f"{model}({provider})"))
+            options.append(option)
+
+        parameters.append(
+            ToolParameter(
+                name="model",
+                label=I18nObject(en_US="Model", zh_Hans="Model"),
+                human_description=I18nObject(
+                    en_US="All available ASR models. You can config model in the Model Provider of Settings.",
+                    zh_Hans="所有可用的 ASR 模型。你可以在设置中的模型供应商里配置。",
+                ),
+                type=ToolParameter.ToolParameterType.SELECT,
+                form=ToolParameter.ToolParameterForm.FORM,
+                required=True,
+                options=options,
+            )
+        )
+        return parameters

api/core/tools/builtin_tool/providers/audio/tools/asr.yaml

@@ -0,0 +1,22 @@
+identity:
+  name: asr
+  author: hjlarry
+  label:
+    en_US: Speech To Text
+description:
+  human:
+    en_US: Convert audio file to text.
+    zh_Hans: 将音频文件转换为文本。
+  llm: Convert audio file to text.
+parameters:
+  - name: audio_file
+    type: file
+    required: true
+    label:
+      en_US: Audio File
+      zh_Hans: 音频文件
+    human_description:
+      en_US: The audio file to be converted.
+      zh_Hans: 要转换的音频文件。
+    llm_description: The audio file to be converted.
+    form: llm

api/core/tools/builtin_tool/providers/audio/tools/tts.py

@@ -0,0 +1,87 @@
+import io
+from collections.abc import Generator
+from typing import Any
+
+from core.model_manager import ModelManager
+from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
+from core.tools.builtin_tool.tool import BuiltinTool
+from core.tools.entities.common_entities import I18nObject
+from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolParameterOption
+from services.model_provider_service import ModelProviderService
+
+
+class TTSTool(BuiltinTool):
+    def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Generator[ToolInvokeMessage, None, None]:
+        provider, model = tool_parameters.get("model").split("#")  # type: ignore
+        voice = tool_parameters.get(f"voice#{provider}#{model}")
+        model_manager = ModelManager()
+        model_instance = model_manager.get_model_instance(
+            tenant_id=self.runtime.tenant_id,
+            provider=provider,
+            model_type=ModelType.TTS,
+            model=model,
+        )
+        tts = model_instance.invoke_tts(
+            content_text=tool_parameters.get("text"),  # type: ignore
+            user=user_id,
+            tenant_id=self.runtime.tenant_id,
+            voice=voice,  # type: ignore
+        )
+        buffer = io.BytesIO()
+        for chunk in tts:
+            buffer.write(chunk)
+
+        wav_bytes = buffer.getvalue()
+        yield self.create_text_message("Audio generated successfully")
+        yield self.create_blob_message(
+            blob=wav_bytes,
+            meta={"mime_type": "audio/x-wav"},
+        )
+
+    def get_available_models(self) -> list[tuple[str, str, list[Any]]]:
+        model_provider_service = ModelProviderService()
+        models = model_provider_service.get_models_by_model_type(tenant_id=self.runtime.tenant_id, model_type="tts")
+        items = []
+        for provider_model in models:
+            provider = provider_model.provider
+            for model in provider_model.models:
+                voices = model.model_properties.get(ModelPropertyKey.VOICES, [])
+                items.append((provider, model.model, voices))
+        return items
+
+    def get_runtime_parameters(self) -> list[ToolParameter]:
+        parameters = []
+
+        options = []
+        for provider, model, voices in self.get_available_models():
+            option = ToolParameterOption(value=f"{provider}#{model}", label=I18nObject(en_US=f"{model}({provider})"))
+            options.append(option)
+            parameters.append(
+                ToolParameter(
+                    name=f"voice#{provider}#{model}",
+                    label=I18nObject(en_US=f"Voice of {model}({provider})"),
+                    type=ToolParameter.ToolParameterType.SELECT,
+                    form=ToolParameter.ToolParameterForm.FORM,
+                    options=[
+                        ToolParameterOption(value=voice.get("mode"), label=I18nObject(en_US=voice.get("name")))
+                        for voice in voices
+                    ],
+                )
+            )
+
+        parameters.insert(
+            0,
+            ToolParameter(
+                name="model",
+                label=I18nObject(en_US="Model", zh_Hans="Model"),
+                human_description=I18nObject(
+                    en_US="All available TTS models. You can config model in the Model Provider of Settings.",
+                    zh_Hans="所有可用的 TTS 模型。你可以在设置中的模型供应商里配置。",
+                ),
+                type=ToolParameter.ToolParameterType.SELECT,
+                form=ToolParameter.ToolParameterForm.FORM,
+                required=True,
+                options=options,
+            ),
+        )
+        return parameters

api/core/tools/builtin_tool/providers/audio/tools/tts.yaml

@@ -0,0 +1,22 @@
+identity:
+  name: tts
+  author: hjlarry
+  label:
+    en_US: Text To Speech
+description:
+  human:
+    en_US: Convert text to audio file.
+    zh_Hans: 将文本转换为音频文件。
+  llm: Convert text to audio file.
+parameters:
+  - name: text
+    type: string
+    required: true
+    label:
+      en_US: Text
+      zh_Hans: 文本
+    human_description:
+      en_US: The text to be converted.
+      zh_Hans: 要转换的文本。
+    llm_description: The text to be converted.
+    form: llm

api/core/tools/builtin_tool/providers/webscraper/tools/webscraper.py

@@ -0,0 +1,36 @@
+from collections.abc import Generator
+from typing import Any
+
+from core.tools.builtin_tool.tool import BuiltinTool
+from core.tools.entities.tool_entities import ToolInvokeMessage
+from core.tools.errors import ToolInvokeError
+from core.tools.utils.web_reader_tool import get_url
+
+
+class WebscraperTool(BuiltinTool):
+    def _invoke(
+        self,
+        user_id: str,
+        tool_parameters: dict[str, Any],
+    ) -> Generator[ToolInvokeMessage, None, None]:
+        """
+        invoke tools
+        """
+        try:
+            url = tool_parameters.get("url", "")
+            user_agent = tool_parameters.get("user_agent", "")
+            if not url:
+                yield self.create_text_message("Please input url")
+                return
+
+            # get webpage
+            result = get_url(url, user_agent=user_agent)
+
+            if tool_parameters.get("generate_summary"):
+                # summarize and return
+                yield self.create_text_message(self.summary(user_id=user_id, content=result))
+            else:
+                # return full webpage
+                yield self.create_text_message(result)
+        except Exception as e:
+            raise ToolInvokeError(str(e))