Feature: resolve web links in messages to embed text into prompt

src/hooks/useSubmit.ts

@@ -4,7 +4,7 @@ import { useTranslation } from 'react-i18next';
 import { ChatInterface, MessageInterface } from '@type/chat';
 import { getChatCompletion, getChatCompletionStream } from '@api/api';
 import { parseEventSource } from '@api/helper';
-import { limitMessageTokens, updateTotalTokenUsed } from '@utils/messageUtils';
+import { limitMessageTokens, resolveWebLinks, updateTotalTokenUsed } from '@utils/messageUtils';
 import { _defaultChatConfig } from '@constants/chat';
 import { officialAPIEndpoint } from '@constants/auth';
 
@@ -70,13 +70,15 @@ const useSubmit = () => {
       if (chats[currentChatIndex].messages.length === 0)
         throw new Error('No messages submitted!');
 
-      const messages = limitMessageTokens(
+      let messages = limitMessageTokens(
         chats[currentChatIndex].messages,
         chats[currentChatIndex].config.max_tokens,
         chats[currentChatIndex].config.model
       );
       if (messages.length === 0) throw new Error('Message exceed max token!');
 
+      messages = await resolveWebLinks(messages);
+
       // no api key (free)
       if (!apiKey || apiKey.length === 0) {
         // official endpoint

src/utils/messageUtils.ts

@@ -87,6 +87,49 @@ export const limitMessageTokens = (
   return limitedMessages;
 };
 
+
+export const resolveWebLinks = async (messages: MessageInterface[]): Promise<MessageInterface[]> => {
+  const resolvedMessages: MessageInterface[] = [];
+
+  const fetchWebContent = async (link: string): Promise<string> => {
+    try {
+      const response = await fetch(link);
+      const resolvedContent = await response.text();
+      return resolvedContent;
+    } catch (error) {
+      console.error(`Failed to fetch web content from ${link}:`, error);
+      return '';
+    }
+  };
+
+  const replaceWebLinks = async (content: string): Promise<string> => {
+    const linkRegex = /(http[s]?:\/\/[^\s]+)/g;
+    const matches = content.match(linkRegex);
+
+    if (!matches) {
+      return content;
+    }
+
+    for (const match of matches) {
+      try {
+        const resolvedContent = await fetchWebContent(match);
+        content = content.replace(match, resolvedContent);
+      } catch (error) {
+        console.error(`Failed to fetch web content from ${match}:`, error);
+      }
+    }
+
+    return content;
+  };
+
+  for (const message of messages) {
+    const resolvedContent = await replaceWebLinks(message.content);
+    resolvedMessages.push({ ...message, content: resolvedContent });
+  }
+
+  return resolvedMessages;
+};
+
 export const updateTotalTokenUsed = (
   model: ModelOptions,
   promptMessages: MessageInterface[],