feat: add IoD search

src/web/web.ts

@@ -8,6 +8,7 @@ import { getWebsiteFromQuery, processSingleWebsite } from "./website"
 import { searxngSearch } from "./search-engines/searxng"
 import { braveAPISearch } from "./search-engines/brave-api"
 import { webBaiduSearch } from "./search-engines/baidu"
+import { searchIod } from "./iod"
 
 const getHostName = (url: string) => {
   try {
@@ -37,30 +38,66 @@ const searchWeb = (provider: string, query: string) => {
   }
 }
 
-export const getSystemPromptForWeb = async (query: string) => {
+export const getSystemPromptForWeb = async (
+  query: string,
+  keywords: string[] = [],
+  webSearch = true,
+  iodSearch = false
+) => {
   try {
-
     const websiteVisit = getWebsiteFromQuery(query)
-    let search: {
-      url: any;
-      content: string;
+    let webSearchResults: {
+      url: any
+      content: string
     }[] = []
+    // let search_results_web = ""
 
-    const isVisitSpecificWebsite = await getIsVisitSpecificWebsite()
+    if (webSearch) {
+      const isVisitSpecificWebsite = await getIsVisitSpecificWebsite()
 
-    if (isVisitSpecificWebsite && websiteVisit.hasUrl) {
+      if (isVisitSpecificWebsite && websiteVisit.hasUrl) {
+        const url = websiteVisit.url
+        const queryWithoutUrl = websiteVisit.queryWithouUrls
+        webSearchResults = await processSingleWebsite(url, queryWithoutUrl)
+      } else {
+        const searchProvider = await getSearchProvider()
+        webSearchResults = await searchWeb(searchProvider, query)
+      }
 
-      const url = websiteVisit.url
-      const queryWithoutUrl = websiteVisit.queryWithouUrls
-      search = await processSingleWebsite(url, queryWithoutUrl)
-
-    } else {
-      const searchProvider = await getSearchProvider()
-      search = await searchWeb(searchProvider, query)
+      // search_results_web = webSearchResults
+      //   .map(
+      //     (result, idx) =>
+      //       `<result source="${result.url}" id="${idx}">${result.content}</result>`
+      //   )
+      //   .join("\n")
     }
 
+    let iodSearchResults: {
+      doId: string
+      name: string
+      url?: string
+      // pdf_url?: string
+      description: string
+    }[] = []
+    // let search_results_iod = ""
 
-    const search_results = search
+    if (iodSearch) {
+      iodSearchResults = await searchIod(query, keywords)
+      // search_results_iod = iodSearchResults
+      //   .map(
+      //     (result, idx) =>
+      //       `<result source="${result.url}" id="${idx}">${result.content}</result>`
+      //   )
+      //   .join("\n")
+    }
+
+    const search_results =  iodSearchResults.map((res) => ({
+      url: `${res.doId}: ${res.name}`,
+      content: res.description
+    }))
+      .concat(
+        webSearchResults
+      )
       .map(
         (result, idx) =>
           `<result source="${result.url}" id="${idx}">${result.content}</result>`
@@ -77,13 +114,14 @@ export const getSystemPromptForWeb = async (query: string) => {
 
     return {
       prompt,
-      source: search.map((result) => {
+      webSources: webSearchResults.map((result) => {
         return {
           url: result.url,
           name: getHostName(result.url),
           type: "url"
         }
-      })
+      }),
+      iodSources: iodSearchResults,
     }
   } catch (e) {
     console.error(e)