page-assist/src/web/web.ts

import { getWebSearchPrompt } from "~/services/ollama"
import { webGoogleSearch } from "./search-engines/google"
import { webDuckDuckGoSearch } from "./search-engines/duckduckgo"
import { getIsVisitSpecificWebsite, getSearchProvider } from "@/services/search"
import { webSogouSearch } from "./search-engines/sogou"
import { webBraveSearch } from "./search-engines/brave"
import { getWebsiteFromQuery, processSingleWebsite } from "./website"
import { searxngSearch } from "./search-engines/searxng"
import { braveAPISearch } from "./search-engines/brave-api"
import { webBaiduSearch } from "./search-engines/baidu"
import { searchIod } from "./iod"
import type { IodRegistryEntry } from "~/types/iod"

const getHostName = (url: string) => {
  try {
    const hostname = new URL(url).hostname
    return hostname
  } catch (e) {
    return ""
  }
}

const searchWeb = (provider: string, query: string) => {
  switch (provider) {
    case "duckduckgo":
      return webDuckDuckGoSearch(query)
    case "sogou":
      return webSogouSearch(query)
    case "brave":
      return webBraveSearch(query)
    case "searxng":
      return searxngSearch(query)
    case "brave-api":
      return braveAPISearch(query)
    case "baidu":
      return webBaiduSearch(query)
    default:
      return webGoogleSearch(query)
  }
}

export const getSystemPromptForWeb = async (
  query: string,
  keywords: string[] = [],
  webSearch = true,
  iodSearch = false
) => {
  try {
    const websiteVisit = getWebsiteFromQuery(query)
    let webSearchResults: {
      url: any
      content: string
    }[] = []
    // let search_results_web = ""

    if (webSearch) {
      const isVisitSpecificWebsite = await getIsVisitSpecificWebsite()

      if (isVisitSpecificWebsite && websiteVisit.hasUrl) {
        const url = websiteVisit.url
        const queryWithoutUrl = websiteVisit.queryWithouUrls
        webSearchResults = await processSingleWebsite(url, queryWithoutUrl)
      } else {
        const searchProvider = await getSearchProvider()
        webSearchResults = await searchWeb(searchProvider, query)
      }

      // search_results_web = webSearchResults
      //   .map(
      //     (result, idx) =>
      //       `<result source="${result.url}" id="${idx}">${result.content}</result>`
      //   )
      //   .join("\n")
    }

    let iodSearchResults: IodRegistryEntry[] = []
    // let search_results_iod = ""

    if (iodSearch) {
      iodSearchResults = await searchIod(query, keywords)
      // search_results_iod = iodSearchResults
      //   .map(
      //     (result, idx) =>
      //       `<result source="${result.url}" id="${idx}">${result.content}</result>`
      //   )
      //   .join("\n")
    }
    const iod_search_results =  iodSearchResults.map((res) => ({
      url: `${res.doId}: ${res.name}`,
      content: res.description
    })).map(
      (result, idx) =>
        `<result source="${result.url}" id="${idx}">${result.content}</result>`
    )
    .join("\n");
    console.log("iod_search_result:"+iod_search_results);
    const web_search_results = webSearchResults.map(
      (result, idx) =>
        `<result source="${result.url}" id="${idx}">${result.content}</result>`
    )
    .join("\n");
    const search_results = (iodSearch?"<数联网搜索结果>"+iod_search_results+"</数联网搜索结果>":"")
      + (webSearch?"<万维网搜索结果>"+web_search_results+"</万维网搜索结果>":"");

    const current_date_time = new Date().toLocaleString()

    const system = await getWebSearchPrompt()

    const prompt = system
      .replace("{current_date_time}", current_date_time)
      .replace("{search_results}", search_results)

    return {
      prompt,
      webSources: webSearchResults.map((result) => {
        return {
          url: result.url,
          name: getHostName(result.url),
          type: "url"
        }
      }),
      iodSources: iodSearchResults,
    }
  } catch (e) {
    console.error(e)
    return {
      prompt: "",
      source: []
    }
  }
}