update configs
This commit is contained in:
CaiHQ
178
src/web/iod.ts
178
src/web/iod.ts
@@ -26,14 +26,65 @@ export const tokenizeInput = function (input: string): string[] {
|
||||
}
|
||||
//doipUrl = tcp://reg01.public.internetofdata.cn:21037
|
||||
export const iodConfig = {
|
||||
"gatewayUrl": "tcp://021.node.internetapi.cn:21052",
|
||||
"gatewayUrl": "tcp://reg01.public.internetofdata.cn:21037",
|
||||
"registry":"data/Registry",
|
||||
"localRepository":"data/Repository",
|
||||
"doBrowser":"http://021.node.internetapi.cn:21030/SCIDE/SCManager"
|
||||
}
|
||||
export const iodConfigLocal = {
|
||||
"gatewayUrl": "tcp://127.0.0.1:21036",
|
||||
"registry":"bdware/Registry",
|
||||
"localRepository":"bdtest.local/myrepo1",
|
||||
"doBrowser":"http://127.0.0.1:21030/SCIDE/SCManager"
|
||||
}
|
||||
function inGrepList(str: string){
|
||||
return "什么|问题|需要|合适|设计|考虑|合作|精度|传感器|最新|研究|药物".indexOf(str)!=-1;
|
||||
}
|
||||
|
||||
export const makeSearchParamsWithDataType = function(count: number, keyword: string| string[], dataType: string){
|
||||
const searchMode = [];
|
||||
searchMode.push({"key":"data_type", "type":"MUST", "value":dataType})
|
||||
if (typeof keyword === 'string') {
|
||||
// 如果 keyword 是字符串，则直接添加一个 searchMode 条目
|
||||
searchMode.push({
|
||||
key: "description",
|
||||
type: "MUST",
|
||||
value: keyword
|
||||
});
|
||||
} else if (Array.isArray(keyword)) {
|
||||
// 如果 keyword 是数组，则为每个元素添加一个 searchMode 条目
|
||||
keyword.forEach(str => {
|
||||
if (!inGrepList(str))
|
||||
searchMode.push({
|
||||
key: "description",
|
||||
type: "SHOULD",
|
||||
value: str
|
||||
});
|
||||
});
|
||||
}
|
||||
return {
|
||||
action: "executeContract",
|
||||
contractID: "BDBrowser",
|
||||
operation: "sendRequestDirectly",
|
||||
arg: {
|
||||
id: iodConfig.registry,
|
||||
//doipUrl:"tcp://127.0.0.1:21039",
|
||||
doipUrl: iodConfig.gatewayUrl,
|
||||
op: "Search",
|
||||
vars:{
|
||||
timeout:15000
|
||||
},
|
||||
attributes: {
|
||||
offset: 0,
|
||||
count,
|
||||
bodyBase64Encoded: false,
|
||||
searchMode:searchMode
|
||||
},
|
||||
body: ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const makeRegSearchParams = function(count: number, keyword: string| string[]){
|
||||
const searchMode = [];
|
||||
if (typeof keyword === 'string') {
|
||||
@@ -185,91 +236,61 @@ export async function localIodSearch(
|
||||
const abortController = new AbortController();
|
||||
setTimeout(() => abortController.abort(), 10000);
|
||||
const params = makeRegSearchParams(TOTAL_SEARCH_RESULTS, keywords);
|
||||
console.log('params------->',params)
|
||||
const dataParams = makeSearchParamsWithDataType(TOTAL_SEARCH_RESULTS,keywords, "data");
|
||||
const scenarioParams = makeSearchParamsWithDataType(TOTAL_SEARCH_RESULTS,keywords, "scenario");
|
||||
const orgParams = makeSearchParamsWithDataType(TOTAL_SEARCH_RESULTS,keywords, "organization");
|
||||
|
||||
try {
|
||||
const response = await fetch(iodConfig.doBrowser, {
|
||||
method: "POST",
|
||||
body: JSON.stringify(params),
|
||||
signal: abortController.signal
|
||||
});
|
||||
console.log('params------->',params)
|
||||
const requests = [
|
||||
fetch(iodConfig.doBrowser,{method: "POST", body: JSON.stringify(dataParams),signal: abortController.signal}),
|
||||
fetch(iodConfig.doBrowser,{method: "POST", body: JSON.stringify(scenarioParams),signal: abortController.signal}),
|
||||
fetch(iodConfig.doBrowser,{method: "POST", body: JSON.stringify(orgParams),signal: abortController.signal})
|
||||
];
|
||||
//TODO @Zhaoweijie， 这三类分别是数据、场景、团队的搜索请求。
|
||||
const responses = await Promise.all(requests);
|
||||
const results = await Promise.all(responses.map(res => res.json()));
|
||||
const allResults: IodRegistryEntry[] = [];
|
||||
for (const res of results) {
|
||||
// 检查顶层状态
|
||||
if (res.status !== "Success") {
|
||||
continue; // 跳过失败的请求
|
||||
}
|
||||
let body;
|
||||
try {
|
||||
body = JSON.parse(res.result.body);
|
||||
} catch (e) {
|
||||
console.warn("Failed to parse result.body as JSON", e);
|
||||
continue;
|
||||
}
|
||||
|
||||
const res = await response.json();
|
||||
if (res.status !== "Success") {
|
||||
return [];
|
||||
if (body.code !== 0) {
|
||||
continue;
|
||||
}
|
||||
const entries: IodRegistryEntry[] = body.data?.results || [];
|
||||
// 数据清洗：补全 url 和 doId
|
||||
for (const r of entries) {
|
||||
r.url = r.url || r.pdf_url;
|
||||
r.doId = r.doId || r.doid;
|
||||
}
|
||||
// 合并到总结果
|
||||
allResults.push(...entries);
|
||||
}
|
||||
|
||||
const body = JSON.parse(res.result.body);
|
||||
if (body.code !== 0) {
|
||||
return [];
|
||||
const seenDoIds = new Set<string>();
|
||||
const prunedResults: IodRegistryEntry[] = [];
|
||||
for (const r of allResults) {
|
||||
if (r.doId && !seenDoIds.has(r.doId)) {
|
||||
seenDoIds.add(r.doId);
|
||||
prunedResults.push(r);
|
||||
}
|
||||
}
|
||||
|
||||
let results: IodRegistryEntry[] = body.data?.results || [];
|
||||
for (const r of results) {
|
||||
r.url = r.url || r.pdf_url;
|
||||
}
|
||||
for (const r of results) {
|
||||
r.doId = r.doId || r.doid;
|
||||
}
|
||||
|
||||
// results 根据 doId 去重
|
||||
const map = new Map<string, IodRegistryEntry>();
|
||||
for (const r of results) {
|
||||
map.set(r.doId, r);
|
||||
}
|
||||
return Array.from(map.values());
|
||||
return prunedResults;
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
return [];
|
||||
}
|
||||
/*
|
||||
const results = (
|
||||
await Promise.all(
|
||||
keywords.map(async (keyword) => {
|
||||
const abortController = new AbortController()
|
||||
setTimeout(() => abortController.abort(), 10000)
|
||||
//http://47.93.156.31:21033/SCIDE/SCManager
|
||||
|
||||
const params = makeRegSearchParams(TOTAL_SEARCH_RESULTS, keyword)
|
||||
return fetch(iodConfig.doBrowser, {
|
||||
method: "POST",
|
||||
body: JSON.stringify(params),
|
||||
signal: abortController.signal
|
||||
})
|
||||
.then((response) => response.json())
|
||||
.then((res) => {
|
||||
if (res.status !== "Success") {
|
||||
console.log(res)
|
||||
return []
|
||||
}
|
||||
const body = JSON.parse(res.result.body)
|
||||
if (body.code !== 0) {
|
||||
console.log(body)
|
||||
return []
|
||||
}
|
||||
const results: IodRegistryEntry[] = body.data?.results || []
|
||||
for (const r of results) {
|
||||
r.url = r.url || r.pdf_url
|
||||
}
|
||||
for (const r of results) {
|
||||
r.doId = r.doId || r.doid
|
||||
}
|
||||
return results
|
||||
})
|
||||
.catch((e) => {
|
||||
console.log(e)
|
||||
return []
|
||||
})
|
||||
})
|
||||
)
|
||||
).flat()
|
||||
|
||||
// results 根据 doId 去重
|
||||
const map = new Map<string, IodRegistryEntry>()
|
||||
for (const r of results) {
|
||||
map.set(r.doId, r)
|
||||
}
|
||||
return Array.from(map.values())
|
||||
*/
|
||||
|
||||
|
||||
}
|
||||
|
||||
const ARXIV_URL_PATTERN = /^https?:\/\/arxiv\.org\//
|
||||
@@ -280,6 +301,7 @@ export const searchIod = async (query: string, keywords: string[]) => {
|
||||
|
||||
const isSimpleMode = await getIsSimpleInternetSearch()
|
||||
console.log("searchMode:"+isSimpleMode+"\n kw:"+JSON.stringify(keywords)+"\n"+" ->searchResult:\n"+JSON.stringify(searchResults))
|
||||
console.log("pruned Search Result:"+JSON.stringify(searchResults.map(r=>r.doId+" "+r.name)))
|
||||
if (isSimpleMode) {
|
||||
await getOllamaURL()
|
||||
return searchResults
|
||||
|
||||
Reference in New Issue
Block a user