Update dependencies and fix import paths
This commit is contained in:
55
src/libs/process-knowledge.ts
Normal file
55
src/libs/process-knowledge.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
import { getKnowledgeById, updateKnowledgeStatus } from "@/db/knowledge"
|
||||
import { PageAssistPDFUrlLoader } from "@/loader/pdf-url"
|
||||
import {
|
||||
defaultEmbeddingChunkOverlap,
|
||||
defaultEmbeddingChunkSize
|
||||
} from "@/services/ollama"
|
||||
import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama"
|
||||
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
|
||||
import { PageAssistVectorStore } from "./PageAssistVectorStore"
|
||||
|
||||
export const processKnowledge = async (msg: any, id: string): Promise<void> => {
|
||||
console.log(`Processing knowledge with id: ${id}`)
|
||||
try {
|
||||
const knowledge = await getKnowledgeById(id)
|
||||
|
||||
if (!knowledge) {
|
||||
console.error(`Knowledge with id ${id} not found`)
|
||||
return
|
||||
}
|
||||
|
||||
await updateKnowledgeStatus(id, "processing")
|
||||
|
||||
const ollamaEmbedding = new OllamaEmbeddings({
|
||||
model: knowledge.embedding_model
|
||||
})
|
||||
const chunkSize = await defaultEmbeddingChunkSize()
|
||||
const chunkOverlap = await defaultEmbeddingChunkOverlap()
|
||||
const textSplitter = new RecursiveCharacterTextSplitter({
|
||||
chunkSize,
|
||||
chunkOverlap
|
||||
})
|
||||
|
||||
for (const doc of knowledge.source) {
|
||||
if (doc.type === "pdf" || doc.type === "application/pdf") {
|
||||
const loader = new PageAssistPDFUrlLoader({
|
||||
name: doc.filename,
|
||||
url: doc.content
|
||||
})
|
||||
let docs = await loader.load()
|
||||
const chunks = await textSplitter.splitDocuments(docs)
|
||||
await PageAssistVectorStore.fromDocuments(chunks, ollamaEmbedding, {
|
||||
knownledge_id: knowledge.id,
|
||||
file_id: doc.source_id
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
await updateKnowledgeStatus(id, "finished")
|
||||
} catch (error) {
|
||||
console.error(`Error processing knowledge with id: ${id}`, error)
|
||||
await updateKnowledgeStatus(id, "failed")
|
||||
} finally {
|
||||
console.log(`Finished processing knowledge with id: ${id}`)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user