Add d3-dsv and @types/d3-dsv dependencies

src/loader/txt.ts

@@ -0,0 +1,57 @@
+import { BaseDocumentLoader } from "langchain/document_loaders/base"
+import { Document } from "@langchain/core/documents"
+export interface WebLoaderParams {
+  url: string
+  name: string
+}
+
+export class PageAssisTXTUrlLoader
+  extends BaseDocumentLoader
+  implements WebLoaderParams
+{
+  pdf: { content: string; page: number }[]
+  url: string
+  name: string
+
+  constructor({ url, name }: WebLoaderParams) {
+    super()
+    this.url = url
+    this.name = name
+  }
+
+  public async parse(raw: string): Promise<string[]> {
+    return [raw]
+  }
+  async load(): Promise<Document<Record<string, any>>[]> {
+    const res = await fetch(this.url)
+
+    if (!res.ok) {
+      throw new Error(`Failed to fetch ${this.url}`)
+    }
+
+    const raw = await res.text()
+
+    const parsed = await this.parse(raw)
+    let metadata = { source: this.name, type: "csv" }
+    parsed.forEach((pageContent, i) => {
+      if (typeof pageContent !== "string") {
+        throw new Error(
+          `Expected string, at position ${i} got ${typeof pageContent}`
+        )
+      }
+    })
+    return parsed.map(
+      (pageContent, i) =>
+        new Document({
+          pageContent,
+          metadata:
+            parsed.length === 1
+              ? metadata
+              : {
+                  ...metadata,
+                  line: i + 1
+                }
+        })
+    )
+  }
+}