feat: add epub file reader support

.gitignore

@@ -40,3 +40,6 @@ dist/
 
 # vs code
 .vscode/launch.json
+
+# jetbrains ide
+.idea/

README.md

@@ -97,7 +97,7 @@ export const runtime = "nodejs"; // default
 /** @type {import('next').NextConfig} */
 const nextConfig = {
  experimental: {
- serverComponentsExternalPackages: ["pdf-parse"], // Puts pdf-parse in actual NodeJS mode with NextJS App Router
+ serverComponentsExternalPackages: ["pdf-parse", "@gxl/epub-parser"], // Puts pdf-parse and @gxl/epub-parser in actual NodeJS mode with NextJS App Router
  },
 };
 

examples/epub.ts

@@ -0,0 +1,18 @@
+import { EpubReader, VectorStoreIndex } from "llamaindex";
+
+async function main() {
+ // Load PDF
+ const reader = new EpubReader();
+ const documents = await reader.loadData("data/wells.epub");
+ // Split text and create embeddings. Store them in a VectorStoreIndex
+ const index = await VectorStoreIndex.fromDocuments(documents);
+
+ // Query the index
+ const queryEngine = index.asQueryEngine();
+ const response = await queryEngine.query("What is this about?");
+
+ // Output response
+ console.log(response.toString());
+}
+
+main().catch(console.error);

packages/core/package.json

@@ -4,6 +4,7 @@
  "license": "MIT",
  "dependencies": {
  "@anthropic-ai/sdk": "^0.9.1",
+ "@gxl/epub-parser": "2.0.4",
  "@notionhq/client": "^2.2.13",
  "@xenova/transformers": "^2.8.0",
  "crypto-js": "^4.2.0",

packages/core/src/index.ts

@@ -20,6 +20,7 @@ export * from "./embeddings";
 export * from "./indices";
 export * from "./llm/LLM";
 export * from "./readers/CSVReader";
+export * from "./readers/EpubReader";
 export * from "./readers/HTMLReader";
 export * from "./readers/MarkdownReader";
 export * from "./readers/NotionReader";

packages/core/src/readers/EpubReader.ts

@@ -0,0 +1,63 @@
+import { parseEpub } from "@gxl/epub-parser";
+import { Document } from "../Node";
+import { GenericFileSystem } from "../storage/FileSystem";
+import { DEFAULT_FS } from "../storage/constants";
+import { BaseReader } from "./base";
+/**
+ * Read the text of a Epub file
+ */
+export class EpubReader implements BaseReader {
+ async loadData(
+ file: string,
+ fs: GenericFileSystem = DEFAULT_FS,
+ ): Promise<Document[]> {
+ const dataBuffer = (await fs.readFile(file)) as any;
+ const book = await parseEpub(dataBuffer, {
+ type: "buffer",
+ expand: true,
+ });
+ const sections = book.sections ?? [];
+ const header = `${book.info?.author}\n${book.info?.publisher}\n${book.info?.title}`;
+ const options = this.getOptions();
+ const results = await Promise.all(
+ sections.map((section) => {
+ return new Promise(async (resolve) => {
+ const parsed = await this.parseContent(section.htmlString, options);
+ resolve(parsed);
+ });
+ }),
+ );
+ return [
+ new Document({ text: `${header}\n${results.join("\n")}`, id_: file }),
+ ];
+ }
+
+ /**
+ * Wrapper for string-strip-html usage.
+ * @param html Raw HTML content to be parsed.
+ * @param options An object of options for the underlying library
+ * @see getOptions
+ * @returns The HTML content, stripped of unwanted tags and attributes
+ */
+ async parseContent(html: string, options: any = {}): Promise<string> {
+ const { stripHtml } = await import("string-strip-html"); // ESM only
+ return stripHtml(html).result;
+ }
+
+ /**
+ * Wrapper for our configuration options passed to string-strip-html library
+ * @see https://codsen.com/os/string-strip-html/examples
+ * @returns An object of options for the underlying library
+ */
+ getOptions() {
+ return {
+ skipHtmlDecoding: true,
+ stripTogetherWithTheirContents: [
+ "script", // default
+ "style", // default
+ "xml", // default
+ "head", // <-- custom-added
+ ],
+ };
+ }
+}

packages/core/src/readers/SimpleDirectoryReader.ts

@@ -4,6 +4,7 @@ import { CompleteFileSystem, walk } from "../storage/FileSystem";
 import { DEFAULT_FS } from "../storage/constants";
 import { PapaCSVReader } from "./CSVReader";
 import { DocxReader } from "./DocxReader";
+import { EpubReader } from "./EpubReader";
 import { HTMLReader } from "./HTMLReader";
 import { MarkdownReader } from "./MarkdownReader";
 import { PDFReader } from "./PDFReader";
@@ -42,6 +43,7 @@ export const FILE_EXT_TO_READER: Record<string, BaseReader> = {
  docx: new DocxReader(),
  htm: new HTMLReader(),
  html: new HTMLReader(),
+ epub: new EpubReader(),
 };
 
 export type SimpleDirectoryReaderLoadDataProps = {