diff --git a/file2text/loader.py b/file2text/loader.py index 88facfa..38cf5a2 100644 --- a/file2text/loader.py +++ b/file2text/loader.py @@ -3,16 +3,10 @@ import codecs from datetime import datetime from langchain_community.document_loaders.csv_loader import CSVLoader from langchain_community.document_loaders.text import TextLoader -# from langchain_community.document_loaders.epub import UnstructuredEPubLoader -# from langchain_community.document_loaders.pdf import UnstructuredPDFLoader -# from langchain_community.document_loaders import PyPDFLoader -# from langchain_community.document_loaders import UnstructuredWordDocumentLoader -from langchain_community.document_loaders.chm import UnstructuredCHMLoader -from langchain_community.document_loaders import UnstructuredExcelLoader -from langchain_community.document_loaders import UnstructuredPowerPointLoader -from langchain_text_splitters import RecursiveCharacterTextSplitter from docx import Document # import chm +from ebooklib import epub +from bs4 import BeautifulSoup from pptx import Presentation from openpyxl import load_workbook import mobi @@ -29,7 +23,7 @@ class MyMobiLoader: content=f.read() return html2text.html2text(content) -class MyChmLoader(UnstructuredCHMLoader): +class MyChmLoader: def __init__(self, file_path): self.filepath = file_path self.chm_file = None @@ -114,11 +108,28 @@ class MyExcelLoader: return '\n'.join(content) class MyEpubLoader: + def __init__(self, file_path): + self.filepath = file_path + self.book = None + + def load(self): + """Reads the EPUB file and returns all text content as a string.""" + self.book = epub.read_epub(self.filepath) + content = [] + + for item in self.book.get_items(): + if isinstance(item, epub.EpubHtml): + soup = BeautifulSoup(item.get_content(), 'html.parser') + text = soup.get_text() + content.append(text.strip()) + + return '\n\n'.join(content) + +class MyTextLoader(TextLoader): def load(self): docs = super().load() - print(len(docs)) - return ' '.join([d.page_content for d in docs]) - + return '\n'.join([d.page_content for d in docs]) + def fileloader(file_path): # Load the PDF file and split the data into chunks data = None @@ -140,7 +151,7 @@ def fileloader(file_path): elif file_path.lower().endswith('.mobi'): loader = MyMobiLoader(file_path=file_path) else: - loader = TextLoader(file_path=file_path) + loader = MyTextLoader(file_path=file_path) data = loader.load() return data diff --git a/requirements.txt b/requirements.txt index 35cc8f7..ba5cbc3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,7 @@ langchain_community unstructured -pypdf -# rapidocr-onnxruntime 从pdf文件中的图像中获取文字 pillow -pypandoc -# epub文件需安装pandoc, 在macos上执行:brew install pandoc +# rapidocr-onnxruntime 从pdf文件中的图像中获取文字 rapidocr-onnxruntime mobi html2text @@ -13,4 +10,6 @@ python-docx python-pptx openpyxl PyPDF2 +ebooklib +beautifulsoup4