This commit is contained in:
yumoqing 2025-05-13 16:41:41 +08:00
parent 8d3e724322
commit 034fb514c4
2 changed files with 27 additions and 17 deletions

View File

@ -3,16 +3,10 @@ import codecs
from datetime import datetime from datetime import datetime
from langchain_community.document_loaders.csv_loader import CSVLoader from langchain_community.document_loaders.csv_loader import CSVLoader
from langchain_community.document_loaders.text import TextLoader from langchain_community.document_loaders.text import TextLoader
# from langchain_community.document_loaders.epub import UnstructuredEPubLoader
# from langchain_community.document_loaders.pdf import UnstructuredPDFLoader
# from langchain_community.document_loaders import PyPDFLoader
# from langchain_community.document_loaders import UnstructuredWordDocumentLoader
from langchain_community.document_loaders.chm import UnstructuredCHMLoader
from langchain_community.document_loaders import UnstructuredExcelLoader
from langchain_community.document_loaders import UnstructuredPowerPointLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from docx import Document from docx import Document
# import chm # import chm
from ebooklib import epub
from bs4 import BeautifulSoup
from pptx import Presentation from pptx import Presentation
from openpyxl import load_workbook from openpyxl import load_workbook
import mobi import mobi
@ -29,7 +23,7 @@ class MyMobiLoader:
content=f.read() content=f.read()
return html2text.html2text(content) return html2text.html2text(content)
class MyChmLoader(UnstructuredCHMLoader): class MyChmLoader:
def __init__(self, file_path): def __init__(self, file_path):
self.filepath = file_path self.filepath = file_path
self.chm_file = None self.chm_file = None
@ -114,11 +108,28 @@ class MyExcelLoader:
return '\n'.join(content) return '\n'.join(content)
class MyEpubLoader: class MyEpubLoader:
def __init__(self, file_path):
self.filepath = file_path
self.book = None
def load(self):
"""Reads the EPUB file and returns all text content as a string."""
self.book = epub.read_epub(self.filepath)
content = []
for item in self.book.get_items():
if isinstance(item, epub.EpubHtml):
soup = BeautifulSoup(item.get_content(), 'html.parser')
text = soup.get_text()
content.append(text.strip())
return '\n\n'.join(content)
class MyTextLoader(TextLoader):
def load(self): def load(self):
docs = super().load() docs = super().load()
print(len(docs)) return '\n'.join([d.page_content for d in docs])
return ' '.join([d.page_content for d in docs])
def fileloader(file_path): def fileloader(file_path):
# Load the PDF file and split the data into chunks # Load the PDF file and split the data into chunks
data = None data = None
@ -140,7 +151,7 @@ def fileloader(file_path):
elif file_path.lower().endswith('.mobi'): elif file_path.lower().endswith('.mobi'):
loader = MyMobiLoader(file_path=file_path) loader = MyMobiLoader(file_path=file_path)
else: else:
loader = TextLoader(file_path=file_path) loader = MyTextLoader(file_path=file_path)
data = loader.load() data = loader.load()
return data return data

View File

@ -1,10 +1,7 @@
langchain_community langchain_community
unstructured unstructured
pypdf
# rapidocr-onnxruntime 从pdf文件中的图像中获取文字
pillow pillow
pypandoc # rapidocr-onnxruntime 从pdf文件中的图像中获取文字
# epub文件需安装pandoc 在macos上执行brew install pandoc
rapidocr-onnxruntime rapidocr-onnxruntime
mobi mobi
html2text html2text
@ -13,4 +10,6 @@ python-docx
python-pptx python-pptx
openpyxl openpyxl
PyPDF2 PyPDF2
ebooklib
beautifulsoup4