This commit is contained in:
yumoqing 2025-05-13 16:41:41 +08:00
parent 8d3e724322
commit 034fb514c4
2 changed files with 27 additions and 17 deletions

View File

@ -3,16 +3,10 @@ import codecs
from datetime import datetime
from langchain_community.document_loaders.csv_loader import CSVLoader
from langchain_community.document_loaders.text import TextLoader
# from langchain_community.document_loaders.epub import UnstructuredEPubLoader
# from langchain_community.document_loaders.pdf import UnstructuredPDFLoader
# from langchain_community.document_loaders import PyPDFLoader
# from langchain_community.document_loaders import UnstructuredWordDocumentLoader
from langchain_community.document_loaders.chm import UnstructuredCHMLoader
from langchain_community.document_loaders import UnstructuredExcelLoader
from langchain_community.document_loaders import UnstructuredPowerPointLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from docx import Document
# import chm
from ebooklib import epub
from bs4 import BeautifulSoup
from pptx import Presentation
from openpyxl import load_workbook
import mobi
@ -29,7 +23,7 @@ class MyMobiLoader:
content=f.read()
return html2text.html2text(content)
class MyChmLoader(UnstructuredCHMLoader):
class MyChmLoader:
def __init__(self, file_path):
self.filepath = file_path
self.chm_file = None
@ -114,11 +108,28 @@ class MyExcelLoader:
return '\n'.join(content)
class MyEpubLoader:
def __init__(self, file_path):
self.filepath = file_path
self.book = None
def load(self):
"""Reads the EPUB file and returns all text content as a string."""
self.book = epub.read_epub(self.filepath)
content = []
for item in self.book.get_items():
if isinstance(item, epub.EpubHtml):
soup = BeautifulSoup(item.get_content(), 'html.parser')
text = soup.get_text()
content.append(text.strip())
return '\n\n'.join(content)
class MyTextLoader(TextLoader):
def load(self):
docs = super().load()
print(len(docs))
return ' '.join([d.page_content for d in docs])
return '\n'.join([d.page_content for d in docs])
def fileloader(file_path):
# Load the PDF file and split the data into chunks
data = None
@ -140,7 +151,7 @@ def fileloader(file_path):
elif file_path.lower().endswith('.mobi'):
loader = MyMobiLoader(file_path=file_path)
else:
loader = TextLoader(file_path=file_path)
loader = MyTextLoader(file_path=file_path)
data = loader.load()
return data

View File

@ -1,10 +1,7 @@
langchain_community
unstructured
pypdf
# rapidocr-onnxruntime 从pdf文件中的图像中获取文字
pillow
pypandoc
# epub文件需安装pandoc 在macos上执行brew install pandoc
# rapidocr-onnxruntime 从pdf文件中的图像中获取文字
rapidocr-onnxruntime
mobi
html2text
@ -13,4 +10,6 @@ python-docx
python-pptx
openpyxl
PyPDF2
ebooklib
beautifulsoup4