bugfix
This commit is contained in:
parent
8d3e724322
commit
034fb514c4
@ -3,16 +3,10 @@ import codecs
|
||||
from datetime import datetime
|
||||
from langchain_community.document_loaders.csv_loader import CSVLoader
|
||||
from langchain_community.document_loaders.text import TextLoader
|
||||
# from langchain_community.document_loaders.epub import UnstructuredEPubLoader
|
||||
# from langchain_community.document_loaders.pdf import UnstructuredPDFLoader
|
||||
# from langchain_community.document_loaders import PyPDFLoader
|
||||
# from langchain_community.document_loaders import UnstructuredWordDocumentLoader
|
||||
from langchain_community.document_loaders.chm import UnstructuredCHMLoader
|
||||
from langchain_community.document_loaders import UnstructuredExcelLoader
|
||||
from langchain_community.document_loaders import UnstructuredPowerPointLoader
|
||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
from docx import Document
|
||||
# import chm
|
||||
from ebooklib import epub
|
||||
from bs4 import BeautifulSoup
|
||||
from pptx import Presentation
|
||||
from openpyxl import load_workbook
|
||||
import mobi
|
||||
@ -29,7 +23,7 @@ class MyMobiLoader:
|
||||
content=f.read()
|
||||
return html2text.html2text(content)
|
||||
|
||||
class MyChmLoader(UnstructuredCHMLoader):
|
||||
class MyChmLoader:
|
||||
def __init__(self, file_path):
|
||||
self.filepath = file_path
|
||||
self.chm_file = None
|
||||
@ -114,11 +108,28 @@ class MyExcelLoader:
|
||||
return '\n'.join(content)
|
||||
|
||||
class MyEpubLoader:
|
||||
def __init__(self, file_path):
|
||||
self.filepath = file_path
|
||||
self.book = None
|
||||
|
||||
def load(self):
|
||||
"""Reads the EPUB file and returns all text content as a string."""
|
||||
self.book = epub.read_epub(self.filepath)
|
||||
content = []
|
||||
|
||||
for item in self.book.get_items():
|
||||
if isinstance(item, epub.EpubHtml):
|
||||
soup = BeautifulSoup(item.get_content(), 'html.parser')
|
||||
text = soup.get_text()
|
||||
content.append(text.strip())
|
||||
|
||||
return '\n\n'.join(content)
|
||||
|
||||
class MyTextLoader(TextLoader):
|
||||
def load(self):
|
||||
docs = super().load()
|
||||
print(len(docs))
|
||||
return ' '.join([d.page_content for d in docs])
|
||||
|
||||
return '\n'.join([d.page_content for d in docs])
|
||||
|
||||
def fileloader(file_path):
|
||||
# Load the PDF file and split the data into chunks
|
||||
data = None
|
||||
@ -140,7 +151,7 @@ def fileloader(file_path):
|
||||
elif file_path.lower().endswith('.mobi'):
|
||||
loader = MyMobiLoader(file_path=file_path)
|
||||
else:
|
||||
loader = TextLoader(file_path=file_path)
|
||||
loader = MyTextLoader(file_path=file_path)
|
||||
data = loader.load()
|
||||
return data
|
||||
|
||||
|
@ -1,10 +1,7 @@
|
||||
langchain_community
|
||||
unstructured
|
||||
pypdf
|
||||
# rapidocr-onnxruntime 从pdf文件中的图像中获取文字
|
||||
pillow
|
||||
pypandoc
|
||||
# epub文件需安装pandoc, 在macos上执行:brew install pandoc
|
||||
# rapidocr-onnxruntime 从pdf文件中的图像中获取文字
|
||||
rapidocr-onnxruntime
|
||||
mobi
|
||||
html2text
|
||||
@ -13,4 +10,6 @@ python-docx
|
||||
python-pptx
|
||||
openpyxl
|
||||
PyPDF2
|
||||
ebooklib
|
||||
beautifulsoup4
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user