bugfix
This commit is contained in:
parent
8d3e724322
commit
034fb514c4
@ -3,16 +3,10 @@ import codecs
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from langchain_community.document_loaders.csv_loader import CSVLoader
|
from langchain_community.document_loaders.csv_loader import CSVLoader
|
||||||
from langchain_community.document_loaders.text import TextLoader
|
from langchain_community.document_loaders.text import TextLoader
|
||||||
# from langchain_community.document_loaders.epub import UnstructuredEPubLoader
|
|
||||||
# from langchain_community.document_loaders.pdf import UnstructuredPDFLoader
|
|
||||||
# from langchain_community.document_loaders import PyPDFLoader
|
|
||||||
# from langchain_community.document_loaders import UnstructuredWordDocumentLoader
|
|
||||||
from langchain_community.document_loaders.chm import UnstructuredCHMLoader
|
|
||||||
from langchain_community.document_loaders import UnstructuredExcelLoader
|
|
||||||
from langchain_community.document_loaders import UnstructuredPowerPointLoader
|
|
||||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
||||||
from docx import Document
|
from docx import Document
|
||||||
# import chm
|
# import chm
|
||||||
|
from ebooklib import epub
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
from pptx import Presentation
|
from pptx import Presentation
|
||||||
from openpyxl import load_workbook
|
from openpyxl import load_workbook
|
||||||
import mobi
|
import mobi
|
||||||
@ -29,7 +23,7 @@ class MyMobiLoader:
|
|||||||
content=f.read()
|
content=f.read()
|
||||||
return html2text.html2text(content)
|
return html2text.html2text(content)
|
||||||
|
|
||||||
class MyChmLoader(UnstructuredCHMLoader):
|
class MyChmLoader:
|
||||||
def __init__(self, file_path):
|
def __init__(self, file_path):
|
||||||
self.filepath = file_path
|
self.filepath = file_path
|
||||||
self.chm_file = None
|
self.chm_file = None
|
||||||
@ -114,11 +108,28 @@ class MyExcelLoader:
|
|||||||
return '\n'.join(content)
|
return '\n'.join(content)
|
||||||
|
|
||||||
class MyEpubLoader:
|
class MyEpubLoader:
|
||||||
|
def __init__(self, file_path):
|
||||||
|
self.filepath = file_path
|
||||||
|
self.book = None
|
||||||
|
|
||||||
|
def load(self):
|
||||||
|
"""Reads the EPUB file and returns all text content as a string."""
|
||||||
|
self.book = epub.read_epub(self.filepath)
|
||||||
|
content = []
|
||||||
|
|
||||||
|
for item in self.book.get_items():
|
||||||
|
if isinstance(item, epub.EpubHtml):
|
||||||
|
soup = BeautifulSoup(item.get_content(), 'html.parser')
|
||||||
|
text = soup.get_text()
|
||||||
|
content.append(text.strip())
|
||||||
|
|
||||||
|
return '\n\n'.join(content)
|
||||||
|
|
||||||
|
class MyTextLoader(TextLoader):
|
||||||
def load(self):
|
def load(self):
|
||||||
docs = super().load()
|
docs = super().load()
|
||||||
print(len(docs))
|
return '\n'.join([d.page_content for d in docs])
|
||||||
return ' '.join([d.page_content for d in docs])
|
|
||||||
|
|
||||||
def fileloader(file_path):
|
def fileloader(file_path):
|
||||||
# Load the PDF file and split the data into chunks
|
# Load the PDF file and split the data into chunks
|
||||||
data = None
|
data = None
|
||||||
@ -140,7 +151,7 @@ def fileloader(file_path):
|
|||||||
elif file_path.lower().endswith('.mobi'):
|
elif file_path.lower().endswith('.mobi'):
|
||||||
loader = MyMobiLoader(file_path=file_path)
|
loader = MyMobiLoader(file_path=file_path)
|
||||||
else:
|
else:
|
||||||
loader = TextLoader(file_path=file_path)
|
loader = MyTextLoader(file_path=file_path)
|
||||||
data = loader.load()
|
data = loader.load()
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
@ -1,10 +1,7 @@
|
|||||||
langchain_community
|
langchain_community
|
||||||
unstructured
|
unstructured
|
||||||
pypdf
|
|
||||||
# rapidocr-onnxruntime 从pdf文件中的图像中获取文字
|
|
||||||
pillow
|
pillow
|
||||||
pypandoc
|
# rapidocr-onnxruntime 从pdf文件中的图像中获取文字
|
||||||
# epub文件需安装pandoc, 在macos上执行:brew install pandoc
|
|
||||||
rapidocr-onnxruntime
|
rapidocr-onnxruntime
|
||||||
mobi
|
mobi
|
||||||
html2text
|
html2text
|
||||||
@ -13,4 +10,6 @@ python-docx
|
|||||||
python-pptx
|
python-pptx
|
||||||
openpyxl
|
openpyxl
|
||||||
PyPDF2
|
PyPDF2
|
||||||
|
ebooklib
|
||||||
|
beautifulsoup4
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user