This commit is contained in:
yumoqing 2025-05-13 14:20:13 +08:00
parent 94c6636667
commit 7c0d105d85
3 changed files with 129 additions and 28 deletions

View File

@ -1,3 +1,48 @@
# file2text
read file and convert it's content to text
read file and convert it's content to text, this modulde suport a lot of file types
## Docx file
## Excel file
## powerpoint file
```
python-pptx
```
## text file
## Pdf file
## Epub file
## Mobi file
## CHM file
1. Install libchm (if not already installed):
On Ubuntu/Debian:
```
sudo apt install libchm-bin libchm-dev
```
On macOS (with Homebrew):
```
brew install chmlib
```
2. Install Python binding:
Unfortunately, python-chm is not always available via PyPI. Instead, you might need to install it via your system package manager or from source.
On Ubuntu/Debian:
```
sudo apt install python3-chm
```
If using macOS, you might need to build a wrapper manually (let me know if you need help with that).
📝 Notes:
This extracts plain HTML text. You can clean it up using BeautifulSoup if needed.
CHM files can contain a huge number of pages. Consider filtering by file path if needed.
Would you like a version that returns cleaned text only (no HTML tags)?

View File

@ -2,16 +2,21 @@ import os
import codecs
from datetime import datetime
from langchain_community.document_loaders.csv_loader import CSVLoader
from langchain_community.document_loaders.epub import UnstructuredEPubLoader
from langchain_community.document_loaders.text import TextLoader
# from langchain_community.document_loaders.epub import UnstructuredEPubLoader
# from langchain_community.document_loaders.pdf import UnstructuredPDFLoader
from langchain_community.document_loaders import PyPDFLoader
# from langchain_community.document_loaders import PyPDFLoader
# from langchain_community.document_loaders import UnstructuredWordDocumentLoader
from langchain_community.document_loaders.chm import UnstructuredCHMLoader
from langchain_community.document_loaders import UnstructuredWordDocumentLoader
from langchain_community.document_loaders import UnstructuredExcelLoader
from langchain_community.document_loaders import UnstructuredPowerPointLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from docx import Document
# import chm
from pptx import Presentation
from openpyxl import load_workbook
import mobi
import PyPDF2
import html2text
class MyMobiLoader:
@ -25,42 +30,90 @@ class MyMobiLoader:
return html2text.html2text(content)
class MyChmLoader(UnstructuredCHMLoader):
def load(self):
docs = super().load()
return ' '.join([d.page_content for d in docs])
class MyPdfLoader(PyPDFLoader):
def __init__(self, file_path):
self.filepath = file_path
self.chm_file = None
def load(self):
"""Reads the CHM file and returns concatenated text from all text/html entries."""
self.chm_file = chm.CHMFile(self.filepath)
content = []
def callback(chm_item):
if chm_item[0].endswith(('.html', '.htm', '.txt')):
try:
data = self.chm_file.read_file(chm_item[0])
if data:
content.append(data.decode('utf-8', errors='ignore'))
except Exception:
pass
return True
self.chm_file.walk(callback)
return '\n'.join(content)
class MyPdfLoader:
def __init__(self, file_path, **kw):
super().__init__(file_path=file_path, extract_images=True, **kw)
self.filepath = file_path
self.reader = None
def load(self):
docs = super().load()
"""Reads the PDF file and returns all text as a single string."""
text = ''
for d in docs:
text += ' '.join(d.page_content.split('\t'))
with open(self.filepath, 'rb') as file:
self.reader = PyPDF2.PdfReader(file)
for page in self.reader.pages:
text += page.extract_text() or ''
return ' '.join(text.split('\t'))
class MyWordLoader:
def __init__(self, file_path):
self.filepath = file_path
self.document = None
def load(self):
"""Reads the .docx file and returns the full text as a single string."""
self.document = Document(self.filepath)
text = '\n'.join([para.text for para in self.document.paragraphs])
return text
class MyWordLoader(UnstructuredWordDocumentLoader):
def load(self):
docs = super().load()
return ' '.join([d.page_content for d in docs])
class MyPptLoader:
def __init__(self, file_path):
self.filepath = file_path
class MyPptLoader(UnstructuredPowerPointLoader):
def load(self):
docs = super().load()
return ' '.join([d.page_content for d in docs])
def load(self):
prs = Presentation(self.filepath)
text = []
for slide in prs.slides:
for shape in slide.shapes:
if hasattr(shape, "text"):
text.append(shape.text)
return "\n".join(text)
class MyCsvLoader(CSVLoader):
def load(self):
docs = super().load()
return ' '.join([d.page_content for d in docs])
class MyExcelLoader(UnstructuredExcelLoader):
def load(self):
docs = super().load()
return ' '.join([d.page_content for d in docs])
class MyExcelLoader:
def __init__(self, file_path):
self.filepath = file_path
self.workbook = None
class MyEpubLoader(UnstructuredEPubLoader):
def load(self):
"""Reads all sheets in the Excel file and returns the content as a string."""
self.workbook = load_workbook(filename=self.filepath, data_only=True)
content = []
for sheet in self.workbook.worksheets:
content.append(f"--- Sheet: {sheet.title} ---")
for row in sheet.iter_rows(values_only=True):
row_text = '\t'.join(str(cell) if cell is not None else '' for cell in row)
content.append(row_text)
return '\n'.join(content)
class MyEpubLoader:
def load(self):
docs = super().load()
print(len(docs))

View File

@ -9,5 +9,8 @@ rapidocr-onnxruntime
mobi
html2text
chm
docx
python-docx
python-pptx
openpyxl
PyPDF2