bugfix
This commit is contained in:
parent
94c6636667
commit
7c0d105d85
47
README.md
47
README.md
@ -1,3 +1,48 @@
|
||||
# file2text
|
||||
|
||||
read file and convert it's content to text
|
||||
read file and convert it's content to text, this modulde suport a lot of file types
|
||||
|
||||
## Docx file
|
||||
## Excel file
|
||||
## powerpoint file
|
||||
```
|
||||
python-pptx
|
||||
```
|
||||
## text file
|
||||
## Pdf file
|
||||
## Epub file
|
||||
## Mobi file
|
||||
|
||||
## CHM file
|
||||
1. Install libchm (if not already installed):
|
||||
|
||||
On Ubuntu/Debian:
|
||||
|
||||
```
|
||||
sudo apt install libchm-bin libchm-dev
|
||||
```
|
||||
|
||||
On macOS (with Homebrew):
|
||||
|
||||
```
|
||||
brew install chmlib
|
||||
```
|
||||
|
||||
2. Install Python binding:
|
||||
|
||||
Unfortunately, python-chm is not always available via PyPI. Instead, you might need to install it via your system package manager or from source.
|
||||
|
||||
On Ubuntu/Debian:
|
||||
|
||||
```
|
||||
sudo apt install python3-chm
|
||||
```
|
||||
|
||||
If using macOS, you might need to build a wrapper manually (let me know if you need help with that).
|
||||
📝 Notes:
|
||||
|
||||
This extracts plain HTML text. You can clean it up using BeautifulSoup if needed.
|
||||
|
||||
CHM files can contain a huge number of pages. Consider filtering by file path if needed.
|
||||
|
||||
Would you like a version that returns cleaned text only (no HTML tags)?
|
||||
|
@ -2,16 +2,21 @@ import os
|
||||
import codecs
|
||||
from datetime import datetime
|
||||
from langchain_community.document_loaders.csv_loader import CSVLoader
|
||||
from langchain_community.document_loaders.epub import UnstructuredEPubLoader
|
||||
from langchain_community.document_loaders.text import TextLoader
|
||||
# from langchain_community.document_loaders.epub import UnstructuredEPubLoader
|
||||
# from langchain_community.document_loaders.pdf import UnstructuredPDFLoader
|
||||
from langchain_community.document_loaders import PyPDFLoader
|
||||
# from langchain_community.document_loaders import PyPDFLoader
|
||||
# from langchain_community.document_loaders import UnstructuredWordDocumentLoader
|
||||
from langchain_community.document_loaders.chm import UnstructuredCHMLoader
|
||||
from langchain_community.document_loaders import UnstructuredWordDocumentLoader
|
||||
from langchain_community.document_loaders import UnstructuredExcelLoader
|
||||
from langchain_community.document_loaders import UnstructuredPowerPointLoader
|
||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
from docx import Document
|
||||
# import chm
|
||||
from pptx import Presentation
|
||||
from openpyxl import load_workbook
|
||||
import mobi
|
||||
import PyPDF2
|
||||
import html2text
|
||||
|
||||
class MyMobiLoader:
|
||||
@ -25,42 +30,90 @@ class MyMobiLoader:
|
||||
return html2text.html2text(content)
|
||||
|
||||
class MyChmLoader(UnstructuredCHMLoader):
|
||||
def load(self):
|
||||
docs = super().load()
|
||||
return ' '.join([d.page_content for d in docs])
|
||||
|
||||
class MyPdfLoader(PyPDFLoader):
|
||||
def __init__(self, file_path):
|
||||
self.filepath = file_path
|
||||
self.chm_file = None
|
||||
|
||||
def load(self):
|
||||
"""Reads the CHM file and returns concatenated text from all text/html entries."""
|
||||
self.chm_file = chm.CHMFile(self.filepath)
|
||||
content = []
|
||||
|
||||
def callback(chm_item):
|
||||
if chm_item[0].endswith(('.html', '.htm', '.txt')):
|
||||
try:
|
||||
data = self.chm_file.read_file(chm_item[0])
|
||||
if data:
|
||||
content.append(data.decode('utf-8', errors='ignore'))
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
|
||||
self.chm_file.walk(callback)
|
||||
return '\n'.join(content)
|
||||
|
||||
class MyPdfLoader:
|
||||
def __init__(self, file_path, **kw):
|
||||
super().__init__(file_path=file_path, extract_images=True, **kw)
|
||||
|
||||
self.filepath = file_path
|
||||
self.reader = None
|
||||
|
||||
def load(self):
|
||||
docs = super().load()
|
||||
"""Reads the PDF file and returns all text as a single string."""
|
||||
text = ''
|
||||
for d in docs:
|
||||
text += ' '.join(d.page_content.split('\t'))
|
||||
with open(self.filepath, 'rb') as file:
|
||||
self.reader = PyPDF2.PdfReader(file)
|
||||
for page in self.reader.pages:
|
||||
text += page.extract_text() or ''
|
||||
return ' '.join(text.split('\t'))
|
||||
|
||||
class MyWordLoader:
|
||||
def __init__(self, file_path):
|
||||
self.filepath = file_path
|
||||
self.document = None
|
||||
|
||||
def load(self):
|
||||
"""Reads the .docx file and returns the full text as a single string."""
|
||||
self.document = Document(self.filepath)
|
||||
text = '\n'.join([para.text for para in self.document.paragraphs])
|
||||
return text
|
||||
|
||||
class MyWordLoader(UnstructuredWordDocumentLoader):
|
||||
def load(self):
|
||||
docs = super().load()
|
||||
return ' '.join([d.page_content for d in docs])
|
||||
class MyPptLoader:
|
||||
def __init__(self, file_path):
|
||||
self.filepath = file_path
|
||||
|
||||
class MyPptLoader(UnstructuredPowerPointLoader):
|
||||
def load(self):
|
||||
docs = super().load()
|
||||
return ' '.join([d.page_content for d in docs])
|
||||
def load(self):
|
||||
prs = Presentation(self.filepath)
|
||||
text = []
|
||||
for slide in prs.slides:
|
||||
for shape in slide.shapes:
|
||||
if hasattr(shape, "text"):
|
||||
text.append(shape.text)
|
||||
return "\n".join(text)
|
||||
|
||||
class MyCsvLoader(CSVLoader):
|
||||
def load(self):
|
||||
docs = super().load()
|
||||
return ' '.join([d.page_content for d in docs])
|
||||
|
||||
class MyExcelLoader(UnstructuredExcelLoader):
|
||||
def load(self):
|
||||
docs = super().load()
|
||||
return ' '.join([d.page_content for d in docs])
|
||||
class MyExcelLoader:
|
||||
def __init__(self, file_path):
|
||||
self.filepath = file_path
|
||||
self.workbook = None
|
||||
|
||||
class MyEpubLoader(UnstructuredEPubLoader):
|
||||
def load(self):
|
||||
"""Reads all sheets in the Excel file and returns the content as a string."""
|
||||
self.workbook = load_workbook(filename=self.filepath, data_only=True)
|
||||
content = []
|
||||
|
||||
for sheet in self.workbook.worksheets:
|
||||
content.append(f"--- Sheet: {sheet.title} ---")
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
row_text = '\t'.join(str(cell) if cell is not None else '' for cell in row)
|
||||
content.append(row_text)
|
||||
|
||||
return '\n'.join(content)
|
||||
|
||||
class MyEpubLoader:
|
||||
def load(self):
|
||||
docs = super().load()
|
||||
print(len(docs))
|
||||
|
@ -9,5 +9,8 @@ rapidocr-onnxruntime
|
||||
mobi
|
||||
html2text
|
||||
chm
|
||||
docx
|
||||
python-docx
|
||||
python-pptx
|
||||
openpyxl
|
||||
PyPDF2
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user