bugfix
This commit is contained in:
parent
94c6636667
commit
7c0d105d85
47
README.md
47
README.md
@ -1,3 +1,48 @@
|
|||||||
# file2text
|
# file2text
|
||||||
|
|
||||||
read file and convert it's content to text
|
read file and convert it's content to text, this modulde suport a lot of file types
|
||||||
|
|
||||||
|
## Docx file
|
||||||
|
## Excel file
|
||||||
|
## powerpoint file
|
||||||
|
```
|
||||||
|
python-pptx
|
||||||
|
```
|
||||||
|
## text file
|
||||||
|
## Pdf file
|
||||||
|
## Epub file
|
||||||
|
## Mobi file
|
||||||
|
|
||||||
|
## CHM file
|
||||||
|
1. Install libchm (if not already installed):
|
||||||
|
|
||||||
|
On Ubuntu/Debian:
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo apt install libchm-bin libchm-dev
|
||||||
|
```
|
||||||
|
|
||||||
|
On macOS (with Homebrew):
|
||||||
|
|
||||||
|
```
|
||||||
|
brew install chmlib
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Install Python binding:
|
||||||
|
|
||||||
|
Unfortunately, python-chm is not always available via PyPI. Instead, you might need to install it via your system package manager or from source.
|
||||||
|
|
||||||
|
On Ubuntu/Debian:
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo apt install python3-chm
|
||||||
|
```
|
||||||
|
|
||||||
|
If using macOS, you might need to build a wrapper manually (let me know if you need help with that).
|
||||||
|
📝 Notes:
|
||||||
|
|
||||||
|
This extracts plain HTML text. You can clean it up using BeautifulSoup if needed.
|
||||||
|
|
||||||
|
CHM files can contain a huge number of pages. Consider filtering by file path if needed.
|
||||||
|
|
||||||
|
Would you like a version that returns cleaned text only (no HTML tags)?
|
||||||
|
@ -2,16 +2,21 @@ import os
|
|||||||
import codecs
|
import codecs
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from langchain_community.document_loaders.csv_loader import CSVLoader
|
from langchain_community.document_loaders.csv_loader import CSVLoader
|
||||||
from langchain_community.document_loaders.epub import UnstructuredEPubLoader
|
|
||||||
from langchain_community.document_loaders.text import TextLoader
|
from langchain_community.document_loaders.text import TextLoader
|
||||||
|
# from langchain_community.document_loaders.epub import UnstructuredEPubLoader
|
||||||
# from langchain_community.document_loaders.pdf import UnstructuredPDFLoader
|
# from langchain_community.document_loaders.pdf import UnstructuredPDFLoader
|
||||||
from langchain_community.document_loaders import PyPDFLoader
|
# from langchain_community.document_loaders import PyPDFLoader
|
||||||
|
# from langchain_community.document_loaders import UnstructuredWordDocumentLoader
|
||||||
from langchain_community.document_loaders.chm import UnstructuredCHMLoader
|
from langchain_community.document_loaders.chm import UnstructuredCHMLoader
|
||||||
from langchain_community.document_loaders import UnstructuredWordDocumentLoader
|
|
||||||
from langchain_community.document_loaders import UnstructuredExcelLoader
|
from langchain_community.document_loaders import UnstructuredExcelLoader
|
||||||
from langchain_community.document_loaders import UnstructuredPowerPointLoader
|
from langchain_community.document_loaders import UnstructuredPowerPointLoader
|
||||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||||
|
from docx import Document
|
||||||
|
# import chm
|
||||||
|
from pptx import Presentation
|
||||||
|
from openpyxl import load_workbook
|
||||||
import mobi
|
import mobi
|
||||||
|
import PyPDF2
|
||||||
import html2text
|
import html2text
|
||||||
|
|
||||||
class MyMobiLoader:
|
class MyMobiLoader:
|
||||||
@ -25,42 +30,90 @@ class MyMobiLoader:
|
|||||||
return html2text.html2text(content)
|
return html2text.html2text(content)
|
||||||
|
|
||||||
class MyChmLoader(UnstructuredCHMLoader):
|
class MyChmLoader(UnstructuredCHMLoader):
|
||||||
def load(self):
|
def __init__(self, file_path):
|
||||||
docs = super().load()
|
self.filepath = file_path
|
||||||
return ' '.join([d.page_content for d in docs])
|
self.chm_file = None
|
||||||
|
|
||||||
class MyPdfLoader(PyPDFLoader):
|
def load(self):
|
||||||
|
"""Reads the CHM file and returns concatenated text from all text/html entries."""
|
||||||
|
self.chm_file = chm.CHMFile(self.filepath)
|
||||||
|
content = []
|
||||||
|
|
||||||
|
def callback(chm_item):
|
||||||
|
if chm_item[0].endswith(('.html', '.htm', '.txt')):
|
||||||
|
try:
|
||||||
|
data = self.chm_file.read_file(chm_item[0])
|
||||||
|
if data:
|
||||||
|
content.append(data.decode('utf-8', errors='ignore'))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return True
|
||||||
|
|
||||||
|
self.chm_file.walk(callback)
|
||||||
|
return '\n'.join(content)
|
||||||
|
|
||||||
|
class MyPdfLoader:
|
||||||
def __init__(self, file_path, **kw):
|
def __init__(self, file_path, **kw):
|
||||||
super().__init__(file_path=file_path, extract_images=True, **kw)
|
self.filepath = file_path
|
||||||
|
self.reader = None
|
||||||
|
|
||||||
def load(self):
|
def load(self):
|
||||||
docs = super().load()
|
"""Reads the PDF file and returns all text as a single string."""
|
||||||
text = ''
|
text = ''
|
||||||
for d in docs:
|
with open(self.filepath, 'rb') as file:
|
||||||
text += ' '.join(d.page_content.split('\t'))
|
self.reader = PyPDF2.PdfReader(file)
|
||||||
|
for page in self.reader.pages:
|
||||||
|
text += page.extract_text() or ''
|
||||||
|
return ' '.join(text.split('\t'))
|
||||||
|
|
||||||
|
class MyWordLoader:
|
||||||
|
def __init__(self, file_path):
|
||||||
|
self.filepath = file_path
|
||||||
|
self.document = None
|
||||||
|
|
||||||
|
def load(self):
|
||||||
|
"""Reads the .docx file and returns the full text as a single string."""
|
||||||
|
self.document = Document(self.filepath)
|
||||||
|
text = '\n'.join([para.text for para in self.document.paragraphs])
|
||||||
return text
|
return text
|
||||||
|
|
||||||
class MyWordLoader(UnstructuredWordDocumentLoader):
|
class MyPptLoader:
|
||||||
def load(self):
|
def __init__(self, file_path):
|
||||||
docs = super().load()
|
self.filepath = file_path
|
||||||
return ' '.join([d.page_content for d in docs])
|
|
||||||
|
|
||||||
class MyPptLoader(UnstructuredPowerPointLoader):
|
def load(self):
|
||||||
def load(self):
|
prs = Presentation(self.filepath)
|
||||||
docs = super().load()
|
text = []
|
||||||
return ' '.join([d.page_content for d in docs])
|
for slide in prs.slides:
|
||||||
|
for shape in slide.shapes:
|
||||||
|
if hasattr(shape, "text"):
|
||||||
|
text.append(shape.text)
|
||||||
|
return "\n".join(text)
|
||||||
|
|
||||||
class MyCsvLoader(CSVLoader):
|
class MyCsvLoader(CSVLoader):
|
||||||
def load(self):
|
def load(self):
|
||||||
docs = super().load()
|
docs = super().load()
|
||||||
return ' '.join([d.page_content for d in docs])
|
return ' '.join([d.page_content for d in docs])
|
||||||
|
|
||||||
class MyExcelLoader(UnstructuredExcelLoader):
|
class MyExcelLoader:
|
||||||
def load(self):
|
def __init__(self, file_path):
|
||||||
docs = super().load()
|
self.filepath = file_path
|
||||||
return ' '.join([d.page_content for d in docs])
|
self.workbook = None
|
||||||
|
|
||||||
class MyEpubLoader(UnstructuredEPubLoader):
|
def load(self):
|
||||||
|
"""Reads all sheets in the Excel file and returns the content as a string."""
|
||||||
|
self.workbook = load_workbook(filename=self.filepath, data_only=True)
|
||||||
|
content = []
|
||||||
|
|
||||||
|
for sheet in self.workbook.worksheets:
|
||||||
|
content.append(f"--- Sheet: {sheet.title} ---")
|
||||||
|
for row in sheet.iter_rows(values_only=True):
|
||||||
|
row_text = '\t'.join(str(cell) if cell is not None else '' for cell in row)
|
||||||
|
content.append(row_text)
|
||||||
|
|
||||||
|
return '\n'.join(content)
|
||||||
|
|
||||||
|
class MyEpubLoader:
|
||||||
def load(self):
|
def load(self):
|
||||||
docs = super().load()
|
docs = super().load()
|
||||||
print(len(docs))
|
print(len(docs))
|
||||||
|
@ -9,5 +9,8 @@ rapidocr-onnxruntime
|
|||||||
mobi
|
mobi
|
||||||
html2text
|
html2text
|
||||||
chm
|
chm
|
||||||
docx
|
python-docx
|
||||||
|
python-pptx
|
||||||
|
openpyxl
|
||||||
|
PyPDF2
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user