diff --git a/README.md b/README.md index d989cf3..586e1ad 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,48 @@ # file2text -read file and convert it's content to text \ No newline at end of file +read file and convert it's content to text, this modulde suport a lot of file types + +## Docx file +## Excel file +## powerpoint file +``` +python-pptx +``` +## text file +## Pdf file +## Epub file +## Mobi file + +## CHM file +1. Install libchm (if not already installed): + +On Ubuntu/Debian: + +``` +sudo apt install libchm-bin libchm-dev +``` + +On macOS (with Homebrew): + +``` +brew install chmlib +``` + +2. Install Python binding: + +Unfortunately, python-chm is not always available via PyPI. Instead, you might need to install it via your system package manager or from source. + +On Ubuntu/Debian: + +``` +sudo apt install python3-chm +``` + +If using macOS, you might need to build a wrapper manually (let me know if you need help with that). +📝 Notes: + + This extracts plain HTML text. You can clean it up using BeautifulSoup if needed. + + CHM files can contain a huge number of pages. Consider filtering by file path if needed. + +Would you like a version that returns cleaned text only (no HTML tags)? diff --git a/file2text/loader.py b/file2text/loader.py index c07962f..88facfa 100644 --- a/file2text/loader.py +++ b/file2text/loader.py @@ -2,16 +2,21 @@ import os import codecs from datetime import datetime from langchain_community.document_loaders.csv_loader import CSVLoader -from langchain_community.document_loaders.epub import UnstructuredEPubLoader from langchain_community.document_loaders.text import TextLoader +# from langchain_community.document_loaders.epub import UnstructuredEPubLoader # from langchain_community.document_loaders.pdf import UnstructuredPDFLoader -from langchain_community.document_loaders import PyPDFLoader +# from langchain_community.document_loaders import PyPDFLoader +# from langchain_community.document_loaders import UnstructuredWordDocumentLoader from langchain_community.document_loaders.chm import UnstructuredCHMLoader -from langchain_community.document_loaders import UnstructuredWordDocumentLoader from langchain_community.document_loaders import UnstructuredExcelLoader from langchain_community.document_loaders import UnstructuredPowerPointLoader from langchain_text_splitters import RecursiveCharacterTextSplitter +from docx import Document +# import chm +from pptx import Presentation +from openpyxl import load_workbook import mobi +import PyPDF2 import html2text class MyMobiLoader: @@ -25,42 +30,90 @@ class MyMobiLoader: return html2text.html2text(content) class MyChmLoader(UnstructuredCHMLoader): - def load(self): - docs = super().load() - return ' '.join([d.page_content for d in docs]) - -class MyPdfLoader(PyPDFLoader): + def __init__(self, file_path): + self.filepath = file_path + self.chm_file = None + + def load(self): + """Reads the CHM file and returns concatenated text from all text/html entries.""" + self.chm_file = chm.CHMFile(self.filepath) + content = [] + + def callback(chm_item): + if chm_item[0].endswith(('.html', '.htm', '.txt')): + try: + data = self.chm_file.read_file(chm_item[0]) + if data: + content.append(data.decode('utf-8', errors='ignore')) + except Exception: + pass + return True + + self.chm_file.walk(callback) + return '\n'.join(content) + +class MyPdfLoader: def __init__(self, file_path, **kw): - super().__init__(file_path=file_path, extract_images=True, **kw) - + self.filepath = file_path + self.reader = None + def load(self): - docs = super().load() + """Reads the PDF file and returns all text as a single string.""" text = '' - for d in docs: - text += ' '.join(d.page_content.split('\t')) + with open(self.filepath, 'rb') as file: + self.reader = PyPDF2.PdfReader(file) + for page in self.reader.pages: + text += page.extract_text() or '' + return ' '.join(text.split('\t')) + +class MyWordLoader: + def __init__(self, file_path): + self.filepath = file_path + self.document = None + + def load(self): + """Reads the .docx file and returns the full text as a single string.""" + self.document = Document(self.filepath) + text = '\n'.join([para.text for para in self.document.paragraphs]) return text -class MyWordLoader(UnstructuredWordDocumentLoader): - def load(self): - docs = super().load() - return ' '.join([d.page_content for d in docs]) +class MyPptLoader: + def __init__(self, file_path): + self.filepath = file_path -class MyPptLoader(UnstructuredPowerPointLoader): - def load(self): - docs = super().load() - return ' '.join([d.page_content for d in docs]) + def load(self): + prs = Presentation(self.filepath) + text = [] + for slide in prs.slides: + for shape in slide.shapes: + if hasattr(shape, "text"): + text.append(shape.text) + return "\n".join(text) class MyCsvLoader(CSVLoader): def load(self): docs = super().load() return ' '.join([d.page_content for d in docs]) -class MyExcelLoader(UnstructuredExcelLoader): - def load(self): - docs = super().load() - return ' '.join([d.page_content for d in docs]) +class MyExcelLoader: + def __init__(self, file_path): + self.filepath = file_path + self.workbook = None -class MyEpubLoader(UnstructuredEPubLoader): + def load(self): + """Reads all sheets in the Excel file and returns the content as a string.""" + self.workbook = load_workbook(filename=self.filepath, data_only=True) + content = [] + + for sheet in self.workbook.worksheets: + content.append(f"--- Sheet: {sheet.title} ---") + for row in sheet.iter_rows(values_only=True): + row_text = '\t'.join(str(cell) if cell is not None else '' for cell in row) + content.append(row_text) + + return '\n'.join(content) + +class MyEpubLoader: def load(self): docs = super().load() print(len(docs)) diff --git a/requirements.txt b/requirements.txt index bd8440b..35cc8f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,8 @@ rapidocr-onnxruntime mobi html2text chm -docx +python-docx +python-pptx +openpyxl +PyPDF2