import os import codecs from datetime import datetime from langchain_community.document_loaders.csv_loader import CSVLoader from langchain_community.document_loaders.text import TextLoader from docx import Document import python_doc import CHM from ebooklib import epub from bs4 import BeautifulSoup from pptx import Presentation from openpyxl import load_workbook import mobi import PyPDF2 import html2text class MyMobiLoader: def __init__(self, file_path): self.file_path = file_path def load(self): tempdir, filepath = mobi.extract(self.file_path) with codecs.open(filepath, "r", "utf-8") as f: content=f.read() return html2text.html2text(content) class MyChmLoader: def __init__(self, file_path): self.filepath = file_path self.chm_file = None def load(self): """Reads the CHM file and returns concatenated text from all text/html entries.""" self.chm_file = CHM.CHMFile(self.filepath) content = [] def callback(chm_item): if chm_item[0].endswith(('.html', '.htm', '.txt')): try: data = self.chm_file.read_file(chm_item[0]) if data: content.append(data.decode('utf-8', errors='ignore')) except Exception: pass return True self.chm_file.walk(callback) return '\n'.join(content) class MyPdfLoader: def __init__(self, file_path, **kw): self.filepath = file_path self.reader = None def load(self): """Reads the PDF file and returns all text as a single string.""" text = '' with open(self.filepath, 'rb') as file: self.reader = PyPDF2.PdfReader(file) for page in self.reader.pages: text += page.extract_text() or '' return ' '.join(text.split('\t')) class MyDocLoader: def __init__(self, file_path): self.filepath = file_path def load(self): """Extract plain text from a .doc file using python-doc.""" with open(self.filepath, 'rb') as f: document = python_doc.Document(f) return document.text() class MyDocxLoader: def __init__(self, file_path): self.filepath = file_path self.document = None def load(self): """Reads the .docx file and returns the full text as a single string.""" self.document = Document(self.filepath) text = '\n'.join([para.text for para in self.document.paragraphs]) return text class MyPptLoader: def __init__(self, file_path): self.filepath = file_path def load(self): prs = Presentation(self.filepath) text = [] for slide in prs.slides: for shape in slide.shapes: if hasattr(shape, "text"): text.append(shape.text) return "\n".join(text) class MyCsvLoader(CSVLoader): def load(self): docs = super().load() return ' '.join([d.page_content for d in docs]) class MyExcelLoader: def __init__(self, file_path): self.filepath = file_path self.workbook = None def load(self): """Reads all sheets in the Excel file and returns the content as a string.""" self.workbook = load_workbook(filename=self.filepath, data_only=True) content = [] for sheet in self.workbook.worksheets: content.append(f"--- Sheet: {sheet.title} ---") for row in sheet.iter_rows(values_only=True): row_text = '\t'.join(str(cell) if cell is not None else '' for cell in row) content.append(row_text) return '\n'.join(content) class MyEpubLoader: def __init__(self, file_path): self.filepath = file_path self.book = None def load(self): """Reads the EPUB file and returns all text content as a string.""" self.book = epub.read_epub(self.filepath) content = [] for item in self.book.get_items(): if isinstance(item, epub.EpubHtml): soup = BeautifulSoup(item.get_content(), 'html.parser') text = soup.get_text() content.append(text.strip()) return '\n\n'.join(content) class MyTextLoader(TextLoader): def load(self): docs = super().load() return '\n'.join([d.page_content for d in docs]) def fileloader(file_path): # Load the PDF file and split the data into chunks data = None if file_path.lower().endswith('.pdf'): loader = MyPdfLoader(file_path=file_path) elif file_path.lower().endswith('.docx'): loader = MyDocxLoader(file_path=file_path) elif file_path.lower().endswith('.doc'): loader = MyDocLoader(file_path=file_path) elif file_path.lower().endswith('.pptx') or file_path.lower().endswith('.pptx'): loader = MyPptLoader(file_path=file_path) elif file_path.lower().endswith('.xlsx') or file_path.lower().endswith('.xls'): loader = MyExcelLoader(file_path=file_path) elif file_path.lower().endswith('.csv'): loader = MyCsvLoader(file_path=file_path) elif file_path.lower().endswith('.epub'): loader = MyEpubLoader(file_path=file_path) elif file_path.lower().endswith('.chm'): loader = MyChmLoader(file_path=file_path) elif file_path.lower().endswith('.mobi'): loader = MyMobiLoader(file_path=file_path) else: loader = MyTextLoader(file_path=file_path) data = loader.load() return data if __name__ == '__main__': import sys if len(sys.argv) < 2: print(f'{sys.argv[0]} file\nload a file and get its text') sys.exit(1) text = fileloader(sys.argv[1]) print(f'{text}')