From 935119ef44f0b2bba3ad871ec8dd1f3eb16b5b3e Mon Sep 17 00:00:00 2001 From: yumoqing Date: Sat, 24 May 2025 13:10:30 +0800 Subject: [PATCH] bugfix --- file2text/loader.py | 259 ++++++++++++++++++++------------------------ 1 file changed, 116 insertions(+), 143 deletions(-) diff --git a/file2text/loader.py b/file2text/loader.py index 2c15753..f87bb58 100644 --- a/file2text/loader.py +++ b/file2text/loader.py @@ -12,169 +12,142 @@ import mobi from pypdf import PdfReader import html2text -class MyMobiLoader: +class BaseLoader: def __init__(self, file_path): - self.file_path = file_path - + self.filepath = file_path + def load(self): - tempdir, filepath = mobi.extract(self.file_path) + pts = [] + for i, t in self.load_pagetext(): + pts.append(' '.join(t.split('\t'))) + return '\n'.join(pts) + + def load_pagetext(self): + raise Exception('Not implement') + +class MyMobiLoader(BaseLoader): + def load_pagetext(self): + tempdir, filepath = mobi.extract(self.filepath) with codecs.open(filepath, "r", "utf-8") as f: content=f.read() - return html2text.html2text(content) - -class MyChmLoader: - def __init__(self, file_path): - self.filepath = file_path - self.chm_file = None - - def load(self): - """Reads the CHM file and returns concatenated text from all text/html entries.""" - self.chm_file = CHM.CHMFile(self.filepath) - content = [] - - def callback(chm_item): - if chm_item[0].endswith(('.html', '.htm', '.txt')): - try: - data = self.chm_file.read_file(chm_item[0]) - if data: - content.append(data.decode('utf-8', errors='ignore')) - except Exception: - pass - return True - - self.chm_file.walk(callback) - return '\n'.join(content) - -class MyPdfLoader: - def __init__(self, file_path, **kw): - self.filepath = file_path - - def load(self): + yield (0, html2text.html2text(content)) + +class MyPdfLoader(BaseLoader): + def load_pagetext(self): """Reads the PDF file and returns all text as a single string.""" - text = '' reader = PdfReader(self.filepath) pts = [] - for page in reader.pages: + for i,page in enumerate(reader.pages): t = page.extract_text() or '' - pts.append(' '.join(t.split('\t'))) - return ' '.join(pts) + yield (i, t) -class MyDocLoader: - def __init__(self, file_path): - self.filepath = file_path +class MyDocxLoader(BaseLoader): + def load_pagetext(self): + """Reads the .docx file and returns the full text as a single string.""" + docx = Document(self.filepath) + for i, para in enumerate(docx.paragraphs): + yield (i,para.text) - def load(self): - """Extract plain text from a .doc file using python-doc.""" - with open(self.filepath, 'rb') as f: - document = python_doc.Document(f) - return document.text() +class MyPptLoader(BaseLoader): + def load_pagetext(self): + prs = Presentation(self.filepath) + for i, slide in enumerate(prs.slides): + txts = [] + for shape in slide.shapes: + if hasattr(shape, "text"): + txts.append(shape.text) + yield (i,'\n'.join(txts)) -class MyDocxLoader: +class MyCsvLoader(BaseLoader): + def load_pagetext(self): + loader = CSVLoader(self.filepath) + docs = loader.load() + for i, d in enumerate(docs): + dat = (i, d.page_content) + yield dat + +class MyExcelLoader(BaseLoader): + def load_pagetext(self): + """Reads all sheets in the Excel file and returns the content as a string.""" + self.workbook = load_workbook(filename=self.filepath, data_only=True) + content = [] + + for i, sheet in enumerate(self.workbook.worksheets): + txts = [] + for row in sheet.iter_rows(values_only=True): + row_text = '\t'.join(str(cell) if cell is not None else '' for cell in row) + txts.append(row_text) + yield(i, '\n'.join(txts)) + + return content + +class MyEpubLoader(BaseLoader): def __init__(self, file_path): self.filepath = file_path - self.document = None + self.book = None + + def load_pagetext(self): + """Reads the EPUB file and returns all text content as a string.""" + self.book = epub.read_epub(self.filepath) + for i, item in enumerate(self.book.get_items()): + if isinstance(item, epub.EpubHtml): + soup = BeautifulSoup(item.get_content(), 'html.parser') + text = soup.get_text() + yield(i, text.strip()) + + +class MyTextLoader(BaseLoader): + def load_pagetext(self): + loader = TextLoader(self.filepath) + docs = loader.load() + for i, d in enumerate(docs): + dat = (i, d.page_content) + yield dat + +class File2Text: + all_loaders = { + 'docx':MyDocxLoader, + 'pptx':MyPptLoader, + 'csv':MyCsvLoader, + 'xlsx':MyExcelLoader, + 'pdf':MyPdfLoader, + 'epub':MyEpubLoader, + 'mobi':MyMobiLoader, + 'txt':MyTextLoader + } + def __init__(self, filepath): + self.filepath = filepath + + def load_pagetext(self): + k = self.filepath.lower().split('.')[-1] + klass = self.all_loaders.get(k, MyTextLoader) + loader = klass(self.filepath) + for d in loader.load_pagetext(): + yield d def load(self): - """Reads the .docx file and returns the full text as a single string.""" - self.document = Document(self.filepath) - text = '\n'.join([para.text for para in self.document.paragraphs]) - return text - -class MyPptLoader: - def __init__(self, file_path): - self.filepath = file_path - - def load(self): - prs = Presentation(self.filepath) - text = [] - for slide in prs.slides: - for shape in slide.shapes: - if hasattr(shape, "text"): - text.append(shape.text) - return "\n".join(text) - -class MyCsvLoader(CSVLoader): - def load(self): - docs = super().load() - return ' '.join([d.page_content for d in docs]) - -class MyExcelLoader: - def __init__(self, file_path): - self.filepath = file_path - self.workbook = None - - def load(self): - """Reads all sheets in the Excel file and returns the content as a string.""" - self.workbook = load_workbook(filename=self.filepath, data_only=True) - content = [] - - for sheet in self.workbook.worksheets: - content.append(f"--- Sheet: {sheet.title} ---") - for row in sheet.iter_rows(values_only=True): - row_text = '\t'.join(str(cell) if cell is not None else '' for cell in row) - content.append(row_text) - - return '\n'.join(content) - -class MyEpubLoader: - def __init__(self, file_path): - self.filepath = file_path - self.book = None - - def load(self): - """Reads the EPUB file and returns all text content as a string.""" - self.book = epub.read_epub(self.filepath) - content = [] - - for item in self.book.get_items(): - if isinstance(item, epub.EpubHtml): - soup = BeautifulSoup(item.get_content(), 'html.parser') - text = soup.get_text() - content.append(text.strip()) - - return '\n\n'.join(content) - -class MyTextLoader(TextLoader): - def load(self): - docs = super().load() - return '\n'.join([d.page_content for d in docs]) - + k = self.filepath.lower().split('.')[-1] + klass = self.all_loaders.get(k, MyTextLoader) + loader = klass(self.filepath) + return loader.load() + def fileloader(file_path): - # Load the PDF file and split the data into chunks - data = None - if file_path.lower().endswith('.pdf'): - loader = MyPdfLoader(file_path=file_path) - elif file_path.lower().endswith('.docx'): - loader = MyDocxLoader(file_path=file_path) - elif file_path.lower().endswith('.doc'): - raise Exception(f'not supported file({file_path}') - # loader = MyDocLoader(file_path=file_path) - elif file_path.lower().endswith('.pptx') or file_path.lower().endswith('.pptx'): - loader = MyPptLoader(file_path=file_path) - elif file_path.lower().endswith('.xlsx') or file_path.lower().endswith('.xls'): - loader = MyExcelLoader(file_path=file_path) - elif file_path.lower().endswith('.csv'): - loader = MyCsvLoader(file_path=file_path) - elif file_path.lower().endswith('.epub'): - loader = MyEpubLoader(file_path=file_path) - elif file_path.lower().endswith('.chm'): - raise Exception(f'not supported file({file_path}') - # loader = MyChmLoader(file_path=file_path) - elif file_path.lower().endswith('.mobi'): - loader = MyMobiLoader(file_path=file_path) - else: - loader = MyTextLoader(file_path=file_path) - data = loader.load() - return data + loader = File2Text(file_path) + return loader.load() + +def filepageloader(file_path): + loader = File2Text(file_path) + for d in loader.load_pagetext(): + yield d if __name__ == '__main__': import sys - from appPublic.textsplit import split_text_with_dialog_preserved if len(sys.argv) < 2: print(f'{sys.argv[0]} file\nload a file and get its text') sys.exit(1) text = fileloader(sys.argv[1]) - txtlst = split_text_with_dialog_preserved(text) - for txt in txtlst: - print(f'{txt}') - + print(f'{text=}') + """ + for i, txt in filepageloader(sys.argv[1]): + print(f'page:{i}\n{txt}\n=======\n') + """