bugfix
This commit is contained in:
parent
61fddf4bd9
commit
935119ef44
@ -12,169 +12,142 @@ import mobi
|
|||||||
from pypdf import PdfReader
|
from pypdf import PdfReader
|
||||||
import html2text
|
import html2text
|
||||||
|
|
||||||
class MyMobiLoader:
|
class BaseLoader:
|
||||||
def __init__(self, file_path):
|
def __init__(self, file_path):
|
||||||
self.file_path = file_path
|
self.filepath = file_path
|
||||||
|
|
||||||
def load(self):
|
def load(self):
|
||||||
tempdir, filepath = mobi.extract(self.file_path)
|
pts = []
|
||||||
|
for i, t in self.load_pagetext():
|
||||||
|
pts.append(' '.join(t.split('\t')))
|
||||||
|
return '\n'.join(pts)
|
||||||
|
|
||||||
|
def load_pagetext(self):
|
||||||
|
raise Exception('Not implement')
|
||||||
|
|
||||||
|
class MyMobiLoader(BaseLoader):
|
||||||
|
def load_pagetext(self):
|
||||||
|
tempdir, filepath = mobi.extract(self.filepath)
|
||||||
with codecs.open(filepath, "r", "utf-8") as f:
|
with codecs.open(filepath, "r", "utf-8") as f:
|
||||||
content=f.read()
|
content=f.read()
|
||||||
return html2text.html2text(content)
|
yield (0, html2text.html2text(content))
|
||||||
|
|
||||||
class MyChmLoader:
|
class MyPdfLoader(BaseLoader):
|
||||||
def __init__(self, file_path):
|
def load_pagetext(self):
|
||||||
self.filepath = file_path
|
|
||||||
self.chm_file = None
|
|
||||||
|
|
||||||
def load(self):
|
|
||||||
"""Reads the CHM file and returns concatenated text from all text/html entries."""
|
|
||||||
self.chm_file = CHM.CHMFile(self.filepath)
|
|
||||||
content = []
|
|
||||||
|
|
||||||
def callback(chm_item):
|
|
||||||
if chm_item[0].endswith(('.html', '.htm', '.txt')):
|
|
||||||
try:
|
|
||||||
data = self.chm_file.read_file(chm_item[0])
|
|
||||||
if data:
|
|
||||||
content.append(data.decode('utf-8', errors='ignore'))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
return True
|
|
||||||
|
|
||||||
self.chm_file.walk(callback)
|
|
||||||
return '\n'.join(content)
|
|
||||||
|
|
||||||
class MyPdfLoader:
|
|
||||||
def __init__(self, file_path, **kw):
|
|
||||||
self.filepath = file_path
|
|
||||||
|
|
||||||
def load(self):
|
|
||||||
"""Reads the PDF file and returns all text as a single string."""
|
"""Reads the PDF file and returns all text as a single string."""
|
||||||
text = ''
|
|
||||||
reader = PdfReader(self.filepath)
|
reader = PdfReader(self.filepath)
|
||||||
pts = []
|
pts = []
|
||||||
for page in reader.pages:
|
for i,page in enumerate(reader.pages):
|
||||||
t = page.extract_text() or ''
|
t = page.extract_text() or ''
|
||||||
pts.append(' '.join(t.split('\t')))
|
yield (i, t)
|
||||||
return ' '.join(pts)
|
|
||||||
|
|
||||||
class MyDocLoader:
|
class MyDocxLoader(BaseLoader):
|
||||||
def __init__(self, file_path):
|
def load_pagetext(self):
|
||||||
self.filepath = file_path
|
"""Reads the .docx file and returns the full text as a single string."""
|
||||||
|
docx = Document(self.filepath)
|
||||||
|
for i, para in enumerate(docx.paragraphs):
|
||||||
|
yield (i,para.text)
|
||||||
|
|
||||||
def load(self):
|
class MyPptLoader(BaseLoader):
|
||||||
"""Extract plain text from a .doc file using python-doc."""
|
def load_pagetext(self):
|
||||||
with open(self.filepath, 'rb') as f:
|
prs = Presentation(self.filepath)
|
||||||
document = python_doc.Document(f)
|
for i, slide in enumerate(prs.slides):
|
||||||
return document.text()
|
txts = []
|
||||||
|
for shape in slide.shapes:
|
||||||
|
if hasattr(shape, "text"):
|
||||||
|
txts.append(shape.text)
|
||||||
|
yield (i,'\n'.join(txts))
|
||||||
|
|
||||||
class MyDocxLoader:
|
class MyCsvLoader(BaseLoader):
|
||||||
|
def load_pagetext(self):
|
||||||
|
loader = CSVLoader(self.filepath)
|
||||||
|
docs = loader.load()
|
||||||
|
for i, d in enumerate(docs):
|
||||||
|
dat = (i, d.page_content)
|
||||||
|
yield dat
|
||||||
|
|
||||||
|
class MyExcelLoader(BaseLoader):
|
||||||
|
def load_pagetext(self):
|
||||||
|
"""Reads all sheets in the Excel file and returns the content as a string."""
|
||||||
|
self.workbook = load_workbook(filename=self.filepath, data_only=True)
|
||||||
|
content = []
|
||||||
|
|
||||||
|
for i, sheet in enumerate(self.workbook.worksheets):
|
||||||
|
txts = []
|
||||||
|
for row in sheet.iter_rows(values_only=True):
|
||||||
|
row_text = '\t'.join(str(cell) if cell is not None else '' for cell in row)
|
||||||
|
txts.append(row_text)
|
||||||
|
yield(i, '\n'.join(txts))
|
||||||
|
|
||||||
|
return content
|
||||||
|
|
||||||
|
class MyEpubLoader(BaseLoader):
|
||||||
def __init__(self, file_path):
|
def __init__(self, file_path):
|
||||||
self.filepath = file_path
|
self.filepath = file_path
|
||||||
self.document = None
|
self.book = None
|
||||||
|
|
||||||
|
def load_pagetext(self):
|
||||||
|
"""Reads the EPUB file and returns all text content as a string."""
|
||||||
|
self.book = epub.read_epub(self.filepath)
|
||||||
|
for i, item in enumerate(self.book.get_items()):
|
||||||
|
if isinstance(item, epub.EpubHtml):
|
||||||
|
soup = BeautifulSoup(item.get_content(), 'html.parser')
|
||||||
|
text = soup.get_text()
|
||||||
|
yield(i, text.strip())
|
||||||
|
|
||||||
|
|
||||||
|
class MyTextLoader(BaseLoader):
|
||||||
|
def load_pagetext(self):
|
||||||
|
loader = TextLoader(self.filepath)
|
||||||
|
docs = loader.load()
|
||||||
|
for i, d in enumerate(docs):
|
||||||
|
dat = (i, d.page_content)
|
||||||
|
yield dat
|
||||||
|
|
||||||
|
class File2Text:
|
||||||
|
all_loaders = {
|
||||||
|
'docx':MyDocxLoader,
|
||||||
|
'pptx':MyPptLoader,
|
||||||
|
'csv':MyCsvLoader,
|
||||||
|
'xlsx':MyExcelLoader,
|
||||||
|
'pdf':MyPdfLoader,
|
||||||
|
'epub':MyEpubLoader,
|
||||||
|
'mobi':MyMobiLoader,
|
||||||
|
'txt':MyTextLoader
|
||||||
|
}
|
||||||
|
def __init__(self, filepath):
|
||||||
|
self.filepath = filepath
|
||||||
|
|
||||||
|
def load_pagetext(self):
|
||||||
|
k = self.filepath.lower().split('.')[-1]
|
||||||
|
klass = self.all_loaders.get(k, MyTextLoader)
|
||||||
|
loader = klass(self.filepath)
|
||||||
|
for d in loader.load_pagetext():
|
||||||
|
yield d
|
||||||
|
|
||||||
def load(self):
|
def load(self):
|
||||||
"""Reads the .docx file and returns the full text as a single string."""
|
k = self.filepath.lower().split('.')[-1]
|
||||||
self.document = Document(self.filepath)
|
klass = self.all_loaders.get(k, MyTextLoader)
|
||||||
text = '\n'.join([para.text for para in self.document.paragraphs])
|
loader = klass(self.filepath)
|
||||||
return text
|
return loader.load()
|
||||||
|
|
||||||
class MyPptLoader:
|
|
||||||
def __init__(self, file_path):
|
|
||||||
self.filepath = file_path
|
|
||||||
|
|
||||||
def load(self):
|
|
||||||
prs = Presentation(self.filepath)
|
|
||||||
text = []
|
|
||||||
for slide in prs.slides:
|
|
||||||
for shape in slide.shapes:
|
|
||||||
if hasattr(shape, "text"):
|
|
||||||
text.append(shape.text)
|
|
||||||
return "\n".join(text)
|
|
||||||
|
|
||||||
class MyCsvLoader(CSVLoader):
|
|
||||||
def load(self):
|
|
||||||
docs = super().load()
|
|
||||||
return ' '.join([d.page_content for d in docs])
|
|
||||||
|
|
||||||
class MyExcelLoader:
|
|
||||||
def __init__(self, file_path):
|
|
||||||
self.filepath = file_path
|
|
||||||
self.workbook = None
|
|
||||||
|
|
||||||
def load(self):
|
|
||||||
"""Reads all sheets in the Excel file and returns the content as a string."""
|
|
||||||
self.workbook = load_workbook(filename=self.filepath, data_only=True)
|
|
||||||
content = []
|
|
||||||
|
|
||||||
for sheet in self.workbook.worksheets:
|
|
||||||
content.append(f"--- Sheet: {sheet.title} ---")
|
|
||||||
for row in sheet.iter_rows(values_only=True):
|
|
||||||
row_text = '\t'.join(str(cell) if cell is not None else '' for cell in row)
|
|
||||||
content.append(row_text)
|
|
||||||
|
|
||||||
return '\n'.join(content)
|
|
||||||
|
|
||||||
class MyEpubLoader:
|
|
||||||
def __init__(self, file_path):
|
|
||||||
self.filepath = file_path
|
|
||||||
self.book = None
|
|
||||||
|
|
||||||
def load(self):
|
|
||||||
"""Reads the EPUB file and returns all text content as a string."""
|
|
||||||
self.book = epub.read_epub(self.filepath)
|
|
||||||
content = []
|
|
||||||
|
|
||||||
for item in self.book.get_items():
|
|
||||||
if isinstance(item, epub.EpubHtml):
|
|
||||||
soup = BeautifulSoup(item.get_content(), 'html.parser')
|
|
||||||
text = soup.get_text()
|
|
||||||
content.append(text.strip())
|
|
||||||
|
|
||||||
return '\n\n'.join(content)
|
|
||||||
|
|
||||||
class MyTextLoader(TextLoader):
|
|
||||||
def load(self):
|
|
||||||
docs = super().load()
|
|
||||||
return '\n'.join([d.page_content for d in docs])
|
|
||||||
|
|
||||||
def fileloader(file_path):
|
def fileloader(file_path):
|
||||||
# Load the PDF file and split the data into chunks
|
loader = File2Text(file_path)
|
||||||
data = None
|
return loader.load()
|
||||||
if file_path.lower().endswith('.pdf'):
|
|
||||||
loader = MyPdfLoader(file_path=file_path)
|
def filepageloader(file_path):
|
||||||
elif file_path.lower().endswith('.docx'):
|
loader = File2Text(file_path)
|
||||||
loader = MyDocxLoader(file_path=file_path)
|
for d in loader.load_pagetext():
|
||||||
elif file_path.lower().endswith('.doc'):
|
yield d
|
||||||
raise Exception(f'not supported file({file_path}')
|
|
||||||
# loader = MyDocLoader(file_path=file_path)
|
|
||||||
elif file_path.lower().endswith('.pptx') or file_path.lower().endswith('.pptx'):
|
|
||||||
loader = MyPptLoader(file_path=file_path)
|
|
||||||
elif file_path.lower().endswith('.xlsx') or file_path.lower().endswith('.xls'):
|
|
||||||
loader = MyExcelLoader(file_path=file_path)
|
|
||||||
elif file_path.lower().endswith('.csv'):
|
|
||||||
loader = MyCsvLoader(file_path=file_path)
|
|
||||||
elif file_path.lower().endswith('.epub'):
|
|
||||||
loader = MyEpubLoader(file_path=file_path)
|
|
||||||
elif file_path.lower().endswith('.chm'):
|
|
||||||
raise Exception(f'not supported file({file_path}')
|
|
||||||
# loader = MyChmLoader(file_path=file_path)
|
|
||||||
elif file_path.lower().endswith('.mobi'):
|
|
||||||
loader = MyMobiLoader(file_path=file_path)
|
|
||||||
else:
|
|
||||||
loader = MyTextLoader(file_path=file_path)
|
|
||||||
data = loader.load()
|
|
||||||
return data
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
import sys
|
import sys
|
||||||
from appPublic.textsplit import split_text_with_dialog_preserved
|
|
||||||
if len(sys.argv) < 2:
|
if len(sys.argv) < 2:
|
||||||
print(f'{sys.argv[0]} file\nload a file and get its text')
|
print(f'{sys.argv[0]} file\nload a file and get its text')
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
text = fileloader(sys.argv[1])
|
text = fileloader(sys.argv[1])
|
||||||
txtlst = split_text_with_dialog_preserved(text)
|
print(f'{text=}')
|
||||||
for txt in txtlst:
|
"""
|
||||||
print(f'{txt}')
|
for i, txt in filepageloader(sys.argv[1]):
|
||||||
|
print(f'page:{i}\n{txt}\n=======\n')
|
||||||
|
"""
|
||||||
|
Loading…
Reference in New Issue
Block a user