sqlor/dataloader/dataloader.py

70 lines
1.6 KiB
Python
Raw Normal View History

2023-04-12 09:50:54 +08:00
from appPublic.jsonConfig import getConfig
2019-08-21 09:26:46 +08:00
import openpyxl as xlsx
import asyncio
2023-04-11 15:12:10 +08:00
from sqlor.dbpools import DBPools
2023-04-19 14:50:25 +08:00
import aiomysql
import aiopg
2023-04-12 09:50:54 +08:00
from typeconv import convrec
2019-08-21 09:26:46 +08:00
class CBObject:
2023-04-11 15:12:10 +08:00
def __init__(self,db, name):
self.db = db
2019-08-21 09:26:46 +08:00
self.tbl = name
2023-04-11 15:12:10 +08:00
async def handle(self,ws):
db = DBPools()
async with db.sqlorContext(self.db) as sor:
2023-05-11 20:27:38 +08:00
delete_sql = "TRUNCATE TABLE %s" % self.tbl
2023-05-11 16:43:14 +08:00
await sor.sqlExe(delete_sql, {})
2023-04-12 09:50:54 +08:00
info = await sor.I(self.tbl)
2023-04-11 15:12:10 +08:00
for rec in getRecord(ws):
2023-04-12 09:50:54 +08:00
r = [ v for v in rec.values() if v is not None ]
if len(r) == 0:
continue
rec = convrec(info, rec)
await sor.C(self.tbl, rec)
2019-08-21 09:26:46 +08:00
2023-04-12 09:50:54 +08:00
def getRecord(ws):
2019-08-21 09:26:46 +08:00
names = []
types = []
for i,r in enumerate(ws.rows):
if i==0:
for j,c in enumerate(r):
nt = c.value.split(':')
if len(nt) < 2:
nt.append('')
n,t = nt
names.append(n)
types.append(t)
else:
dic = {}
for j,c in enumerate(r):
v = c.value
dic[names[j]] = v
2023-04-12 09:50:54 +08:00
yield dic
2019-08-21 09:26:46 +08:00
2023-04-12 09:50:54 +08:00
async def loaddatainexcel(xlsxfile):
2019-08-21 09:26:46 +08:00
wb = xlsx.load_workbook(xlsxfile)
2023-04-12 09:50:54 +08:00
print(f'{wb.sheetnames=}')
dbname = [ i[2:-2] for i in wb.sheetnames if i.startswith('__')][0]
print(f'{dbname=}')
2019-08-21 09:26:46 +08:00
for name in wb.sheetnames:
2023-04-11 15:12:10 +08:00
if name.startswith('__'):
continue
2019-08-21 09:26:46 +08:00
ws = wb[name]
2023-04-11 15:12:10 +08:00
cbobj = CBObject(dbname, name)
await cbobj.handle(ws)
2019-08-21 09:26:46 +08:00
if __name__ == '__main__':
import sys
2023-04-12 09:50:54 +08:00
import os
p = os.getcwd()
config = getConfig(p)
print(f'{config.databases=},cwd={p}')
2023-04-11 15:12:10 +08:00
DBPools(config.databases)
2019-08-21 09:26:46 +08:00
if len(sys.argv) < 2:
print('%s xlsxfile' % sys.argv[0])
sys.exit(1)
loop = asyncio.get_event_loop()
2023-04-12 09:50:54 +08:00
loop.run_until_complete(loaddatainexcel(sys.argv[1]))