From d18adf7aee06c118289c0c7edfa61954e1ff8c74 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Tue, 16 Jul 2019 16:33:07 +0800 Subject: [PATCH] init --- README.md | 3 + appPublic/CSVData.py | 65 +++++ appPublic/Config.py | 37 +++ appPublic/Distribution.py | 57 ++++ appPublic/ExecFile.py | 107 ++++++++ appPublic/FiniteStateMachine.py | 53 ++++ appPublic/MiniI18N.py | 238 +++++++++++++++++ appPublic/ObjectCache.py | 53 ++++ appPublic/ProbiablitySet.py | 29 ++ appPublic/PublicData.py | 18 ++ appPublic/RSAutils.py | 89 +++++++ appPublic/SQLite3Utils.py | 220 +++++++++++++++ appPublic/Singleton.py | 42 +++ appPublic/__init__.py | 0 appPublic/apsw.version.SQLite3Utils.apsw.py | 242 +++++++++++++++++ appPublic/argsConvert.py | 182 +++++++++++++ appPublic/csv_Data.py | 54 ++++ appPublic/datamapping.py | 40 +++ appPublic/dbpool.py | 45 ++++ appPublic/dictExt.py | 20 ++ appPublic/dictObject.py | 64 +++++ appPublic/easyExcel.py | 71 +++++ appPublic/exceldata.py | 281 ++++++++++++++++++++ appPublic/excelwriter.py | 144 ++++++++++ appPublic/folderUtils.py | 176 ++++++++++++ appPublic/genetic.py | 44 +++ appPublic/jsonConfig.py | 64 +++++ appPublic/jsonIO.py | 38 +++ appPublic/localefunc.py | 43 +++ appPublic/macAddress.py | 38 +++ appPublic/myImport.py | 6 + appPublic/myjson.py | 18 ++ appPublic/mylog.py | 89 +++++++ appPublic/objectAction.py | 62 +++++ appPublic/pickleUtils.py | 16 ++ appPublic/rc4.py | 250 +++++++++++++++++ appPublic/receiveMail.py | 137 ++++++++++ appPublic/restrictedEnv.py | 34 +++ appPublic/rsa.py | 170 ++++++++++++ appPublic/rsaPeer.py | 125 +++++++++ appPublic/sockPackage.py | 134 ++++++++++ appPublic/strUtils.py | 20 ++ appPublic/testdict.py | 6 + appPublic/timeUtils.py | 157 +++++++++++ appPublic/timecost.py | 38 +++ appPublic/unicoding.py | 39 +++ appPublic/uniqueID.py | 20 ++ appPublic/workers.py | 250 +++++++++++++++++ setup.py | 43 +++ 49 files changed, 4171 insertions(+) create mode 100644 README.md create mode 100755 appPublic/CSVData.py create mode 100755 appPublic/Config.py create mode 100755 appPublic/Distribution.py create mode 100755 appPublic/ExecFile.py create mode 100755 appPublic/FiniteStateMachine.py create mode 100755 appPublic/MiniI18N.py create mode 100755 appPublic/ObjectCache.py create mode 100755 appPublic/ProbiablitySet.py create mode 100755 appPublic/PublicData.py create mode 100755 appPublic/RSAutils.py create mode 100755 appPublic/SQLite3Utils.py create mode 100755 appPublic/Singleton.py create mode 100755 appPublic/__init__.py create mode 100755 appPublic/apsw.version.SQLite3Utils.apsw.py create mode 100755 appPublic/argsConvert.py create mode 100755 appPublic/csv_Data.py create mode 100755 appPublic/datamapping.py create mode 100755 appPublic/dbpool.py create mode 100755 appPublic/dictExt.py create mode 100755 appPublic/dictObject.py create mode 100755 appPublic/easyExcel.py create mode 100755 appPublic/exceldata.py create mode 100755 appPublic/excelwriter.py create mode 100755 appPublic/folderUtils.py create mode 100755 appPublic/genetic.py create mode 100755 appPublic/jsonConfig.py create mode 100755 appPublic/jsonIO.py create mode 100755 appPublic/localefunc.py create mode 100755 appPublic/macAddress.py create mode 100755 appPublic/myImport.py create mode 100755 appPublic/myjson.py create mode 100755 appPublic/mylog.py create mode 100755 appPublic/objectAction.py create mode 100755 appPublic/pickleUtils.py create mode 100755 appPublic/rc4.py create mode 100755 appPublic/receiveMail.py create mode 100755 appPublic/restrictedEnv.py create mode 100755 appPublic/rsa.py create mode 100755 appPublic/rsaPeer.py create mode 100755 appPublic/sockPackage.py create mode 100755 appPublic/strUtils.py create mode 100755 appPublic/testdict.py create mode 100755 appPublic/timeUtils.py create mode 100755 appPublic/timecost.py create mode 100755 appPublic/unicoding.py create mode 100755 appPublic/uniqueID.py create mode 100755 appPublic/workers.py create mode 100755 setup.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..cf78d89 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# appPublic +a set of icommon modules for python development + diff --git a/appPublic/CSVData.py b/appPublic/CSVData.py new file mode 100755 index 0000000..989b89d --- /dev/null +++ b/appPublic/CSVData.py @@ -0,0 +1,65 @@ +import csv + +class CSVData: + def __init__(self,csvfile,names = None,headline = 0,dataline = 1): + self.csvfile = csvfile + self.names = names + self.headline = headline + self.dataline = dataline + + def read(self): + f = open(self.csvfile,'rb') + reader = csv.reader(f) + fields = None + if self.names is not None: + fields = self.names + data = [] + lno = 0 + for l in reader: + if fields is None and lno == self.headline: + fields = [f for f in l] + if lno >= self.dataline: + rec = {} + for i in range(len(fields)): + rec[fields[i]] = l[i] + data.append(rec) + lno += 1 + f.close() + return data + + def iterRead(self): + self.fd = open(self.csvfile,'r') + try: + reader = csv.reader(self.fd) + fields = None + if self.names is not None: + fields = self.names + lno = 0 + self.onBegin() + for l in reader: + if fields is None and lno == self.headline: + fields = [f for f in l] + if lno >= self.dataline: + rec = {} + for i in range(len(fields)): + rec[fields[i]] = l[i] + self.onRecord(rec) + lno += 1 + self.fd.close() + self.onFinish() + except exception as e: + fd.close() + raise e + + def onReadBegin(self): + pass + def onRecord(self,rec): + print(rec) + + def onFinish(self): + print("onFinish() called") + +if __name__ == '__main__': + import sys + cd = CSVData(sys.argv[1],names = ['st_date','open_price','max_price','min_price','close_price','volume','adj_price']) + cd.iterRead() diff --git a/appPublic/Config.py b/appPublic/Config.py new file mode 100755 index 0000000..26c4774 --- /dev/null +++ b/appPublic/Config.py @@ -0,0 +1,37 @@ +# Config.py +# Copyright (c) 2009 longtop Co. +# See LICENSE for details. +# author: yumoqing@gmail.com +# created date: 2009-02-01 +# last modified date: 2009-02-05 + +import os,sys +from appPublic.ExecFile import ExecFile +from appPublic.dictObject import DictObject +from appPublic.Singleton import Singleton +from zope.interface import implements +CONFIG_FILE = 'conf/config.ini' +from folderUtils import ProgramPath +class Node(object) : + pass + +class Config: + + __metaclass = Singleton + def __init__(self,configpath=None): + if configpath is None: + ps = CONFIG_FILE.split('/') + configpath = os.path.join(ProgramPath(),*ps) + self.configfile = configpath + self.__execfile = ExecFile(self,path=configpath) + self.__execfile.set('Node',Node) + self.__execfile.set('DictObject',DictObject) + self.__execfile.set('dict',DictObject) + r,msg = self.__execfile.run() + if not r: + print(r,msg) + +def getConfig(path=None): + conf = Config(path) + return conf + diff --git a/appPublic/Distribution.py b/appPublic/Distribution.py new file mode 100755 index 0000000..96fbede --- /dev/null +++ b/appPublic/Distribution.py @@ -0,0 +1,57 @@ +import numpy as np +import random + +class Distribution: + def __init__(self,vl,pl=[],dtype=np.int): + self.dtype = dtype + self.cnt = len(vl) + vlen = len(vl) + plen = len(pl) + sp = sum(pl) + if plen > vlen: + pl = pl[:vlen] + plen = vlen + if sp > 1.0: + raise Exception("Probability > 1") + if plen < vlen: + pv = 1.0 - sp + n = vlen - plen + p = pv / n + for i in range(n-1,vlen): + pl.append(p) + self.vl = vl + self.pl = pl + vp = [] + vp = [1,] + i = vlen - 1 + while i > 0: + vp.append(vp[vlen - i - 1] - pl[i]) + i -= 1 + vp.reverse() + self.vp = vp + + def distri(self,value): + for v in self.vl: + print(v,(len([i for i in value if v == i])*1.0) /(len(value)*1.0)) + + def func(self,v): + i = 0 + while i < self.cnt: + if v < self.vp[i]: + return self.vl[i] + i += 1 + return self.vp[self.cnt -1] + + def generator(self,count): + a = np.random.rand(count) + func = np.frompyfunc(self.func,1,1) + return func(a) + +if __name__ == '__main__': + d1 = Distribution([1,2,3,4],[.5,.24]) + v = d1.generator(100000) + print('v=',v) + + print(d1.distri(v)) + v = d1.generator(2000000) + print(d1.distri(v)) diff --git a/appPublic/ExecFile.py b/appPublic/ExecFile.py new file mode 100755 index 0000000..3346813 --- /dev/null +++ b/appPublic/ExecFile.py @@ -0,0 +1,107 @@ +# ExecFile.py +# usage : +# r = ExecFile() +# r.set('a','bbbb') +# r.run('test/cards.ini') +# r.cards +# +import os,sys + +class DictConfig(dict): + def __init__(self,dic=None,path=None,str=None,namespace={}): + dict.__init__(self) + self.namespace=namespace + if dic is not None and type(dic) == dict: + self.__dict__.update(dic) + self.__subConfig() + if path is not None: + self.__path = path + self.__load(path) + if str is not None: + self.__confstr = str + try: + exec(str,self.namespace,self.__dict__) + self.__subConfig() + except: + pass + def keys(self): + return self.__dict__.keys() + + def __getitem__(self,n): + return self.__dict__[n] + + def __getattr__(self,name): + if self.__dict__.has_key(name): + return self.__dict__[name] + raise AttributeError(name) + + def __subConfig(self): + for n in self.__dict__.keys(): + if type(self.__dict__[n]) == dict: + self.__dict__[n] = DictConfig(dic=self.__dict__[n]) + elif type(self.__dict__[n]) == type([]): + a = [] + for i in self.__dict__[n]: + if type(i) == dict: + a.append(DictConfig(dic=i)) + else: + a.append(i) + self.__dict__[n] = a + elif type(self.__dict__[n]) == type(()): + a = [] + for i in self.__dict__[n]: + if type(i) == dict: + a.append(DictConfig(dic=i)) + else: + a.append(i) + self.__dict__[n] = tuple(a) + + def __load(self,path): + d = {} + c = {} + f = open(path,'r') + buf = f.read() + f.close() + try: + exec(buf,self.namespace,namespace) + #print d + #print "c=",c + self.__dict__.update(c) + #print self.__dict__ + self.__subConfig() + return True + except Exception as e: + print(self.__path,e) + return False + +class ExecFile(object) : + def __init__(self,obj=None,path=None,namespace={}): + self.namespace = namespace + if obj == None: + obj = self + self.__object = obj + #self.namespace.update(self.__object.__dict__) + self.__file = path + + def set(self,name,v) : + setattr(self.__object,name,v) + + def get(self,name,default=None) : + return getattr(self.__object,name,default) + + def run(self,path=None) : + if path!=None: + self.__file = path + if self.__file is None: + raise Exception('exec file is none') + f = open(self.__file,'r') + buf = f.read() + f.close() + try : + exec(buf,globals(),self.__object.__dict__) + except Exception as e: + print("ExecFile()",e) + return (False,e) + return (True,'') + + diff --git a/appPublic/FiniteStateMachine.py b/appPublic/FiniteStateMachine.py new file mode 100755 index 0000000..e906a6a --- /dev/null +++ b/appPublic/FiniteStateMachine.py @@ -0,0 +1,53 @@ +# FiniteStateMachine.py +## a virtual State object of FSM +# +class BaseFSM(object): + def enterState(self, obj): + raise NotImplementedError() + + def execState(self, obj): + raise NotImplementedError() + + def exitState(self, obj): + raise NotImplementedError() + +## a FMS Manager +# only need one Manager for a FSM +class FSMManager(object): + def __init__(self): + self._fsms = {} + + def addState(self,state,fsm): + self._fsms[state] = fsm + + def delState(self,state): + del self._fsms[state] + + def getFSM(self, state): + return self._fsms[state] + + def frame(self, objs, state): + for obj in objs: + if state == obj.curr_state: + obj.keepState() + else: + obj.changeState(state, self._fsms[state]) + +## the object with has a Finite State Machine +# +class FSMObject(object): + def attachFSM(self,state,fsm): + self.fsm_state_object = fsm + self.fsm_cur_state = state + + def changeState(self,new_state,newfsm): + self.fsm_cur_state = new_state + self.fsm_state_object.exitState(self) + self.fsm_state_object = new_fsm + self.fsm_state_object.enterState(self) + self.fsm_state_object.execState(self) + + def keepState(self): + self.fsm_state_object.execState(self) + + \ No newline at end of file diff --git a/appPublic/MiniI18N.py b/appPublic/MiniI18N.py new file mode 100755 index 0000000..62f7dea --- /dev/null +++ b/appPublic/MiniI18N.py @@ -0,0 +1,238 @@ +import os,re,sys +import codecs +from appPublic.folderUtils import _mkdir +from appPublic.Singleton import SingletonDecorator +from appPublic.folderUtils import ProgramPath +import threading +import time + +import locale + +comment_re = re.compile(r'\s*#.*') +msg_re = re.compile(r'\s*([^:]*)\s*:\s*([^\s].*)') + +def dictModify(d, md) : + for i in md.keys() : + if md[i]!=None : + d[i] = md[i] + return d + +convert_pairs = {':':'\\x3A', + '\n':'\\x0A', + '\r':'\\x0D', +} + +def charEncode(s) : + r = '' + v = s.split('\\') + s = '\\\\'.join(v) + for i in convert_pairs.keys() : + v = s.split(i) + s = convert_pairs[i].join(v) + # print 'i=',i,'iv=',convert_pairs[i],'s=',s + return s + +def charDecode(s) : + for i in convert_pairs.items() : + v = s.split(i[1]) + s = i[0].join(v) + v = s.split('\\\\') + s = '\\'.join(v) + return s + +def getTextDictFromLines(lines) : + d = {} + for l in lines : + l = ''.join(l.split('\r')) + if comment_re.match(l) : + continue + m = msg_re.match(l) + if m : + grp = m.groups() + d[charDecode(grp[0])] = charDecode(grp[1]) + return d + +def getFirstLang(lang) : + s = lang.split(',') + return s[0] + +@SingletonDecorator +class MiniI18N: + """ + """ + def __init__(self,path,lang=None,coding='utf8') : + self.path = path + l = locale.getdefaultlocale() + self.curLang = l[0] + self.coding = coding + self.id = 'i18n' + self.langTextDict = {} + self.messages = {} + self.setupMiniI18N() + self.missed_pt = None + self.translated_pt = None + self.header_pt = None + self.footer_pt = None + self.show_pt=None + self.clientLangs = {} + self.languageMapping = {} + self.timeout = 600 + + def __call__(self,msg,lang=None) : + """ + """ + if type(msg) == type(b''): + msg = msg.decode(self.coding) + return self.getLangText(msg,lang) + + def setLangMapping(self,lang,path): + self.languageMapping[lang] = path + + def getLangMapping(self,lang): + return self.languageMapping.get(lang,lang) + + def setTimeout(self,timeout=600): + self.timeout = timeout + + def delClientLangs(self): + t = threading.currentThread() + tim = time.time() - self.timeout + [ self.clientLangs.pop(k,None) for k in self.clientLangs.keys() if self.clientLangs[k]['timestamp'] < tim ] + + def getLangDict(self,lang): + return self.langTextDict.get('lang',{}) + + def getLangText(self,msg,lang=None) : + """ + """ + if lang==None : + lang = self.getCurrentLang() + if lang not in self.langTextDict.keys() : + self.langTextDict[lang] = {} + if msg not in self.messages.keys() : + self.messages[msg] = '' + dict = self.langTextDict[lang] + if msg not in dict.keys() : + return msg + return dict[msg] + + def getMissTextList(self,lang=None) : + """ + """ + if lang==None : + lang = self.getCurrentLang() + if lang not in self.langTextDict.keys() : + self.langTextDict[lang] = {} + texts = [] + + keys = list(self.messages.keys()) + keys.sort() + for i in keys : + if i not in self.langTextDict[lang].keys() : + texts.append(charEncode(i) + ':' ) + + s = '\n'.join(texts) + return s + + def getTranslatedTextList(self,lang=None) : + """ + """ + if lang==None : + lang = self.getCurrentLang() + if lang not in self.langTextDict.keys() : + self.langTextDict[lang] = {} + texts = [] + keys = list(self.langTextDict[lang].keys()) + keys.sort() + for i in keys : + texts.append(charEncode(i) + ':' + charEncode(self.langTextDict[lang][i])) + + s = '\n'.join(texts) + return s + + def I18nTranslating(self,newtexts,lang=None,submit='') : + """ + """ + if lang==None : + lang = self.getCurrentLang() + if lang not in self.langTextDict.keys() : + self.langTextDict[lang] = {} + + textDict = getTextDictFromLines(newtexts.split('\n')) + d = {} + if lang in self.langTextDict : + d = self.langTextDict[lang] + self.langTextDict[lang].update(textDict) + for i in textDict.keys() : + self.messages[i] = '' + self.writeTranslateMessage() + + def writeUntranslatedMessages(self) : + p = os.path.join(self.path,'i18n') + langs = [] + + for f in os.listdir(p) : + if os.path.isdir(os.path.join(p,f)) : + langs.append(f) + for lang in langs: + p1 = os.path.join(self.path,'i18n',lang) + if not os.path.exists(p1) : + _mkdir(p1) + p2 = os.path.join(p1,'unmsg.txt') + f = codecs.open(p2,'w',self.coding) + f.write(self.getMissTextList(lang)) + f.close() + + + def writeTranslateMessage(self) : + p1 = os.path.join(self.path,'i18n',self.getCurrentLang()) + if not os.path.exists(p1) : + _mkdir(p1) + p2 = os.path.join(p1,'msg.txt') + f = codecs.open(p2,'w',self.ccoding) + f.write(self.getTranslatedTextList()) + f.close() + + def setupMiniI18N(self) : + """ + """ + + p = os.path.join(self.path,'i18n') + langs = [] + + for f in os.listdir(p) : + if os.path.isdir(os.path.join(p,f)) : + langs.append(f) + for dir in langs : + p1 = os.path.join(p,dir,'msg.txt') + if os.path.exists(p1) : + f = codecs.open(p1,'r',self.coding) + textDict = getTextDictFromLines(f.readlines()) + f.close() + d = {} + if dir in self.langTextDict : + d = self.langTextDict[dir] + self.langTextDict[dir] = textDict + for i in textDict.keys() : + self.messages[i] = '' + + self._p_changed = 1 + + def setCurrentLang(self,lang): + lang = self.getLangMapping(lang) + t = time.time() + threadid = threading.currentThread() + a = dict(timestamp=t,lang=lang) + self.clientLangs[threadid] = a + + def getCurrentLang(self) : + """ + """ + threadid = threading.currentThread() + return self.clientLangs[threadid]['lang'] + +def getI18N(coding='utf8'): + path = ProgramPath() + i18n = MiniI18N(path,coding) + return i18n + diff --git a/appPublic/ObjectCache.py b/appPublic/ObjectCache.py new file mode 100755 index 0000000..381b18a --- /dev/null +++ b/appPublic/ObjectCache.py @@ -0,0 +1,53 @@ +# !/usr/bin/env python +# +# ObjectsCache is a Objects cache +# the Object has to have a method "get_size" to tell +# the cacher counting the objects size + +class ObjectCache(dict) : + def __init__(self,maxsize=10000000,*args) : + super(ObjectsCache,self).__init__(*args) + self.maxsize = maxsize + self.size = 0 + self._shadow = {} + + def __setitem__(self,key,item) : + try : + size = item.get_size() + self.size += size + except : + return + if self.size >= self.maxsize : + tmp = [(t,key) for key,(t,size) in self._shadow.iteritems() ] + tmp.sort() + for i in xrange(len(tmp)//2) : + del self[tmp[i][i]] + del tmp + super(ObjectCache,self).__setitem__(key,item) + self._shadow[key] = [time.time(),size] + + def __getitem__(self,key) : + try : + item = super(ObjectCache,self).__getitem__(key) + except : + raise + else : + self._shadow[key][0] = time.time() + return item + + def get(self,key,default=None) : + if self.has_key(key) : + return self[key] + else : + return default + + def __delitem__(self,key) : + try : + super(ObjectCache,self).__delitem__(key) + except : + raise + else : + self.size -= self._shadow[key][1] + del self._shadow[key] + + diff --git a/appPublic/ProbiablitySet.py b/appPublic/ProbiablitySet.py new file mode 100755 index 0000000..efc7399 --- /dev/null +++ b/appPublic/ProbiablitySet.py @@ -0,0 +1,29 @@ +import numpy as np +import random +from Distribution import Distribution +class ProbabilitySet: + def __init__(self,dset,p): + self.dset = dset + self.probiablity = p + + def dataset(self): + dl = Distribution([1,0],[self.probiablity,]) + v = dl.generator(len(self.dset)) + m = [] + for i in range(len(self.dset)): + if v[i] == 1: + m.append(self.dset) + return m + +if __name__ == '__main__': + from timecost import TimeCost + dset = range(10000000) + pp = [0.1,0.22,0.334,0.88,0.93] + tc = TimeCost() + for p in pp: + tc.begin('dataset() %f' % p) + ps = ProbablitySet(dset,p) + d = ps.dataset() + tc.end('dataset() %f' % p) + + tc.show() diff --git a/appPublic/PublicData.py b/appPublic/PublicData.py new file mode 100755 index 0000000..b52be2d --- /dev/null +++ b/appPublic/PublicData.py @@ -0,0 +1,18 @@ +import os,sys +from folderUtils import ProgramPath + +class PublicData : + def __init__(self) : + p = ProgramPath() + # print p + self.ProgramPath = os.path.dirname(ProgramPath()) + + def set(self,name,value) : + setattr(self,name,value) + + def get(self,name,default=None) : + return getattr(self,name,default) + +public_data = PublicData() + +# print 'ProgramPath=',public_data.get('ProgramPath',None) diff --git a/appPublic/RSAutils.py b/appPublic/RSAutils.py new file mode 100755 index 0000000..3eb4908 --- /dev/null +++ b/appPublic/RSAutils.py @@ -0,0 +1,89 @@ +import codecs + +from Crypto.PublicKey import RSA +from Crypto.Cipher import PKCS1_OAEP +from Crypto.Cipher import KPCS1_V1_5 as V1_5 +from Crypto.Signature import PKCS1_v1_5 +from Crypto.Hash import SHA512, SHA384, SHA256, SHA, MD5 +from Crypto import Random +from base64 import b64encode, b64decode +hash = "SHA-256" + +def readPublickey(fname): + with codecs.open(fname,'r','utf8') as f: + b = f.read() + k = RSA.importKey(b) + return k + return None + +def readPrivatekey(fname,pwd): + with codecs.open(fname,'r','utf8') as f: + b = f.read() + k = RSA.importKey(b,pwd) + return k + return None + +def newkeys(keysize): + random_generator = Random.new().read + key = RSA.generate(keysize, random_generator) + private, public = key, key.publickey() + return public, private + +def importKey(externKey): + return RSA.importKey(externKey) + +def getpublickey(priv_key): + return priv_key.publickey() + +def encrypt(message, pub_key): + cipher = PKCS1_OAEP.new(pub_key) + return cipher.encrypt(message) + +def decrypt(ciphertext, priv_key): + try: + cipher = PKCS1_OAEP.new(priv_key) + return cipher.decrypt(ciphertext) + except Exception as e: + print('e=',e) + cipher = V1_5.new(priv_key) + return cipher.decrypt(ciphertext) + +def sign(message, priv_key, hashAlg = "SHA-256"): + global hash + hash = hashAlg + signer = PKCS1_v1_5.new(priv_key) + + if (hash == "SHA-512"): + digest = SHA512.new() + elif (hash == "SHA-384"): + digest = SHA384.new() + elif (hash == "SHA-256"): + digest = SHA256.new() + elif (hash == "SHA-1"): + digest = SHA.new() + else: + digest = MD5.new() + digest.update(message) + return signer.sign(digest) + +def verify(message, signature, pub_key): + signer = PKCS1_v1_5.new(pub_key) + if (hash == "SHA-512"): + digest = SHA512.new() + elif (hash == "SHA-384"): + digest = SHA384.new() + elif (hash == "SHA-256"): + digest = SHA256.new() + elif (hash == "SHA-1"): + digest = SHA.new() + else: + digest = MD5.new() + digest.update(message) + return signer.verify(digest, signature) + +if __name__ == '__main__': + cipher="""WaMlLEYnhBk+kTDyN/4OJmQf4ccNdk6USgtKpb7eHsYsotq4iyXi3N5hB1E/PqrPSmca1AMDLUcumwIrLeGLT9it3eTBQgl1YQAsmPxa6lF/rDOZoLbwD5sJ6ab/0/fuM4GbotqN5/d0MeuOSELoo8cFWw+7XpRxn9EMYnw5SzsjDQRWxXjZptoaGa/8pBBkDmgLqINif9EWV+8899xqTd0e9w1Gqb7wbt/elRNVBpgsSuSZb+dtBlvNUjuTms8BETSRai5vhXetK26Ms8hrayiy38n7wwEKE8fZ9iFzLtwa6xbhD5KudWbKJFFOZAfpzWttGMwWlISbGQigcW4+Bg==""" + key = readPrivatekey('d:/dev/mecp/conf/RSA.private.key','ymq123') + t = decrypt(cipher,key) + print('t=',t) + \ No newline at end of file diff --git a/appPublic/SQLite3Utils.py b/appPublic/SQLite3Utils.py new file mode 100755 index 0000000..9a4038d --- /dev/null +++ b/appPublic/SQLite3Utils.py @@ -0,0 +1,220 @@ +import os,sys +import thread +from sqlite3 import dbapi2 as sqlite +import time +from localefunc import * +from folderUtils import mkdir +from PublicData import public_data +from mylog import mylog + +def logit(s) : + mylog('%s:%s' % (__file__,s)) + +class Record : + def __init__(self,data,localize=False) : + + for i in data.keys() : + d = data[i] + if localize and type(d)==type('') : + d = localeString(d) + setattr(self,i.lower(),d) + + def __getattr__(self,name) : + name = name.lower() + try : + return getattr(self,name) + except : + raise AttributeError(name) + + def __str__(self) : + a = self.__dict__ + f = [] + for i in a.keys() : + f.append("%s : %s" % (i,str(a[i]))) + return '[%s]' % '\n'.join(f) + +def str2unicode(s) : + if type(s) == type('') : + try : + ret = unicode(s,local_encoding) + return ret + except : + try : + ret = unicode(s,'utf8') + return ret + except : + return buffer(s) + return s + +def unicode2str(s) : + t = type(s) + if t == type(5) : + return long(s) + if t == type(buffer('')) : + return str(s) + if t == type(u"w") : + return s.encode('utf8') + return s + +def argConvert(args) : + if args==None : + return None + t = type(args) + if t==type(()) or t==type([]) : + return [str2unicode(i) for i in args] + if t==type({}) : + for i in args.keys() : + args[i] = str2unicode(args[i]) + return args + return args + +class SQLite3 : + + def __init__(self,dbpath,localize=False) : + self.__dict__['threadMap'] = {} + self.__dict__['localize'] = localize + self.__dict__['dbpath'] = dbpath + self.results = None + self.con = None + self.cursor = None + self.sqlcmd = '' + self._connection(dbpath) + + def _connection(self,dbpath=None) : + if dbpath!=None : + self.dbpath = dbpath + self.con = sqlite.connect(self.dbpath) + self.cursor = self.con.cursor() + self.result = None + self.sqlcmd = '' + + def __setattr__(self, name, value): + id = thread.get_ident() + if not self.__dict__['threadMap'].has_key(id): + self.__dict__['threadMap'][id] = {} + self.threadMap[id][name] = value + + def __getattr__(self, name): + id = thread.get_ident() + + if not self.__dict__['threadMap'].has_key(id) : + self.__dict__['threadMap'][id] = {} + if self.__dict__['threadMap'][id].has_key(name) : + return self.__dict__['threadMap'][id][name] + raise AttributeError(name) + + def tables(self) : + self.SQL("select * from sqlite_master where type='table'") + r = self.FETCH() + ts = [] + while r : + ts.append(r.name) + r = self.FETCH() + return ts + + def columns(self,tablenmae) : + self.SQL('select * from %s' % tablename) + self.desc = self.results.getdescription() + return desc + + def FETCHALL(self) : + all=[] + r = True + r = self.cursor.fetchall() + return r + + def _eatCursorNext(self) : + if self.cursor==None : + return None + r = 1 + while r : + try : + r = self.cursor.next() + except : + return + + def SQL(self,cmd,args=(),retry=0) : + if self.con==None : + print("self.con==None",cmd) + self._connection() + return self.SQL(cmd,args,retry) + return -1 + self._eatCursorNext() + args = argConvert(args) + self.lastSQL = cmd + self.desc = None + try : + if len(cmd.split(';'))>1 : + self.results = self.cursor.executescript(cmd) + else : + self.results = self.cursor.execute(cmd,args) + return True + except Exception as e: + print('execute:',cmd,'error',e) + self.results = None + raise + return True + + def FETCH(self) : + if self.results == None : + return None + if self.desc == None : + try : + self.desc = self.results.description + + except Exception as e: + print("fetch error",self.lastSQL,e) + raise + try : + desc = self.desc + d = self.results.next() + data = {} + for i in range(len(d)) : + data[desc[i][0]] = unicode2str(d[i]) + return Record(data,self.localize) + except StopIteration : + return None + except Exception as e: + print("error happen",e,self,lastSQL) + raise + + def COMMIT(self) : + self.SQL('PRAGMA case_sensitive_like = 1') + try : + self.cursor.fetchall() + except : + pass + + def ROLLBACK(self) : + self.SQL('ROLLBACK') + + def BEGIN(self) : + # self.SQL('BEGIN') + return + + def CLOSE(self) : + self.con = None + self.cursor = None + + +def getDataBase(name) : + a_name='db_%s' % name + db = public_data.get(a_name,None) + if db==None : + dbpath = public_data.get('dbpath_%s' % name,None) + if dbpath==None : + p = public_data.get('ProgramPath',None) + if p==None: + raise Exception('public_data must has a "ProgramPath" variable') + p1 = os.path.join(p,'var') + mkdir(p1) + dbpath = os.path.join(p1,'%s.db3' % name) + public_data.set('dbpath_%s' % name,dbpath) + db = SQLite3(dbpath) + public_data.set(a_name,db) + try : + con = db.con + except : + dbpath = public_data.get('dbpath_%s' % name,None) + db._connection(dbpath) + return db diff --git a/appPublic/Singleton.py b/appPublic/Singleton.py new file mode 100755 index 0000000..9157c98 --- /dev/null +++ b/appPublic/Singleton.py @@ -0,0 +1,42 @@ +# +from appPublic.dictObject import DictObject + +class SingletonDecorator: + def __init__(self,klass): + self.klass = klass + self.instance = None + def __call__(self,*args,**kwds): + if self.instance == None: + self.instance = self.klass(*args,**kwds) + return self.instance + +@SingletonDecorator +class GlobalEnv(DictObject): + pass + +if __name__ == '__main__': + @SingletonDecorator + class Child(object): + def __init__(self,name): + print("clild.init") + self.name = name + def __str__(self): + return 'HAHA：' + self.name + def __expr__(self): + print(self.name) + + @SingletonDecorator + class Handle(object): + def __init__(self,name): + self.name = name + def __expr__(self): + print(self.name) + + c = Child('me') + d = Child('he') + + print(str(c),str(d)) + e = Handle('hammer') + f = Handle('nail'); + print(str(e),str(f)) + diff --git a/appPublic/__init__.py b/appPublic/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/appPublic/apsw.version.SQLite3Utils.apsw.py b/appPublic/apsw.version.SQLite3Utils.apsw.py new file mode 100755 index 0000000..14ef831 --- /dev/null +++ b/appPublic/apsw.version.SQLite3Utils.apsw.py @@ -0,0 +1,242 @@ +import os,sys +import thread +import apsw +import time +from localefunc import * +from folderUtils import mkdir +from PublicData import public_data +from mylog import mylog + +def logit(s) : + mylog('%s:%s' % (__file__,s)) + +class Record : + def __init__(self,data,localize=False) : + + for i in data.keys() : + d = data[i] + if localize and type(d)==type('') : + d = localeString(d) + setattr(self,i.lower(),d) + + def __getattr__(self,name) : + dict = self.__dict__ + name = name.lower() + if name in dict.keys() : + return dict[name] + else : + raise AttributeError(name) + + def __str__(self) : + a = self.__dict__ + f = [] + for i in a.keys() : + f.append("%s : %s" % (i,str(a[i]))) + return '[%s]' % '\n'.join(f) + +def str2unicode(s) : + if type(s) == type('') : + try : + ret = unicode(s,local_encoding) + return ret + except : + try : + ret = unicode(s,'utf8') + return ret + except : + return buffer(s) + return s + +def unicode2str(s) : + t = type(s) + if t == type(5) : + return long(s) + if t == type(buffer('')) : + return str(s) + if t == type(u"w") : + return s.encode('utf8') + return s + +def argConvert(args) : + if args==None : + return None + t = type(args) + if t==type(()) or t==type([]) : + return [str2unicode(i) for i in args] + if t==type({}) : + for i in args.keys() : + args[i] = str2unicode(args[i]) + return args + return args + +class SQLite3 : + + def __init__(self,dbpath,localize=False) : + self.__dict__['threadMap'] = {} + self.__dict__['localize'] = localize + self.__dict__['dbpath'] = dbpath + self.results = None + self.con = None + self.cursor = None + self.sqlcmd = '' + self._connection(dbpath) + + def _connection(self,dbpath=None) : + if dbpath!=None : + self.dbpath = dbpath + self.con = apsw.Connection(self.dbpath) + self.cursor = self.con.cursor() + self.result = None + self.sqlcmd = '' + + def __setattr__(self, name, value): + id = thread.get_ident() + if not self.__dict__['threadMap'].has_key(id): + self.__dict__['threadMap'][id] = {} + self.threadMap[id][name] = value + + def __getattr__(self, name): + id = thread.get_ident() + + if not self.__dict__['threadMap'].has_key(id) : + self.__dict__['threadMap'][id] = {} + if self.__dict__['threadMap'][id].has_key(name) : + return self.__dict__['threadMap'][id][name] + raise AttributeError(name) + + def tables(self) : + self.SQL("select * from sqlite_master where type='table'") + r = self.FETCH() + ts = [] + while r : + ts.append(r.name) + r = self.FETCH() + return ts + + def columns(self,tablenmae) : + self.SQL('select * from %s' % tablename) + self.desc = self.results.getdescription() + return desc + + def FETCHALL(self) : + all=[] + r = True + while r: + r = self.FETCH() + if r : + all.append(r) + return all + + def _eatCursorNext(self) : + if self.cursor==None : + return None + r = 1 + while r : + try : + r = self.cursor.next() + except : + return + + def SQL(self,cmd,args=(),retry=0) : + if self.con==None : + print("self.con==None") + return -1 + self._eatCursorNext() + args = argConvert(args) + self.lastSQL = cmd + self.desc = None + try : + self.results = self.cursor.execute(cmd,args) + return True + except apsw.IncompleteExecutionError as e : + r = True + while r : + r = self.FETCH() + return self.SQL(cmd,args,retry + 1) + except apsw.BusyError as e: + if retry >20 : + print("execute:(",cmd,")Busy, error",e) + self.results=None + raise e + time.sleep(0.01) + return self.SQL(cmd,args,retry + 1) + except apsw.LockedError as e : + if retry >20 : + print("execute:(",cmd,")Locked, error",e) + self.results = None + raise e + time.sleep(0.01) + return self.SQL(cmd,args,retry + 1) + except apsw.ThreadingViolationError as e : + self._connection() + return self.SQL(cmd,args,retry+1) + except apsw.Error as e: + print('execute:',cmd,'error',e) + self.results = None + raise + print("sql finish") + return True + + def FETCH(self) : + if self.results == None : + print("FETCH error,results==None") + return None + if self.desc == None : + try : + self.desc = self.results.getdescription() + except apsw.ExecutionCompleteError as e : + # print("getdescription error",self.lastSQL,e) + return None + except Exception as e: + print("fetch error",self.lastSQL,e) + raise + try : + desc = self.desc + d = self.results.next() + data = {} + for i in range(len(d)) : + data[desc[i][0]] = unicode2str(d[i]) + return Record(data,self.localize) + except StopIteration : + # print("StopIteration error") + return None + except apsw.Error as e: + print("error happen",e,self,lastSQL) + raise + + def COMMIT(self) : + self.SQL('COMMIT') + + def ROLLBACK(self) : + self.SQL('ROLLBACK') + + def BEGIN(self) : + self.SQL('BEGIN') + return + + def CLOSE(self) : + self.con = None + self.cursor = None + + +def getDataBase(name) : + a_name='db_%s' % name + db = public_data.get(a_name,None) + if db==None : + dbpath = public_data.get('dbpath_%s' % name,None) + if dbpath==None : + p = public_data.get('ProgramPath',None) + if p==None: + raise Exception('public_data must has a "ProgramPath" variable') + p1 = os.path.join(p,'var') + mkdir(p1) + dbpath = os.path.join(p1,'%s.db3' % name) + public_data.set('dbpath_%s' % name,dbpath) + db = SQLite3(dbpath) + public_data.set(a_name,db) + try : + con = db.con + except : + dbpath = public_data.get('dbpath_%s' % name,None) + db._connection(dbpath) + return db diff --git a/appPublic/argsConvert.py b/appPublic/argsConvert.py new file mode 100755 index 0000000..640486f --- /dev/null +++ b/appPublic/argsConvert.py @@ -0,0 +1,182 @@ +# -*- coding:utf8 -*- +import re +class ConvertException(Exception): + pass + +class ArgsConvert(object): + def __init__(self,preString,subfixString,coding='utf-8'): + self.preString = preString + self.subfixString = subfixString + self.coding=coding + sl1 = [ u'\\' + c for c in self.preString ] + sl2 = [ u'\\' + c for c in self.subfixString ] + ps = u''.join(sl1) + ss = u''.join(sl2) + re1 = ps + r"[_a-zA-Z_\u4e00-\u9fa5][a-zA-Z_0-9\u4e00-\u9fa5\,\.\'\{\}\[\]\(\)\-\+\*\/]*" + ss + self.re1 = re1 + # print( self.re1,len(self.re1),len(re1),type(self.re1)) + + def convert(self,obj,namespace,default=''): + """ obj can be a string,[],or dictionary """ + if type(obj) == type(u''): + return self.convertUnicode(obj,namespace,default) + if type(obj) == type(''): + return self.convertString(obj,namespace,default) + if type(obj) == type([]): + ret = [] + for o in obj: + ret.append(self.convert(o,namespace,default)) + return ret + if type(obj) == type({}): + ret = {} + for k in obj.keys(): + ret.update({k:self.convert(obj.get(k),namespace,default)}) + return ret + # print( type(obj),"not converted") + return obj + + def findAllVariables(self,src): + r = [] + for ph in re.findall(self.re1,src): + dl = self.getVarName(ph) + r.append(dl) + return r + + def getVarName(self,vs): + return vs[len(self.preString):-len(self.subfixString)] + + def getVarValue(self,var,namespace,default): + v = default + try: + v = eval(var,namespace) + except Exception as e: + v = namespace.get(var,default) + return v + + def convertUnicode(self,s,namespace,default): + args = re.findall(self.re1,s) + for arg in args: + dl = s.split(arg) + var = self.getVarName(arg) + v = self.getVarValue(var,namespace,default) + if type(v) != type(u''): + v = str(v) + s = v.join(dl) + return s + + def convertString(self,s,namespace,default): + ret = self.convertUnicode(s,namespace,default) + return ret + +class ConditionConvert(object): + def __init__(self,pString = u'$<',sString=u'>$',coding='utf-8'): + self.coding = coding + self.pString = pString + self.sString = sString + pS = ''.join([u'\\'+i for i in self.pString ]) + sS = ''.join([u'\\'+i for i in self.sString ]) + self.re1 = re.compile(u'(' + pS + '/?' + u'[_a-zA-Z_\u4e00-\u9fa5][a-zA-Z_0-9\u4e00-\u9fa5\,\.\'\{\}\[\]\(\)\-\+\*\/]*' + sS + u')') + self.buffer1 = [] + def convert(self,obj,namespace): + """ obj can be a string,[],or dictionary """ + if type(obj) == type(u''): + return self.convertUnicode(obj,namespace) + if type(obj) == type(''): + return self.convertString(obj,namespace) + if type(obj) == type([]): + ret = [] + for o in obj: + ret.append(self.convert(o,namespace)) + return ret + if type(obj) == type({}): + ret = {} + for k in obj.keys(): + ret.update({k:self.convert(obj.get(k),namespace)}) + return ret + # print( type(obj),"not converted") + return obj + + def getVarName(self,vs): + return vs[len(self.pString):-len(self.sString)] + + def getVarValue(self,var,namespace): + v = None + try: + v = eval(var,namespace) + except Exception as e: + v = namespace.get(var,None) + return v + + def convertList(self,alist,namespace): + ret = [] + handleList = alist + while len(handleList) > 0: + i = handleList[0] + handleList = handleList[1:] + if len(self.re1.findall(i)) < 1: + ret.append(i) + else: + name = self.getVarName(i) + + if name[0] == u'/': + name = name[1:] + if len(self.buffer1) < 1: + raise ConvertException('name(%s) not match' % name) + if self.buffer1[-1] != name: + raise ConvertException('name(%s) not match(%s)' % (self.buffer1[-1],name)) + val = self.getVarValue(name,namespace) + self.buffer1 = self.buffer1[:-1] + if val is not None: + return u''.join(ret),handleList + else: + return u'',handleList + else: + self.buffer1.append(name) + subStr,handleList = self.convertList(handleList,namespace) + ret.append(subStr) + if len(self.buffer1)>0: + raise ConvertException('name(s)(%s) not closed' % ','.join(self.buffer1)) + return u''.join(ret),[] + + def convertUnicode(self,s,namespace): + ret = [] + parts = self.re1.split(s) + s,b = self.convertList(parts,namespace) + return s + + def convertString(self,s,namespace): + ret = self.convertUnicode(s,namespace) + return ret + + +if __name__ == '__main__': + """ + ns = { + 'a':12, + 'b':'of', + 'c':'abc', + u'是':'is', + 'd':{ + 'a':'doc', + 'b':'gg', + } + } + AC = ArgsConvert('%{','}%') + s1 = u"%{a}% is a number,%{d['b']}% is %{是}% undefined,%{c}% is %{d['a']+'(rr)'}% string" + arglist=['this is a descrciption %{b}% selling book',123,'ereg%{a}%,%{c}%'] + argdict={ + 'my':arglist, + 'b':s1 + } + print(f(s1,'<=>',AC.convert(s1,ns))) + print(f(argdict,'<=>',AC.convert(argdict,ns))) + """ + cc = ConditionConvert() + s2 = u"Begin $$this is $$ba = 100 $$condition out$$ end" + s3 = """select * from RPT_BONDRATINGS +where 1=1 +$$and ratingtype=${rtype}$$$ +$$and bond_id = ${bond}$$$""" + print(f("result=",cc.convert(s2,{'ba':23}))) + print(f("result = ",cc.convert(s3,{'bond':'943','rtype':'1'}))) + diff --git a/appPublic/csv_Data.py b/appPublic/csv_Data.py new file mode 100755 index 0000000..4f950d9 --- /dev/null +++ b/appPublic/csv_Data.py @@ -0,0 +1,54 @@ +import codecs +import csv +class Reader: + def __init__(self,f,delimiter): + self.f = f + self.delimiter = delimiter + self.line = 0 + + def __iter__(self): + return self + + def next(self): + l = self.f.readline() + if l == '': + raise StopIteration() + while l[-1] in [ '\n','\r']: + l = l[:-1] + r = [ i if i != '' else None for i in l.split(self.delimiter) ] + self.line = self.line + 1 + return r + +class CSVData: + def __init__(self,filename,coding='utf8',delimiter=','): + self.filename = filename + self.coding = coding + self.f = codecs.open(filename,'rb',self.coding) + self.reader = Reader(self.f,delimiter) + self.fields = self.reader.next() + + def __del__(self): + self.f.close() + + def __iter__(self): + return self + + def next(self): + try: + r = self.reader.next() + if len(r) != len(self.fields): + print("length diff",len(r),len(self.fields),"at line %d" % self.reader.line) + raise StopIteration() + d = {} + [d.update({self.fields[i]:r[i]}) for i in range(len(self.fields))] + return d + except: + raise StopIteration() + +if __name__ == '__main__': + import sys + cd = CSVData(sys.argv[1]) + for r in cd: + print(r) + + diff --git a/appPublic/datamapping.py b/appPublic/datamapping.py new file mode 100755 index 0000000..337c801 --- /dev/null +++ b/appPublic/datamapping.py @@ -0,0 +1,40 @@ +#dataMapping +from appPublic.dictObject import DictObject + +def keyMapping(dic,mappingtab,keepmiss=True): + ret = {} + keys = [ k for k in dic.keys()] + if not keepmiss: + keys = [ k for k in dic.keys() if k in mappingtab.keys() ] + [ ret.update({mappingtab.get(k,k):dic[k]}) for k in keys ] + return DictObject(**ret) + + +def valueMapping(dic,mappingtab): + """ + mappingtab format: + { + "field1":{ + "a":"1", + "b":"2", + "__default__":"5" + }, + "field2":{ + "a":"3", + "b":"4" + } + } + field1,field2 is in dic.keys() + """ + ret = {} + for k in dic.keys(): + mt = mappingtab.get(k,None) + if mt is None: + ret[k] = dic[k] + else: + dv = mt.get('__default__',dic[k]) + v = mt.get(dic[k],dv) + ret[k] = v + + return DictObject(**ret) + \ No newline at end of file diff --git a/appPublic/dbpool.py b/appPublic/dbpool.py new file mode 100755 index 0000000..b74def4 --- /dev/null +++ b/appPublic/dbpool.py @@ -0,0 +1,45 @@ +from twisted.enterprise.adbapi import ConnectionPool,Transaction + +#import pymssql +#import ibm_db +#import cx_Oracle + +def dbPool(dbdef): + pool = ConnectionPool(dbdef.driver,**dbdef.kwargs) + return pool + +def dbConnect(pool): + conn = pool.connect() + #conn.as_dict = True + return conn + +def dbClose(pool,conn): + pool.disconnect(conn) + +def dbCursor(pool,conn): + return Transaction(pool,conn) + +if __name__ == '__main__': + from appPublic.dictObject import DictObject + dbdict= { + "driver":"pymssql", + "kwargs":{ + "cp_max":30, + "cp_min":10, + "host":"localhost", + "user":"sa", + "password":"ymq123", + "database":"dzh" + } + } + dbdef = DictObject(**dbdict) + pool = dbPool(dbdef) + conn = dbConnect(pool) + cur = dbCursor(pool,conn) + + for i in range(20): + r = cur.execute("insert into dbo.TQ_QT_INDEX (ID,TRADEDATE,SECODE,TMSTAMP) values(%d,'%s','0','11:22')" % (i,"2015-12-31")) + cur.close() + conn.commit() + dbClose(pool,conn) + pool.close() \ No newline at end of file diff --git a/appPublic/dictExt.py b/appPublic/dictExt.py new file mode 100755 index 0000000..4230614 --- /dev/null +++ b/appPublic/dictExt.py @@ -0,0 +1,20 @@ + +def dictExtend(s,addon): + ret = {} + ret.update(s) + skeys = ret.keys() + for k,v in addon.items(): + if k not in skeys: + ret[k] = v + continue + if type(v)!=type(ret[k]): + ret[k] = v + continue + if type(v)==type({}): + ret[k] = dictExtend(ret[k],v) + continue + if type(v)==type([]): + ret[k] = ret[k] + v + continue + ret[k] = v + return ret diff --git a/appPublic/dictObject.py b/appPublic/dictObject.py new file mode 100755 index 0000000..a92df4c --- /dev/null +++ b/appPublic/dictObject.py @@ -0,0 +1,64 @@ +def multiDict2Dict(md): + ns = {} + for k,v in md.items(): + ov = ns.get(k,None) + if ov is None: + ns[k] = v + elif type(ov) == type([]): + ov.append(v) + ns[k] = ov + else: + ns[k] = [ov,v] + return ns + +class DictObject(dict): + def __init__(self,**args): + try: + dict.__init__(self,**args) + for k,v in self.items(): + self.__setattr__(k,self._DOitem(v)) + except Exception as e: + print("DictObject.__init__()",e,args) + raise e + @classmethod + def isMe(self,name): + return name == 'DictObject' + + def __getattr__(self,name): + if name in self: + return self[name] + return None + + def __setattr__(self,name,v): + self[name] = v + + def _DOArray(self,a): + b = [ self._DOitem(i) for i in a ] + return b + + def _DOitem(self, i): + if type(i) is type({}): + return DictObject(**i) + if type(i) is type([]): + return self._DOArray(i) + return i + +def dictObjectFactory(_klassName__,**kwargs): + def findSubclass(_klassName__,klass): + for k in klass.__subclasses__(): + if k.isMe(_klassName__): + return k + k1 = findSubclass(_klassName__,k) + if k1 is not None: + return k1 + return None + try: + if _klassName__=='DictObject': + return DictObject(**kwargs) + k = findSubclass(_klassName__,DictObject) + if k is None: + return DictObject(**kwargs) + return k(**kwargs) + except Exception as e: + print("dictObjectFactory()",e,_klassName__) + raise e diff --git a/appPublic/easyExcel.py b/appPublic/easyExcel.py new file mode 100755 index 0000000..051f2f1 --- /dev/null +++ b/appPublic/easyExcel.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from win32com.client import Dispatch +import win32com.client + +class EasyExcel: + """A utility to make it easier to get at Excel. Remembering + to save the data is your problem, as is error handling. + Operates on one workbook at a time.""" + + def __init__(self, filename=None): + self.xlApp = win32com.client.Dispatch('Excel.Application') + if filename: + self.filename = filename + self.xlBook = self.xlApp.Workbooks.Open(filename) + #self.xlBook.Visible = False + else: + self.xlBook = self.xlApp.Workbooks.Add() + self.filename = '' + + def save(self, newfilename=None): + if newfilename: + self.filename = newfilename + self.xlBook.SaveAs(newfilename) + else: + self.xlBook.Save() + + def setSheetName(self,sheet,name): + self.xlBook.Worksheets(sheet).Name = name + + + def newSheet(self,sheetName): + pass + + def close(self): + self.xlBook.Close(SaveChanges=0) + self.xlApp.Quit() + del self.xlApp + + def getCell(self, sheet, row, col): + "Get value of one cell" + sht = self.xlBook.Worksheets(sheet) + return sht.Cells(row, col).Value + + def setCell(self, sheet, row, col, value): + "set value of one cell" + sht = self.xlBook.Worksheets(sheet) + sht.Cells(row, col).Value = value + + def getRange(self, sheet, row1, col1, row2, col2): + "return a 2d array (i.e. tuple of tuples)" + sht = self.xlBook.Worksheets(sheet) + return sht.Range(sht.Cells(row1, col1), sht.Cells(row2, col2)).Value + + def addPicture(self, sheet, pictureName, Left, Top, Width, Height): + "Insert a picture in sheet" + sht = self.xlBook.Worksheets(sheet) + sht.Shapes.AddPicture(pictureName, 1, 1, Left, Top, Width, Height) + + def cpSheet(self, before): + "copy sheet" + shts = self.xlBook.Worksheets + shts(1).Copy(None,shts(1)) + +if __name__ == "__main__": + PNFILE = r'c:\screenshot.bmp' + xls = EasyExcel(r'D:\test.xls') + xls.addPicture('Sheet1', PNFILE, 20,20,1000,1000) + xls.cpSheet('Sheet1') + xls.save() + xls.close() \ No newline at end of file diff --git a/appPublic/exceldata.py b/appPublic/exceldata.py new file mode 100755 index 0000000..76de2c2 --- /dev/null +++ b/appPublic/exceldata.py @@ -0,0 +1,281 @@ +import xlrd +import os +import sys +import datetime + +from appPublic.strUtils import * + +TCS={ + 'int':int, + 'float':float, + 'str':str, +} + +def isEmptyCell(cell): + return cell.ctype == xlrd.XL_CELL_EMPTY + +def isCommentValue(v): + if type(v)==type('') and v[0] == '#': + return True + return False + +def purekey(k): + return k.split(':')[0] + +def castedValue(v,k): + ki = k.split(':') + if len(ki)<2 or v is None: + return v + ki = ki[1:] + if 'list' in ki: + if type(v) == type(''): + v = v.split(',') + elif type(v) != type([]): + v = [v] + + for k,tc in TCS.items(): + if k in ki: + if type(v) == type([]): + return [ tc(i) for i in v ] + else: + return tc(v) + return v + +class ExcelData: + _indictors = { + ':__dict__':'ff', + ':__list__':'ff', + ':__include__':'ff', + } + + def __init__(self,xlsfile,encoding='UTF8',startrow=0,startcol=0): + self._book = xlrd.open_workbook(xlsfile) + self.encoding = encoding + self._filename = xlsfile + self.startrow=0 + self.startcol=0 + self._dataset = self.dataload() + + def __del__(self): + del self._book + del self._dataset + + def cellvalue(self,sheet,x,y): + if sheet.cell_type(x,y)==xlrd.XL_CELL_EMPTY: + return None + if sheet.cell_type(x,y)==xlrd.XL_CELL_DATE: + y,m,d,hh,mm,ss = xlrd.xldate_as_tuple(sheet.cell_value(x,y),self._book.datemode) + return datetime.date(y,m,d) + s = sheet.cell_value(x,y) + return self.trimedValue(s) + + def isCommentCell(self,cell): + if isEmptyCell(cell): + return False + v = self.trimedValue(cell.value) + return isCommentValue(v) + + def dateMode(self): + return self._book.datemode + + def trimedValue(self,v): + if type(v) == type(u' '): + v = v.encode(self.encoding) + if type(v) == type(''): + v = lrtrim(v) + return v + def dataload(self): + dat = {} + for name in self._book.sheet_names(): + sheet = self._book.sheet_by_name(name) + #name = name.encode(self.encoding) + dat[self.trimedValue(name)] = self.loadSheetData(sheet) + return dat + + def findDataRange(self,sheet,pos,maxr): + x,y = pos + j = y + 1 + while j < sheet.ncols: + if isEmptyCell(sheet.cell(x,j)) or self.isCommentCell(sheet.cell(x,y)): + maxy = j + break + j += 1 + i = x + 1 + maxx = maxr + while i < maxr: + if not isEmptyCell(sheet.cell(i,y)): + maxx = i + break + i += 1 + return maxx + + def loadSheetData(self,sheet): + return self.loadSheetDataRange(sheet,(self.startrow,self.startcol),sheet.nrows) + + def include(self,filename,id): + try: + sub = ExcelData(filename,self.encoding) + except Exception as e: + print(e,filename) + return None + if id is None: + return sub.dict() + env = {'data':sub.dict()} + try: + exec("""resutl__ = data%s""" % id,globals(),env) + except Exception as e: + print(e,id) + return None + return env['resutl__'] + + def loadSingleData(self,sheet,pos): + x,y = pos + if sheet.ncols==y: + v = self.cellvalue(sheet,x,y) + if isCommentValue(v): + return None + return v + ret = [] + while y < sheet.ncols: + v = self.cellvalue(sheet,x,y) + if v is None: + break + if isCommentValue(v): + break + ret.append(v) + y += 1 + + if len(ret) < 1: + return None + + if len(ret)<2: + return ret[0] + if ret[0] == '__include__': + if len(ret)<2: + print("include mode error: __include__ filename id") + return None + id = None + if len(ret)>=3: + id = ret[2] + return self.include(ret[1],id) + return ret + + def loadDictData(self,sheet,pos,maxr): + ret = {} + x,y = pos + while x < maxr: + mr = self.findDataRange(sheet,(x,y),maxr) + #print "loadDictData:debug:",x,y,maxr,mr + k = self.cellvalue(sheet,x,y) + if isCommentValue(k): + x = x + 1 + continue + if k is not None: + if 'records' in k.split(':'): + v = self.loadRecords(sheet,(x,y+1),maxr) + else: + v = self.loadSheetDataRange(sheet,(x,y+1),mr) + ret[purekey(k)] = castedValue(v,k) + x = mr + + return ret + + def loadSheetDataRange(self,sheet,pos,maxr): + x,y = pos + #print "debug1:",pos,maxr + if maxr - x < 1 : + #print "debug1-1:",pos,maxr + return None + if isEmptyCell(sheet.cell(x,y)): + #print "debug1-2:",pos,maxr + return None + + cv = self.cellvalue(sheet,x,y) + #print cv + if isCommentValue(cv): + pos = (x+1,y) + return self.loadSheetDataRange(sheet,pos,maxr) + + if cv == '__include__': + return self.include(self.cellvalue(sheet,x,y+1),self.cellvalue(sheet,x,y+2)) + + if cv == '__dict__': + #print "cv==__dict__" + i = x + 1 + vs = [] + while i < maxr: + v = self.cellvalue(sheet,i,y) + if v == '__dict__': + vs.append(self.loadDictData(sheet,(x+1,y),i)) + x = i + i += 1 + vs.append(self.loadDictData(sheet,(x+1,y),i)) + if len(vs) < 1: + return None + if len(vs) < 2: + return vs[0] + return vs + return self.loadDictData(sheet,(x+1,y),maxr) + + if cv == '__list__': + i = x + 1 + vs = [] + while i < maxr: + v = self.loadSingleData(sheet,(i,y)) + vs.append(v) + i += 1 + return vs + + if maxr - x < 2: + v = self.loadSingleData(sheet,(x,y)) + return v + + return self.loadRecords(sheet,pos,maxr) + + def loadRecords(self,sheet,pos,maxr): + x,y = pos + v = self.cellvalue(sheet,x,y) + if v==None or isCommentValue(v): + return self.loadRecords(sheet,(x+1,y),maxr) + + data = [] + i = x + 1 + j = y + keys = [ self.trimedValue(k.value) for k in sheet.row(x)[y:] ] + while i < maxr: + d = {} + j = y + while j < sheet.ncols: + k = self.cellvalue(sheet,x,j) + if k is None or isCommentValue(k): + break + if sheet.cell_type(x,j) == xlrd.XL_CELL_EMPTY: + break + v = self.cellvalue(sheet,i,j) + if sheet.cell_type(x,j) != xlrd.XL_CELL_EMPTY: + d[purekey(k)] = castedValue(v,k) + j += 1 + data.append(d) + i += 1 + return data + + def dict(self): + return self._dataset + +class ExcelDataL(ExcelData): + def dataload(self): + ret = [] + for name in self._book.sheet_names(): + dat = {} + sheet = self._book.sheet_by_name(name) + name = name.encode(self.encoding) + dat[name] = self.loadSheetData(sheet) + ret.append(dat) + return ret + +if __name__ == '__main__': + if len(sys.argv)<2: + print("Usage:\n%s execlfile" % sys.argv[0]) + sys.exit(1) + ed = ExcelData(sys.argv[1]) + print(ed.dict()) diff --git a/appPublic/excelwriter.py b/appPublic/excelwriter.py new file mode 100755 index 0000000..5ba30da --- /dev/null +++ b/appPublic/excelwriter.py @@ -0,0 +1,144 @@ +import xlwt +from appPublic.strUtils import * + +class ExcelWriter: + def __init__(self,encoding='gb2312'): + self.encoding = encoding + + def writeV(self,sheet,x,y,v): + if type(v) == type([]): + return self.writeList(sheet,x,y,v) + + if type(v) == type({}): + return self.writeDict(sheet,x,y,v) + + if type(v) not in (type({}),type([])): + if type(v) == type(' '): + v = lrtrim(v) + sheet.write(x,y,v) + return 1 + + def write(self,excelfile,dictdata): + wb = xlwt.Workbook(encoding=self.encoding) + for tbl in dictdata.keys(): + ws = wb.add_sheet(tbl,cell_overwrite_ok=True) + self.writeV(ws,0,0,dictdata[tbl]) + wb.save(excelfile) + + def createRecordTitle(self,ws,x,y,title,poss,isList=False): + if isList: + poss['__list__'][title] = True + + if title in poss.keys(): + return + + if len(poss.keys()) > 1: + d_ = {} + for k,v in poss.items(): + if k != '__list__': + d_[k] = v + y = max(d_.values()) + 1 + # ws.write(x,y,title) + poss[title] = y + + def writeRecordTitle(self,ws,x,poss): + for k in poss.keys(): + if k == '__list__': + continue + if k in poss['__list__'].keys(): + ws.write(x,poss[k],k+':list') + else: + ws.write(x,poss[k],k) + + def writeRecords(self,ws,x,y,alist): + ox = x + oy = y + poss = {'__list__':{}} + x = ox + 1 + for r in alist: + for k,v in r.items(): + isList = False + if type(v) == type([]): + isList = True + v = ','.join(v) + self.createRecordTitle(ws,ox,oy,k,poss,isList) + ws.write(x,poss[k],v) + x = x + 1 + self.writeRecordTitle(ws,ox,poss) + return x - ox + + def isRecords(self,alist): + records = True + for r in alist: + if type(r) != type({}): + return False + for k,v in r.items(): + if type(v) == type({}): + return False + if type(v) == type([]): + for c in v: + if type(c) in [type([]),type({})]: + return False + return True + + def writeDict(self,ws,x,y,adict): + ox = x + ws.write(x,y,'__dict__') + x = x + 1 + for k in adict.keys(): + ws.write(x,y,k) + cnt = self.writeV(ws,x,y+1,adict[k]) + x = x + cnt + # print "writeV return ",cnt,"handled key=",k,"next row=",x + return x - ox + + def writeList(self,ws,x,y,alist,singlecell=False): + if self.isRecords(alist): + return self.writeRecords(ws,x,y,alist) + + ox = x + if singlecell is True: + s = ','.join([ str(i) for i in alist ]) + ws.write(x,y,s) + return 1 + multiline = False + for d in alist: + if type(d) == type({}): + multiline=True + if multiline is True: + for i in alist: + if type(i) == type({}): + rows = self.writeDict(ws,x,y,i) + elif type(i) == type([]): + rows = self.writeMultiLineList(ws,x,y,i) + else: + ws.write(x,y,i) + rows = 1 + x = x + rows + return x - ox + else: + for i in alist: + if type(i) == type([]): + self.writeList(ws,x,y,i,singlecell=True) + else: + ws.write(x,y,i) + y = y + 1 + return 1 + + def writeMultiLineList(self,ws,x,y,alist): + ox = x + ws.write(x,y,'__list__') + x = x + 1 + for i in alist: + ws.write(x,y,i) + x = x + 1 + return x - os + +if __name__ == '__main__': + data = { + 'my1':['23423','423424','t334t3',2332,'erfverfefew'], + 'my2':[{'aaa':1,'bbb':'bbb'},{'aaa':1,'bbb':'bbb'}], + } + w = ExcelWriter() + w.write('d:\\text.xls',data) + \ No newline at end of file diff --git a/appPublic/folderUtils.py b/appPublic/folderUtils.py new file mode 100755 index 0000000..86a8a43 --- /dev/null +++ b/appPublic/folderUtils.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- + +import os +import sys +import stat +import os.path +import platform +import time +""" +import win32api +""" +import sys + + +def startsWith(text,s): + return text[:len(s)] == s + +def endsWith(text,s): + return text[-len(s):] == s + +def ProgramPath(): + filename = sys.argv[0] + if getattr(sys,'frozen',False): + filename = sys.executable + p = os.path.dirname(os.path.abspath(filename)) + return p + +def timestamp2datatiemStr(ts): + t = time.localtime(ts) + return '%04d-%02d-%-02d %02d:%02d:%02d' % (t.tm_year,t.tm_mon,t.tm_mday,t.tm_hour,t.tm_min,t.tm_sec) + +""" +def findAllDrives(): + Drives=[] + # print "Searching for drives..." + drives=win32api.GetLogicalDriveStrings().split(":") + for i in drives: + # print "i=",i,":" + dr=i[-1].lower() + if dr.isalpha(): + dr+=":\\" + inf=None + try: + inf=win32api.GetVolumeInformation(dr) + except: + pass # Removable drive, not ready + # You'll still get the drive letter, but inf will be None + Drives.append([dr,inf]) + return Drives +""" + +## list all folder name under folder named by path +# +def folderList(path) : + for name in os.listdir(path) : + full_name = os.path.join(path,name) + if os.path.isdir(full_name): + yield full_name + +def listFile(folder,suffixs=[],rescursive=False): + subffixs = [ i.lower() for i in suffixs ] + for f in os.listdir(folder): + p = os.path.join(folder,f) + if rescursive and os.path.isdir(p): + for p1 in listFile(p,suffixs=suffixs,rescursive=True): + yield p1 + if os.path.isfile(p): + e = p.lower() + if suffixs == [] : + yield p + for s in subffixs: + if e.endswith(s): + yield p + +def folderInfo(root,uri=''): + relpath = uri + if uri[1]=='/': + relpath = uri[1:] + + path = os.path.join(root,*relpath.split('/')) + ret = [] + for name in os.listdir(path): + full_name = os.path.join(path,name) + s = os.stat(full_name) + if stat.S_ISDIR(s.st_mode): + ret.append( { + 'id':relpath + '/' + name, + 'name':name, + 'path':relpath, + 'type':'dir', + 'size':s.st_size, + 'mtime':timestamp2datatiemStr(s.st_mtime), + }) + if stat.S_ISREG(s.st_mode): + ret.append( { + 'id':relpath + '/' + name, + 'name':name, + 'path':relpath, + 'type':'file', + 'size':s.st_size, + 'mtime':timestamp2datatiemStr(s.st_mtime), + }) + return ret + + +def rmdir_recursive(dir): + """Remove a directory, and all its contents if it is not already empty.""" + for name in os.listdir(dir): + full_name = os.path.join(dir, name) + # on Windows, if we don't have write permission we can't remove + # the file/directory either, so turn that on + if not os.access(full_name, os.W_OK): + os.chmod(full_name, 0o600) + if os.path.isdir(full_name): + rmdir_recursive(full_name) + else: + os.remove(full_name) + os.rmdir(dir) + +def _mkdir(newdir) : + """works the way a good mkdir should :) + - already exists, silently complete + - regular file in the way, raise an exception + - parent directory(ies) does not exist, make them as well + """ + if os.path.isdir(newdir): + pass + elif os.path.isfile(newdir): + raise OSError("a file with the same name as the desired " \ + "dir, '%s', already exists." % newdir) + else: + head, tail = os.path.split(newdir) + if head and not os.path.isdir(head): + _mkdir(head) + #print "_mkdir %s" % repr(newdir) + if tail: + os.mkdir(newdir) + +def _copyfile(fp,dir) : + fs = open(fp,'rb') + name = os.path.basename(fp) + newfp = os.path.join(dir,getFileName(name,dir)) + f = open(newfp,'wb') + while True : + data = fs.read(65536) + if not data : + break + f.write(data) + fs.close() + f.close() + return True + +def _copydir(fp,dir,topdistinct) : + name = os.path.basename(fp) + newname = getFileName(name,dir) + debug(newname) + newfp = os.path.join(dir,newname) + _mkdir(newfp) + if fp==topdistinct : + return True + + flist = os.listdir(fp) + for name in flist : + full_name = os.path.join(fp,name) + if os.path.isdir(full_name) : + p = os.path.join(dir,name) + _copydir(full_name,newfp,topdistinct) + else : + if os.path.isfile(full_name) : + _copyfile(full_name,newfp) + return True + +mkdir=_mkdir +copyfile = _copyfile +copydir = _copydir +rmdir = rmdir_recursive diff --git a/appPublic/genetic.py b/appPublic/genetic.py new file mode 100755 index 0000000..35c796a --- /dev/null +++ b/appPublic/genetic.py @@ -0,0 +1,44 @@ +class Genetic: + """ + A Base class for genetical objects, + all the instances can inherite attributes from its parent. + """ + def __init__(self): + self.__parent__ = None + self.__children__ = [] + #print dir(self) + + def __getattr__(self,n): + d = self.__dict__ + if n in d.keys(): + return d[n] + p = self.__parent__ #d['__parent__'] + if p is not None: + return getattr(p,n) + raise AttributeError(n) + + def addChild(self,c): + self.__children__.append(c) + c.__parent__ = self + + def setParent(self,p): + p.addChild(self) + +if __name__ == '__main__': + class A(Genetic): + def __init__(self,a1,a2): + Genetic.__init__(self) + self.a1 = a1 + self.a2 = a2 + + class B(Genetic): + def __init__(self,b): + Genetic.__init__(self) + self.b = b + gp = A(1,2) + p = B(3) + c = A(4,5) + gc = B(6) + gc.setParent(c) + c.setParent(p) + p.setParent(gp) diff --git a/appPublic/jsonConfig.py b/appPublic/jsonConfig.py new file mode 100755 index 0000000..879389a --- /dev/null +++ b/appPublic/jsonConfig.py @@ -0,0 +1,64 @@ +import os,sys +import json +from appPublic.dictObject import DictObject +from appPublic.Singleton import SingletonDecorator +from appPublic.folderUtils import ProgramPath +from appPublic.argsConvert import ArgsConvert + +def key2ansi(dict): + #print dict + return dict + a = {} + for k,v in dict.items(): + k = k.encode('utf-8') + #if type(v) == type(u" "): + # v = v.encode('utf-8') + a[k] = v + + return a + +class JsonObject(DictObject): + """ + JsonObject class load json from a json file + """ + def __init__(self,jsonholder,keytype='ansi',NS=None): + self.__jsonholder__ = jsonholder + self.NS = NS + jhtype = type(jsonholder) + if jhtype == type("") or jhtype == type(u''): + f = open(jsonholder,'r') + else: + f = jsonholder + try: + a = json.load(f) + except Exception as e: + print("exception:",self.__jsonholder__,e) + raise e + finally: + if type(jsonholder) == type(""): + f.close() + + if self.NS is not None: + ac = ArgsConvert('$[',']$') + a = ac.convert(a,self.NS) + DictObject.__init__(self,**a) + +@SingletonDecorator +class JsonConfig(JsonObject): + pass +def getConfig(path=None,NS=None): + if path==None: + path = ProgramPath() + cfname = os.path.abspath(os.path.join(path,"conf","config.json")) + # print __name__,cfname + a = JsonConfig(cfname,NS=NS) + return a + +if __name__ == '__main__': + conf = JsonConfig(sys.argv[1]) + #print conf.db,conf.sql + #conf1 = JsonConfig(sys.argv[1],keytype='unicode') + conf1 = JsonConfig(sys.argv[1],keytype='ansi') + + print("conf=",dir(conf)) + print("conf1=",dir(conf1) ) diff --git a/appPublic/jsonIO.py b/appPublic/jsonIO.py new file mode 100755 index 0000000..d6f9b66 --- /dev/null +++ b/appPublic/jsonIO.py @@ -0,0 +1,38 @@ +import json + +def uni_str(a, encoding): + if a is None: + return None + if isinstance(a, (list, tuple)): + s = [] + for i, k in enumerate(a): + s.append(uni_str(k, encoding)) + return s + elif isinstance(a, dict): + s = {} + for i, k in enumerate(a.items()): + key, value = k + s[uni_str(key, encoding)] = uni_str(value, encoding) + return s + elif isinstance(a, bool): + return a + elif isinstance(a, unicode): + return a + elif isinstance(a, str) or (hasattr(a, '__str__') and callable(getattr(a, '__str__'))): + if getattr(a, '__str__'): + a = str(a) + return unicode(a, encoding) + else: + return a + +def success(data): + return dict(success=True,data=data) + +def error(errors): + return dict(success=False,errors=errors) + +def jsonEncode(data,encode='utf-8'): + return json.dumps(uni_str(data, encode)) + +def jsonDecode(jsonstring): + return json.loads(jsonstring) diff --git a/appPublic/localefunc.py b/appPublic/localefunc.py new file mode 100755 index 0000000..aab14e4 --- /dev/null +++ b/appPublic/localefunc.py @@ -0,0 +1,43 @@ +# this function will fix a bug for open a file with a not english name. +# + +import sys +import locale + +language, local_encoding = locale.getdefaultlocale() +if sys.platform == 'win32': + import locale, codecs + local_encoding = locale.getdefaultlocale()[1] + if local_encoding.startswith('cp'): # "cp***" ? + try: + codecs.lookup(local_encoding) + except LookupError: + import encodings + encodings._cache[local_encoding] = encodings._unknown + encodings.aliases.aliases[local_encoding] = 'mbcs' + +def locale_open(filename,mode='rb') : + return open(filename.encode(local_encoding),mode) + +def localeString(s) : + try : + return unicode(s,'utf-8').encode(local_encoding) + + except : + return s + +def utf8String(s) : + try : + return unicode(s,local_encoding).encode('utf-8') + except : + return s + +def charsetString(s,charset) : + try : + return unicode(s,local_encoding).encode(charset) + except : + try : + return unicode(s,'utf-8').encode(charset) + except : + return s + diff --git a/appPublic/macAddress.py b/appPublic/macAddress.py new file mode 100755 index 0000000..8c3af37 --- /dev/null +++ b/appPublic/macAddress.py @@ -0,0 +1,38 @@ +#! /usr/bin/env python +import locale +import psutil +import socket + + +def getAllAddress(): + iocounts = psutil.net_io_counters(pernic=True) + ns = [ k for k in iocounts.keys() if iocounts[k].bytes_sent>0 and iocounts[k].bytes_recv>0 ] + stats = psutil.net_if_stats() + stat = [ i for i in stats.keys() if i in ns ] + hds = psutil.net_if_addrs() + for n,v in hds.items(): + if n not in stat: + continue + for i in v: + if i.family == socket.AF_INET: + yield n,i.address + +def getAllMacAddress(): + coding = locale.getdefaultlocale()[1] + iocounts = psutil.net_io_counters(pernic=True) + ns = [ k for k in iocounts.keys() if iocounts[k].bytes_sent>0 and iocounts[k].bytes_recv>0 ] + stats = psutil.net_if_stats() + stat = [ i for i in stats.keys() if i in ns ] + hds = psutil.net_if_addrs() + for n,v in hds.items(): + if n not in stat: + continue + for i in v: + if i.family == socket.AF_PACKET: + yield n,i.address + +if __name__ == '__main__': + def test(): + for i in getAllAddress(): + print("mac=",i) + test() diff --git a/appPublic/myImport.py b/appPublic/myImport.py new file mode 100755 index 0000000..ccf4c09 --- /dev/null +++ b/appPublic/myImport.py @@ -0,0 +1,6 @@ +def myImport(modulename): + modules = modulename.split('.') + if len(modules) > 1: + a = __import__(modules[0]) + return eval('a.' + '.'.join(modules[1:])) + return __import__(modulename) \ No newline at end of file diff --git a/appPublic/myjson.py b/appPublic/myjson.py new file mode 100755 index 0000000..ed6f638 --- /dev/null +++ b/appPublic/myjson.py @@ -0,0 +1,18 @@ +import ujson as json +import codecs + +def loadf(fn,coding='utf8'): + f = codecs.open(fn,'r',coding) + d = json.load(f) + f.close() + return d + +def dumpf(obj,fn,coding='utf8'): + f = codecs.open(fn,'w',coding) + json.dump(obj,f) + f.close() + +load = json.load +dump = json.dump +loads = json.loads +dumps = json.dumps \ No newline at end of file diff --git a/appPublic/mylog.py b/appPublic/mylog.py new file mode 100755 index 0000000..0ffdf99 --- /dev/null +++ b/appPublic/mylog.py @@ -0,0 +1,89 @@ +import os +from datetime import datetime +from PublicData import public_data +from folderUtils import mkdir + +myLogPath = '.' + +AllCatelogs=['SYSError', + 'SYSWarn', + 'APPError', + 'APPWarn', + 'APPInfo', + 'DEBUG1', + 'DEBUG2', + 'DEBUG3', + 'DEBUG4', + 'DEBUG5', +] + +class MyLog : + def __init__(self,path) : + self.setLogPath(path) + + def setLogPath(self,path='.') : + self.myLogPath = path + logp=os.path.join(path,'log') + mkdir(logp) + + def __call__(self,msg='') : + p = os.path.join(self.myLogPath,'log','my.log') + f = open(p,'a') + d = datetime.now() + f.write('%04d-%02d-%02d %02d:%02d:%02d %s\n' % ( d.year,d.month,d.day,d.hour,d.minute,d.second,msg)) + f.close() + +class LogMan : + def __init__(self) : + self.logers = {} + self.catelogs = AllCatelogs + + def addCatelog(self,catelog) : + if catelog not in self.catelogs : + self.catelogs.append(catelog) + + def addLoger(self,name,func,catelog) : + if type(catelog)!=type([]) : + catelog = [catelog] + catelog = [ i for i in catelog if i in self.catelogs ] + log = { + 'name':name, + 'func':func, + 'catelog':catelog, + } + self.logers[name] = log + + def delLoger(self,name) : + if name in self.logers.keys() : + del self.logers[name] + + def setCatelog(self,name,catelog) : + if type(catelog)!=type([]) : + catelog = [catelog] + catelog = [ i for i in catelog if i in self.catelogs ] + if name in self.logers.keys() : + log = self.logers[name] + log['catelog'] = catelog + self.logers[name] = log + + def __call__(self,msg='',catelog='APPInfo') : + for name,loger in self.logers.items() : + c = loger['catelog'] + if type(c)!=type([]) : + c = [c] + if catelog in c : + f = loger['func'] + f(msg) + +def mylog(s,catelog='APPInfo') : + logman = public_data.get('mylog',None) + if logman==None : + path = public_data.get('ProgramPath',None) + if path==None : + raise Exception('ProgramPath Not found in "public_data"') + log = MyLog(path) + logman = LogMan() + logman.addLoger('mylog',log,AllCatelogs) + public_data.set('mylog',logman) + return logman(s,catelog) + diff --git a/appPublic/objectAction.py b/appPublic/objectAction.py new file mode 100755 index 0000000..125fed7 --- /dev/null +++ b/appPublic/objectAction.py @@ -0,0 +1,62 @@ +from appPublic.Singleton import SingletonDecorator + +@SingletonDecorator +class ObjectAction(object): + def __init__(self): + self.actionList = {} + + def init(self,id,action): + idA = self.actionList.get(id,None) + if idA is None: + idA = self.actionList[id] = {} + self.actionList[id][action] = [] + + def add(self,id,action,func): + idA = self.actionList.get(id,None) + if idA is None: + idA = self.actionList[id] = {} + fL = idA.get(action,None) + if fL is None: + fL = self.actionList[id][action] = [] + self.actionList[id][action].append(func) + + def execute(self,id,action,data,callback=None): + if action in ['#','*']: + return data + idA = self.actionList.get(id,None) + if idA is None: + return data + fL = idA.get(action,[]) + fL += idA.get('*',[]) + for f in fL: + data = f(id,action,data) + if len(fL)==0: + for f in idA.get('#',[]): + data = f(id,action,data) + if callback is not None: + callback(data) + return data + +if __name__ == '__main__': + def f(id,act,data): + return data + + def f1(id,act,data): + return data + + def f2(id,act,data): + return data + + def add(): + oa = ObjectAction() + oa.add('test','b',f) + #oa.add('test','*',f1) + oa.add('test','#',f2) + + def exe(): + oa = ObjectAction() + oa.execute('test','a','data1') + + add() + exe() + diff --git a/appPublic/pickleUtils.py b/appPublic/pickleUtils.py new file mode 100755 index 0000000..b764167 --- /dev/null +++ b/appPublic/pickleUtils.py @@ -0,0 +1,16 @@ +import pickle + +def saveData(fn,*args): + f = open(fn,'wb') + a = [ pickle.dump(arg,f) for arg in args ] + f.close() + +def loadData(fn,cnt): + a = [None] * cnt + try: + f = open(fn,'rb') + a = [ pickle.load(f) for i in range(cnt) ] + f.close() + return a + except: + return a diff --git a/appPublic/rc4.py b/appPublic/rc4.py new file mode 100755 index 0000000..0ae4ba4 --- /dev/null +++ b/appPublic/rc4.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +import random, base64 +from hashlib import sha1 + +class RC4: + def __init__(self,key='1234567890',data_coding='utf8'): + self.bcoding = 'iso-8859-1' + self.dcoding = data_coding + self.key = key.encode(self.bcoding) + self.salt = b'AFUqx9WZuI32lnHk' + + def _crypt(self,data,key): + """RC4 algorithm return bytes""" + if type(data)==type(''): + data = data.encode(self.dcoding) + x = 0 + box = [i for i in range(256) ] + for i in range(256): + x = (x + box[i] + key[i % len(key)]) % 256 + box[i], box[x] = box[x], box[i] + x = y = 0 + out = [] + for char in data: + x = (x + 1) % 256 + y = (y + box[x]) % 256 + box[x], box[y] = box[y], box[x] + out.append(chr(char ^ box[(box[x] + box[y]) % 256])) + + return ''.join(out).encode(self.bcoding) + + def encode(self,data, encode=base64.b64encode, salt_length=16): + """RC4 encryption with random salt and final encoding""" + #salt = '' + #for n in range(salt_length): + # salt += chr(random.randrange(256)) + #salt = salt.encode(self.bcoding) + a = sha1(self.key + self.salt) + k = a.digest() + data = self.salt + self._crypt(data, k) + + if encode: + data = encode(data) + return data.decode(self.dcoding) + + def decode(self,data, decode=base64.b64decode, salt_length=16): + """RC4 decryption of encoded data""" + if decode: + data = decode(data) + salt = data[:salt_length] + a = sha1(self.key + self.salt) + k = a.digest() #.decode('iso-8859-1') + r = self._crypt(data[salt_length:], k) + return r.decode(self.dcoding) + + +if __name__=='__main__': + # 需要加密的数据 + data = '''hello python 爱的实打实大师大师大师的发送到发送到而非个人格个二哥而而二哥而个人各位,UDP是一种无连接对等通信协议，没有服务器和客户端概念，通信的任何一方均可通过通信原语直接和其他方通信 + +HOME FAQ DOCS DOWNLOAD + + +index +next | +previous | +Twisted 18.9.0 documentation » Twisted Names (DNS) » Developer Guides » + +Creating a custom server +The builtin DNS server plugin is useful, but the beauty of Twisted Names is that you can build your own custom servers and clients using the names components. +In this section you will learn about the components required to build a simple DNS server. +You will then learn how to create a custom DNS server which calculates responses dynamically. +A simple forwarding DNS server +Lets start by creating a simple forwarding DNS server, which forwards all requests to an upstream server (or servers). +simple_server.py +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An example of a simple non-authoritative DNS server. +""" + +from twisted.internet import reactor +from twisted.names import client, dns, server + + +def main(): + """ + Run the server. + """ + factory = server.DNSServerFactory( + clients=[client.Resolver(resolv='/etc/resolv.conf')] + ) + + protocol = dns.DNSDatagramProtocol(controller=factory) + + reactor.listenUDP(10053, protocol) + reactor.listenTCP(10053, factory) + + reactor.run() + + +if __name__ == '__main__': + raise SystemExit(main()) +In this example we are passing a client.Resolver instance to the DNSServerFactory and we are configuring that client to use the upstream DNS servers which are specified in a local resolv.conf file. +Also note that we start the server listening on both UDP and TCP ports. This is a standard requirement for DNS servers. +You can test the server using dig. For example: +$ dig -p 10053 @127.0.0.1 example.com SOA +short +sns.dns.icann.org. noc.dns.icann.org. 2013102791 7200 3600 1209600 3600 +A server which computes responses dynamically +Now suppose we want to create a bespoke DNS server which responds to certain hostname queries by dynamically calculating the resulting IP address, while passing all other queries to another DNS server. Queries for hostnames matching the pattern workstation{0-9}+ will result in an IP address where the last octet matches the workstation number. +We’ll write a custom resolver which we insert before the standard client resolver. The custom resolver will be queried first. +Here’s the code: +override_server.py +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +An example demonstrating how to create a custom DNS server. + +The server will calculate the responses to A queries where the name begins with +the word "workstation". + +Other queries will be handled by a fallback resolver. + +eg + python doc/names/howto/listings/names/override_server.py + + $ dig -p 10053 @localhost workstation1.example.com A +short + 172.0.2.1 +""" + +from twisted.internet import reactor, defer +from twisted.names import client, dns, error, server + + + +class DynamicResolver(object): + """ + A resolver which calculates the answers to certain queries based on the + query type and name. + """ + _pattern = 'workstation' + _network = '172.0.2' + + def _dynamicResponseRequired(self, query): + """ + Check the query to determine if a dynamic response is required. + """ + if query.type == dns.A: + labels = query.name.name.split('.') + if labels[0].startswith(self._pattern): + return True + + return False + + + def _doDynamicResponse(self, query): + """ + Calculate the response to a query. + """ + name = query.name.name + labels = name.split('.') + parts = labels[0].split(self._pattern) + lastOctet = int(parts[1]) + answer = dns.RRHeader( + name=name, + payload=dns.Record_A(address=b'%s.%s' % (self._network, lastOctet))) + answers = [answer] + authority = [] + additional = [] + return answers, authority, additional + + + def query(self, query, timeout=None): + """ + Check if the query should be answered dynamically, otherwise dispatch to + the fallback resolver. + """ + if self._dynamicResponseRequired(query): + return defer.succeed(self._doDynamicResponse(query)) + else: + return defer.fail(error.DomainError()) + + + +def main(): + """ + Run the server. + """ + factory = server.DNSServerFactory( + clients=[DynamicResolver(), client.Resolver(resolv='/etc/resolv.conf')] + ) + + protocol = dns.DNSDatagramProtocol(controller=factory) + + reactor.listenUDP(10053, protocol) + reactor.listenTCP(10053, factory) + + reactor.run() + + + +if __name__ == '__main__': + raise SystemExit(main()) +Notice that DynamicResolver.query returns a Deferred. On success, it returns three lists of DNS records (answers, authority, additional), which will be encoded by dns.Message and returned to the client. On failure, it returns a DomainError, which is a signal that the query should be dispatched to the next client resolver in the list. +Note +The fallback behaviour is actually handled by ResolverChain. +ResolverChain is a proxy for other resolvers. It takes a list of IResolver providers and queries each one in turn until it receives an answer, or until the list is exhausted. +Each IResolver in the chain may return a deferred DomainError, which is a signal that ResolverChain should query the next chained resolver. +The DNSServerFactory constructor takes a list of authoritative resolvers, caches and client resolvers and ensures that they are added to the ResolverChain in the correct order. +Let’s use dig to see how this server responds to requests that match the pattern we specified: +$ dig -p 10053 @127.0.0.1 workstation1.example.com A +short +172.0.2.1 + +$ dig -p 10053 @127.0.0.1 workstation100.example.com A +short +172.0.2.100 +And if we issue a request that doesn’t match the pattern: +$ dig -p 10053 @localhost www.example.com A +short +93.184.216.119 +Further Reading +For simplicity, the examples above use the reactor.listenXXX APIs. But your application will be more flexible if you use the Twisted Application APIs, along with the Twisted plugin system and twistd. Read the source code of names.tap to see how the twistd names plugin works. +Table Of Contents +Creating a custom server +A simple forwarding DNS server +A server which computes responses dynamically +Further Reading +Previous topic +Creating and working with a names (DNS) server +Next topic +Examples +This Page +Show Source +Quick search + +Enter search terms or a module, class or function name. + + +Site design +By huw.wilkins. +''' + # 密钥 + key = '123456' + rc4 = RC4(key) + print(data) + # 加码 + encoded_data = rc4.encode(data) + print(encoded_data,len(encoded_data) ) + # 解码 + decoded_data = rc4.decode(encoded_data) + print(decoded_data) \ No newline at end of file diff --git a/appPublic/receiveMail.py b/appPublic/receiveMail.py new file mode 100755 index 0000000..9072c0f --- /dev/null +++ b/appPublic/receiveMail.py @@ -0,0 +1,137 @@ +import poplib,pdb,email,re,time +from email import header +import datetime +import os + +POP_ADDR = r'pop.126.com' +USER = '' +PASS = '' +CONFIG = '' + +def getYear(date): + rslt = re.search(r'\b2\d{3}\b', date) + return int(rslt.group()) + +def getMonth(date): + monthMap = {'Jan':1,'Feb':2,'Mar':3,'Apr':4,'May':5,'Jun':6, + 'Jul':7,'Aug':8,'Sep':9,'Oct':10,'Nov':11,'Dec':12,} + + rslt = re.findall(r'\b\w{3}\b', date) + for i in range(len(rslt)): + month = monthMap.get(rslt[i]) + if None != month: + break + + return month + +def getDay(date): + rslt = re.search(r'\b\d{1,2}\b', date) + return int(rslt.group()) + +def getTime(date): + rslt = re.search(r'\b\d{2}:\d{2}:\d{2}\b', date) + timeList = rslt.group().split(':') + + for i in range(len(timeList)): + timeList[i] = int(timeList[i]) + + return timeList + +def transformDate(date): + rslt = getYear(date) + rslt = rslt * 100 + rslt = rslt + getMonth(date) + rslt = rslt * 100 + rslt = rslt + getDay(date) + + + timeList = getTime(date) + for i in range(len(timeList)): + rslt = rslt * 100 + rslt = rslt + timeList[i] + + return rslt + +def getRecentReadMailTime(): + fp = open(CONFIG, 'r') + rrTime = fp.read() + fp.close() + return rrTime + +def setRecentReadMailTime(): + fp = open(CONFIG, 'w') + fp.write(time.ctime()) + fp.close() + return + +def getTimeEarly(period): + def years(n): + return datetime.timedelta(years=n) + def months(n): + return datetime.timedelta(years=n) + def days(n): + return datetime.timedelta(days=n) + def hours(n): + return datetime.timedelta(hours=n) + def minutes(n): + return datetime.timedelta(minutes=n) + def seconds(n): + return datetime.timedelta(seconds=n) + + funcs={ + 'y':years, + 'm':months, + 'd':days, + 'H':hours, + 'M':minutes, + 'S':seconds, + } + pattern='(\d*)([ymdHMS])' + r=re.compile(pattern) + s = r.findall(period) + t = datetime.datetime.now() + for v,ty in s: + td = funcs[ty](int(v)) + t = t - td + return time.ctime(t.timestamp()) + + +def parseMailContent(msg): + if msg.is_multipart(): + for part in msg.get_payload(): + parseMailContent(part) + else: + bMsgStr = msg.get_payload(decode=True) + charset = msg.get_param('charset') + msgStr = 'Decode Failed' + try: + if None == charset: + msgStr = bMsgStr.decode() + else: + msgStr = bMsgStr.decode(charset) + except: + pass + + print(msgStr) + +def recvEmail(POP_ADDR,USER,PASS,PERIOD,callback): + server = poplib.POP3(POP_ADDR) + server.user(USER) + server.pass_(PASS) + + mailCount,size = server.stat() + mailNoList = list(range(mailCount)) + mailNoList.reverse() + FROMTIME = getTimeEarly(PERIOD) + hisTime = transformDate(FROMTIME) + #pdb.set_trace() + for i in mailNoList: + message = server.retr(i+1)[1] + mail = email.message_from_bytes(b'\n'.join(message)) + + if transformDate(mail.get('Date')) > hisTime: + if not callback(mail): + break + #parseMailContent(mail) + else: + break diff --git a/appPublic/restrictedEnv.py b/appPublic/restrictedEnv.py new file mode 100755 index 0000000..1ef2bcd --- /dev/null +++ b/appPublic/restrictedEnv.py @@ -0,0 +1,34 @@ +import appPublic.timeUtils as tu +import datetime as dt + +class RestrictedEnv: + def __init__(self): + self.reg('today',self.today) + self.reg('date',self.date) + self.reg('datetime',self.datetime) + self.reg('now',dt.datetime.now) + + def reg(self,k,v): + self.__dict__[k] = v + + def run(self,dstr): + dstr = '__tempkey__ = %s' % dstr + exec(dstr,globals(),self.__dict__) + return self.__tempkey__ + + def today(self): + now = dt.datetime.now() + return tu.ymdDate(now.year,now.month,now.day) + + def date(self,dstr): + return tu.str2Date(dstr) + + def datetime(self,dstr): + return tu.str2Datetime(dstr) + +if __name__ == '__main__': + ns = RestrictedEnv() + a = ns.run('today()') + b = ns.run("date('2011-10-31')") + c = ns.run('datetime("2012-03-12 10:22:22")') + d = ns.run('now()') diff --git a/appPublic/rsa.py b/appPublic/rsa.py new file mode 100755 index 0000000..ad20fc1 --- /dev/null +++ b/appPublic/rsa.py @@ -0,0 +1,170 @@ +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import padding +from cryptography.hazmat.primitives import serialization +from cryptography.exceptions import InvalidSignature +import base64 + +class RSA: + def __init__(self): + pass + + def write_privatekey(self,private_key,fname,password=None): + pwd = password + pem = '' + if pwd is not None: + pwd = bytes(pwd,encoding='utf8') if not isinstance(pwd, bytes) else pwd + pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.BestAvailableEncryption(pwd) + ) + else: + pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + + + with open(fname,'w') as f: + text = pem.decode('utf8') + f.write(text) + + def write_publickey(self,public_key,fname): + pem = public_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo + ) + + with open(fname,'w') as f: + text = pem.decode('utf8') + f.write(text) + + def read_privatekey(self,fname,password=None): + pwd = password + if password is not None: + pwd = bytes(password,encoding='utf8') if not isinstance(password, bytes) else password + with open(fname, "rb") as key_file: + key = serialization.load_pem_private_key( + key_file.read(), + password=pwd, + backend=default_backend() + ) + return key + + def read_publickey(self,fname): + with open(fname,'r') as f: + public_key_pem_export = f.read() + public_key_pem_export = bytes(public_key_pem_export,encoding='utf8') if not isinstance(public_key_pem_export, bytes) else public_key_pem_export + return serialization.load_pem_public_key(data=public_key_pem_export,backend=default_backend()) + + def create_privatekey(self): + return rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + backend=default_backend() + ) + + def create_publickey(self,private_key): + return private_key.public_key() + + def encode(self,public_key,text): + message_bytes = bytes(text, encoding='utf8') if not isinstance(text, bytes) else text + return str(base64.b64encode(public_key.encrypt(message_bytes, + padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), + label=None + ) + )), encoding='utf-8') + + def decode(self,private_key,cipher): + cipher = cipher.encode('utf8') if not isinstance(cipher, bytes) else cipher + ciphertext_decoded = base64.b64decode(cipher) + plain_text = private_key.decrypt( + ciphertext_decoded,padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), + label=None + ) + ) + return str(plain_text, encoding='utf8') + + def sign(self,private_key,message): + data_to_sign = bytes(message, encoding='utf8') if not isinstance( + message, + bytes + ) else message + signer = private_key.signer( + padding.PSS( + mgf=padding.MGF1(hashes.SHA256()), + salt_length=padding.PSS.MAX_LENGTH + ), + hashes.SHA256() + ) + signer.update(data_to_sign) + signature = str( + base64.b64encode(signer.finalize()), + encoding='utf8' + ) + return signature + + def check_sign(self,public_key,plain_text,signature): + try: + plain_text_bytes = bytes( + plain_text, + encoding='utf8' + ) if not isinstance(plain_text, bytes) else plain_text + signature = base64.b64decode( + signature + ) if not isinstance(signature, bytes) else signature + verifier = public_key.verifier( + signature, + padding.PSS( + mgf=padding.MGF1(hashes.SHA256()), + salt_length=padding.PSS.MAX_LENGTH + ), + hashes.SHA256() + ) + verifier.update(plain_text_bytes) + verifier.verify() + return True + except InvalidSignature as e: + return False + +if __name__ == '__main__': + Recv_cipher=b'JHHDhjaHLeLRopobfiJvWtVn8Bu3/3AsV6Xr8MwHBBEli7v+oHRNH2dcAfVa7VcdBlKlr7W+hDDAxlex3/OzwyRp4R5DcDsepTLPaG+nKK6zj0MGkvEJ6iNpABO9uohskFXPBuO6t3+G6cKRMMeIU7g7oSJqlbKHJyRmd9j8OHS2fFYL331oRhJvyuJe5zrdxHEOez+XEt2AbuYi7WFFVlM/DvX/tjAG3SHXr14GlJYGbuNR2LNIapAMBSt7GDQ/LrzLv54ysE3OZpVFnOszVt5ythiDPoImnpJ990Fb/1yd7goPZ5NA8cSKCu7dDV42JWcj44JHrIfNsMR7aG9QxQ==' + r = RSA() + mpri = r.create_privatekey() + mpub = r.create_publickey(mpri) + + zpri = r.create_privatekey() + zpub = r.create_publickey(zpri) + + text = 'this is a test data, aaa' + cipher = r.encode(mpub,text) + signature = r.sign(zpri,text) + + ntext = r.decode(mpri,cipher) + check = r.check_sign(zpub,ntext,signature) + print(text,ntext,check) + + ypri = r.read_privatekey('d:/dev/mecp/conf/RSA.private.key','ymq123') + ypub = r.read_publickey('d:/dev/mecp/conf/RSA.public.key') + + x = r.encode(ypub,'root:ymq123') + print('root:ymq123 encode=',x,len(x),len(Recv_cipher)) + orgtext=r.decode(ypri,Recv_cipher) + print(orgtext) + + r.write_publickey(ypub,'./test.public.key') + r.write_privatekey(ypri,'./test.private.key','ymq123') + ypri = r.read_privatekey('./test.private.key','ymq123') + ypub = r.read_publickey('./test.public.key') + + x = r.encode(ypub,text) + ntext = r.decode(ypri,x) + print(text,'<==>',ntext) + \ No newline at end of file diff --git a/appPublic/rsaPeer.py b/appPublic/rsaPeer.py new file mode 100755 index 0000000..5933141 --- /dev/null +++ b/appPublic/rsaPeer.py @@ -0,0 +1,125 @@ +from appPublic.rsa import RSA +from appPublic.rc4 import RC4 +import ujson as json +import random + +class DeliverPacket: + def __init__(self,sender,c,k,s): + self.sender = sender + self.c = c + self.k = k + self.s = s + + def pack(self): + d = { + "sender":self.sender, + "c":self.c, + "k":self.k, + "s":self.s, + } + return json.dumps(d) + + def unpack(self,body): + d = json.loads(body) + self.sender = d.sender + self.c = d['c'] + self.k = d['k'] + self.s = d['s'] + +class RSAPeer: + def __init__(self,myid,myPrikey,pearPubKey=None): + self.myid = myid + self.mypri = myPrikey + self.peerpub = pearPubKey + self.rsa = RSA() + + def getPeerPublicKey(self,id): + pass + + def _genSystematicKey(self): + t = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890~!@#$%^&*' + kl = random.randint(10,15) + ky = [] + klen = len(t) - 1 + for k in range(kl): + i = random.randint(0,klen) + # print(k,klen,i) + ky.append(t[i]) + return ''.join(ky) + + def encode(self,text): + """ + return a json text + json ojbect have three addt: + k:encrypted rc4 key + s:signature + c:ciphertext encrypted by key + """ + d = {"id":self.myid,"data":text} + text = json.dumps(d) + sk = self._genSystematicKey() + rc4 = RC4(sk) + c = rc4.encode(text) + s = self.rsa.sign(self.mypri,sk) + if self.peerpub is None: + return None + k = self.rsa.encode(self.peerpub,sk) + d = { + 'c':c, + 'k':k, + 's':s + } + return json.dumps(d) + + def decode(self,body): + """ + cipher a json text + json ojbect have three addt: + k:encrypted rc4 key + s:signature + c:ciphertext encrypted by key + """ + d = json.loads(body) + signature = d['s'] + sk = self.rsa.decode(self.mypri,d['k']) + # print('sk=',sk,'k=',d['k'],type(d['k'])) + rc4 = RC4(sk) + t = rc4.decode(d['c']) + d = json.loads(t) + ret = d['data'] + if self.peerpub is not None and not self.rsa.check_sign(self.peerpub,sk,signature): + return None + if self.peerpub is None: + peerpub = self.getPeerPublicKey(d['id']) + if peerpub is None: + return None + if not self.rsa.check_sign(peerpub,sk,signature): + return None + return ret + +if __name__ == '__main__': + r = RSA() + mary_pri = r.create_privatekey() + mary_pub = r.create_publickey(mary_pri) + + john_pri = r.create_privatekey() + john_pub = r.create_publickey(john_pri) + + john_rp = RSAPeer(john_pri,mary_pub) + mary_rp = RSAPeer(mary_pri,john_pub) + txt = '''hello python 爱的实打实大师大师大师的发送到发送到而非个人格个二哥而而二哥而个人各位,UDP是一种无连接对等通信协议，没有服务器和客户端概念，通信的任何一方均可通过通信原语直接和其他方通信 + +HOME FAQ DOCS DOWNLOAD + + +index +next | +previous | +Twisted 18.9.0 documentation » Twisted Names (DNS) » Developer Guides » ''' + c = john_rp.encode(txt) + newtxt = mary_rp.decode(c) + print(txt) + print('<===>') + print(c) + print('<===>') + print(newtxt) \ No newline at end of file diff --git a/appPublic/sockPackage.py b/appPublic/sockPackage.py new file mode 100755 index 0000000..4e3c4bb --- /dev/null +++ b/appPublic/sockPackage.py @@ -0,0 +1,134 @@ +import os +import time +import threading +import sys +from socket import AF_INET,SOCK_STREAM,socket +from mylog import mylog + +def logit(s) : + mylog(__file__ + ':' + s) + +class background(threading.Thread) : + def __init__(self,func,kw) : + threading.Thread.__init__(self) + self.func = func + self.kw = kw + + def run(self) : + if self.func!=None : + self.func(**self.kw) + return + +def BackgroundCall(func,datas) : + b=background(func,datas) + b.start() + return + +class SocketServerError(Exception) : + pass + +class SocketClientError(Exception) : + pass + +class SocketServer(threading.Thread) : + + def __init__(self,host,port,max_connect=10,callee=None) : + threading.Thread.__init__(self, name = 'SocketServer') + self.setDaemon(False) + self.host = host + self.port = int(port) + self.max_c = max_connect + self.ready = False + self.keep_running = 0 + self.callee = callee + self.setSocketServer() + + def setSocketServer(self) : + try : + self.sock = socket(AF_INET,SOCK_STREAM) + self.sock.bind((self.host,self.port)) + self.sock.listen(self.max_c) + self.ready = True + except Exception as e: + logit('setSocketServer() Error:%s\nhost=%s,port=%d' % (e,self.host,self.port)) + pass + + def run(self) : + if not self.ready : + raise SocketServerError('not ready') + callee = self.callee + if self.callee!=None : + callee = self.callee + self.keep_running = 1 + while self.keep_running : + conn,addr = self.sock.accept() + BackgroundCall(callee,{'conn':conn,'addr':addr}) + # conn.close() + + def stop(self) : + self.keep_running = 0 + + def callee(self,conn,addr) : + while 1 : + d = conn.recv(1024) + if d==None : + break + conn.send(d) + con.close() + +class SocketClient : + + def __init__(self,host,port) : + self.host = host + self.port = port + self.ready = False + self.connect() + + # if tim ==0 not blocking + def timeout(self,tim) : + if self.ready : + self.sock.setblocking(tim>0) + if tim>0 : + self.sock.settimeout(tim) + + def connect(self) : + try : + self.sock = socket(AF_INET,SOCK_STREAM) + self.sock.connect((self.host,self.port)) + self.ready = True + except Exception as e: + self.ready = False + logit('Socket connect error,%s\nhost=%s,port=%s' % (e,self.host,self.port)) + raise SocketClientError('connect error') + + def read(self,size) : + try : + data = self.sock.recv(size) + return data + except Exception as e: + logit('recv error,%s' % e) + raise SocketClientError('recv error') + + def write(self,data) : + try : + self.sock.send(data) + except Exception as e: + logit('recv error,%s' % e) + raise SocketClientError('send error') + + def close(self) : + self.sock.close() + self.ready = False + +if __name__ == '__main__' : + s = SocketServer('localhost',12232) + s.start() + time.sleep(5) + while 1 : + c = SocketClient('localhost',12232) + msg = 'msg1' + print("send:",msg) + c.write(msg) + d = c.read(1024) + print("get:",d) + time.sleep(1) diff --git a/appPublic/strUtils.py b/appPublic/strUtils.py new file mode 100755 index 0000000..abaa82f --- /dev/null +++ b/appPublic/strUtils.py @@ -0,0 +1,20 @@ +# strUtils + +def rtrim(ss): + s = ss + if s=='': + return s + while s[-1] == ' ': + s = s[:-1] + return s +def ltrim(ss): + s = ss + if s=='': + return s + while s[0] == ' ': + s = s[1:] + return s +def lrtrim(ss): + s = ltrim(ss) + s = rtrim(s) + return s diff --git a/appPublic/testdict.py b/appPublic/testdict.py new file mode 100755 index 0000000..251e953 --- /dev/null +++ b/appPublic/testdict.py @@ -0,0 +1,6 @@ +import ExecFile +c = ExecFile.DictConfig(path='./config.dict') + +print(c.d.b[1].c,c.d.c.a,c.d.c.b,c.d.c.c[3].f) +print(c.d.c.c[1]) +print(c.d.c.d) diff --git a/appPublic/timeUtils.py b/appPublic/timeUtils.py new file mode 100755 index 0000000..7e50f8a --- /dev/null +++ b/appPublic/timeUtils.py @@ -0,0 +1,157 @@ +import os,sys +import time +import datetime + +leapMonthDays = [0,31,29,31,30,31,30,31,31,30,31,30,31] +unleapMonthDays = [0,31,28,31,30,31,30,31,31,30,31,30,31] + +def curDatetime(): + return datetime.datetime.now() + +def curDateString(): + d = curDatetime() + return '%04d-%02d-%02d' %(d.year,d.month,d.day) + +def timestampstr(): + d = curDatetime() + return '%04d-%02d-%02d %02d:%02d:%02d.%03d' % (d.year, + d.month, + d.day, + d.hour, + d.minute, + d.second, + d.microsecond/1000) + +def isMonthLastDay(d): + dd = datetime.timedelta(1) + d1 = d + dd + if d1.month != d.month: + return True + return False + +def isLearYear(year): + if year % 4 == 0 and year % 100 == 0 and not (year % 400 == 0): + return True + return False + +def timestamp(dt): + return int(time.mktime((dt.year,dt.month,dt.day,dt.hour,dt.minute,dt.second,dt.microsecond,0,0))) + +def timeStampSecond(dt): + return int(time.mktime((dt.year,dt.month,dt.day,dt.hour,dt.minute,dt.second,0,0,0))) + +def addSeconds(dt,s): + ndt = dt + datetime.timedelta(0,s) + return ndt + +def monthMaxDay(y,m): + dt = ymdDate(y,m,1) + if isLeapYear(dt): + return leapMonthDays[m] + return unleapMonthDays[m] + +def date2str(dt=None): + if dt is None: + dt = curDatetime() + return '%04d-%02d-%-02d' % (dt.year,dt.month,dt.day) + +def time2str(dt): + return '%02d:%02d:%02d' % (dt.hour,dt,minute,dt.second) + +def str2Date(dstr): + try: + haha = dstr.split(' ') + y,m,d = haha[0].split('-') + H = M = S = 0 + if len(haha) > 1: + H,M,S = haha[1].split(':') + return ymdDate(int(y),int(m),int(d),int(H),int(M),int(S)) + except Exception as e: + print(e) + return None + +def ymdDate(y,m,d,H=0,M=0,S=0): + return datetime.datetime(y,m,d,H,M,S) + +def str2Datetime(dstr): + d,t = dstr.split(' ') + y,m,d = d.split('-') + H,M,S = t.split(':') + return datetime.datetime(int(y),int(m),int(d),int(H),int(M),int(S)) + +def addMonths(dt,months): + y = dt.year() + m = dt.month() + d = dt.day() + mm = m % 12 + md = m / 12 + if md != 0: + y += md + m = mm + maxd = monthMaxDay(y,m) + if d > maxd: + d = maxd + return ymdDate(y,m,d) + +def addYears(dt,years): + y = dt.year() + years + m = dt.month() + d = dt.day() + maxd = monthMaxDay(y,m) + if d > maxd: + d = maxd + return ymdDate(y,m,d) + +def dateAdd(dt,days=0,months=0,years=0): + if days != 0: + dd = datetime.timedelta(days) + dt = dt + dd + if months != 0: + dt = addMonths(dt,months) + if years != 0: + dt = addYears(dt,years) + return dt + +def firstSunday(dt): + f = dt.weekday() + if f<6: + return dt + datetime.timedelta(7 - f) + return dt + +DTFORMAT = '%Y%m%d %H%M%S' +def getCurrentTimeStamp() : + t = time.localtime() + return TimeStamp(t) + +def TimeStamp(t) : + return time.strftime(DTFORMAT,t) + +def StepedTimestamp(baseTs,ts,step) : + if step<2 : + return ts + offs = int(timestampSub(ts,baseTs)) + step = int(step) + r,m = divmod(offs,step) + if m < step/2 : + return timestampAdd(baseTs,step * r) + else : + return timestampAdd(baseTs,step * (r+1)) + +def timestampAdd(ts1,ts2) : + t1 = time.strptime(ts1,DTFORMAT) + tf = time.mktime(t1) + if type(ts2)=='' : + t2 = time.strptime(ts2,DTFORMAT) + ts2 = time.mktime(t2) + tf += ts2 + t = time.localtime(tf) + return TimeStamp(t) + +def timestampSub(ts1,ts2) : + t1 = time.strptime(ts1,DTFORMAT) + t2 = time.strptime(ts2,DTFORMAT) + ret = time.mktime(t1) - time.mktime(t2) + return int(ret) + +def timestamp2dt(t): + return datetime.datetime.fromtimestamp(t) diff --git a/appPublic/timecost.py b/appPublic/timecost.py new file mode 100755 index 0000000..778c991 --- /dev/null +++ b/appPublic/timecost.py @@ -0,0 +1,38 @@ +import time +import datetime + +BEGIN=0 +END=1 + +def datetimeStr(t): + dt = time.localtime(t) + return time.strftime('%Y-%m-%d %H:%M:%S',dt) + +class TimeCost: + def __init__(self): + self.timerecord = {} + + def begin(self,name): + self.timerecord[name] = {'begin':time.time()} + + def end(self,name): + if name not in self.timerecord.keys(): + self.timerecord[name] = {'begin':time.time()} + d = self.timerecord[name] + d['end'] = time.time() + + def getTimeCost(self,name): + return self.timerecord[name]['end'] - self.timerecord[name]['begin'] + + def getTimeBegin(self,name): + return self.timerecord[name]['begin'] + + def getTimeEnd(self,name): + return self.timerecord[name]['end'] + + def show(self): + for name in self.timerecord.keys(): + d = self.timerecord[name] + cost = d['end'] - d['begin'] + print(name,'begin:',datetimeStr(d['begin']),'end:',datetimeStr(d['end']),'cost:%f seconds' % cost) + diff --git a/appPublic/unicoding.py b/appPublic/unicoding.py new file mode 100755 index 0000000..9cacba9 --- /dev/null +++ b/appPublic/unicoding.py @@ -0,0 +1,39 @@ +#unidict.py +import locale + +def unicoding(d,coding='utf8'): + if type(d) == type(''): + return d + if type(d) == type(b''): + try: + if coding is not Noene: + return d.decode(coding) + else: + return d.decode(locale.getdefaultlocale()[1]) + except: + try: + return d.decode(locale.getdefaultlocale()[1]) + except: + try: + return d.decode('utf8') + except: + return d + return d + +def uObject(obj,coding='utf8'): + otype = type(obj) + if otype == type(u''): + return obj + if otype == type({}): + return uDict(obj,coding) + if otype == type([]): + return [uObject(i,coding) for i in obj ] + if hasattr(obj,'decode'): + return obj.decode(coding) + return obj + +def uDict(dict,coding='utf8'): + d = {} + for k,v in dict.items(): + d[uObject(k)] = uObject(v) + return d diff --git a/appPublic/uniqueID.py b/appPublic/uniqueID.py new file mode 100755 index 0000000..4b623d0 --- /dev/null +++ b/appPublic/uniqueID.py @@ -0,0 +1,20 @@ +import uuid + +node = None + +def setNode(n='ff001122334455'): + global node + if len(n)>12: + return + for c in n: + if c not in ['0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f']: + return + node = eval('0x' + n + 'L') + +def getID(): + global node + if node is None: + node = uuid.getnode() + u = uuid.uuid1(node) + return u.hex + diff --git a/appPublic/workers.py b/appPublic/workers.py new file mode 100755 index 0000000..d84ffa1 --- /dev/null +++ b/appPublic/workers.py @@ -0,0 +1,250 @@ +import os +import sys +import time +import pyodbc as db +from multiprocessing import Process, Queue,Pipe,freeze_support,cpu_count +from threading import Thread +import logging + +from appPublic.folderUtils import ProgramPath +FINISH_JOB=0 +EXIT=4 +ABORT_JOB=5 +COMMIT_JOB=6 +DO_JOB=1 +ERROR=2 + +OK=0 +def debug(s): + fn = os.path.join(ProgramPath(),"worker_child.log") + f = open(fn,"a") + f.write("%s\n" % s) + f.close() + pass #print s + +logger = logging.getLogger(__name__) + +class Worker(Process): + def __init__(self,inQ,outQ): + Process.__init__(self) + self.inQ = inQ + self.outQ = outQ + self.done = False + self.aborted = False + self.task_no = 0 + + def abort_task(self): + pass + + def finish_task(self): + pass + + def job_begin(self): + pass + + def job_end(self): + pass + + def abort(self): + self.aborted = True + + def run(self): + self.job_begin() + self.job() + if self.aborted: + self.abort_task() + self.outQ.put([ABORT_JOB,self.name,self.task_no,None]) + debug("process=%s,aborted at task_no=%d" % (self.name,self.task_no)) + #self.inQ.cancel_join_thread() + else: + self.finish_task() + self.outQ.put([FINISH_JOB,self.name,self.task_no,None]) + debug("process=%s,end at task_no=%d" % (self.name,self.task_no)) + self.job_end() + debug("process %s end" % self.name) + sys.exit(0) + + def job(self): + debug("process %s running" % (self.name)) + while not self.done: + cmd,task_no,args = self.inQ.get() + self.task_no = task_no + debug("process=%s,cmd=%d,task_no=%d" % (self.name,cmd,self.task_no)) + if cmd==ABORT_JOB: + logger.info("process %s aborted" % self.name) + self.aborted = True + self.done = True + continue + if cmd==FINISH_JOB: + logger.info("process %s finished" % self.name) + self.done = True + continue + if cmd!=DO_JOB: + continue + try: + d = self.task(args) + self.outQ.put([cmd,self.name,self.task_no,d]) + debug("process=%s,cmd=%d,task_no=%d,responed" % (self.name,cmd,self.task_no)) + except Exception as e: + debug("ERROR:%s" % e) + self.outQ.put([ERROR,self.name,self.task_no,"Error"]) + debug("process=%s,cmd=%d,task_no=%d,error" % (self.name,cmd,self.task_no)) + self.abort() + + def task(self,args): + return None + +class Workers(Thread): + def __init__(self,worker_cnt,Wklass,argv): + Thread.__init__(self) + self.worker_cnt = worker_cnt + self.aborted = False + self.workers = [] + self.taskQ = Queue() + self.doneQ = Queue() + self.follows = [] + self.task_cnt = 0 + self.resp_cnt = 0 + self.task_no = 0 + self.max_task_no_resp = 0 + + i = 0 + while i < self.worker_cnt: + w = Wklass(self.taskQ,self.doneQ,*argv) + self.workers.append(w) + w.start() + i += 1 + + def __del__(self): + self.cleanTaskQ() + self.cleanDoneQ() + self.taskQ.close() + self.doneQ.close() + + def eraseDeadProcess(self): + d = [ p for p in self.workers if p.is_alive() ] + self.workers = d + self.worker_cnt = len(d) + + def addFollowWorkers(self,workers): + self.follows.append(workers) + + def isFollowDone(self): + for w in self.follows: + if not w.isDone(): + #logger.info("%s, follow %s still alive" % (self.name,w.name)) + return False + return True + + def isFinished(self): + for w in self.workers: + if w.is_alive(): + #logger.info("%s, process %s still alive" % (self.name,w.name)) + return False + return True + + def isDone(self): + if not self.isFollowDone(): + return False + if not self.isFinished(): + return False + return True + + def isAborted(self): + return self.aborted + + def run(self): + while not self.isFinished(): + #logger.info("thread %s, task_cnt=%d,resp_cnt = %d,aborted=%s" % (self.name,self.task_cnt,self.resp_cnt,str(self.aborted))) + status,proName,task_no,data = self.getResult() + if status is not None: + self.handleResult(status,proName,data) + for w in self.follows: + if w.isAborted(): + self.abortTask() + self.eraseDeadProcess() + #time.sleep(0.01) + logger.info("thread %s end .................." % (self.name)) + + def handleResult(self,status,proName,data): + #logger.info("thread=%s,status=%s,proName=%s" % (self.name,str(status),proName)) + if status == FINISH_JOB or status == ABORT_JOB: + for p in self.workers: + if p.name == proName and p.is_alive(): + p.join() + #logger.info("%s,finished job,status=%d" % (self.name,status)) + return + if status == ERROR and not self.aborted: + #logger.info("thread %s, error" % (self.name)) + self.abortTask() + logger.info("%s,error job,status=%d" % (self.name,status)) + return + if status == DO_JOB: + self.resp_cnt += 1 + for i in self.follows: + #logger.info("task hand to %s" % (i.name)) + i.addTask(data,waitting=True) + + def done(self): + while self.max_task_no_resp < self.task_no: + self.eraseDeadProcess() + if self.worker_cnt == 0: + return + time.sleep(0.01) + + for w in self.workers: + self.task_no += 1 + self.taskQ.put([FINISH_JOB,self.task_no,None],False) + + for w in self.follows: + w.done() + + def addTask(self,args,waitting=False): + try: + if self.isAborted(): + return False + self.task_no += 1 + self.taskQ.put([DO_JOB,self.task_no,args],waitting) + self.task_cnt += 1 + return True + except Exception as e: + logger.info("thread (%s)error:%s" % (self.name,mstr(e))) + return False + + def abortTask(self): + if self.aborted: + return + self.aborted = True + self.cleanTaskQ() + for w in self.workers: + self.task_no += 1 + self.taskQ.put([ABORT_JOB,self.task_no,None],False) + for w in self.follows: + w.abortTask() + + def cleanTaskQ(self): + r = True + while r: + try: + self.taskQ.get(False) + except: + r = False + + def cleanDoneQ(self): + r = True + while r: + try: + self.doneQ.get(False) + except: + r = False + + def getResult(self,waitting=True): + try: + status,proName,task_no,data = self.doneQ.get(waitting) + if task_no > self.max_task_no_resp: + self.max_task_no_resp = task_no + #logger.info("%s status=%s,%s,%d" % (self.name,str(status),proName,task_no)) + return (status,proName,task_no,data) + except Exception as e: + logger.info("thread %s error:%s" % (self.name,str(e))) + return None,None,None,None diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..691a519 --- /dev/null +++ b/setup.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + + +from distutils.core import setup +from setuptools import setup, find_packages + +# usage: +# python setup.py bdist_wininst generate a window executable file +# python setup.py bdist_egg generate a egg file +# Release information about eway + +version = "5.0.16" +description = "appPublic" +author = "yumoqing" +email = "yumoqing@icloud.com" + +packages=find_packages() +package_data = {} + +setup( + name="appPublic", + version=version, + + # uncomment the following lines if you fill them out in release.py + description=description, + author=author, + author_email=email, + + install_requires=[ + ], + packages=packages, + package_data=package_data, + keywords = [ + ], + classifiers = [ + 'Development Status :: 3 - Alpha', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Framework :: utils', + ], + platforms= 'any' +)