webgame 日志脚本-最终版
0、testlogfor.sh
#!/bin/shecho `date`export PATH="/usr/kerberos/bin:$PATH"export KRB5CCNAME=/tmp/krb5cc_pub_$$trap kdestroy 0 1 2 3 5 15kinit -k -t /etc/krb5.keytabk=0while((k++<1))do dd=`date +%Y-%m-%d -d " $k days ago"` echo $dd ssh root@10.3.17.26 "cat /data/remote-log-server/log/promition/${dd}/promition.success.log.${dd}-* > /data/remote-log-server/log/promition/${dd}/promition.success.log.${dd};" scp root@10.3.17.26:/data/remote-log-server/log/promition/$dd/promition.success.log.${dd} /data/remote-log-server/log/gamelogin/wqo/test/promition.success.log.${dd} ssh root@10.3.17.26 "rm -f /data/remote-log-server/log/promition/$dd/promition.success.log.${dd};" echo "************ $dd promition.success end**************" scp root@10.3.17.26:/data/remote-log-server/log/gamelogin/${dd}/gamelogin.log /data/remote-log-server/log/gamelogin/wqo/test/gamelogin.log.${dd} echo "************ $dd gamelogin end**************"done#!/bin/shcd /data/remote-log-server/log/gamelogin/wqo/testif [ -f "flux-rate.log" ] ; then echo "*****flux-rate.log is existed**********" > flux-rate.logfiecho "*********start********"m=0while((m++<1))do##dd=`date +%Y-%m-%d -d " $m days ago " `dd="2011-08-24"if [ ! -f "gamelogin.log.$dd" ] ; then echo "*****gamelogin.log.$dd not existed*******" exitfi if [ ! -f "glogin.log.$dd" ] ; then echo "***********glogin.log.$dd not exist, start to create it**************" awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3 "|" $4}}' gamelogin.log.$dd | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log.$dd echo "***********glogin.log.$dd end***********" else echo "************glogin.log.$dd is existed********************"fiif [ ! -f "glogin-uid.$dd" ] ; then echo "********************glogin-uid.$dd not exist, create id***********" awk 'BEGIN {FS="|"}; {print $2}' glogin.log.$dd | sort | uniq > glogin-uid.$ddelse echo "********************glogin-uid.$dd is existed******"fi################################################################echo "****for promition*****"k=0while((k++<1))do((n=k+m))##tt=`date +%Y-%m-%d -d " $n days ago " `tt="2011-08-10" if [ -f "promition.success.log.$tt" ] ; then if [ ! -f "prom-suc.log.$tt" ] ; then echo "************prom-suc.log.$tt not exist , start to create it***********************" awk 'BEGIN {FS=","}; {print $6 " " $7 " " $5 }' promition.success.log.$tt | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log.$tt echo "***********prom-suc.log.$tt end***************" else echo "***********prom-suc.log.$tt is existed**********************" fi if [ ! -f "prom-suc-uid.$tt" ] ; then echo "***************prom-suc-uid.$tt is not exited, create**********************" awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log.$tt | sort | uniq > prom-suc-uid.$tt else echo "****************prom-suc-uid.$tt is existed********************" fi cat glogin-uid.$dd | wc -l > tmp-glog-prom-uid.log cat prom-suc-uid.$tt | wc -l >> tmp-glog-prom-uid.log cat prom-suc-uid.$tt glogin-uid.$dd | sort | uniq -d > tmp-match-uid.log cat tmp-match-uid.log | wc -l >> tmp-glog-prom-uid.log echo $dd >>tmp-glog-prom-uid.log echo $tt >>tmp-glog-prom-uid.log cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log awk 'BEGIN {FS=" "}; {print $5 "," $4 "," $3/$2 "," $3 "," $2 "," $1 }' tmp-flux-rate.log >> flux-rate.log echo "***********flux-rate.log end************" rm -f tmp* else echo "********promition.success.log.$tt not exit!***************" fidonedone##cat flux-rate.log | sort > webgame-total.csvecho "*****************total end**********************"echo "****************category start*****************************"##sh testcatfor.shecho "****************category end ******************************"echo "***************service start ******************************"##sh testservicefor.shecho "***************service end *****************************"echo "****************start insert data into db*****************************"##/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_total webgame-total.csv##/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_category webgame-cat.csv##/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_service webgame-cat-service.csvecho "****************data end*********************"#!/bin/shstart=`date`echo $startcd /data/remote-log-server/log/gamelogin/wqo/testif [ -f "flux-rate.log" ] ; then echo "*****flux-rate.log is existed**********" > flux-rate.logfiecho "*********start********"m=0while((m++<1))dodd=`date +%Y-%m-%d -d " $m days ago " `if [ ! -f "gamelogin.log.$dd" ] ; then echo "*****gamelogin.log.$dd not existed*******" exitfi if [ ! -f "glogin.log.$dd" ] ; then echo "***********glogin.log.$dd not exist, start to create it**************" awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3 "|" $4}}' gamelogin.log.$dd | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log.$dd echo "***********glogin.log.$dd end***********" else echo "************glogin.log.$dd is existed********************"fiif [ ! -f "glogin-uid.$dd" ] ; then echo "********************glogin-uid.$dd not exist, create id***********" awk 'BEGIN {FS="|"}; {print $2}' glogin.log.$dd | sort | uniq > glogin-uid.$ddelse echo "********************glogin-uid.$dd is existed******"fi################################################################echo "****for promition*****"k=0while((k++<30))do((n=k+m))tt=`date +%Y-%m-%d -d " $n days ago " ` if [ -f "promition.success.log.$tt" ] ; then if [ ! -f "prom-suc.log.$tt" ] ; then echo "************prom-suc.log.$tt not exist , start to create it***********************" awk 'BEGIN {FS=","}; {print $6 " " $7 " " $5 }' promition.success.log.$tt | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log.$tt echo "***********prom-suc.log.$tt end***************" else echo "***********prom-suc.log.$tt is existed**********************" fi if [ ! -f "prom-suc-uid.$tt" ] ; then echo "***************prom-suc-uid.$tt is not exited, create**********************" awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log.$tt | sort | uniq > prom-suc-uid.$tt else echo "****************prom-suc-uid.$tt is existed********************" fi cat glogin-uid.$dd | wc -l > tmp-glog-prom-uid.log cat prom-suc-uid.$tt | wc -l >> tmp-glog-prom-uid.log cat prom-suc-uid.$tt glogin-uid.$dd | sort | uniq -d > tmp-match-uid.log cat tmp-match-uid.log | wc -l >> tmp-glog-prom-uid.log echo $dd >>tmp-glog-prom-uid.log echo $tt >>tmp-glog-prom-uid.log cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log awk 'BEGIN {FS=" "}; {print $5 "," $4 "," $3/$2 "," $3 "," $2 "," $1 }' tmp-flux-rate.log >> flux-rate.log echo "***********flux-rate.log end************" rm -f tmp* else echo "********promition.success.log.$tt not exit!***************" fidonedonecat flux-rate.log | sort > webgame-total.csvecho "*****************total end**********************"echo "****************category start*****************************"sh testcatfor.shecho "****************category end ******************************"echo "***************service start ******************************"sh testservicefor.shecho "***************service end *****************************"echo "****************start insert data into db*****************************"/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_total webgame-total.csv/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_category webgame-cat.csv/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_service webgame-cat-service.csvecho "****************data end*********************"end=`date`echo $end#!/bin/shif [ -f "flux-cat-rate.log" ]; then echo "****flux-cat-rate.log is existed******" > flux-cat-rate.logfiecho "**********analysis category and service number**************"n=0while((n++<1))do tt=`date +%Y-%m-%d -d " $n days ago" ` if [ ! -f "glogin.log.$tt" ] ; then echo "********glogin.log.$tt not existed********************" fi echo "****glogin.log.$tt is existed********" m=0 while((m++<=30)) do ((p=m+n)) dd=`date +%Y-%m-%d -d " $p days ago " ` if [ ! -f "prom-suc.log.$dd" ] ; then echo "********prom-suc.log.$dd not existed****************" else echo "***********prom-suc.log.$dd is existed*******************" awk 'BEGIN {FS="|"}; {print $3 "|" $4} ' prom-suc.log.$dd | sort | uniq > tmp-pcat.log awk 'BEGIN {FS="|"}; {print $1 } ' tmp-pcat.log | sort | uniq > tmp-pcat-name.log catnum=`cat tmp-pcat-name.log | wc -l` echo "**$catnum**" #####for###### k=0 while((k++<${catnum})) do catcontent=`sed -n "${k} p" tmp-pcat-name.log` echo "---------$catcontent------------" cat prom-suc.log.$dd | grep $catcontent > tmp-prom-suc-cat.log awk 'BEGIN {FS="|"}; {print $2 "|" $3}' tmp-prom-suc-cat.log | sort | uniq > tmp-prom-suc-cat-uid.log ####tt=`date +%Y-%m-%d -d " 1 days ago" ` cat glogin.log.$tt | grep $catcontent > tmp-glogin-cat.log awk 'BEGIN {FS="|"}; {print $2 "|" $3}' tmp-glogin-cat.log | sort | uniq > tmp-glogin-cat-uid.log cat tmp-prom-suc-cat-uid.log | wc -l > tmp-cat-prom-glogin-uid.log cat tmp-glogin-cat-uid.log | wc -l >> tmp-cat-prom-glogin-uid.log cat tmp-prom-suc-cat-uid.log tmp-glogin-cat-uid.log | sort | uniq -d | wc -l >> tmp-cat-prom-glogin-uid.log echo $catcontent >> tmp-cat-prom-glogin-uid.log echo $dd >> tmp-cat-prom-glogin-uid.log echo $tt >> tmp-cat-prom-glogin-uid.log cat tmp-cat-prom-glogin-uid.log | xargs echo > tmp-cat-rate.log awk 'BEGIN {FS=" "}; {print $4 "," $5 "," $6 "," $3/$1 "," $3 "," $1 "," $2 }' tmp-cat-rate.log >> flux-cat-rate.log done echo "**********flux-cat-rate.log***************" fi donedonecat flux-cat-rate.log | sort > webgame-cat.csv#!/bin/shif [ -f "flux-service-rate.log" ]; then echo "****flux-service-rate.log is existed******" > flux-service-rate.logfiecho "**********analysis category and service number**************"n=0while((n++<1))do tt=`date +%Y-%m-%d -d " $n days ago" ` if [ ! -f "glogin.log.$tt" ] ; then echo "********glogin.log.$tt not existed********************" fi echo "****glogin.log.$tt is existed********" m=0 while((m++<30)) do ((p=m+n)) dd=`date +%Y-%m-%d -d " $p days ago " ` if [ ! -f "prom-suc.log.$dd" ] ; then echo "********prom-suc.log.$dd not existed****************" else echo "***********prom-suc.log.$dd is existed*******************" awk 'BEGIN {FS="|"}; {print $3 "|" $4} ' prom-suc.log.$dd | sort | uniq > tmp-pcat.log awk 'BEGIN {FS="|"}; {print $1 } ' tmp-pcat.log | sort | uniq > tmp-pcat-name.log catnum=`cat tmp-pcat-name.log | wc -l` echo "**$catnum**" #####for###### k=0 while((k++<${catnum})) do catcontent=`sed -n "${k} p" tmp-pcat-name.log` echo "---------$catcontent------------" cat prom-suc.log.$dd | grep $catcontent > tmp-prom-suc-cat.log awk 'BEGIN {FS="|"}; {print $2 "|" $3 "|" $4}' tmp-prom-suc-cat.log | sort | uniq > tmp-prom-suc-cat-service-uid.log awk 'BEGIN {FS="|"}; {print $3}' tmp-prom-suc-cat-service-uid.log | sort | uniq > tmp-pcat-service-name.log servicenum=`cat tmp-pcat-service-name.log | wc -l` echo "+++++$servicenum++" ######for service########## g=0 while((g++<${servicenum})) do servicecontent=`sed -n "${g} p" tmp-pcat-service-name.log` echo "---------------$servicecontent------------" cat glogin.log.$tt | grep $catcontent > tmp-glogin-catser.log ###### echo "*****11111********" cat tmp-glogin-catser.log | grep $servicecontent > tmp-glogin-cat-service.log ##### echo "*****22222*********" awk 'BEGIN {FS="|"}; {print $2 "|" $3 "|" $4 }' tmp-glogin-cat-service.log | sort | uniq > tmp-glogin-cat-service-uid.log cat tmp-prom-suc-cat-service-uid.log | grep $servicecontent | wc -l > tmp-service-prom-glogin-uid.log cat tmp-glogin-cat-service-uid.log | wc -l >> tmp-service-prom-glogin-uid.log cat tmp-prom-suc-cat-service-uid.log tmp-glogin-cat-service-uid.log | sort | uniq -d | wc -l >> tmp-service-prom-glogin-uid.log echo $catcontent >> tmp-service-prom-glogin-uid.log echo $servicecontent >> tmp-service-prom-glogin-uid.log echo $dd >> tmp-service-prom-glogin-uid.log echo $tt >> tmp-service-prom-glogin-uid.log cat tmp-service-prom-glogin-uid.log | xargs echo > tmp-service-rate.log awk 'BEGIN {FS=" "}; {print $4 "," $5 "," $6 "," $7 "," $3/$1 "," $3 "," $1 "," $2 }' tmp-service-rate.log >> flux-service-rate.log done done echo "**********flux-service-rate.log***************" fi donedonecat flux-service-rate.log | sort > webgame-cat-service.csv#!/usr/bin/python#python_webgame.pyimport sys, csvimport MySQLdbdef main(user, pwd, db, table, csvfile): try: conn = getconn(user, pwd, db) except MySQLdb.Error, e: print "Error %d: %s" % (e.args[0], e.args[1]) sys.exit(1) cursor = conn.cursor() loadcsv(cursor, table, csvfile) cursor.close() conn.close()def getconn(user, pwd, db): conn = MySQLdb.connect(host = "10.3.18.158", user = user, passwd = pwd, db = db) return conndef nullify(L): """Convert empty strings in the given list to None.""" #helper function def f(x): if(x == ""): return None else: return x return [ f(x) for x in L ]def loadcsv(cursor, table, filename): """ Open a csc file and load it into a sql table. Assumptions: - the first line in the file is a header """ f = open(filename) c = csv.reader(f) header = c.next() numfields = len(header) query = buildInsertCmd(table, numfields) f.seek(0) for line in c: if len(line)<1: continue vals = nullify(line) cursor.execute(query, vals) returndef buildInsertCmd(table, numfields): """ example: >>>buildInserCmd("foo", 3) 'insert int foo values (%s,%s,%s)' """ assert(numfields > 0) placeholders = (numfields-1) * "%s, " + "%s" query = ("insert into %s" % table) + (" values (null, %s)" % placeholders) return queryif __name__ == '__main__': args = sys.argv[1:] if(len(args) < 5): print "error: arguments: user "password" db table csvfile" sys.exit(1) main(*args) 1 楼 printfabcd 2011-11-05 export PATH="/usr/kerberos/bin:$PATH"