1. CMD
right_cmd_command_regit.bat
@echo off reg add "HKCR\*\shell\ms-dos" /ve /d DOS_COMMAND /f reg add "HKCR\*\shell\ms-dos\command" /ve /d "cmd.exe /k cd %%1" /f reg add "HKCR\Folder\shell\ms-dos" /ve /d DOS_COMMAND /f reg add "HKCR\Folder\shell\ms-dos\command" /ve /d "cmd.exe /k cd %%1" /f
2.Shell
2.1 pmdownload
#!/bin/sh # ------------------------------------------------------------------------------- # pmdownload - # This file contains PM download script sample # # Note: The output is written to file "pmdownload.out" # FTP parameters # # Example: # hostName=192.168.1.1 # userName=user1 # password=123 # remoteDir=/home/user1/pmfiles hostName=ftp_hostname userName=ftp_username password=ftp_password remoteDir=ftp_remote_directory csvSeparator="," # ------------------------------------------------------------------------------- # Getting CMS bin directory pmcmd=$0 cmsbin=`dirname $pmcmd` if [ $pmcmd = $cmsbin ]; then cmsbin="." fi cd $cmsbin # ------------------------------------------------------------------------------- # Get yesterday, today and day before yesterday date strings year=`date +%Y` month=`date +%m` day=`date +%d` day=`expr "$day" - 1` if [ $day -eq 0 ]; then month=`expr "$month" - 1` month="0${month}" if [ $month -eq 0 ]; then month=12 year=`expr "$year" - 1` fi day=`cal $month $year | grep . | fmt -1 | tail -1` fi if [ $day -lt 10 ]; then day="0${day}" fi year=`echo ${year} | sed 's/^20//'` # ------------------------------------------------------------------------------- # Get day before yesterday date strings year2=`date +%Y` month2=`date +%m` day2=`date +%d` day2=`expr "$day2" - 2` if [ $day2 -eq 0 ]; then month2=`expr "$month2" - 1` month2="0${month2}" if [ $month2 -eq 0 ]; then month2=12 year2=`expr "$year2" - 1` fi day2=`cal $month2 $year2 | grep . | fmt -1 | tail -1` fi if [ $day2 -lt 10 ]; then day2="0${day2}" fi year2=`echo ${year2} | sed 's/^20//'` # -------------------------------------- # yesterdayDate format -> MM/DD/YY yesterdayDate=${month}/${day}/${year} # -------------------------------------- # todayDate format -> MM/DD/YY todayDate=`date +'%m/%d/%y'` # -------------------------------------- # beforeYesterdayDate format -> MM/DD/YY beforeYesterdayDate=${month2}/${day2}/${year2} # -------------------------------------- # 15-MIN Start and End time for getpm.sh script startTime15min="${yesterdayDate} 00:00:00" endTime15min="${yesterdayDate} 23:59:59" # -------------------------------------- # 1-DAY Start and End time for getpm.sh script startTime1day="${beforeYesterdayDate} 00:00:00" endTime1day="${beforeYesterdayDate} 23:59:59" # -------------------------------------- # 15-MIN fileDate format -> MMDDYY fileDate=${month}${day}${year} # -------------------------------------- # 1-DAY fileDate format -> MMDDYY fileDate2=${month2}${day2}${year2} # -------------------------------------- # Hostname file_hostname=`hostname` # ------------------------------------------------------------------------------- # Output files outputFiles() { filesize=0 if [ -f pmdownload.out ]; then filesize=`du -k pmdownload.out|awk '{print $1}'` if [ "$filesize" -gt 300 ]; then if [ -f pmdownload.out4 ]; then cp pmdownload.out4 pmdownload.out5 fi if [ -f pmdownload.out3 ]; then cp pmdownload.out3 pmdownload.out4 fi if [ -f pmdownload.out2 ]; then cp pmdownload.out2 pmdownload.out3 fi if [ -f pmdownload.out1 ]; then cp pmdownload.out1 pmdownload.out2 fi cp pmdownload.out pmdownload.out1 rm -f pmdownload.out fi fi } # ------------------------------------------------------------------------------ # Get 15-MIN PM data getpm_15MIN() { pmtype=$1 pmtype1=`echo $pmtype | sed 's/15MIN//'` ##echo "pmtype1 =" $pmtype1 pmlocation=$2 if [ "$pmlocation" = "" ] then pmloc="BOTH" fi if [ "$pmlocation" = "NEND" ] then pmloc="2" fi if [ "$pmlocation" = "FEND" ] then pmloc="1" fi outputFiles # ---------------------------------------------------- # Execute getpm to retrieve 15-MIN PM data timeStamp=`date +'%m%d%y_%H%M%S'` timenow=`date +'%m/%d/%y %H:%M:%S'` filenamePM="pm_"$pmtype1"_15MIN_${timeStamp}.csv" ##echo " FilenamePM = " $filenamePM if [ "$pmloc" = "BOTH" ] then pmcmd="${cmsbin}/getpm.sh -p $pmtype" pmcmd="${pmcmd} -s \"${startTime15min}\" -e \"${endTime15min}\"" pmcmd="${pmcmd} -f csv -o $filenamePM -rs $csvSeparator" fi if [ "$pmloc" != "BOTH" ] then pmcmd="${cmsbin}/getpm.sh -p $pmtype" pmcmd="${pmcmd} -s \"${startTime15min}\" -e \"${endTime15min}\"" pmcmd="${pmcmd} -l $pmloc -f csv -o $filenamePM -rs $csvSeparator" fi echo "">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "# Execute getpm $1 15-MIN at ${timenow}">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "">>pmdownload.out echo ">>${pmcmd}" >>pmdownload.out if [ "$pmloc" = "BOTH" ] then ${cmsbin}/getpm.sh -p $pmtype -s "${startTime15min}" -e "${endTime15min}" \ -f csv -o $filenamePM -rs $csvSeparator >>pmdownload.out fi if [ "$pmloc" != "BOTH" ] then ${cmsbin}/getpm.sh -p $pmtype -s "${startTime15min}" -e "${endTime15min}" \ -l $pmloc -f csv -o $filenamePM -rs $csvSeparator >>pmdownload.out fi # ---------------------------------------------------- # Transfer the 15-MIN PM output file using cftp script if [ -f $filenamePM ]; then filenameFTP="${file_hostname}_pm_${pmtype1}_15MIN_${fileDate}_000000.csv" echo "">>pmdownload.out echo ">>Rename file ${filenamePM} to ${filenameFTP}">>pmdownload.out cp ${filenamePM} ${filenameFTP} rm ${filenamePM} localFile="${cmsbin}/${filenameFTP}" newlocalFile="newlocalFile" awk "NR==1" ${localFile}>${newlocalFile} if [ "$pmtype1" = "GEPort15Min" ]||[ "$pmtype1" = "10GEPort15Min" ]||[ "$pmtype1" = "EthDsl15Min" ]||[ "$pmtype1" = "PON15Min" ] then tac ${localFile} | awk -F$csvSeparator '!_[$3,$6,$10]++' | tac >>${newlocalFile} localFile="${newlocalFile}" fi if [ "$pmtype1" = "DSL15Min" ] then tac ${localFile} | awk -F$csvSeparator '!_[$3,$43,$45]++' | tac >>${newlocalFile} localFile="${newlocalFile}" fi if [ "$pmtype1" = "ERPS15Min" ] then tac ${localFile} | awk -F$csvSeparator '!_[$3,$6,$27]++' | tac >>${newlocalFile} localFile="${newlocalFile}" fi if [ "$pmtype1" = "ONT15Min" ]||[ "$pmtype1" = "OntDs115Min" ]||[ "$pmtype1" = "ONTPWE315Min" ] then tac ${localFile} | awk -F$csvSeparator '!_[$3,$5,$8]++' | tac >>${newlocalFile} localFile="${newlocalFile}" fi if [ "$pmtype1" = "OntEthFe15Min" ]||[ "$pmtype1" = "OntEthGe15Min" ]||[ "$pmtype1" = "OntEthHpna15Min" ] then tac ${localFile} | awk -F$csvSeparator '!_[$3,$6,$9]++' | tac >>${newlocalFile} localFile="${newlocalFile}" fi remoteFile="${remoteDir}/${filenameFTP}" ftpcmd="${cmsbin}/cftp -h $hostName -u $userName -p $password" ftpcmd="${ftpcmd} $localFile $remoteFile" timenow=`date +'%m/%d/%y %H:%M:%S'` echo "">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "# Transfer $1 15-MIN file at ${timenow}">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "">>pmdownload.out echo ">>${ftpcmd}" >>pmdownload.out ${ftpcmd} >>pmdownload.out rm -f $localFile rm -f $newlocalFile fi } # ------------------------------------------------------------------------------ # Get both 1-DAY PM data getpm_1DAY() { pmtype=$1 pmtype1=`echo $pmtype | sed 's/1DAY//'` ##echo "pmtype1 = " $pmtype1 pmlocation=$2 if [ "$pmlocation" = "" ] then pmloc="BOTH" fi if [ "$pmlocation" = "NEND" ] then pmloc="2" fi if [ "$pmlocation" = "FEND" ] then pmloc="1" fi outputFiles # ---------------------------------------------------- # Execute getpm to retrieve 1-DAY PM data timeStamp=`date +'%m%d%y_%H%M%S'` timenow=`date +'%m/%d/%y %H:%M:%S'` filenamePM="pm_"$pmtype1"_1DAY_${timeStamp}.csv" ##echo "filenamePM = " $filenamePM if [ "$pmloc" = "BOTH" ] then pmcmd="${cmsbin}/getpm.sh -p $pmtype" pmcmd="${pmcmd} -s \"${startTime1day}\" -e \"${endTime1day}\"" pmcmd="${pmcmd} -f csv -o $filenamePM -rs $csvSeparator" fi if [ "$pmloc" != "BOTH" ] then pmcmd="${cmsbin}/getpm.sh -p $pmtype" pmcmd="${pmcmd} -s \"${startTime1day}\" -e \"${endTime1day}\"" pmcmd="${pmcmd} -l $pmloc -f csv -o $filenamePM -rs $csvSeparator" fi echo "">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "# Execute getpm $1 1-DAY at ${timenow}">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "">>pmdownload.out echo ">>${pmcmd}" >>pmdownload.out if [ "$pmloc" = "BOTH" ] then ${cmsbin}/getpm.sh -p $pmtype -s "${startTime1day}" -e "${endTime1day}" \ -f csv -o $filenamePM -rs $csvSeparator >>pmdownload.out fi if [ "$pmloc" != "BOTH" ] then ${cmsbin}/getpm.sh -p $pmtype -s "${startTime1day}" -e "${endTime1day}" \ -l $pmloc -f csv -o $filenamePM -rs $csvSeparator >>pmdownload.out fi # ---------------------------------------------------- # Transfer the 1-DAY PM output file using cftp script if [ -f $filenamePM ]; then filenameFTP="${file_hostname}_pm_${pmtype1}_1DAY_${fileDate2}_000000.csv" echo "">>pmdownload.out echo ">>Rename file ${filenamePM} to ${filenameFTP}">>pmdownload.out cp ${filenamePM} ${filenameFTP} rm ${filenamePM} localFile="${cmsbin}/${filenameFTP}" remoteFile="${remoteDir}/${filenameFTP}" ftpcmd="${cmsbin}/cftp -h $hostName -u $userName -p $password" ftpcmd="${ftpcmd} $localFile $remoteFile" timenow=`date +'%m/%d/%y %H:%M:%S'` echo "">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "# Transfer $1 1-DAY file at ${timenow}">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "">>pmdownload.out echo ">>${ftpcmd}" >>pmdownload.out ${ftpcmd} >>pmdownload.out rm -f $localFile fi } getpm_adsl() { pmtype=$1 pmlocation=$2 if [ "$pmlocation" = "" ] then pmloc="BOTH" fi if [ "$pmlocation" = "NEND" ] then pmloc="2" fi if [ "$pmlocation" = "FEND" ] then pmloc="1" fi outputFiles # ---------------------------------------------------- # Execute getpm to retrieve 15-MIN PM data timeStamp=`date +'%m%d%y_%H%M%S'` timenow=`date +'%m/%d/%y %H:%M:%S'` filenamePM="pm_${pmtype}_${timeStamp}.csv" echo " FilenamePM = " $filenamePM if [ "$pmloc" = "BOTH" ] then pmcmd="${cmsbin}/getpm.sh -p $pmtype" pmcmd="${pmcmd} -s \"${startTime15min}\" -e \"${endTime15min}\"" pmcmd="${pmcmd} -f csv -o $filenamePM -rs $csvSeparator" fi if [ "$pmloc" != "BOTH" ] then pmcmd="${cmsbin}/getpm.sh -p $pmtype" pmcmd="${pmcmd} -s \"${startTime15min}\" -e \"${endTime15min}\"" pmcmd="${pmcmd} -l $pmloc -f csv -o $filenamePM -rs $csvSeparator" fi echo "">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "# Execute getpm $1 15-MIN at ${timenow}">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "">>pmdownload.out echo ">>${pmcmd}" >>pmdownload.out if [ "$pmloc" = "BOTH" ] then ${cmsbin}/getpm.sh -p $pmtype -s "${startTime15min}" -e "${endTime15min}" \ -f csv -o $filenamePM -rs $csvSeparator >>pmdownload.out fi if [ "$pmloc" != "BOTH" ] then ${cmsbin}/getpm.sh -p $pmtype -s "${startTime15min}" -e "${endTime15min}" \ -l $pmloc -f csv -o $filenamePM -rs $csvSeparator >>pmdownload.out fi # ---------------------------------------------------- # Transfer the 15-MIN PM output file using cftp script if [ -f $filenamePM ]; then filenameFTP="${file_hostname}_pm_${pmtype1}_15MIN_${fileDate}_000000.csv" echo "">>pmdownload.out echo ">>Rename file ${filenamePM} to ${filenameFTP}">>pmdownload.out cp ${filenamePM} ${filenameFTP} rm ${filenamePM} localFile="${cmsbin}/${filenameFTP}" remoteFile="${remoteDir}/${filenameFTP}" ftpcmd="${cmsbin}/cftp -h $hostName -u $userName -p $password" ftpcmd="${ftpcmd} $localFile $remoteFile" timenow=`date +'%m/%d/%y %H:%M:%S'` echo "">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "# Transfer $1 15-MIN file at ${timenow}">>pmdownload.out echo "# ---------------------------------------------------">>pmdownload.out echo "">>pmdownload.out echo ">>${ftpcmd}" >>pmdownload.out ${ftpcmd} >>pmdownload.out rm -f $localFile fi } # ------------------------------------------------------------------------------- # Download PM data getpmdata1Day() { #This will download both NEND and FEND PM in the same file getpm_1DAY $1 #This will download only NEND PM #getpm_1DAY $1 NEND #This will download only FEND PM #getpm_1DAY $1 FEND } getpmdataadsl() { #This will download both NEND and FEND PM in the same file getpm_adsl $1 #This will download only NEND PM #getpm_adsl $1 NEND #This will download only FEND PM #getpm_adsl $1 FEND } getpmdata15Min() { #This will download both NEND and FEND PM in the same file getpm_15MIN $1 #This will download only NEND PM #getpm_15MIN $1 NEND #This will download only FEND PM #getpm_15MIN $1 FEND } # ------------------------------------------------------------------------------- # Start downloading PM data from each facility type # To disable the download, add "#" character before "getpmdata" getpmdata1Day OC481DAY getpmdata1Day OC121DAY getpmdata1Day OC31DAY getpmdata1Day STS48c1DAY getpmdata1Day STS12c1DAY getpmdata1Day STS3c1DAY getpmdata1Day STS11DAY getpmdata1Day DS31DAY getpmdata1Day DS11DAY getpmdata1Day IMA1DAY getpmdata1Day IMALink1DAY getpmdata1Day ADSL1DAY getpmdata1Day HDSL1DAY getpmdata1Day XDSL1DAY getpmdata1Day XDSLGroup1DAY getpmdata15Min OC4815MIN getpmdata15Min OC1215MIN getpmdata15Min OC315MIN getpmdata15Min STS48c15MIN getpmdata15Min STS12c15MIN getpmdata15Min STS3c15MIN getpmdata15Min STS115MIN getpmdata15Min DS315MIN getpmdata15Min DS115MIN getpmdata15Min IMA15MIN getpmdata15Min IMALink15MIN getpmdata15Min ADSL15MIN getpmdata15Min HDSL15MIN getpmdata15Min XDSL15MIN getpmdata15Min XDSLGroup15MIN ## By default the following are 15 Min Data only getpmdata15Min ETHERNET getpmdataadsl ADSLCALLSTATUS getpmdataadsl XDSLCALLSTATUS getpmdataadsl XDSLGROUPCALLSTATUS #------------------------------------------ ##PM for E5312/E5400/E7 getpmdata1Day GEPort1Day getpmdata1Day 10GEPort1Day getpmdata1Day ERPS1Day getpmdata1Day OntEthGe1Day getpmdata1Day OntEthFe1Day getpmdata1Day OntEthHpna1Day getpmdata1Day OntDs11Day getpmdata1Day ONT1Day getpmdata1Day ONTPWE31Day getpmdata1Day EthDsl1Day getpmdata1Day DSL1Day getpmdata1Day PON1Day getpmdata15Min GEPort15Min getpmdata15Min 10GEPort15Min getpmdata15Min ERPS15Min getpmdata15Min OntEthGe15Min getpmdata15Min OntEthFe15Min getpmdata15Min OntEthHpna15Min getpmdata15Min OntDs115Min getpmdata15Min ONT15Min getpmdata15Min ONTPWE315Min getpmdata15Min EthDsl15Min getpmdata15Min DSL15Min getpmdata15Min PON15Min
2.2 login.expect
#!/usr/bin/expect set host [lindex $argv 0 ] set user [lindex $argv 1 ] set password [lindex $argv 2 ] set prompt [lindex $argv 3 ] set timeout 10 spawn ssh -p 1035 $user@$host expect { "*yes/no*" {send "yes\r";exp_continue} "*assword:" {send "$password\r"} } interact
2.3 fulfill pmgcs data
2.3.1 daily_pm_check_missing_data_and_recover.sh
#!/bin/bash REDSHIFT_HOST="xxx" REDSHIFT_PORT=5439 REDSHIFT_DB="xxx" REDSHIFT_USER="xxx" REDSHIFT_PSWD="xxx" start_date=$1 if [ -z $start_date ];then start_date=`date --date="-7 day" +%Y-%m-%d` fi #end_date=$(date -d "$start_date 1 day" +%Y-%m-%d) #end_date=`date --date="-1 day" +%Y-%m-%d` end_date=`date +%Y-%m-%d` start_time=$(date -d "$start_date" +%s) end_time=$(date -d $end_date +%s) echo "start_date: $start_date end_date:$end_date" echo "start_time: $start_time end_time: $end_time" rm -rf missing_pm.sql cp missing_pm.sql.tmp missing_pm.sql sed -i "s/START_TIME/${start_time}/g" missing_pm.sql sed -i "s/END_TIME/${end_time}/g" missing_pm.sql #unload_sql="`cat missing_pm.sql`" #echo $unload_sql export PGPASSWORD=$REDSHIFT_PSWD psql -h $REDSHIFT_HOST -p $REDSHIFT_PORT -d $REDSHIFT_DB -U $REDSHIFT_USER -f missing_pm.sql >result.txt netops@tonyben-dev:~/pmgcs$ ca daily_pm_check_missing_data_and_recover.sh ca: command not found netops@tonyben-dev:~/pmgcs$ cat daily_pm_check_missing_data_and_recover.sh #!/bin/bash BASEDIR=$(dirname $0) log="$BASEDIR/daily_pm_check_missing_data_and_recover.`date +'%Y_%m_%d'`.log" cd $BASEDIR if [ -f /tmp/daily_pm_check_missing_data_and_recover.lock ];then echo "$0 is in process , will exist"|tee -a $log exit fi touch /tmp/daily_pm_check_missing_data_and_recover.lock echo "[`date +'%Y-%m-%d %H:%M:%S'`] Begin to load missing devices"|tee -a $log ./check_pm_missing_data.sh|tee -a $log for org in `cat result.txt|awk '{print $1}'|sort|uniq` do if [ -n "$org" ] && [ "$org" -eq "$org" ] 2>/dev/null; then echo "$org need process" cat result.txt|grep $org>_result_${org} fi done rm _result_50 echo "[`date +'%Y-%m-%d %H:%M:%S'`] Begin to process missing devices"|tee -a $log for s in `ls _result_*` do while [ `ps -ef|grep pm_missing_data_move.py|grep -v grep|wc -l` -gt 3 ] do sleep 1s done if [ -f $s ];then if [ `ps -ef|grep -v grep|grep $s|wc -l` -eq 0 ];then mv $s run_$s nohup python pm_missing_data_move.py -e [email protected] --password xxx --filename run_$s && mv run_$s done_${s} & fi fi # nohup python pm_missing_data_move.py -e [email protected] --password xxx --filename $s & done #python pm_missing_data_move.py -e [email protected] --password xxx $1|tee -a $log echo "[`date +'%Y-%m-%d %H:%M:%S'`] End"|tee -a $log rm -rf result.txt.bak #cp result.txt result.txt.bak mv result.txt result.txt.`date +'%Y_%m_%d'` log_file="s3_daily_pm_check_missing_data_`date +'%Y_%m_%d'`.log" cat ${log_file}*|grep match|awk '{print $5,$7,$6}'>tmp.log sed -i 's/orgid//g' tmp.log sed -i 's/file_name//g' tmp.log sed -i 's/date_time//g' tmp.log sed -i 's/://g' tmp.log parm="fullsync" if [ -z $1 ];then parm="delta" fi subject="[`date +'%Y-%m-%d %H:%M:%S'`] [$parm] Daily PM Files RE-SYNC STATICS" cat tmp.log |awk '{print $1}'|sort|uniq -c |sort -n>tmp body="" while IFS= read -r line do body="$body<li>$line</li>" done < tmp #python sendEmail.py -e [email protected] -s "$subject" -b "$body" -a tmp.log --password xxx python sendEmail.py -e [email protected] -s "$subject" -b "$body" --password xxx mv tmp.log tmp.log.`date +'%Y_%m_%d'` tar -zcvf s3_daily_pm_check_missing_data_`date +'%Y_%m_%d_%H_%M'`.log.tar.gz ${log_file}* rm -rf ${log_file}* rm -rf /tmp/daily_pm_check_missing_data_and_recover.lock
2.3.2 check_pm_missing_data.sh
#!/bin/bash REDSHIFT_HOST="xx" REDSHIFT_PORT=5439 REDSHIFT_DB="xx" REDSHIFT_USER="masteruser" REDSHIFT_PSWD="xx start_date=$1 if [ -z $start_date ];then start_date=`date --date="-7 day" +%Y-%m-%d` fi #end_date=$(date -d "$start_date 1 day" +%Y-%m-%d) #end_date=`date --date="-1 day" +%Y-%m-%d` end_date=`date +%Y-%m-%d` start_time=$(date -d "$start_date" +%s) end_time=$(date -d $end_date +%s) echo "start_date: $start_date end_date:$end_date" echo "start_time: $start_time end_time: $end_time" rm -rf missing_pm.sql cp missing_pm.sql.tmp missing_pm.sql sed -i "s/START_TIME/${start_time}/g" missing_pm.sql sed -i "s/END_TIME/${end_time}/g" missing_pm.sql #unload_sql="`cat missing_pm.sql`" #echo $unload_sql export PGPASSWORD=$REDSHIFT_PSWD psql -h $REDSHIFT_HOST -p $REDSHIFT_PORT -d $REDSHIFT_DB -U $REDSHIFT_USER -f missing_pm.sql >result.txt
2.3.3 fulfill_data.sh
#!/bin/bash orgid=$1 device=$2 datelist=$3 process_file=$4 thread=$5 bucket_prefix="s3://gcs.calix.com-wifi-pm-per-" log_file="s3_daily_pm_check_missing_data_`date +'%Y_%m_%d'`.log" if [ -x $orgid ];then echo "No OrgID" exit fi if [ -x $device ];then echo "No Device" exit fi #echo "Call fulfill_data $orgid $device $process_file $thread" #echo "Begin process org: $orgid ,device: $device ,missing date:$datelist"|tee -a $log_file for bucket in radio sta do for folder in backup do for f in `aws s3 ls ${bucket_prefix}$bucket/$folder/$orgid/$device|awk '{print $4}'` do fnocsv=${f%%.csv} cdevice=`echo $fnocsv|awk -F '_' '{print $1}'` time_long=`echo $fnocsv|awk -F '_' '{print $2}'` date_str=$(date -d @$time_long +'%Y-%m-%d') if [ `echo $datelist|grep $date_str|wc -l` -eq 1 ];then echo "[$process_file][$thread] match ${bucket_prefix}$bucket/$folder/$orgid/${f} $date_str">>$log_file # echo "[$process_file][$thread] aws s3 mv ${bucket_prefix}$bucket/$folder/$orgid/${f} ${bucket_prefix}$bucket/$orgid/${f}"|tee -a $log_file aws s3 mv ${bucket_prefix}${bucket}/backup/$orgid/${f} ${bucket_prefix}${bucket}/$orgid/${f} aws s3 mv ${bucket_prefix}${bucket}/dupcheck/$orgid/${f} ${bucket_prefix}${bucket}/$orgid/${f} # aws s3 mv ${bucket_prefix}sta/backup/$orgid/${f} ${bucket_prefix}sta/$orgid/${f} # aws s3 mv ${bucket_prefix}sta/dupcheck/$orgid/${f} ${bucket_prefix}sta/$orgid/${f} fi done done done
2.3.4 pm_missing_data_move.py
import os,sys import logging.handlers import boto3 import botocore.session from concurrent import futures import subprocess import datetime import logging import argparse import smtplib import threading from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from urllib import unquote pool = futures.ThreadPoolExecutor(max_workers=10) device_pool = futures.ThreadPoolExecutor(max_workers=10) bucket_pref="gcs.calix.com-wifi-pm-per-" logger = logging.getLogger() static_map={} def move_file(file_name,orgid): for b in ['radio','sta']: for folder in ['dupcheck','backup']: bucket_name="%s%s"%(bucket_pref,b) sfile="s3://%s/%s/%s/%s" %(bucket_name,folder,orgid,file_name) dfile="s3://%s/%s/%s" %(bucket_name,orgid,file_name) logger.error("aws s3 mv %s %s",sfile,dfile) subprocess.call(["aws", "s3","mv",sfile,dfile]) def process_device(orgid,device,time_list): logger.error("%s Begin process org:%s ,device:%s missing date:%s"%(filename,orgid,device,','.join(time_list))) subprocess.call(['./fulfill_data.sh',orgid,device,','.join(time_list),filename,threading.currentThread().getName()]) return def process_org_data(orgid,device_map): logger.error("%s--Begin process org:%s ,with devices:%s --" %(filename,orgid,len(device_map))) for device in device_map.keys(): device_pool.submit(process_device,orgid,device,device_map[device]) #process_device(orgid,device,device_map[device]) def init_logger(): """ initial logger info :return: """ handler = logging.StreamHandler(sys.stderr) fmt = '%(asctime)s-[%(filename)s:%(lineno)s]-[%(threadName)s]- %(message)s' formatter = logging.Formatter(fmt) handler.setFormatter(formatter) logger.setLevel(logging.ERROR) logger.addHandler(handler) def generate_body(): body = """<h4>Daily PM Files RE-SYNC STATICS</h4>""" body+="<br><br><hr><div>" body+="<table>" body+="<tr><th>Org</th><th>Device</th><th>Date Time</th><th>File Name</th></tr>" for orgid in static_map.keys(): device_map=static_map[orgid] cnt=0 for device in device_map.keys(): list=device_map[device] for detail in list: cnt+=1 body+="<tr><td> %s </td><td> %s </td><td> %s </td><td> %s </td></tr>"\ %(orgid,device,detail['date_time'],detail['file_name']) body+="<tr><td>Total</td><td colspan=3>devices: %s total number:%s</td></tr>"%(len(device_map),cnt) body+="</table>" return body def generate_static(args): logging.error(static_map) #if len(static_map)==0: # logging.error("No need to send email") # return subject = "[%s] Daily PM Files RE-SYNC STATICS[%s]"%(datetime.datetime.now().strftime('%y-%m-%d'), "FullSync" if args.fullsync else "Delta") body = generate_body() emails = args.email smtp_host = args.smtpHost smtp_port = args.smtpPort username = args.username password = args.password sent_from = args.sentFrom if emails: mail_server = smtplib.SMTP() try: msg = MIMEMultipart() msg['From'] = sent_from msg['Subject'] = subject msg.attach(MIMEText(body, 'html')) mail_server.connect(smtp_host, smtp_port) mail_server.ehlo() mail_server.starttls() mail_server.ehlo() if password and smtp_host != 'eng-smtp.calix.local': mail_server.login(username, unquote(password)) for recipient in emails: logger.error("send email to %s", recipient) msg['To'] = recipient mail_server.sendmail(sent_from, recipient, msg.as_string()) except Exception as err: logger.error("send email failed:%s", err) return False finally: if mail_server: mail_server.close() return True return False if __name__ == '__main__': init_logger() parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-e', '--email', action='append', help="To send multiple emails --email <email-address-1> --email <email-address-2> ...") parser.add_argument('--smtpHost', type=str, help="Host of SMTP server", required=False, default="outlook.office365.com") parser.add_argument('--smtpPort', type=int, help="Port of SMTP server", required=False, default=587) parser.add_argument('--sentFrom', type=str, help="outlook email", required=False, default="[email protected]") parser.add_argument('--username', type=str, help="outlook username", required=False, default="[email protected]") parser.add_argument('--password', type=str, default='xx', help="outlook password", required=False) parser.add_argument('--fullsync',action='store_true',help="full sync all results",required=False) parser.add_argument('--filename',type=str,default="result.txt",help="full sync all results",required=False) args = parser.parse_args() global filename filename=args.filename logger.error(args) org_map={} history_map={} if not args.fullsync: if os.path.isfile('result.txt.bak'): with open('result.txt.bak') as fp: line = fp.readline() cnt = 1 while line: line = fp.readline() cnt += 1 if cnt>2 and line.lstrip().find('|')>0: tmp=line.split('|') orgid=tmp[0].strip() device=tmp[1].strip() date_time=tmp[2].split()[0].strip() _tmp_str='%s-%s-%s'%(orgid,device,date_time) history_map[_tmp_str]=_tmp_str with open(filename) as fp: line = fp.readline() #cnt = 1 while line: line = fp.readline() # cnt += 1 # if cnt>2 and line.lstrip().find('|')>0: if line.lstrip().find('|')>0 and line.split('|')[0].strip()!='orgid': tmp=line.split('|') orgid=tmp[0].strip() device=tmp[1].strip() date_time=tmp[2].split()[0].strip() _tmp_str='%s-%s-%s'%(orgid,device,date_time) if history_map.has_key(_tmp_str): logger.info("orgid:%s device:%s date_time:%s had executed on last job,ignore",orgid,device,date_time) continue device_map=org_map[orgid] if org_map.has_key(orgid) else {} time_list=device_map[device] if device_map.has_key(device) else [] time_list.append(date_time) device_map[device]=time_list org_map[orgid]=device_map if org_map.has_key('50'): org_map.pop('50') for org in org_map.keys(): process_org_data(org,org_map[org]) pool.shutdown(wait=True) device_pool.shutdown(wait=True) #generate_static(args)
2.3.5 sendEmail.py
#!/usr/bin/python # -*- coding: UTF-8 -*- import os,sys import json import urllib2 import argparse import logging import logging.handlers import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.multipart import MIMEBase from email import Encoders from urllib import unquote logger = logging.getLogger() def get_response(url): jsonObj = None try: req = urllib2.Request(url) res_data = urllib2.urlopen(req) jsonObj = json.loads(res_data.read()) except Exception as e: logger.error("get response error", e) finally: return jsonObj def generate_body(): body = """<h4>Simulator Damo Org Data</h4>""" with open('log.log') as fp: line = fp.readline() cnt = 1 while line: body+=line+"<br>" line = fp.readline() cnt += 1 return body def send_email(args): subject = args.subject body = args.body emails = args.email smtp_host = args.smtpHost smtp_port = args.smtpPort username = args.username password = args.password sent_from = args.sentFrom if emails: mail_server = smtplib.SMTP() try: msg = MIMEMultipart() msg['From'] = sent_from msg['Subject'] = subject msg.attach(MIMEText(body, 'html')) if args.attachment: for name in args.attachment: part = MIMEBase('application', "octet-stream") part.set_payload(open(name, "rb").read()) Encoders.encode_base64(part) part.add_header('Content-Disposition', 'attachment; filename="%s"'%(name)) msg.attach(part) mail_server.connect(smtp_host, smtp_port) mail_server.ehlo() mail_server.starttls() mail_server.ehlo() if password and smtp_host != 'eng-smtp.calix.local': mail_server.login(username, unquote(password)) for recipient in emails: logger.info("send email to %s", recipient) msg['To'] = recipient mail_server.sendmail(sent_from, recipient, msg.as_string()) except Exception as err: logger.error("send email failed:%s", err) return False finally: if mail_server: mail_server.close() return True return False def init_log(): """ init logger :param log_file: :return: """ logger.setLevel(logging.DEBUG) handler = logging.StreamHandler(sys.stderr) fmt = '%(asctime)s-[%(filename)s:%(lineno)s]-[%(threadName)s]- %(message)s' formatter = logging.Formatter(fmt) handler.setFormatter(formatter) logger.addHandler(handler) def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-e', '--email', action='append', help="To send multiple emails --email <email-address-1> --email <email-address-2> ...") parser.add_argument('--smtpHost', type=str, help="Host of SMTP server", required=False, default="outlook.office365.com") parser.add_argument('--smtpPort', type=int, help="Port of SMTP server", required=False, default=587) parser.add_argument('--sentFrom', type=str, help="outlook email", required=False, default="xx) parser.add_argument('--username', type=str, help="outlook username", required=False, default="xx) parser.add_argument('--password', type=str, default='xx', help="outlook password", required=False) parser.add_argument('-s','--subject',default='test subject') parser.add_argument('-b','--body',default='test body') parser.add_argument('-a','--attachment',action='append') args = parser.parse_args() init_log() logger.debug("*" * 20) logger.debug(args) logger.debug("*" * 20) send_email(args) if __name__ == '__main__': main()
2.3.6 netstat_check.sh
#!/bin/bash ss -s echo "-----------------------------------------" netstat -n | awk '/^tcp/ {++state[$NF]} END {for(key in state) print key,"\t",state[key]}' echo "-----------------------------------------" lsof |grep s3_daily_pm_check_missing_dat|awk '{print $1}'|sort|uniq -c|sort -n echo "-----------------------------------------" lsof |grep s3_daily_pm_check_missing_dat