python 서버상태 모니터링 세트

하드, 디스크, 프로세스, 로드, 네트워크, tcp, 메모리, 스왑

한방에

#!/usr/bin/env python
# -*- coding: utf-8 -*-
 
import os
import os.path
import sys
import subprocess
import commands
import time
import mysql.connector # apt-get install python-mysql.connector
import socket
 
'''
[ mysql 테이블 ]
CREATE TABLE `server` (
  `idx` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `host` varchar(255) NOT NULL DEFAULT '',
  `diskTotal` bigint(20) unsigned NOT NULL DEFAULT '0',
  `diskUse` bigint(20) unsigned NOT NULL DEFAULT '0',
  `diskFree` bigint(20) unsigned NOT NULL DEFAULT '0',
  `process` bigint(20) unsigned NOT NULL DEFAULT '0',
  `loadAvg1` float unsigned NOT NULL DEFAULT '0',
  `loadAvg5` float unsigned NOT NULL DEFAULT '0',
  `loadAvg15` float unsigned NOT NULL DEFAULT '0',
  `netRxTotal` bigint(20) unsigned NOT NULL DEFAULT '0',
  `netRxUse` bigint(20) unsigned NOT NULL DEFAULT '0',
  `netTxTotal` bigint(20) unsigned NOT NULL DEFAULT '0',
  `netTxUse` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpESTABLISHED` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpSYN_SENT` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpSYN_RECV` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpFIN_WAIT1` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpFIN_WAIT2` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpTIME_WAIT` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpCLOSE` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpCLOSE_WAIT` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpLAST_ACK` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpLISTEN` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpCLOSING` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpUNKNOWN` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memTotal` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memUsed` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memFree` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memShared` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memBuffers` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memCached` bigint(20) unsigned NOT NULL DEFAULT '0',
  `swapTotal` bigint(20) unsigned NOT NULL DEFAULT '0',
  `swapUsed` bigint(20) unsigned NOT NULL DEFAULT '0',
  `swapFree` bigint(20) unsigned NOT NULL DEFAULT '0',
  `insertDateTime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
  `bootDateTime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
  PRIMARY KEY (`idx`),
  UNIQUE KEY `idx_UNIQUE` (`idx`),
  KEY `hostName_idx` (`host`),
  KEY `insertDateTime_idx` (`insertDateTime`),
  KEY `searchA_idx` (`host`),
  KEY `searchB_idx` (`host`,`insertDateTime`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;


CREATE TABLE `serverMin` (
  `idx` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `host` varchar(255) NOT NULL DEFAULT '',
  `diskTotal` bigint(20) unsigned NOT NULL DEFAULT '0',
  `diskUse` bigint(20) unsigned NOT NULL DEFAULT '0',
  `diskFree` bigint(20) unsigned NOT NULL DEFAULT '0',
  `process` bigint(20) unsigned NOT NULL DEFAULT '0',
  `loadAvg1` float unsigned NOT NULL DEFAULT '0',
  `loadAvg5` float unsigned NOT NULL DEFAULT '0',
  `loadAvg15` float unsigned NOT NULL DEFAULT '0',
  `netRxTotal` bigint(20) unsigned NOT NULL DEFAULT '0',
  `netRxUse` bigint(20) unsigned NOT NULL DEFAULT '0',
  `netTxTotal` bigint(20) unsigned NOT NULL DEFAULT '0',
  `netTxUse` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpESTABLISHED` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpSYN_SENT` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpSYN_RECV` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpFIN_WAIT1` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpFIN_WAIT2` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpTIME_WAIT` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpCLOSE` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpCLOSE_WAIT` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpLAST_ACK` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpLISTEN` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpCLOSING` bigint(20) unsigned NOT NULL DEFAULT '0',
  `tcpUNKNOWN` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memTotal` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memUsed` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memFree` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memShared` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memBuffers` bigint(20) unsigned NOT NULL DEFAULT '0',
  `memCached` bigint(20) unsigned NOT NULL DEFAULT '0',
  `swapTotal` bigint(20) unsigned NOT NULL DEFAULT '0',
  `swapUsed` bigint(20) unsigned NOT NULL DEFAULT '0',
  `swapFree` bigint(20) unsigned NOT NULL DEFAULT '0',
  `insertDateTime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
  `bootDateTime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
  PRIMARY KEY (`idx`),
  UNIQUE KEY `idx_UNIQUE` (`idx`),
  KEY `hostName_idx` (`host`),
  KEY `insertDateTime_idx` (`insertDateTime`),
  KEY `searchA_idx` (`host`),
  KEY `searchB_idx` (`host`,`insertDateTime`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
'''
 
'''
중복실행 방지코드
'''
isRun = commands.getstatusoutput('ps -aux | grep "' + __file__ + '" | grep -v "grep" | awk "{print \$2}" | wc -l')
 
if int(isRun[1]) > 1:
    print "already run"
    sys.exit()
 
'''
라이브러리
'''
 
def mysqlConnect():
    if hasattr(mysqlConnect, 'conn') == False or mysqlConnect.conn.is_connected() == False:
        mysqlConnect.conn = mysql.connector.connect(user='id', password='pw', host='host', database='server')
        # mysqlConnect.conn = mysql.connector.connect(user='id', password='pw', host='host', database='server')
        mysqlConnect.cursor = mysqlConnect.conn.cursor(buffered=True)
 
    return mysqlConnect
 
def mysqlQuery(queryStr):
    print queryStr
 
    mysqlInit = mysqlConnect();
    mysqlInit.cursor.execute(queryStr)
    mysqlInit.conn.commit()
 
    return mysqlInit.cursor
 
 
def mysqlQueryFetchOne(queryStr):
 
    cursor = mysqlQuery(queryStr)
 
    fetch = cursor.fetchone()
 
    if fetch == None:
        return None
 
    desc = cursor.description
 
    returnArr = {}
 
    for (name, value) in zip(desc, fetch):
        returnArr[name[0]] = value
 
    return returnArr
 
 
def strArrConcat(strArr):
    from cStringIO import StringIO
 
    strIo = StringIO()
 
    for string in strArr:
        strIo.write(str(string))
 
    return strIo.getvalue()
 
 
 
'''
환경변수
'''
envHostName = socket.gethostname()
envNow = time.localtime()
envDateTime = strArrConcat([
    envNow[0]
    ,'-'
    ,envNow[1]
    ,'-'
    ,envNow[2]
    ,' '
    ,envNow[3]
    ,':'
    ,envNow[4]
    ,':00'
    #,envNow[5]
])
 
 
statCheckDic = {}
statCheckDic.clear()
 
 
# network traffic (반듯이 1분마다 실행되어야함)
cmdResult = commands.getstatusoutput("cat /proc/net/dev")
 
cmdLineArr = cmdResult[1].split('\n')
 
netRxTotal = long(0)
netRxBefore = long(0)
netTxTotal = long(0)
netTxBefore = long(0)
 
cacheFileName = __file__ + ".cache.traffic"
 
if os.path.isfile(cacheFileName) and os.access(cacheFileName, os.R_OK):
    cacheFile = open(cacheFileName, 'r') 
    cacheFileLine = cacheFile.readline() 
    cacheFile.close()
 
    if cacheFileLine != '' and cacheFileLine.index('|') > 0:
 
        cacheFileLineArr = cacheFileLine.split('|')
    
        netRxBefore = long(cacheFileLineArr[0])
        netTxBefore = long(cacheFileLineArr[1])
 
for line in cmdLineArr:
 
    lineArr = line.split()
 
    if lineArr[1].isdigit():
 
        netRxTotal = netRxTotal + long(lineArr[1])
        #netRxUse = netRxUse + long(lineArr[2])
        netTxTotal = netTxTotal + long(lineArr[9])
        #netTxUse = netTxUse + long(lineArr[10])
 
statCheckDic['netRxTotal'] = int(netRxTotal) / 1000
if netRxBefore > 0:
    statCheckDic['netRxUse'] = int(netRxTotal - netRxBefore) / 1000
 
statCheckDic['netTxTotal'] = int(netTxTotal) / 1000
if netTxBefore > 0:
    statCheckDic['netTxUse'] = int(netTxTotal - netTxBefore) / 1000
 
cacheFile = open(cacheFileName, 'w')
cacheFile.write(str(netRxTotal));
cacheFile.write("|");
cacheFile.write(str(netTxTotal));
cacheFile.close()
 
 
# cpustat
cmdResult = commands.getstatusoutput("cat /proc/stat")
cmdLineArr = cmdResult[1].split('\n')
 
 
for line in cmdLineArr:
 
    lineArr = line.split()
    
    if lineArr[0] == 'btime':
        bootDateTime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(lineArr[1])))
 
 
statCheckDic['bootDateTime'] = bootDateTime
 
 
 
# load avg
cmdResult = commands.getstatusoutput("cat /proc/loadavg")
lineArr = cmdResult[1].split()
 
statCheckDic['loadAvg1'] = lineArr[0]
statCheckDic['loadAvg5'] = lineArr[1]
statCheckDic['loadAvg15'] = lineArr[2]
 
 
# process count
cmdResult = commands.getstatusoutput("ps aux | wc -l")
 
statCheckDic['process'] = cmdResult[1]
 
 
# disk stat
cmdResult = commands.getstatusoutput("df -lm | grep '/dev/'")
cmdLineArr = cmdResult[1].split('\n')
 
diskTotal = 0
diskUse = 0
diskFree = 0
 
for line in cmdLineArr:
 
    lineArr = line.split()
    
    diskTotal = diskTotal + int(lineArr[1])
    diskUse = diskUse + int(lineArr[2])
    diskFree = diskFree + int(lineArr[3])
 
statCheckDic['diskTotal'] = diskTotal
statCheckDic['diskUse'] = diskUse
statCheckDic['diskFree'] = diskFree
 
 
 
 
# network connection stat
cmdResult = commands.getstatusoutput("netstat -an | awk '/tcp/ {print $6}' | sort | uniq -c")
cmdLineArr = cmdResult[1].split('\n')
 
for line in cmdLineArr:
     if line[-11:] == 'ESTABLISHED':
         lineArr = line.split()
         statCheckDic['tcpESTABLISHED'] = lineArr[0]
     elif line[-10:] == 'CLOSE_WAIT':
         lineArr = line.split()
         statCheckDic['tcpCLOSE_WAIT'] = lineArr[0]
     elif line[-9:] == 'FIN_WAIT1':
         lineArr = line.split()
         statCheckDic['tcpFIN_WAIT1'] = lineArr[0]
     elif line[-9:] == 'FIN_WAIT2':
         lineArr = line.split()
         statCheckDic['tcpFIN_WAIT2'] = lineArr[0]
     elif line[-9:] == 'TIME_WAIT':
         lineArr = line.split()
         statCheckDic['tcpTIME_WAIT'] = lineArr[0]
     elif line[-8:] == 'SYN_SENT':
         lineArr = line.split()
         statCheckDic['tcpSYN_SENT'] = lineArr[0]
     elif line[-8:] == 'SYN_RECV':
         lineArr = line.split()
         statCheckDic['tcpSYN_RECV'] = lineArr[0]
     elif line[-8:] == 'LAST_ACK':
         lineArr = line.split()
         statCheckDic['tcpLAST_ACK'] = lineArr[0]
     elif line[-7:] == 'CLOSING':
         lineArr = line.split()
         statCheckDic['tcpCLOSING'] = lineArr[0]
     elif line[-7:] == 'UNKNOWN':
         lineArr = line.split()
         statCheckDic['tcpUNKNOWN'] = lineArr[0]
     elif line[-6:] == 'LISTEN':
         lineArr = line.split()
         statCheckDic['tcpLISTEN'] = lineArr[0]
     elif line[-5:] == 'CLOSE':
         lineArr = line.split()
         statCheckDic['tcpCLOSE'] = lineArr[0]
 
 
 
 
 
# memory stat
cmdResult = commands.getstatusoutput("free | grep -E 'Mem|Swap'")
cmdLineArr = cmdResult[1].split('\n')
 
for line in cmdLineArr:
    if line[:4] == 'Mem:':
        lineArr = line.split()
 
        statCheckDic['memTotal'] = lineArr[1]
        statCheckDic['memUsed'] = lineArr[2]
        statCheckDic['memFree'] = lineArr[3]
        statCheckDic['memShared'] = lineArr[4]
        statCheckDic['memBuffers'] = lineArr[5]
        statCheckDic['memCached'] = lineArr[6]
 
    if line[:5] == 'Swap:':
        lineArr = line.split()
 
        statCheckDic['swapTotal'] = lineArr[1]
        statCheckDic['swapUsed'] = lineArr[2]
        statCheckDic['swapFree'] = lineArr[3]
 
 
# print statCheckDic
 
 
 
querySetArr = []
querySetArr.append("`host` = '" + envHostName + "'")
querySetArr.append("`insertDateTime` = '" + envDateTime + "'")
 
for statCheckKey in statCheckDic:
    statCheckVal = statCheckDic[statCheckKey]
 
    querySetArr.append("`" + str(statCheckKey) + "` = '" + str(statCheckVal) + "'")
 
 
mysqlQuery(strArrConcat([
    "DELETE FROM"
    ," `server` "
 
    ," WHERE "
    ,"`host` = '"
    ,envHostName
    ,"'"
]))
 
mysqlQuery(strArrConcat([
    "INSERT INTO"
    ," `server` "
 
    ," SET "
    ,' , '.join(querySetArr)
]))
 
mysqlQuery(strArrConcat([
    "INSERT INTO"
    ," `serverMin` "
 
    ," SET "
    ,' , '.join(querySetArr)
]))
 
 
 

클론 1분 등록하면

한방에 뙇! (아래 이미지 클릭!)