Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add daemon which periodically pushes process and docker stats to State DB #3525

Merged
merged 21 commits into from
Nov 27, 2019
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
210 changes: 210 additions & 0 deletions files/image_config/procdockerstats/procdockerstats
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
# !/usr/bin/env python
'''
This code is for a specific daemon, process-docker.
Which sends process and docker CPU/memory utilization data to DB every 2 mins.
jleveque marked this conversation as resolved.
Show resolved Hide resolved
'''

import sys
import os
import time
import syslog
import subprocess
import re
import swsssdk
jleveque marked this conversation as resolved.
Show resolved Hide resolved
from datetime import datetime
jleveque marked this conversation as resolved.
Show resolved Hide resolved

VERSION = '1.0'

SYSLOG_IDENTIFIER = "procdockerstats"

REDIS_HOSTIP = "127.0.0.1"

# ========================== Syslog wrappers ==========================
def log_info(msg, also_print_to_console=False):
syslog.openlog(SYSLOG_IDENTIFIER)
syslog.syslog(syslog.LOG_INFO, msg)
syslog.closelog()

def log_warning(msg, also_print_to_console=False):
syslog.openlog(SYSLOG_IDENTIFIER)
syslog.syslog(syslog.LOG_WARNING, msg)
syslog.closelog()

def log_error(msg, also_print_to_console=False):
syslog.openlog(SYSLOG_IDENTIFIER)
syslog.syslog(syslog.LOG_ERR, msg)
syslog.closelog()

# ========================== ProcessDocker class ==========================
class ProcDockerStats:

def __init__(self):
self.state_db = swsssdk.SonicV2Connector(host=REDIS_HOSTIP)
self.state_db.connect("STATE_DB")
pra-moh marked this conversation as resolved.
Show resolved Hide resolved

jleveque marked this conversation as resolved.
Show resolved Hide resolved
def run_command(self, cmd):
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
(stdout, stderr) = proc.communicate()
if proc.returncode != 0:
log_error("Error running command '{}'".format(cmd))
jleveque marked this conversation as resolved.
Show resolved Hide resolved
else:
return stdout

def format_docker_cmd_output(self, cmdout):
lines = re.split("\n", cmdout)
keys = re.split(" +", lines[0])
dict1 = dict()
jleveque marked this conversation as resolved.
Show resolved Hide resolved
dict_list = []
for item in lines[1:]:
values1 = re.split(" +", item)
dict1 = dict(zip(keys, values1))
pra-moh marked this conversation as resolved.
Show resolved Hide resolved
dict_list.append(dict1)
formatted_dict = self.create_docker_dict(dict_list)
return formatted_dict

def format_process_cmd_output(self, cmdout):
lines = re.split("\n", cmdout)
keys = re.split(" +", lines[0])
keylist = list(filter(None, keys))
dict1 = dict()
jleveque marked this conversation as resolved.
Show resolved Hide resolved
dict_list = []
for item in lines[1:]:
values1 = re.split(" +", str(item))
# To remove extra space before UID
val = list(filter(None, values1))
# Merging extra columns created due to space in cmd ouput
val[8:] = [''.join(val[8:])]
jleveque marked this conversation as resolved.
Show resolved Hide resolved
dict1 = dict(zip(keylist, val))
dict_list.append(dict1)
return dict_list

def convert_to_bytes(self, value):
u = re.search('[a-zA-Z]+', value)
v = float(filter(str.isdigit, value))
jleveque marked this conversation as resolved.
Show resolved Hide resolved
unit = u.group(0)
UNITS_B = 'B'
UNITS_KB = 'KB'
UNITS_MB = 'MB'
UNITS_MiB = 'MiB'
UNITS_GiB = 'GiB'
if unit.lower() == UNITS_B.lower():
jleveque marked this conversation as resolved.
Show resolved Hide resolved
return v
elif unit.lower() == UNITS_KB.lower():
v1 = v * 1000
return v1
elif unit.lower() == UNITS_MB.lower():
v1 = v * 1000 * 1000
return v1
elif unit.lower() == UNITS_MiB.lower():
v1 = v * 1024 * 1024
return v1
jleveque marked this conversation as resolved.
Show resolved Hide resolved
elif unit.lower() == UNITS_GiB.lower():
v1 = v * 1024 * 1024 * 1024
return round(v1)

def create_docker_dict(self, dict_list):
dockerdict = {}
for row in dict_list[0:]:
cid = row.get('CONTAINER ID')
if cid:
key = 'Docker_Stats|' + str(cid)
jleveque marked this conversation as resolved.
Show resolved Hide resolved
dockerdict[key] = {}
dockerdict[key]['NAME'] = row.get('NAME')

splitcol = row.get('CPU %')
cpu = re.split("%", str(splitcol))
dockerdict[key]['CPU%'] = str(cpu[0])

splitcol = row.get('MEM USAGE / LIMIT')
memuse = re.split(" / ", str(splitcol))
# converting MiB and GiB to bytes
dockerdict[key]['MEM'] = str(self.convert_to_bytes(memuse[0]))
jleveque marked this conversation as resolved.
Show resolved Hide resolved
dockerdict[key]['MEM_LIMIT'] = str(self.convert_to_bytes(memuse[1]))

splitcol = row.get('MEM %')
mem = re.split("%", str(splitcol))
dockerdict[key]['MEM%'] = str(mem[0])

splitcol = row.get('NET I/O')
netio = re.split(" / ", str(splitcol))
dockerdict[key]['NET_IN'] = str(self.convert_to_bytes(netio[0]))
dockerdict[key]['NET_OUT'] = str(self.convert_to_bytes(netio[1]))

splitcol = row.get('BLOCK I/O')
blockio = re.split(" / ", str(splitcol))
dockerdict[key]['BLOCK_IN'] = str(self.convert_to_bytes(blockio[0]))
dockerdict[key]['BLOCK_OUT'] = str(self.convert_to_bytes(blockio[1]))

dockerdict[key]['PIDS'] = row.get('PIDS')
return dockerdict

def update_dockerstats_command(self):
cmd = "docker stats --no-stream -a"
data = self.run_command(cmd)
if not data:
log_error("'{}' returned null output".format(cmd))
return False
dockerdata = self.format_docker_cmd_output(data)
if not dockerdata:
log_error("formatting for docker output failed")
return False
# wipe out all data from state_db before updating
self.state_db.delete_all_by_pattern('STATE_DB', 'Docker_Stats|*')
for k1,v1 in dockerdata.iteritems():
for k2,v2 in v1.iteritems():
self.update_state_db(k1, k2, v2)
pra-moh marked this conversation as resolved.
Show resolved Hide resolved

def update_processstats_command(self):
data = self.run_command("ps -eo uid,pid,ppid,%mem,%cpu,stime,tty,time,cmd --sort -%cpu | head -5")
processdata = self.format_process_cmd_output(data)
value = ""
# wipe out all data before updating with new values
self.state_db.delete_all_by_pattern('STATE_DB', 'Process_Stats|*')
for row in processdata[0:]:
cid = row.get('PID')
if cid:
value = 'Process_Stats|' + str(cid)
uid = row.get('UID')
self.update_state_db(value, 'UID', uid)
ppid = row.get('PPID')
self.update_state_db(value, 'PPID', ppid)
cpu = row.get('%CPU')
self.update_state_db(value, '%CPU', str(cpu))
mem = row.get('%MEM')
self.update_state_db(value, '%MEM', str(mem))
stime = row.get('STIME')
self.update_state_db(value, 'STIME', stime)
tty = row.get('TT')
self.update_state_db(value, 'TT', tty)
time = row.get('TIME')
self.update_state_db(value, 'TIME', time)
cmd = row.get('CMD')
self.update_state_db(value, 'CMD', cmd)

def update_state_db(self, key1, key2, value2):
self.state_db.set('STATE_DB', key1, key2, value2)

def run(self):
self.update_dockerstats_command()
datetimeobj = datetime.now()
# Adding key to store latest update time.
self.update_state_db('Docker_Stats|LastUpdateTime', 'lastupdate', datetimeobj)
jleveque marked this conversation as resolved.
Show resolved Hide resolved
self.update_processstats_command()
self.update_state_db('Process_Stats|LastUpdateTime', 'lastupdate', datetimeobj)
jleveque marked this conversation as resolved.
Show resolved Hide resolved

# main start
def main():
log_info("process-docker stats daemon starting up..")
if not os.getuid() == 0:
log_error("Must be root to run process-docker daemon")
print "Error: Must be root to run process-docker daemon"
sys.exit(1)
pd = ProcDockerStats()
# Data need to be updated every 2 mins. hence adding delay of 120 seconds
while True:
pd.run()
time.sleep(120)
log_info("process-docker stats daemon exited")

if __name__ == '__main__':
main()
pra-moh marked this conversation as resolved.
Show resolved Hide resolved
jleveque marked this conversation as resolved.
Show resolved Hide resolved
12 changes: 12 additions & 0 deletions files/image_config/procdockerstats/procdockerstats.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
[Unit]
Description=Control Plane ACL configuration daemon
Requires=updategraph.service
After=updategraph.service

[Service]
Type=simple
ExecStart=/usr/bin/procdockerstats

[Install]
WantedBy=multi-user.target