懒狗必备--检测服务器空卡并邮件告知用户

本文最后更新于:2020年12月24日 下午

  • send_email.py
#!/usr/bin/python
# -*-coding:utf-8-*-
import os
import argparse
import configparser
import subprocess
import time
from easydict import EasyDict as edict
import yaml

import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.header import Header

#---------------------------------------------------------------
# 读取配置文件(发、收件人信息)
conf_name = 'email.yml'
self_file_dir = os.path.split(os.path.realpath(__file__))[0]
conf_path = os.path.join(self_file_dir, conf_name)
with open(conf_path, 'r') as f:
    yaml_cfg = edict(yaml.load(f))
#print(yaml_cfg)
smtp_url = yaml_cfg.send.smtp_url
add_from = yaml_cfg.send.send_address
add_from_pass = yaml_cfg.send.send_password
add_to = yaml_cfg.receive.receive_address


#---------------------------------------------------------------
# 初始化邮件
message = MIMEMultipart()
message['Subject'] = Header('Free GPU already....', 'utf-8')
message['From'] = Header(add_from, 'utf-8')
message['To'] = Header(add_to, 'utf-8')

maintext = 'Some free GPUs have been prepared....\n\n'
#a = subprocess.Popen('nvidia-smi', shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE).communicate()[0].decode()
a = subprocess.getstatusoutput('nvidia-smi')[1]
msg = MIMEText(maintext+a, 'plain', 'utf-8')
message.attach(msg)


#---------------------------------------------------------------
# 发送邮件
# Send the message via our own SMTP server, but don't include the
# envelope header.

# pip install nvidia-ml-py==11.450.51
import pynvml
import torch
import time
pynvml.nvmlInit()

cnt = torch.cuda.device_count()
while(1):
    for i in range(cnt):
        handle = pynvml.nvmlDeviceGetHandleByIndex(i)
        meminfo = pynvml.nvmlDeviceGetMemoryInfo(handle)
        memory_total = meminfo.total #第0块显卡总的显存大小
        memory_used = meminfo.used   #这里是字节bytes,所以要想得到以兆M为单位就需要除以1024**2
        
        if memory_used / memory_total < 0.2:
            s = smtplib.SMTP_SSL(smtp_url)
            s.login(add_from, add_from_pass)
            s.sendmail(add_from, [add_to], message.as_string())
            s.quit()
            #print('hello')
            time.sleep(5)
            # 发送完杀掉该进程
            aa = subprocess.getstatusoutput('ps -aux |grep sendmail.py| grep -v grep ')[1]
            for i in aa.split(" "):
            	if i.isdigit():
            		tmp = 'kill ' +i
            		st = subprocess.getstatusoutput(tmp)[1]
            		print(st)
            break
  • email.yml
send:
        smtp_url: smtp.qq.com
        send_address: 462549693@qq.com
        send_password: xxxxxxxxxxxx
receive:
        receive_address: 462549693@qq.com

本博客所有文章除特别声明外,均采用 CC BY-SA 4.0 协议 ,转载请注明出处!