ubuntu 使用Python代码监控GPU和CPU的用量及温度

ubuntu 使用Python代码监控GPU和CPU的用量及温度

问题：

项目中，由于发现存在程序怪掉起不来，检查发现是输入命令nvidia-smi后，不打印NVIDIA显卡的驱动信息。

解决办法1：

reboot重启之后, 再次输入nvidia-smi -l 1后，正常打印。

解决办法2：

猜测原因可能是由于，Ubuntu内核升级导致NVIDIA驱动无法连接到。参考 https://zhuanlan.zhihu.com/p/611276995?utm_id=0 ，产生通过如下方法避免内核自动更新。

通过如下两条命令，并将里面的参数值均改成0

vi /etc/apt/apt.conf.d/10periodic
vi /etc/apt/apt.conf.d/20auto-apgrades

另，

猜测原因：

可能是由于散热问题，导致驱动会频繁崩掉

故，写了一段python代码，每隔5分钟，保存一次GPU和CPU的用量及温度到txt文件中。源码如下：

涉及需要安装的python库
# pip3 install nvidia-ml-py3 -i http://pypi.douban.com/simple --trusted-host pypi.douban.com
# pip3 install psutil

def print_gpu_info():
    # ref: https://blog.csdn.net/zxc120389574/article/details/106220612
    # 获取GPU的用量和温度
    # 这里假设使用的是NVIDIA显卡，需要安装nvidia-ml-py库
    

    print("---------GPU usage and T---------")
    import pynvml
    pynvml.nvmlInit()
    # Use device index to get gpu_device
    gpu_device = pynvml.nvmlDeviceGetHandleByIndex(0)

    # Use gpu_device to get device stats
    memory_info = pynvml.nvmlDeviceGetMemoryInfo(gpu_device)
    memoInfo = "Memory: Used/Total = {}/{}".format(memory_info.used / 1024 / 1024, memory_info.total / 1024 / 1024)

    powerUsage = pynvml.nvmlDeviceGetPowerUsage(gpu_device)
    powerState = pynvml.nvmlDeviceGetPowerState(gpu_device)

    # get GPU temperature
    gpu_device = pynvml.nvmlDeviceGetHandleByIndex(0)
    temperature = pynvml.nvmlDeviceGetTemperature(gpu_device, pynvml.NVML_TEMPERATURE_GPU)
    tempInfo = "Temperature：{0}°C".format(temperature)

    gpu_info ='gpu: ' + memoInfo + '  ' + tempInfo + '  '
    return gpu_info


def print_cpu_info():
    # 获取CPU的用量和温度 ()
    print("---------CPU usage and T---------")
    import psutil
    cpu_percent = psutil.cpu_percent()
    # psutil.sensors_temperatures() 官方文档这有说的只适用于 linux
    # 此程序在ubuntu上验证pass,
    cpu_temperature = psutil.sensors_temperatures()['coretemp'][0].current
    memoInfo = f"CPU utilization: {cpu_percent}%"
    tempInfo = f"CPU temperature: {cpu_temperature}°C"

    cpu_info = '    ||||    cpu: ' + memoInfo + ' ' + tempInfo + ' '
    return cpu_info
    
    
def write2Info(info):
    from datetime import datetime
    t = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
    print('info : ', info)
    info = t + '   '+ info + '\n'
    txtName = './GPUCPU_usage{}.txt'.format(1)
    with open(txtName, 'a', encoding='utf-8') as wf:
        wf.write(info)
        
        
if __name__ == '__main__':
    import time

    while True:
        gpu_info = print_gpu_info()
        cpu_info = print_cpu_info()  
        info = gpu_info + cpu_info
        write2Info(info)
        time.sleep(2)

或 （上下两端代码可能是一样的，只不过下面的代码试运行noproblem）

import psutil
import time

def print_gpu_info():
    #print("---------GPU usage and T---------")
    import pynvml
    pynvml.nvmlInit()
    # Use device index to get gpu_device
    gpu_device = pynvml.nvmlDeviceGetHandleByIndex(0)

    # Use gpu_device to get device stats
    memory_info = pynvml.nvmlDeviceGetMemoryInfo(gpu_device)
    memoInfo = "Memory: Used/Total = {}/{}".format(memory_info.used / 1024 / 1024, memory_info.total / 1024 / 1024)
    
    
    gpu_device = pynvml.nvmlDeviceGetHandleByIndex(0)
    temperature = pynvml.nvmlDeviceGetTemperature(gpu_device, pynvml.NVML_TEMPERATURE_GPU)
    tempInfo = "Temperature：{0}°C".format(temperature)
    # print(tempInfo)

    gpu_info ='gpu: ' + memoInfo + '  ' + tempInfo + '  '
    return gpu_info
    


def print_cpu_info():
    #print("---------CPU usage and T---------")
    import psutil
    cpu_percent = psutil.cpu_percent()
    # psutil.sensors_temperatures() 官方文档这有说的只适用于 linux
    # 此程序在ubuntu上验证pass,
    cpu_temperature = psutil.sensors_temperatures()['coretemp'][0].current
    memoInfo = f"CPU utilization: {cpu_percent}%"
    # print(memoInfo)
    tempInfo = f"CPU temperature: {cpu_temperature}°C"
    # print(tempInfo)

    cpu_info = '    ||||    cpu: ' + memoInfo + ' ' + tempInfo + ' '
    return cpu_info


def write2Info(info):
    from datetime import datetime
    t = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
    print('info : ', info)
    info = t + '   '+ info + '\n'
    txtName = './GPUCPU_usage{}.txt'.format(1)
    with open(txtName, 'a', encoding='utf-8') as wf:
        wf.write(info)


if __name__ == '__main__':
    import time

    while True:
        gpu_info = print_gpu_info()
        cpu_info = print_cpu_info()
        info = gpu_info + cpu_info
        write2Info(info)
        time.sleep(300)

连接

链接：https://pan.baidu.com/s/1ZoPp9mSeydqNuRobkChWsw