Kaggle notebook使用技巧

内容

• 从/kaggle/working/的输出下载文件
• 执行代码后显存变化量CPU/GPU检测
• 使用python Garbage Collection清理运行内存垃圾
• 重置运行内存
• 刷新输出记录

一、下载

1. 从/kaggle/working/的输出下载文件

# 将下载目录添加到压缩文件
from pathlib import Path
import zipfile
img_root = Path('/kaggle/working/img')
with zipfile.ZipFile('img.zip', 'w') as z:
    for img_name in img_root.iterdir():
        z.write(img_name)

# 创建下载链接
import os
os.chdir('/kaggle/working/')
print(os.getcwd())
print(os.listdir("/kaggle/working/"))
from IPython.display import FileLink
FileLink('img.zip')

二、CPU运行内存RAM优化

1. 执行代码后内存变化量检测

import psutil
import os
import time
import sys
import math
from contextlib import contextmanager

@contextmanager
def trace(title):
    t0 = time.time()
    p = psutil.Process(os.getpid())
    m0 = p.memory_info().rss / 2. ** 30
    yield
    m1 = p.memory_info().rss / 2. ** 30
    delta = m1 - m0
    sign = '+' if delta >= 0 else '-'
    delta = math.fabs(delta)
    print(f"[{m1:.1f}GB({sign}{delta:.1f}GB):{time.time() - t0:.1f}sec] {title} ", file=sys.stderr)

# 执行代码后输出变化量
with trace('execute'):
    ## 代码段落 ##

2. 使用python Garbage Collection清理运行内存垃圾

import gc
gc.collect()

3. 重置运行内存RAM

%reset -f

三、GPU显存变化量检测

from contextlib import contextmanager
import math
import os
import subprocess
import sys
import time

import numpy as np
import psutil
import torch


def get_gpu_memory(cmd_path="nvidia-smi",
                   target_properties=("memory.total", "memory.used")):
    """
    ref: https://www.12-technology.com/2022/01/pythongpu.html
    Returns
    -------
    gpu_total : ndarray,  "memory.total"
    gpu_used: ndarray, "memory.used"
    """

    # format option
    format_option = "--format=csv,noheader,nounits"

    cmd = '%s --query-gpu=%s %s' % (cmd_path, ','.join(target_properties), format_option)

    # Command execution in sub-processes
    cmd_res = subprocess.check_output(cmd, shell=True)

    gpu_lines = cmd_res.decode().split('\n')[0].split(', ')

    gpu_total = int(gpu_lines[0]) / 1024
    gpu_used = int(gpu_lines[1]) / 1024

    gpu_total = np.round(gpu_used, 1)
    gpu_used = np.round(gpu_used, 1)
    return gpu_total, gpu_used


class Trace():
    cuda = torch.cuda.is_available()

    @contextmanager
    def timer(self, title):
        t0 = time.time()
        p = psutil.Process(os.getpid())
        cpu_m0 = p.memory_info().rss / 2. ** 30
        if self.cuda: gpu_m0 = get_gpu_memory()[0]
        yield
        cpu_m1 = p.memory_info().rss / 2. ** 30
        if self.cuda: gpu_m1 = get_gpu_memory()[0]

        cpu_delta = cpu_m1 - cpu_m0
        if self.cuda: gpu_delta = gpu_m1 - gpu_m0

        cpu_sign = '+' if cpu_delta >= 0 else '-'
        cpu_delta = math.fabs(cpu_delta)

        if self.cuda: gpu_sign = '+' if gpu_delta >= 0 else '-'
        if self.cuda: gpu_delta = math.fabs(gpu_delta)

        cpu_message = f'{cpu_m1:.1f}GB({cpu_sign}{cpu_delta:.1f}GB)'
        if self.cuda: gpu_message = f'{gpu_m1:.1f}GB({gpu_sign}{gpu_delta:.1f}GB)'

        if self.cuda:
            message = f"[cpu: {cpu_message}, gpu: {gpu_message}: {time.time() - t0:.1f}sec] {title} "
        else:
            message = f"[cpu: {cpu_message}: {time.time() - t0:.1f}sec] {title} "

        print(message, file=sys.stderr)

trace = Trace()
with trace.timer('read train'):
    data = cudf.read_csv('../input/foursquare-location-matching/train.csv')
with trace.timer('del train'):
    del data
    gc.collect()

四、刷新输出

1. tqdm刷新进度

2. sys刷新输出记录

import sys

sys.stdout.write('\r'+str(x))
sys.stdout.flush()