Kaggle notebook tips

content

• Download files from the output of /kaggle/working/
• CPU/GPU detection of video memory change after code execution
• Use python Garbage Collection to clean up running memory garbage
• reset running memory
• flush output record

1. Download

1. Download the file from the output of /kaggle/working/

# 将下载目录添加到压缩文件
from pathlib import Path
import zipfile
img_root = Path('/kaggle/working/img')
with zipfile.ZipFile('img.zip', 'w') as z:
    for img_name in img_root.iterdir():
        z.write(img_name)

# 创建下载链接
import os
os.chdir('/kaggle/working/')
print(os.getcwd())
print(os.listdir("/kaggle/working/"))
from IPython.display import FileLink
FileLink('img.zip')

2. CPU running memory RAM optimization

1. Memory change detection after code execution

import psutil
import os
import time
import sys
import math
from contextlib import contextmanager

@contextmanager
def trace(title):
    t0 = time.time()
    p = psutil.Process(os.getpid())
    m0 = p.memory_info().rss / 2. ** 30
    yield
    m1 = p.memory_info().rss / 2. ** 30
    delta = m1 - m0
    sign = '+' if delta >= 0 else '-'
    delta = math.fabs(delta)
    print(f"[{m1:.1f}GB({sign}{delta:.1f}GB):{time.time() - t0:.1f}sec] {title} ", file=sys.stderr)

# 执行代码后输出变化量
with trace('execute'):
    ## 代码段落 ##

2. Use python Garbage Collection to clean up running memory garbage

import gc
gc.collect()

3. Reset the running memory RAM

%reset -f

3. GPU memory change detection

from contextlib import contextmanager
import math
import os
import subprocess
import sys
import time

import numpy as np
import psutil
import torch


def get_gpu_memory(cmd_path="nvidia-smi",
                   target_properties=("memory.total", "memory.used")):
    """
    ref: https://www.12-technology.com/2022/01/pythongpu.html
    Returns
    -------
    gpu_total : ndarray,  "memory.total"
    gpu_used: ndarray, "memory.used"
    """

    # format option
    format_option = "--format=csv,noheader,nounits"

    cmd = '%s --query-gpu=%s %s' % (cmd_path, ','.join(target_properties), format_option)

    # Command execution in sub-processes
    cmd_res = subprocess.check_output(cmd, shell=True)

    gpu_lines = cmd_res.decode().split('\n')[0].split(', ')

    gpu_total = int(gpu_lines[0]) / 1024
    gpu_used = int(gpu_lines[1]) / 1024

    gpu_total = np.round(gpu_used, 1)
    gpu_used = np.round(gpu_used, 1)
    return gpu_total, gpu_used


class Trace():
    cuda = torch.cuda.is_available()

    @contextmanager
    def timer(self, title):
        t0 = time.time()
        p = psutil.Process(os.getpid())
        cpu_m0 = p.memory_info().rss / 2. ** 30
        if self.cuda: gpu_m0 = get_gpu_memory()[0]
        yield
        cpu_m1 = p.memory_info().rss / 2. ** 30
        if self.cuda: gpu_m1 = get_gpu_memory()[0]

        cpu_delta = cpu_m1 - cpu_m0
        if self.cuda: gpu_delta = gpu_m1 - gpu_m0

        cpu_sign = '+' if cpu_delta >= 0 else '-'
        cpu_delta = math.fabs(cpu_delta)

        if self.cuda: gpu_sign = '+' if gpu_delta >= 0 else '-'
        if self.cuda: gpu_delta = math.fabs(gpu_delta)

        cpu_message = f'{cpu_m1:.1f}GB({cpu_sign}{cpu_delta:.1f}GB)'
        if self.cuda: gpu_message = f'{gpu_m1:.1f}GB({gpu_sign}{gpu_delta:.1f}GB)'

        if self.cuda:
            message = f"[cpu: {cpu_message}, gpu: {gpu_message}: {time.time() - t0:.1f}sec] {title} "
        else:
            message = f"[cpu: {cpu_message}: {time.time() - t0:.1f}sec] {title} "

        print(message, file=sys.stderr)

trace = Trace()
with trace.timer('read train'):
    data = cudf.read_csv('../input/foursquare-location-matching/train.csv')
with trace.timer('del train'):
    del data
    gc.collect()

4. Refresh output

1. tqdm refresh progress

2. sys refresh output record

import sys

sys.stdout.write('\r'+str(x))
sys.stdout.flush()