算法工程师必会：Dockerfile 语法与Dockerfile模板

前言

为何做这个？

• 方便自己快速的建立自己写的算法的GPU或者CPU训练环境的搭建。
  如何使用此文章
• 学习新的技能一般都是先看高人或者前人做的经典实例，实例研究透了，你也是站在巨人肩膀上的人！
  • 先学习别人的模板，再研究语法！
• 适合领域：深度学习，DNN
• 适合gpu环境：linux GPU

Dockerfile 经典模板

GPU深度学习

GPU:cuda10_cudnn7_pytorch

FROM pytorch/pytorch:nightly-devel-cuda10.0-cudnn7
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}

RUN apt-get update -y

RUN pip install numpy scipy matplotlib librosa==0.6.0 tensorflow tensorboardX inflect==0.2.5 Unidecode==1.0.22 jupyter

ADD apex /apex/
WORKDIR /apex/
RUN pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" .

GPU:pytorch0.4_cuda9_cudnn7

FROM pytorch/pytorch:0.4_cuda9_cudnn7
RUN pip install numpy scipy matplotlib librosa==0.6.0 tensorflow tensorboardX inflect==0.2.5 Unidecode==1.0.22 jupyter

GPU:tensorflow:1.12.0-gpu-py3

FROM tensorflow/tensorflow:1.12.0-gpu-py3

# nvidia-docker 1.0
LABEL com.nvidia.volumes.needed="nvidia_driver"
LABEL com.nvidia.cuda.version="${CUDA_VERSION}"

# nvidia-container-runtime
ENV NVIDIA_VISIBLE_DEVICES=all \
    NVIDIA_DRIVER_CAPABILITIES=compute,utility \
    NVIDIA_REQUIRE_CUDA="cuda>=8.0" \
    LANG=C.UTF-8

RUN mkdir /gpt-2
WORKDIR /gpt-2
ADD . /gpt-2
RUN pip3 install librosa

GPU：cuda:9.0_cudnn7_ubuntu16.04_tf1.10

FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
MAINTAINER Mats Rynge <[email protected]>
RUN export DEBIAN_FRONTEND=noninteractive && \
    apt-get update && apt-get upgrade -y --allow-unauthenticated && \
    apt-get install -y --allow-unauthenticated \
        build-essential \
        cmake \
        cuda-drivers \
        curl \
        git \
        libfreetype6-dev \
        libpng12-dev \
        libssl-dev \
        libxpm-dev \
        libzmq3-dev \
        module-init-tools \
        openssh-client \
        openssh-server \
        pkg-config \
        python \
        python-dev \
        python-tk \
        python3 \
        python3-dev \
        python3-tk \
        rsync \
        software-properties-common \
        unzip \
        zip \
        zlib1g-dev \
        openjdk-8-jdk \
        openjdk-8-jre-headless \
        vim \
        wget \
        && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/*

# bazel is required for some TensorFlow projects
RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" >/etc/apt/sources.list.d/bazel.list && \
    curl https://bazel.build/bazel-release.pub.gpg | apt-key add -

RUN export DEBIAN_FRONTEND=noninteractive && \
    apt-get update && \
    apt-get install -y --allow-unauthenticated \
        bazel

RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
    python get-pip.py && \
    rm get-pip.py

RUN pip --no-cache-dir install \
        h5py \
        ipykernel \
        jupyter \
        matplotlib \
        numpy \
        pandas \
        Pillow \
        scipy \
        sklearn \
        && \
    python -m ipykernel.kernelspec

RUN echo "/usr/local/cuda/lib64/" >/etc/ld.so.conf.d/cuda.conf

# For CUDA profiling, TensorFlow requires CUPTI.
RUN echo "/usr/local/cuda/extras/CUPTI/lib64/" >>/etc/ld.so.conf.d/cuda.conf

# Install TensorFlow GPU version.
RUN pip uninstall tensorflow-gpu || true
RUN pip install --upgrade tensorflow-gpu==1.10

# keras
RUN pip install --upgrade keras

GPU:cuda:9.0-cudnn7-runtime-ubuntu16.04

FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
LABEL maintainer "NVIDIA CORPORATION <[email protected]>"

ENV CUDNN_VERSION 7.4.2.24
LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}"

RUN apt-get update && apt-get install -y\
            libcudnn7=$CUDNN_VERSION-1+cuda9.0 && \
    apt-mark hold libcudnn7 && \
    rm -rf /var/lib/apt/lists/*

RUN apt-get update \
    && apt-get install -y tar git curl nano wget dialog net-tools build-essential \
    && apt-get install --no-install-recommends -y -q python3 python3-pip python3-dev \
    && apt-get install -y libxml2-dev libxslt-dev python-dev zlib1g-dev
RUN apt-get update && apt-get install -y  libjpeg-dev libfreetype6-dev zlib1g-dev
RUN apt-get update && apt-get install -y libasound-dev \
           portaudio19-dev libportaudio2 libportaudiocpp0 \
           ffmpeg  wget git
RUN pip3 install --upgrade pip
RUN pip3 install --upgrade setuptools
RUN pip3 install numpy==1.14.0
RUN pip3 install tensorflow-gpu==1.12.0 falcon==1.2.0 inflect==0.2.5 \
       inflect==0.2.5  librosa==0.6.0 matplotlib==2.1.0 \
       scipy==1.0.0 tqdm==4.31.1 Unidecode==1.0.22 \
       pyaudio==0.2.11 sounddevice==0.3.10 keras lws -i https://pypi.douban.com/simple

GPU:cuda:9.2cudnn7-runtime-ubuntu18.04

FROM nvidia/cuda:9.2-runtime-ubuntu18.04
LABEL maintainer "NVIDIA CORPORATION <[email protected]>"
ENV CUDNN_VERSION 7.4.1.5
LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}"
RUN apt-get update && apt-get install -y --no-install-recommends \
            libcudnn7=$CUDNN_VERSION-1+cuda9.2 && \
    apt-mark hold libcudnn7 && \
    rm -rf /var/lib/apt/lists/*
RUN apt-get update \
    && apt-get install -y tar git curl nano wget dialog net-tools build-essential \
    && apt-get install --no-install-recommends -y -q python3 python3-pip python3-dev \
    && apt-get install -y libxml2-dev libxslt-dev python-dev zlib1g-dev
RUN apt-get update && apt-get install -y  libjpeg-dev libfreetype6-dev zlib1g-dev libpng12-dev
RUN apt-get update && apt-get install -y libasound-dev \
           portaudio19-dev libportaudio2 libportaudiocpp0 \
           ffmpeg libav-tools wget git
RUN pip3 install 
RUN pip3 install tensorflow==1.3.0 falcon==1.2.0 inflect==0.2.5 \
       inflect==0.2.5 udioread==2.1.5 librosa==0.6.0 matplotlib==2.1.0 \
       numpy==1.13.3 scipy==1.0.0 tqdm==4.31.1 Unidecode==1.0.22 \
       pyaudio==0.2.11 sounddevice==0.3.10 keras lws -i https://pypi.douban.com/simple

GPU:tensorflow:1.12.0-gpu-py3

ROM tensorflow/tensorflow:1.12.0-gpu-py3

RUN add-apt-repository -y ppa:jonathonf/ffmpeg-4 \
 && apt-get update -qq -y \
 && apt-get install -y libsm6 libxrender1 libxext-dev python3-tk ffmpeg git \
 && apt-get clean \
 && rm -rf /var/lib/apt/lists/*

COPY requirements.txt /opt/
RUN pip3 install --upgrade pip
RUN pip3 --no-cache-dir install -r /opt/requirements.txt && rm /opt/requirements.txt
RUN pip3 install jupyter matplotlib
RUN pip3 install jupyter_http_over_ws

CPU模板

CPU:tensorflow:1.12.0-py3

FROM tensorflow/tensorflow:1.12.0-py3

ENV LANG=C.UTF-8
RUN mkdir /gpt-2
WORKDIR /gpt-2
ADD . /gpt-2
RUN pip3 install librosa

cpu:python:3.6.8-slim

FROM python:3.6.8-slim

WORKDIR /app

RUN apt-get -y update  && apt-get install -y \
  procps \
  python3-dev \
  python-scipy \
  apt-utils \
  python-dev \
  build-essential \
&& rm -rf /var/lib/apt/lists/* \
&&   ln -sf /usr/share/zoneinfo/Asia/ShangHai /etc/localtime \
&&   echo "Asia/Shanghai" > /etc/timezone \
&&   dpkg-reconfigure -f noninteractive tzdata

RUN pip install cython -i https://pypi.tuna.tsinghua.edu.cn/simple

RUN pip install numpy \
 pandas \
 flask \
 jieba \
 neo4j-driver \
 py2neo \
 schema \
 gensim \
 word2vec \
 synonyms \
 pyltp -i https://pypi.tuna.tsinghua.edu.cn/simple

RUN mkdir -p /app/knowledge_graph/import_tool
COPY ./AIops_demov1.2 /app/knowledge_graph/

# Following CMD keeps the container running
# Modify CMD to run the app that you require.
CMD cd /app/knowledge_graph/code && python main.py

EXPOSE 8789

cpu:tensorflow:1.12.0-py3

FROM tensorflow/tensorflow:1.12.0-py3

RUN add-apt-repository -y ppa:jonathonf/ffmpeg-4 \
 && apt-get update -qq -y \
 && apt-get install -y libsm6 libxrender1 libxext-dev python3-tk ffmpeg git \
 && apt-get clean \
 && rm -rf /var/lib/apt/lists/*

COPY requirements.txt /opt/
RUN pip3 install --upgrade pip
RUN pip3 --no-cache-dir install -r /opt/requirements.txt && rm /opt/requirements.txt

WORKDIR "/srv"
CMD ["/bin/bash"]

CPU：

FROM ubuntu:16.04
RUN apt-get update && \
    apt-get install -y --no-install-recommends \
        g++ \
        gcc \
        make \
        automake \
        autoconf \
        bzip2 \
        unzip \
        wget \
        sox \
        libtool \
        git \
        subversion \
        python3 \
        python3-pip \
        python3-dev \
        zlib1g-dev \
        ca-certificates \
        patch \
        ffmpeg \
	vim && \
    rm -rf /var/lib/apt/lists/* && \
    apt-get autoremove -y && \
    apt-get clean

RUN ln -s /usr/bin/python3 /usr/bin/python &&\
    ln -s /usr/bin/pip3 /usr/bin/pip
RUN pip install numpy==1.16.0 \
    scipy==1.1.0 numba==0.37.0  \
    argparse librosa audioread \
    pysoundfile picklable_itertools~=0.1.1 \
    sacred~=0.6.10 tqdm~=4.8.4 q~=2.6 \
    keras==2.1.2 tensorflow==1.8.0 \
    h5py==2.7.1 matplotlib==2.1.1 \
    scikit-image networkx>=2.0 pillow>=4.3.0 \
    imageio>=2.0.1 flask opencv-python==3.4.3.18
    pyworld -i https://pypi.douban.com/simple

cpu:python:3.6-slim-stretch

FROM python:3.6-slim-stretch

RUN apt-get -y update
RUN apt-get install -y --fix-missing \
    build-essential \
    cmake \
    gfortran \
    git \
    wget \
    curl \
    graphicsmagick \
    libgraphicsmagick1-dev \
    libatlas-dev \
    libavcodec-dev \
    libavformat-dev \
    libgtk2.0-dev \
    libjpeg-dev \
    liblapack-dev \
    libswscale-dev \
    pkg-config \
    python3-dev \
    python3-numpy \
    software-properties-common \
    zip \
    && apt-get clean && rm -rf /tmp/* /var/tmp/*
RUN pip3 install -r requirements.txt

docker暴露服务

FROM ubuntu:14.04

RUN apt-get update && apt-get install -y \
    pkg-config \
    libpng-dev \
    libjpeg8-dev \
    libfreetype6-dev \
    libblas-dev \
    liblapack-dev \
    libatlas-base-dev \
    gfortran \
    python \
    python-dev \
    python-pip \
    curl && \
    curl -sL https://deb.nodesource.com/setup_7.x | sudo -E bash - && \
    apt-get install -y nodejs


RUN pip install -U https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.12.1-cp27-none-linux_x86_64.whl

COPY ./server/requirements.txt /tmp/
RUN pip install -r /tmp/requirements.txt

COPY . /src/

WORKDIR /src/static/
RUN npm install && npm run build

WORKDIR /src/server/

EXPOSE 8080
ENTRYPOINT python server.py

dockerfile语法

简单实例

#获取base image
FROM adoptopenjdk/openjdk8:latest 
#类似于执行 linux指令
RUN mkdir /opt/app  
#类似于linux copy指令
COPY demo-docker.jar /opt/app/       
#对外端口
EXPOSE 8080
#执行命令 java -jar /opt/app/demo-docker.jar
CMD ["java", "-jar", "/opt/app/demo-docker.jar"]

设置中文

#设置字体库
RUN apt-get clean && apt-get update

RUN apt-get install -y locales

RUN localedef -c -f UTF-8 -i zh_CN zh_CN.utf8


ENV LANG zh_CN.utf8 

安装ssh服务

RUN apt install -y openssh-server
RUN mkdir -p /var/run/sshd
RUN mkdir root/.ssh

取消pam限制

RUN sed -ri “s/session required pam_loginuid.so/#session required pam_loginuid.so/g” /etc/pam.d/sshd

复制配置文件到相应的位置

ADD authorized_keys /root/.ssh/authorized_keys
ADD run.sh /run.sh
RUN chmod 755 /run.sh

开放端口

EXPOSE 22

设置自启动

CMD ["/run.sh"]

查看专栏详情 立即解锁全部专栏