Skip to content

Commit 2cb6aa7

Browse files
pangyokipangyoki
committed
only save py37 of docker in release version
1 parent 8ccf549 commit 2cb6aa7

File tree

2 files changed

+278
-0
lines changed

2 files changed

+278
-0
lines changed

‎tools/dockerfile/Dockerfile.release16‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# A image for building paddle binaries
2+
# Use cuda devel base image for both cpu and gpu environment
3+
# When you modify it, please be aware of cudnn-runtime version
4+
FROM nvidia/cuda:<baseimg>
5+
MAINTAINER PaddlePaddle Authors <[email protected]>
6+
7+
# ENV variables
8+
ARG WITH_GPU
9+
ARG WITH_AVX
10+
11+
ENV WITH_GPU=${WITH_GPU:-ON}
12+
ENV WITH_AVX=${WITH_AVX:-ON}
13+
14+
ENV HOME /root
15+
# Add bash enhancements
16+
COPY paddle/scripts/docker/root/ /root/
17+
18+
# Prepare packages for Python
19+
RUN apt-get update && \
20+
apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
21+
libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
22+
xz-utils tk-dev libffi-dev liblzma-dev
23+
24+
RUN apt-get update && \
25+
apt-get install -y --allow-downgrades --allow-change-held-packages \
26+
patchelf git python-pip python-dev python-opencv openssh-server bison \
27+
wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \
28+
curl sed grep graphviz libjpeg-dev zlib1g-dev \
29+
python-matplotlib \
30+
automake locales clang-format swig \
31+
liblapack-dev liblapacke-dev \
32+
net-tools libtool module-init-tools && \
33+
apt-get clean -y
34+
35+
RUN wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz -O shellcheck-v0.7.1.linux.x86_64.tar.xz && \
36+
tar -xf shellcheck-v0.7.1.linux.x86_64.tar.xz && cp shellcheck-v0.7.1/shellcheck /usr/bin/shellcheck && \
37+
rm -rf shellcheck-v0.7.1.linux.x86_64.tar.xz shellcheck-v0.7.1
38+
39+
# Downgrade gcc&&g++
40+
<install_gcc>
41+
42+
# install cmake
43+
WORKDIR /home
44+
RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz
45+
ENV PATH=/home/cmake-3.16.0-Linux-x86_64/bin:$PATH
46+
47+
48+
# Install Python3.7
49+
RUN wget -q https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz && \
50+
tar -xzf Python-3.7.0.tgz && cd Python-3.7.0 && \
51+
CFLAGS="-Wformat" ./configure --prefix=/usr/local/python3.7.0 --enable-shared > /dev/null && \
52+
make -j8 > /dev/null && make altinstall > /dev/null && ldconfig
53+
54+
ENV PATH=/usr/local/python3.7.0/include:${PATH}
55+
ENV PATH=/usr/local/python3.7.0/bin:${PATH}
56+
ENV LD_LIBRARY_PATH=/usr/local/python3.7.0/lib:${LD_LIBRARY_PATH}
57+
ENV CPLUS_INCLUDE_PATH=/usr/local/python3.7.0/include/python3.7:$CPLUS_INCLUDE_PATH
58+
RUN ln -sf /usr/local/python3.7.0/bin/python3.7 /usr/local/bin/python3 && ln -sf /usr/local/python3.7.0/bin/python3.7 /usr/bin/python3
59+
RUN mv /usr/bin/python /usr/bin/python.bak && ln -s /usr/local/python3.7.0/bin/python3.7 /usr/local/bin/python && ln -s /usr/local/python3.7.0/bin/python3.7 /usr/bin/python
60+
61+
RUN rm -r /root/python_build
62+
63+
WORKDIR /home
64+
RUN python3.7 -m pip uninstall -y pip setuptools
65+
RUN wget https://files.pythonhosted.org/packages/a7/e0/30642b9c2df516506d40b563b0cbd080c49c6b3f11a70b4c7a670f13a78b/setuptools-50.3.2.zip && apt-get -y install unzip && unzip setuptools-50.3.2.zip
66+
WORKDIR /home/setuptools-50.3.2
67+
RUN python3.7 setup.py build && python3.7 setup.py install
68+
WORKDIR /home
69+
RUN wget https://files.pythonhosted.org/packages/28/af/2c76c8aa46ccdf7578b83d97a11a2d1858794d4be4a1610ade0d30182e8b/pip-20.0.1.tar.gz && tar -zxvf pip-20.0.1.tar.gz
70+
WORKDIR pip-20.0.1
71+
RUN python3.7 setup.py install
72+
73+
WORKDIR /home
74+
RUN rm Python-$version.tgz setuptools-40.6.2.zip setuptools-50.3.2.zip pip-20.0.1.tar.gz && \
75+
rm -r Python-$version setuptools-40.6.2 setuptools-50.3.2 pip-20.0.1
76+
77+
# Install Go and glide
78+
WORKDIR /home
79+
RUN wget -qO- https://paddle-ci.cdn.bcebos.com/go1.8.1.linux-amd64.tar.gz | \
80+
tar -xz -C /usr/local && \
81+
mkdir /root/gopath && \
82+
mkdir /root/gopath/bin && \
83+
mkdir /root/gopath/src
84+
ENV GOROOT=/usr/local/go GOPATH=/root/gopath
85+
# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT.
86+
ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin
87+
# install glide
88+
RUN curl -s -q https://glide.sh/get | sh
89+
90+
# Install TensorRT
91+
# following TensorRT.tar.gz is not the default official one, we do two miny changes:
92+
# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now,
93+
# and its size is only one-third of the official one.
94+
# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle.
95+
# See https://github.com/PaddlePaddle/Paddle/issues/10129 for details.
96+
97+
# Downgrade TensorRT
98+
COPY tools/dockerfile/build_scripts /build_scripts
99+
RUN bash /build_scripts/install_nccl2.sh
100+
RUN rm -rf /build_scripts
101+
102+
# git credential to skip password typing
103+
RUN git config --global credential.helper store
104+
105+
# Fix locales to en_US.UTF-8
106+
RUN localedef -i en_US -f UTF-8 en_US.UTF-8
107+
108+
# specify sphinx version as 1.5.6 and remove -U option for [pip install -U
109+
# sphinx-rtd-theme] since -U option will cause sphinx being updated to newest
110+
# version(1.7.1 for now), which causes building documentation failed.
111+
RUN pip3.7 --no-cache-dir install -U wheel py-cpuinfo==5.0.0 && \
112+
pip3.7 --no-cache-dir install -U docopt PyYAML sphinx==1.5.6 && \
113+
pip3.7 --no-cache-dir install sphinx-rtd-theme==0.1.9 recommonmark
114+
115+
RUN pip3.7 --no-cache-dir install 'pre-commit==1.10.4' 'ipython==5.3.0' && \
116+
pip3.7 --no-cache-dir install 'ipykernel==4.6.0' 'jupyter==1.0.0'
117+
118+
#For docstring checker
119+
RUN pip3.7 --no-cache-dir install pylint pytest astroid isort
120+
121+
RUN pip3.7 --no-cache-dir install coverage
122+
123+
COPY ./python/requirements.txt /root/
124+
RUN pip3.7 --no-cache-dir install -r /root/requirements.txt
125+
126+
# To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use
127+
# the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2
128+
RUN apt-get install -y libssl-dev libffi-dev && apt-get clean -y && \
129+
pip3.7 install --upgrade pip && \
130+
pip3.7 --no-cache-dir install certifi urllib3[secure]
131+
132+
# ar mishandles 4GB files
133+
# https://sourceware.org/bugzilla/show_bug.cgi?id=14625
134+
# remove them when apt-get support 2.27 and higher version
135+
RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/binutils/2.27-9ubuntu1/binutils_2.27.orig.tar.gz && \
136+
tar -xzf binutils_2.27.orig.tar.gz && \
137+
cd binutils-2.27 && \
138+
./configure && make -j && make install && cd .. && rm -rf binutils-2.27 binutils_2.27.orig.tar.gz
139+
140+
RUN apt-get install libprotobuf-dev -y
141+
142+
# Older versions of patchelf limited the size of the files being processed and were fixed in this pr.
143+
# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa
144+
# So install a newer version here.
145+
RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \
146+
dpkg -i patchelf_0.10-2_amd64.deb
147+
148+
# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
149+
RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
150+
CMD source ~/.bashrc
151+
152+
# ccache 3.7.9
153+
RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \
154+
tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \
155+
./configure -prefix=/usr/local/ccache-3.7.9 && \
156+
make -j8 && make install && \
157+
ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache
158+
159+
160+
EXPOSE 22

‎tools/dockerfile/ubuntu16_release.sh‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/bin/bash
2+
3+
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
docker_name=$1
18+
19+
20+
function ref_whl(){
21+
if [[ ${WITH_GPU} == "ON" ]]; then
22+
ref_gpu=gpu-cuda${ref_CUDA_MAJOR}-cudnn${CUDNN_MAJOR}
23+
install_gpu="_gpu"
24+
else
25+
ref_gpu="cpu"
26+
install_gpu=""
27+
fi
28+
29+
if [[ ${WITH_MKL} == "ON" ]]; then
30+
ref_mkl=mkl
31+
else
32+
ref_mkl=openblas
33+
fi
34+
35+
if [[ ${WITH_GPU} != "ON" ]]; then
36+
ref_gcc=""
37+
elif [[ ${gcc_version} == "8.2.0" ]];then
38+
ref_gcc=_gcc8.2
39+
fi
40+
41+
if [[ ${ref_CUDA_MAJOR} == "11.0" ]];then
42+
ref_version=.post110
43+
elif [[ ${ref_CUDA_MAJOR} == "11.2" ]];then
44+
ref_version=.post112
45+
elif [[ ${ref_CUDA_MAJOR} == "10" ]];then
46+
ref_version=.post100
47+
elif [[ ${ref_CUDA_MAJOR} == "10.1" ]];then
48+
ref_version=.post101
49+
elif [[ ${ref_CUDA_MAJOR} == "10.2" && ${PADDLE_VERSION} == "develop" ]];then
50+
ref_version=.post102
51+
elif [[ ${ref_CUDA_MAJOR} == "10.2" && ${PADDLE_VERSION} != "develop" ]];then
52+
ref_version=""
53+
elif [[ ${ref_CUDA_MAJOR} == "9" ]];then
54+
ref_version=.post90
55+
fi
56+
57+
ref_dev=2.1.0.dev0
58+
59+
ref_web="https://paddle-wheel.bj.bcebos.com/${PADDLE_BRANCH}-${ref_gpu}-${ref_mkl}${ref_gcc}"
60+
61+
if [[ ${PADDLE_VERSION} == "develop" && ${WITH_GPU} == "ON" ]]; then
62+
ref_paddle37_whl=paddlepaddle${install_gpu}-${ref_dev}${ref_version}-cp37-cp37m-linux_x86_64.whl
63+
elif [[ ${PADDLE_VERSION} == "develop" && ${WITH_GPU} != "ON" ]]; then
64+
ref_paddle37_whl=paddlepaddle${install_gpu}-${ref_dev}-cp37-cp37m-linux_x86_64.whl
65+
elif [[ ${PADDLE_VERSION} != "develop" && ${WITH_GPU} == "ON" ]]; then
66+
ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}${ref_version}-cp37-cp37m-linux_x86_64.whl
67+
else
68+
ref_paddle37_whl=paddlepaddle${install_gpu}-${PADDLE_VERSION}-cp37-cp37m-linux_x86_64.whl
69+
fi
70+
}
71+
72+
73+
function install_whl(){
74+
dockerfile_line=`wc -l Dockerfile.tmp|awk '{print $1}'`
75+
sed -i "${dockerfile_line}i RUN wget -q ${ref_web}/${ref_paddle37_whl} && pip3.7 install ${ref_paddle37_whl} && rm -f ${ref_paddle37_whl}" Dockerfile.tmp
76+
}
77+
78+
79+
function install_gcc(){
80+
if [ "${gcc_version}" == "8.2.0" ];then
81+
sed -i 's#<install_gcc>#WORKDIR /usr/bin \
82+
COPY tools/dockerfile/build_scripts /build_scripts \
83+
RUN bash /build_scripts/install_gcc.sh gcc82 \&\& rm -rf /build_scripts \
84+
RUN cp gcc gcc.bak \&\& cp g++ g++.bak \&\& rm gcc \&\& rm g++ \
85+
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc \
86+
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ \
87+
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc \
88+
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ \
89+
ENV PATH=/usr/local/gcc-8.2/bin:$PATH #g' Dockerfile.tmp
90+
else
91+
sed -i 's#<install_gcc>#RUN apt-get update \
92+
WORKDIR /usr/bin \
93+
RUN apt install -y gcc g++ #g' Dockerfile.tmp
94+
fi
95+
}
96+
97+
98+
function install_jupyter() {
99+
if [[ ${WITH_NOTEBOOK} == "ON" ]]; then
100+
# install jupyter notebook
101+
fi
102+
}
103+
104+
105+
function make_dockerfile(){
106+
sed "s/<baseimg>/${docker_name}/g" tools/dockerfile/Dockerfile.release16 >Dockerfile.tmp
107+
}
108+
109+
110+
function main(){
111+
make_dockerfile
112+
install_gcc
113+
install_jupyter
114+
ref_whl
115+
install_whl
116+
}
117+
118+
main $@

0 commit comments

Comments
 (0)