From 454c9f4991365970b0f5ff3cd45fc0cab2305d7c Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Sun, 24 Apr 2022 17:16:58 +0200 Subject: [PATCH 01/16] Initial commit of build script --- build_mmdeploy.sh | 346 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 346 insertions(+) create mode 100755 build_mmdeploy.sh diff --git a/build_mmdeploy.sh b/build_mmdeploy.sh new file mode 100755 index 0000000000..1c14a4ea5e --- /dev/null +++ b/build_mmdeploy.sh @@ -0,0 +1,346 @@ +#!/bin/bash +# build_mmdeploy.sh +# Date: 08-03-2022, 24-04-2022 +# +# Run this script to build MMDeploy SDK and prerequisites. +# This script will also setup python venv +# +BUILD_TYPE="Release" +ARCH=$(uname -i) +WITH_PYTHON=1 +WITH_CLEAN=1 +# Get path to script to ensure script runs from 'logends' root +WORKING_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +PPLCV_DIR=${WORKING_DIR}/ppl.cv +MMDEPLOY_DIR=${WORKING_DIR}/MMDeploy +PPLCV_VER="0.6.2" +MMDEPLOY_VER="0.4.0" +INSTALL_PREFIX="/usr/local" +PYTHON_VENV_DIR="mmdeploy" + +appargument1=$1 +#appargument2=$2 + +echo_green() { + if [ -n "$1" ]; then + echo "$(tput setaf 10)$1$(tput sgr 0)" + fi +} +echo_red() { + if [ -n "$1" ]; then + echo "$(tput setaf 1)$1$(tput sgr 0)" + fi +} +echo_blue() { + if [ -n "$1" ]; then + echo "$(tput setaf 4)$1$(tput sgr 0)" + fi +} +contains_element () { + local e match="$1" + shift + for e; do [[ "$e" == "$match" ]] && return 0; done + return 1 +} + +prereqs() { + # spdlog + read -p "Install latest spdlog from source? (y/n)" -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] + then + # remove libspdlog, as it might be an old version + sudo apt-get remove libspdlog-dev -y + + echo "Building and installing latest spdlog from source" + git clone https://github.com/gabime/spdlog.git spdlog + cd spdlog + git pull + git checkout tags/v1.8.1 + mkdir build -p && cd build + # we must build spdlog with -fPIC enabled + cmake .. -DCMAKE_POSITION_INDEPENDENT_CODE=ON && make -j + sudo make install + sudo ldconfig + fi + + # cmake check + echo_green "Check your cmake version:" + cmake --version + read -p "Install latest CMake? (>=3.18 is required) (y/n)" -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] + then + # purge existing + sudo apt-get purge cmake + sudo snap remove cmake + # build cmake from source + sudo apt-get install -y libssl-dev + wget https://github.com/Kitware/CMake/releases/download/v3.23.0/cmake-3.23.0.tar.gz + tar -zxvf cmake-3.23.0.tar.gz + cd cmake-3.23.0 + ./bootstrap + make -j + sudo make install + # + source ~/.bashrc + cmake --version + fi + + # gcc-7 check + echo_green "Check your gcc version:" + gcc --version + read -p "Upgrade to GCC-7? (>=GCC-7 is required) (y/n)" -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] + then + # Add repository if ubuntu < 18.04 + sudo add-apt-repository ppa:ubuntu-toolchain-r/test + sudo apt-get update + sudo apt-get install gcc-7 + sudo apt-get install g++-7 + GCC + fi + + # tensorrt check + echo_green "Check your TensorRT version:" + ## Check if ${TENSORRT_DIR} env variable has been set + if [ -d "${TENSORRT_DIR}" ]; then + echo_green "TENSORRT_DIR env. variable has been set ${TENSORRT_DIR}" + else + echo_red "TENSORRT_DIR env. variable has NOT been set." + if [[ "$ARCH" == aarch64 ]]; then + echo_green "Added TENSORRT_DIR, CUDNN_DIR to env." + echo 'export TENSORRT_DIR="/usr/include/'${ARCH}'-linux-gnu/"' >> ${HOME}/.bashrc + echo 'export CUDNN_DIR="/usr/include/'${ARCH}'-linux-gnu/"' >> ${HOME}/.bashrc + echo 'export LD_LIBRARY_PATH="/usr/lib/'${ARCH}'-linux-gnu/:$LD_LIBRARY_PATH"' >> ${HOME}/.bashrc + source ${HOME}/.bashrc + exec bash + else + echo_red "Please Install TensorRT, CUDNN and add TENSORRT_DIR, CUDNN_DIR to environment variables before running this script!" + exit + fi + fi + + # Determine TensorRT version and set paths accordingly + echo "Checking TensorRT version...Please verify the detected versions below:" + if [[ "$ARCH" == aarch64 ]]; then + cat /usr/include/${ARCH}-linux-gnu/NvInferVersion.h | grep NV_TENSORRT + else + cat ${TENSORRT_DIR}/include/NvInferVersion.h | grep NV_TENSORRT + fi + read -p "Is TensorRT >=8.0.1.6 installed? (Always installed on Jetson) (y/n)" -n 1 -r + echo + if [[ $REPLY =~ ^[Nn]$ ]] + then + echo "Error: You must install TensorRT before installing MMDeploy!" + exit + fi + + # opencv + sudo apt-get install libopencv-dev +} + +py_venv() { + # deactivate venv, if it has already been activated + source deactivate + + #check for python installed version + pyv="$(python3 -V 2>&1)" + pyv_old="Python 3.6" + + if echo "$pyv" | grep -q "$pyv_old"; then + # use python 3.6 + curl https://bootstrap.pypa.io/pip/3.6/get-pip.py -o get-pip.py + else + # use python >=3.7 + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + fi + + # dependencies + sudo apt-get install protobuf-compiler libprotoc-dev libssl-dev curl ninja-build -y + sudo apt-get install libopenblas-dev python3-venv python3-dev python3-setuptools -y + sudo python3 get-pip.py + pip3 install testresources + pip3 install --upgrade setuptools wheel + + read -p "Remove existing Python venv? (y/n)" -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] + then + rm -r ${PYTHON_VENV_DIR} + python3 -m venv ${PYTHON_VENV_DIR} --system-site-packages #system site packages to keep trt from system installation + fi + + source ${PYTHON_VENV_DIR}/bin/activate + python3 get-pip.py + pip3 install testresources + pip3 install --upgrade setuptools wheel + # protofbuf on jetson is quite old - must be upgraded + pip3 install --upgrade protobuf + # Latest pillow is not compatible with mmcv + pip install Pillow==7.0.0 + + if [[ "$ARCH" == aarch64 ]] + then + # TODO Numpy might be installed per default so we should not remove it + pip3 install numpy==1.19.4 + else + pip3 install numpy + fi + pip3 install opencv-python + pip3 install matplotlib + + # pytorch, torchvision, torchaudio + if [[ "$ARCH" == aarch64 ]] + then + # pytorch + wget https://nvidia.box.com/shared/static/fjtbno0vpo676a25cgvuqc1wty0fkkg6.whl -O torch-1.10.0-cp36-cp36m-linux_aarch64.whl + pip3 install torch-1.10.0-cp36-cp36m-linux_aarch64.whl + # torchvision + sudo apt-get install libjpeg-dev zlib1g-dev libpython3-dev libavcodec-dev libavformat-dev libswscale-dev -y + sudo rm -r torchvision + git clone --branch v0.11.1 https://github.com/pytorch/vision torchvision + cd torchvision + export BUILD_VERSION=0.11.1 # where 0.x.0 is the torchvision version + python3 setup.py install + cd ../ + # torchaudio + #sudo apt-get install -y sox libsox-dev libsox-fmt-all + #sudo rm -r torchaudio + #git clone -b v0.10.0 https://github.com/pytorch/audio torchaudio + #cd torchaudio + #git submodule update --init --recursive + #python3 setup.py install + #cd ../ + # mmcv + pip3 uninstall mmcv-full + pip3 install mmcv-full==1.4.1 -f https://download.openmmlab.com/mmcv/dist/cu102/torch1.10.0/index.html + else + pip3 install torch==1.10.0+cu113 torchvision==0.11.1+cu113 torchaudio==0.10.0+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html + # mmcv + pip3 uninstall mmcv-full + pip3 install mmcv-full==1.4.1 -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10.0/index.html + fi + +} + +pplcv() { + ## ppl.cv - install in /usr/local + cd ${WORKING_DIR} + echo_blue "checking out '${PPLCV_DIR}' pkg..." + if [ -d "${PPLCV_DIR}" ]; then + echo_green "Already exists! Checking out the requested version..." + else + git clone https://github.com/openppl-public/ppl.cv.git ${PPLCV_DIR} + fi + cd ${PPLCV_DIR} + git pull + git checkout tags/v${PPLCV_VER} + mkdir build -p && cd build + cmake -DHPCC_USE_CUDA=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} .. && make -j${processor_num} && sudo make install + sudo ldconfig + # pack as tar.gz file + cd .. + tar -zcvf ${WORKING_DIR}/pplcv_${PPLCV_VER}_cuda-${ARCH}-build.tar.gz build/ +} + +mmdeploy(){ + ## h5py (Required by mmdeploy) + ## h5py not directly supported by jetson and must be built/installed manually + sudo apt-get install pkg-config libhdf5-10* libhdf5-dev -y + sudo pip3 install Cython + sudo env H5PY_SETUP_REQUIRES=0 pip3 install -U h5py==2.9.0 + + ## mmdeploy SDK - install in /usr/local + cd ${WORKING_DIR} + echo_blue "checking out '${MMDEPLOY_DIR}' pkg..." + if [ -d "${MMDEPLOY_DIR}" ]; then + echo_green "Already exists! Checking out the requested version..." + else + git clone https://github.com/open-mmlab/mmdeploy.git ${MMDEPLOY_DIR} + fi + cd ${MMDEPLOY_DIR} + git pull + git checkout tags/v${MMDEPLOY_VER} + # reinit submodules + git submodule update --init --recursive + + if [[ $WITH_PYTHON -eq 1 ]] + then + pip install -e . + fi + rm -r build + mkdir build -p && cd build + cmake .. \ + -DMMDEPLOY_BUILD_SDK=ON \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ + -DCMAKE_CXX_COMPILER=g++ \ + -Dpplcv_DIR=/usr/local/lib/cmake/ppl \ + -DMMDEPLOY_TARGET_DEVICES="cuda;cpu" \ + -DMMDEPLOY_TARGET_BACKENDS=trt \ + -DMMDEPLOY_CODEBASES=all \ + -DTENSORRT_DIR=${TENSORRT_DIR} \ + -DCUDNN_DIR=${CUDNN_DIR} + cmake --build . -- -j$(nproc) && sudo cmake --install . + sudo ldconfig + # pack as tar.gz file + cd .. + tar -zcvf ${WORKING_DIR}/mmdeploysdk_${MMDEPLOY_VER}_${ARCH}-build.tar.gz build/ + # Unpack as tar -zxf mmdeploysdk_*.tar.gz --directory MMDeploy-aarch64 + + ## build mmdeploy examples + cp -r ${WORKING_DIR}/MMDeploy/demo/csrc ${WORKING_DIR}/MMDeploy/build/example + cd ${WORKING_DIR}/MMDeploy/build/example + rm -r build + mkdir build -p && cd build + cmake .. -DMMDeploy_DIR=${INSTALL_PREFIX} + make all +} + +all() { + # build all + prereqs + py_venv + pplcv + mmdeploy +} + +##### +# supported package +package_list=( + "all" + "prereqs" + "py_venv" + "pplcv" + "mmdeploy" +) + +##### +# check input argument +if contains_element "$appargument1" "${package_list[@]}"; then + echo_green "Build and install '$appargument1'..." +else + echo_red "Unsupported argument '$appargument1'. Use one of the following:" + for i in ${package_list[@]} + do + echo $i + done + exit +fi + +# prepare build +if [[ $WITH_PYTHON -eq 1 ]] +then + source venv-mmdet/bin/activate +fi + +# remove all build files +if [[ $WITH_CLEAN -eq 1 ]] +then + sudo rm -r ${PPLCV_DIR}/build + sudo rm -r ${MMDEPLOY_DIR}/build +fi + +$appargument1 \ No newline at end of file From 2b0461d25cfd2cf4be5653a721cd343ced7fc7ce Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Sun, 24 Apr 2022 17:37:45 +0200 Subject: [PATCH 02/16] Fixed lint --- build_mmdeploy.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build_mmdeploy.sh b/build_mmdeploy.sh index 1c14a4ea5e..a2dd15098a 100755 --- a/build_mmdeploy.sh +++ b/build_mmdeploy.sh @@ -2,7 +2,7 @@ # build_mmdeploy.sh # Date: 08-03-2022, 24-04-2022 # -# Run this script to build MMDeploy SDK and prerequisites. +# Run this script to build MMDeploy SDK and prerequisites. # This script will also setup python venv # BUILD_TYPE="Release" @@ -128,7 +128,7 @@ prereqs() { if [[ "$ARCH" == aarch64 ]]; then cat /usr/include/${ARCH}-linux-gnu/NvInferVersion.h | grep NV_TENSORRT else - cat ${TENSORRT_DIR}/include/NvInferVersion.h | grep NV_TENSORRT + cat ${TENSORRT_DIR}/include/NvInferVersion.h | grep NV_TENSORRT fi read -p "Is TensorRT >=8.0.1.6 installed? (Always installed on Jetson) (y/n)" -n 1 -r echo @@ -203,7 +203,7 @@ py_venv() { sudo rm -r torchvision git clone --branch v0.11.1 https://github.com/pytorch/vision torchvision cd torchvision - export BUILD_VERSION=0.11.1 # where 0.x.0 is the torchvision version + export BUILD_VERSION=0.11.1 # where 0.x.0 is the torchvision version python3 setup.py install cd ../ # torchaudio @@ -343,4 +343,4 @@ then sudo rm -r ${MMDEPLOY_DIR}/build fi -$appargument1 \ No newline at end of file +$appargument1 From 0ff5c5e2746b0df9b18dc1cf3b94d1632cb5e62e Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Sat, 30 Apr 2022 18:39:36 +0200 Subject: [PATCH 03/16] * Set prefix depending on install arch * Auto detect installed version of cmake, gcc. Update if necessary * Prompt to reinstall venv if exists * Disable .tar packing * Enable MMDEPLOY SDK Python * Misc cleanup --- build_mmdeploy.sh | 175 +++++++++++++++++++++++++++------------------- 1 file changed, 105 insertions(+), 70 deletions(-) diff --git a/build_mmdeploy.sh b/build_mmdeploy.sh index a2dd15098a..23fb068ce7 100755 --- a/build_mmdeploy.sh +++ b/build_mmdeploy.sh @@ -1,23 +1,47 @@ #!/bin/bash -# build_mmdeploy.sh +# build_linux_nvidia.sh # Date: 08-03-2022, 24-04-2022 # -# Run this script to build MMDeploy SDK and prerequisites. +# Run this script to build MMDeploy SDK and necessary prerequisites. # This script will also setup python venv # + +##### +# Build vars BUILD_TYPE="Release" ARCH=$(uname -i) -WITH_PYTHON=1 -WITH_CLEAN=1 -# Get path to script to ensure script runs from 'logends' root -WORKING_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROC_NUM=$(nproc) +# Default GCC +GCC_COMPILER="g++" +##### +# Directories +# WORKING_DIR must correspond to script dir, i.e. MMDeploy root +WORKING_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" PPLCV_DIR=${WORKING_DIR}/ppl.cv MMDEPLOY_DIR=${WORKING_DIR}/MMDeploy + +##### +# Versions PPLCV_VER="0.6.2" MMDEPLOY_VER="0.4.0" -INSTALL_PREFIX="/usr/local" -PYTHON_VENV_DIR="mmdeploy" +CMAKE_VER="3.23.0" + +##### +# Flags +# WITH_PYTHON: Install misc. dependencies in the active venv +WITH_PYTHON=1 +# WITH_CLEAN: Remove build output dirs +WITH_CLEAN=1 + +##### +# Prefix: Set install prefix for ppl.cv, mmdeploy SDK depending on arch +if [[ "$ARCH" == aarch64 ]]; then + INSTALL_PREFIX="/usr/local/aarch64-linux-gnu" +else + INSTALL_PREFIX="/usr/local" +fi +PYTHON_VENV_DIR=${WORKING_DIR}/venv-mmdeploy appargument1=$1 #appargument2=$2 @@ -43,9 +67,13 @@ contains_element () { for e; do [[ "$e" == "$match" ]] && return 0; done return 1 } +function version { + echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; +} prereqs() { # spdlog + echo_green "Checking spdlog version..." read -p "Install latest spdlog from source? (y/n)" -n 1 -r echo if [[ $REPLY =~ ^[Yy]$ ]] @@ -60,47 +88,41 @@ prereqs() { git checkout tags/v1.8.1 mkdir build -p && cd build # we must build spdlog with -fPIC enabled - cmake .. -DCMAKE_POSITION_INDEPENDENT_CODE=ON && make -j + cmake .. -DCMAKE_POSITION_INDEPENDENT_CODE=ON && make -j${PROC_NUM} sudo make install sudo ldconfig fi - # cmake check - echo_green "Check your cmake version:" - cmake --version - read -p "Install latest CMake? (>=3.18 is required) (y/n)" -n 1 -r - echo - if [[ $REPLY =~ ^[Yy]$ ]] - then + # cmake check & install + echo_green "Checking your cmake version..." + CMAKE_DETECT_VER=$(cmake --version | grep -oP '(?<=version).*') + if [ $(version $CMAKE_DETECT_VER) -ge $(version "3.14.0") ]; then + echo_green "Cmake version $CMAKE_DETECT_VER is up to date" + else + echo_green "CMake too old, purging existing cmake and installing ${CMAKE_VER}..." # purge existing sudo apt-get purge cmake sudo snap remove cmake - # build cmake from source - sudo apt-get install -y libssl-dev - wget https://github.com/Kitware/CMake/releases/download/v3.23.0/cmake-3.23.0.tar.gz - tar -zxvf cmake-3.23.0.tar.gz - cd cmake-3.23.0 - ./bootstrap - make -j - sudo make install - # - source ~/.bashrc - cmake --version + # install prebuild + wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-linux-${ARCH}.sh + chmod +x cmake-${CMAKE_VER}-linux-${ARCH}.sh + sudo ./cmake-${CMAKE_VER}-linux-${ARCH}.sh --prefix=/usr --skip-license fi # gcc-7 check - echo_green "Check your gcc version:" - gcc --version - read -p "Upgrade to GCC-7? (>=GCC-7 is required) (y/n)" -n 1 -r - echo - if [[ $REPLY =~ ^[Yy]$ ]] - then + echo_green "Checking your gcc version..." + GCC_DETECT_VER=$(gcc --version | grep -oP '(?<=\)).*' -m1) + if [ $(version $GCC_DETECT_VER) -ge $(version "7.0.0") ]; then + echo_green "GCC version $GCC_DETECT_VER is up to date" + else + echo_green "gcc version too old, installing ${CMAKE_VER}..." + echo_green "Purge existing cmake and install ${GCC_DETECT_VER}..." # Add repository if ubuntu < 18.04 sudo add-apt-repository ppa:ubuntu-toolchain-r/test sudo apt-get update sudo apt-get install gcc-7 sudo apt-get install g++-7 - GCC + GCC_COMPILER="g++-7" fi # tensorrt check @@ -116,6 +138,7 @@ prereqs() { echo 'export CUDNN_DIR="/usr/include/'${ARCH}'-linux-gnu/"' >> ${HOME}/.bashrc echo 'export LD_LIBRARY_PATH="/usr/lib/'${ARCH}'-linux-gnu/:$LD_LIBRARY_PATH"' >> ${HOME}/.bashrc source ${HOME}/.bashrc + echo_green "Please re-run this script for changes to apply!" exec bash else echo_red "Please Install TensorRT, CUDNN and add TENSORRT_DIR, CUDNN_DIR to environment variables before running this script!" @@ -144,7 +167,7 @@ prereqs() { py_venv() { # deactivate venv, if it has already been activated - source deactivate + deactivate #check for python installed version pyv="$(python3 -V 2>&1)" @@ -165,11 +188,15 @@ py_venv() { pip3 install testresources pip3 install --upgrade setuptools wheel - read -p "Remove existing Python venv? (y/n)" -n 1 -r - echo - if [[ $REPLY =~ ^[Yy]$ ]] - then - rm -r ${PYTHON_VENV_DIR} + if [ -d "${PYTHON_VENV_DIR}" ]; then + read -p "Reinstall existing Python venv ${PYTHON_VENV_DIR}? (y/n)" -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] + then + rm -r ${PYTHON_VENV_DIR} + python3 -m venv ${PYTHON_VENV_DIR} --system-site-packages #system site packages to keep trt from system installation + fi + else python3 -m venv ${PYTHON_VENV_DIR} --system-site-packages #system site packages to keep trt from system installation fi @@ -177,14 +204,16 @@ py_venv() { python3 get-pip.py pip3 install testresources pip3 install --upgrade setuptools wheel - # protofbuf on jetson is quite old - must be upgraded - pip3 install --upgrade protobuf - # Latest pillow is not compatible with mmcv + + # Latest PIL is not compatible with mmcv=1.4.1 pip install Pillow==7.0.0 if [[ "$ARCH" == aarch64 ]] then - # TODO Numpy might be installed per default so we should not remove it + # protofbuf on jetson is quite old - must be upgraded + pip3 install --upgrade protobuf + # Install numpy 1.19.4 as newer versions might give "Illegal instruction (core dumped)" on Jetson + # TODO Numpy might be installed per default so we should not remove it. pip3 install numpy==1.19.4 else pip3 install numpy @@ -223,11 +252,13 @@ py_venv() { pip3 uninstall mmcv-full pip3 install mmcv-full==1.4.1 -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10.0/index.html fi - + + # deactivate python venv again + deactivate } pplcv() { - ## ppl.cv - install in /usr/local + ## ppl.cv cd ${WORKING_DIR} echo_blue "checking out '${PPLCV_DIR}' pkg..." if [ -d "${PPLCV_DIR}" ]; then @@ -239,21 +270,16 @@ pplcv() { git pull git checkout tags/v${PPLCV_VER} mkdir build -p && cd build - cmake -DHPCC_USE_CUDA=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} .. && make -j${processor_num} && sudo make install + cmake -DHPCC_USE_CUDA=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} .. && make -j${PROC_NUM} && sudo make install sudo ldconfig - # pack as tar.gz file - cd .. - tar -zcvf ${WORKING_DIR}/pplcv_${PPLCV_VER}_cuda-${ARCH}-build.tar.gz build/ + + # TODO pack prebuild as tar.gz file + #cd .. + #tar -zcvf ${WORKING_DIR}/pplcv_${PPLCV_VER}_cuda-${ARCH}-build.tar.gz build/ } mmdeploy(){ - ## h5py (Required by mmdeploy) - ## h5py not directly supported by jetson and must be built/installed manually - sudo apt-get install pkg-config libhdf5-10* libhdf5-dev -y - sudo pip3 install Cython - sudo env H5PY_SETUP_REQUIRES=0 pip3 install -U h5py==2.9.0 - - ## mmdeploy SDK - install in /usr/local + ## mmdeploy SDK cd ${WORKING_DIR} echo_blue "checking out '${MMDEPLOY_DIR}' pkg..." if [ -d "${MMDEPLOY_DIR}" ]; then @@ -267,27 +293,39 @@ mmdeploy(){ # reinit submodules git submodule update --init --recursive + # python dependencies if [[ $WITH_PYTHON -eq 1 ]] then + source ${PYTHON_VENV_DIR}/bin/activate + + ## h5py (Required by mmdeploy) + ## h5py not directly supported by jetson and must be built/installed manually + sudo apt-get install pkg-config libhdf5-10* libhdf5-dev -y + sudo pip3 install Cython + sudo env H5PY_SETUP_REQUIRES=0 pip3 install -U h5py==2.9.0 + pip install -e . fi - rm -r build + + # build mkdir build -p && cd build cmake .. \ -DMMDEPLOY_BUILD_SDK=ON \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - -DCMAKE_CXX_COMPILER=g++ \ - -Dpplcv_DIR=/usr/local/lib/cmake/ppl \ + -DCMAKE_CXX_COMPILER=${GCC_COMPILER} \ + -Dpplcv_DIR=${INSTALL_PREFIX}/lib/cmake/ppl \ -DMMDEPLOY_TARGET_DEVICES="cuda;cpu" \ -DMMDEPLOY_TARGET_BACKENDS=trt \ -DMMDEPLOY_CODEBASES=all \ + -DMMDEPLOY_BUILD_SDK_PYTHON_API=ON \ -DTENSORRT_DIR=${TENSORRT_DIR} \ -DCUDNN_DIR=${CUDNN_DIR} - cmake --build . -- -j$(nproc) && sudo cmake --install . + cmake --build . -- -j${PROC_NUM} && sudo cmake --install . sudo ldconfig - # pack as tar.gz file - cd .. - tar -zcvf ${WORKING_DIR}/mmdeploysdk_${MMDEPLOY_VER}_${ARCH}-build.tar.gz build/ + + # TODO Pack build output as tar.gz file + #cd .. + # tar -zcvf ${WORKING_DIR}/mmdeploysdk_${MMDEPLOY_VER}_${ARCH}-build.tar.gz build/ # Unpack as tar -zxf mmdeploysdk_*.tar.gz --directory MMDeploy-aarch64 ## build mmdeploy examples @@ -297,6 +335,9 @@ mmdeploy(){ mkdir build -p && cd build cmake .. -DMMDeploy_DIR=${INSTALL_PREFIX} make all + + # deactivate python venv again + deactivate } all() { @@ -330,12 +371,6 @@ else exit fi -# prepare build -if [[ $WITH_PYTHON -eq 1 ]] -then - source venv-mmdet/bin/activate -fi - # remove all build files if [[ $WITH_CLEAN -eq 1 ]] then From 90a505186edc5b9d1a7268d62dee62b963f97951 Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Sat, 30 Apr 2022 18:43:47 +0200 Subject: [PATCH 04/16] Fixed lint --- build_mmdeploy.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build_mmdeploy.sh b/build_mmdeploy.sh index 23fb068ce7..5960039ab9 100755 --- a/build_mmdeploy.sh +++ b/build_mmdeploy.sh @@ -67,8 +67,8 @@ contains_element () { for e; do [[ "$e" == "$match" ]] && return 0; done return 1 } -function version { - echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; +function version { + echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } prereqs() { @@ -213,7 +213,7 @@ py_venv() { # protofbuf on jetson is quite old - must be upgraded pip3 install --upgrade protobuf # Install numpy 1.19.4 as newer versions might give "Illegal instruction (core dumped)" on Jetson - # TODO Numpy might be installed per default so we should not remove it. + # TODO Numpy might be installed per default so we should not remove it. pip3 install numpy==1.19.4 else pip3 install numpy @@ -252,7 +252,7 @@ py_venv() { pip3 uninstall mmcv-full pip3 install mmcv-full==1.4.1 -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10.0/index.html fi - + # deactivate python venv again deactivate } @@ -306,7 +306,7 @@ mmdeploy(){ pip install -e . fi - + # build mkdir build -p && cd build cmake .. \ From 05f67d5db6afc3210c1148c7953e339ef23dfa36 Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Sat, 30 Apr 2022 19:14:32 +0200 Subject: [PATCH 05/16] Renamed script --- build_mmdeploy.sh => build_linux_nvidia.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build_mmdeploy.sh => build_linux_nvidia.sh (100%) diff --git a/build_mmdeploy.sh b/build_linux_nvidia.sh similarity index 100% rename from build_mmdeploy.sh rename to build_linux_nvidia.sh From 11452c5295b75538f919dcf568cadfcde3274f0b Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Fri, 6 May 2022 11:14:36 +0200 Subject: [PATCH 06/16] * Set env variables without restarting script. * OpenCV install optional. * Fix numpy OPENBLAS_CORETYPE on numpy >1.19.4. * Remove specific build files. * Fix prebuild output .tar.gz. * Comments added. exec bash when closing script --- build_linux_nvidia.sh | 73 ++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/build_linux_nvidia.sh b/build_linux_nvidia.sh index 5960039ab9..6d990d3d03 100755 --- a/build_linux_nvidia.sh +++ b/build_linux_nvidia.sh @@ -137,9 +137,11 @@ prereqs() { echo 'export TENSORRT_DIR="/usr/include/'${ARCH}'-linux-gnu/"' >> ${HOME}/.bashrc echo 'export CUDNN_DIR="/usr/include/'${ARCH}'-linux-gnu/"' >> ${HOME}/.bashrc echo 'export LD_LIBRARY_PATH="/usr/lib/'${ARCH}'-linux-gnu/:$LD_LIBRARY_PATH"' >> ${HOME}/.bashrc - source ${HOME}/.bashrc - echo_green "Please re-run this script for changes to apply!" - exec bash + #source ${HOME}/.bashrc + # sourcing in bash script won't set the env. variables so we will set them temporarily + export TENSORRT_DIR="/usr/include/'${ARCH}'-linux-gnu/" + export CUDNN_DIR="/usr/include/'${ARCH}'-linux-gnu/" + export LD_LIBRARY_PATH="/usr/lib/'${ARCH}'-linux-gnu/:$LD_LIBRARY_PATH" else echo_red "Please Install TensorRT, CUDNN and add TENSORRT_DIR, CUDNN_DIR to environment variables before running this script!" exit @@ -161,8 +163,13 @@ prereqs() { exit fi - # opencv - sudo apt-get install libopencv-dev + read -p "Install OpenCV? (Always installed on Jetson) (y/n)" -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] + then + # opencv + sudo apt-get install libopencv-dev + fi } py_venv() { @@ -212,12 +219,14 @@ py_venv() { then # protofbuf on jetson is quite old - must be upgraded pip3 install --upgrade protobuf - # Install numpy 1.19.4 as newer versions might give "Illegal instruction (core dumped)" on Jetson - # TODO Numpy might be installed per default so we should not remove it. - pip3 install numpy==1.19.4 - else - pip3 install numpy + # Install numpy >1.19.4 might give "Illegal instruction (core dumped)" on Jetson + # To solve it, we should set OPENBLAS_CORETYPE + echo 'export OPENBLAS_CORETYPE=ARMV8' >> ~/.bashrc + #source ${HOME}/.bashrc + # sourcing in bash script won't set the env. variables so we will set them temporarily + export OPENBLAS_CORETYPE=ARMV8 fi + pip3 install numpy pip3 install opencv-python pip3 install matplotlib @@ -269,13 +278,21 @@ pplcv() { cd ${PPLCV_DIR} git pull git checkout tags/v${PPLCV_VER} + + # remove all build files + if [[ $WITH_CLEAN -eq 1 ]] + then + sudo rm -r ${PPLCV_DIR}/build + fi + + # build mkdir build -p && cd build - cmake -DHPCC_USE_CUDA=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} .. && make -j${PROC_NUM} && sudo make install + cmake -DHPCC_USE_CUDA=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} .. && make -j${processor_num} && sudo make install sudo ldconfig - # TODO pack prebuild as tar.gz file - #cd .. - #tar -zcvf ${WORKING_DIR}/pplcv_${PPLCV_VER}_cuda-${ARCH}-build.tar.gz build/ + # generate prebuild and pack into .tar.gz + sudo make DESTDIR=./prebuild install + tar -zcvf ${WORKING_DIR}/pplcv_${PPLCV_VER}_cuda-${ARCH}-build.tar.gz -C ./prebuild/ . } mmdeploy(){ @@ -307,6 +324,12 @@ mmdeploy(){ pip install -e . fi + # remove all build files + if [[ $WITH_CLEAN -eq 1 ]] + then + sudo rm -r ${MMDEPLOY_DIR}/build + fi + # build mkdir build -p && cd build cmake .. \ @@ -323,9 +346,9 @@ mmdeploy(){ cmake --build . -- -j${PROC_NUM} && sudo cmake --install . sudo ldconfig - # TODO Pack build output as tar.gz file - #cd .. - # tar -zcvf ${WORKING_DIR}/mmdeploysdk_${MMDEPLOY_VER}_${ARCH}-build.tar.gz build/ + # generate prebuild and pack into .tar.gz + sudo make DESTDIR=./prebuild install + tar -zcvf ${WORKING_DIR}/mmdeploysdk_${MMDEPLOY_VER}_${ARCH}-build.tar.gz -C ./prebuild/ . # Unpack as tar -zxf mmdeploysdk_*.tar.gz --directory MMDeploy-aarch64 ## build mmdeploy examples @@ -333,10 +356,10 @@ mmdeploy(){ cd ${WORKING_DIR}/MMDeploy/build/example rm -r build mkdir build -p && cd build - cmake .. -DMMDeploy_DIR=${INSTALL_PREFIX} + cmake -DMMDeploy_DIR=${INSTALL_PREFIX} .. make all - # deactivate python venv again + # deactivate venv before exit deactivate } @@ -371,11 +394,9 @@ else exit fi -# remove all build files -if [[ $WITH_CLEAN -eq 1 ]] -then - sudo rm -r ${PPLCV_DIR}/build - sudo rm -r ${MMDEPLOY_DIR}/build -fi - $appargument1 + +cd ${WORKING_DIR} + +# update env. variables +exec bash From 0c1b354cf320bdc4533c0146baa207cf6a1cb698 Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Mon, 9 May 2022 07:58:50 +0200 Subject: [PATCH 07/16] * Use root dir as MMDeploy dir. * Use pip3. * Remove explicit MMDeploy checkout version. * Use MMDEPLOY_DIR for examples path --- build_linux_nvidia.sh | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/build_linux_nvidia.sh b/build_linux_nvidia.sh index 6d990d3d03..62bb9b90c7 100755 --- a/build_linux_nvidia.sh +++ b/build_linux_nvidia.sh @@ -19,12 +19,11 @@ GCC_COMPILER="g++" # WORKING_DIR must correspond to script dir, i.e. MMDeploy root WORKING_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" PPLCV_DIR=${WORKING_DIR}/ppl.cv -MMDEPLOY_DIR=${WORKING_DIR}/MMDeploy +MMDEPLOY_DIR=${WORKING_DIR} ##### # Versions PPLCV_VER="0.6.2" -MMDEPLOY_VER="0.4.0" CMAKE_VER="3.23.0" ##### @@ -213,7 +212,7 @@ py_venv() { pip3 install --upgrade setuptools wheel # Latest PIL is not compatible with mmcv=1.4.1 - pip install Pillow==7.0.0 + pip3 install Pillow==7.0.0 if [[ "$ARCH" == aarch64 ]] then @@ -297,16 +296,10 @@ pplcv() { mmdeploy(){ ## mmdeploy SDK - cd ${WORKING_DIR} - echo_blue "checking out '${MMDEPLOY_DIR}' pkg..." - if [ -d "${MMDEPLOY_DIR}" ]; then - echo_green "Already exists! Checking out the requested version..." - else - git clone https://github.com/open-mmlab/mmdeploy.git ${MMDEPLOY_DIR} - fi cd ${MMDEPLOY_DIR} - git pull - git checkout tags/v${MMDEPLOY_VER} + + MMDEPLOY_DETECT_VER=$(cat MMDeploy/mmdeploy/version.py | grep -Eo '[0-9]\.[0-9].[0-9]+') + # reinit submodules git submodule update --init --recursive @@ -352,8 +345,8 @@ mmdeploy(){ # Unpack as tar -zxf mmdeploysdk_*.tar.gz --directory MMDeploy-aarch64 ## build mmdeploy examples - cp -r ${WORKING_DIR}/MMDeploy/demo/csrc ${WORKING_DIR}/MMDeploy/build/example - cd ${WORKING_DIR}/MMDeploy/build/example + cp -r ${MMDEPLOY_DIR}/demo/csrc ${MMDEPLOY_DIR}/build/example + cd ${MMDEPLOY_DIR}/build/example rm -r build mkdir build -p && cd build cmake -DMMDeploy_DIR=${INSTALL_PREFIX} .. From 99deb31ffdd44352dcc6a70aae97eb0727a7ce3e Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Mon, 9 May 2022 08:04:58 +0200 Subject: [PATCH 08/16] Make PyTorch, torchvision, mmcv install optional --- build_linux_nvidia.sh | 63 +++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/build_linux_nvidia.sh b/build_linux_nvidia.sh index 62bb9b90c7..d296b03334 100755 --- a/build_linux_nvidia.sh +++ b/build_linux_nvidia.sh @@ -229,36 +229,41 @@ py_venv() { pip3 install opencv-python pip3 install matplotlib - # pytorch, torchvision, torchaudio - if [[ "$ARCH" == aarch64 ]] + read -p "Install PyTorch, Torchvision, mmcv? (y/n)" -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] then - # pytorch - wget https://nvidia.box.com/shared/static/fjtbno0vpo676a25cgvuqc1wty0fkkg6.whl -O torch-1.10.0-cp36-cp36m-linux_aarch64.whl - pip3 install torch-1.10.0-cp36-cp36m-linux_aarch64.whl - # torchvision - sudo apt-get install libjpeg-dev zlib1g-dev libpython3-dev libavcodec-dev libavformat-dev libswscale-dev -y - sudo rm -r torchvision - git clone --branch v0.11.1 https://github.com/pytorch/vision torchvision - cd torchvision - export BUILD_VERSION=0.11.1 # where 0.x.0 is the torchvision version - python3 setup.py install - cd ../ - # torchaudio - #sudo apt-get install -y sox libsox-dev libsox-fmt-all - #sudo rm -r torchaudio - #git clone -b v0.10.0 https://github.com/pytorch/audio torchaudio - #cd torchaudio - #git submodule update --init --recursive - #python3 setup.py install - #cd ../ - # mmcv - pip3 uninstall mmcv-full - pip3 install mmcv-full==1.4.1 -f https://download.openmmlab.com/mmcv/dist/cu102/torch1.10.0/index.html - else - pip3 install torch==1.10.0+cu113 torchvision==0.11.1+cu113 torchaudio==0.10.0+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html - # mmcv - pip3 uninstall mmcv-full - pip3 install mmcv-full==1.4.1 -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10.0/index.html + # pytorch, torchvision, torchaudio + if [[ "$ARCH" == aarch64 ]] + then + # pytorch + wget https://nvidia.box.com/shared/static/fjtbno0vpo676a25cgvuqc1wty0fkkg6.whl -O torch-1.10.0-cp36-cp36m-linux_aarch64.whl + pip3 install torch-1.10.0-cp36-cp36m-linux_aarch64.whl + # torchvision + sudo apt-get install libjpeg-dev zlib1g-dev libpython3-dev libavcodec-dev libavformat-dev libswscale-dev -y + sudo rm -r torchvision + git clone --branch v0.11.1 https://github.com/pytorch/vision torchvision + cd torchvision + export BUILD_VERSION=0.11.1 # where 0.x.0 is the torchvision version + python3 setup.py install + cd ../ + # torchaudio + #sudo apt-get install -y sox libsox-dev libsox-fmt-all + #sudo rm -r torchaudio + #git clone -b v0.10.0 https://github.com/pytorch/audio torchaudio + #cd torchaudio + #git submodule update --init --recursive + #python3 setup.py install + #cd ../ + # mmcv + pip3 uninstall mmcv-full + pip3 install mmcv-full==1.4.1 -f https://download.openmmlab.com/mmcv/dist/cu102/torch1.10.0/index.html + else + pip3 install torch==1.10.0+cu113 torchvision==0.11.1+cu113 torchaudio==0.10.0+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html + # mmcv + pip3 uninstall mmcv-full + pip3 install mmcv-full==1.4.1 -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10.0/index.html + fi fi # deactivate python venv again From cc4d26acd19ff558ff82aae32369a1b9efd6de9f Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Tue, 17 May 2022 09:01:43 +0200 Subject: [PATCH 09/16] * Do not deactivate python venv. * Make prebuild archive optional. * Add venv* to gitignore --- .gitignore | 1 + build_linux_nvidia.sh | 26 ++++++++++++-------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 3c4bdc851e..80755c38ee 100644 --- a/.gitignore +++ b/.gitignore @@ -91,6 +91,7 @@ venv/ ENV/ env.bak/ venv.bak/ +venv* # Spyder project settings .spyderproject diff --git a/build_linux_nvidia.sh b/build_linux_nvidia.sh index d296b03334..617fb928fd 100755 --- a/build_linux_nvidia.sh +++ b/build_linux_nvidia.sh @@ -32,6 +32,8 @@ CMAKE_VER="3.23.0" WITH_PYTHON=1 # WITH_CLEAN: Remove build output dirs WITH_CLEAN=1 +# WITH_PREBUILD: Generate prebuild archives +WITH_PREBUILD=0 ##### # Prefix: Set install prefix for ppl.cv, mmdeploy SDK depending on arch @@ -172,9 +174,6 @@ prereqs() { } py_venv() { - # deactivate venv, if it has already been activated - deactivate - #check for python installed version pyv="$(python3 -V 2>&1)" pyv_old="Python 3.6" @@ -265,9 +264,6 @@ py_venv() { pip3 install mmcv-full==1.4.1 -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10.0/index.html fi fi - - # deactivate python venv again - deactivate } pplcv() { @@ -295,8 +291,11 @@ pplcv() { sudo ldconfig # generate prebuild and pack into .tar.gz - sudo make DESTDIR=./prebuild install - tar -zcvf ${WORKING_DIR}/pplcv_${PPLCV_VER}_cuda-${ARCH}-build.tar.gz -C ./prebuild/ . + if [[ $WITH_PREBUILD -eq 1 ]] + then + sudo make DESTDIR=./prebuild install + tar -zcvf ${WORKING_DIR}/pplcv_${PPLCV_VER}_cuda-${ARCH}-build.tar.gz -C ./prebuild/ . + fi } mmdeploy(){ @@ -345,9 +344,11 @@ mmdeploy(){ sudo ldconfig # generate prebuild and pack into .tar.gz - sudo make DESTDIR=./prebuild install - tar -zcvf ${WORKING_DIR}/mmdeploysdk_${MMDEPLOY_VER}_${ARCH}-build.tar.gz -C ./prebuild/ . - # Unpack as tar -zxf mmdeploysdk_*.tar.gz --directory MMDeploy-aarch64 + if [[ $WITH_PREBUILD -eq 1 ]] + then + sudo make DESTDIR=./prebuild install + tar -zcvf ${WORKING_DIR}/mmdeploysdk_${MMDEPLOY_VER}_${ARCH}-build.tar.gz -C ./prebuild/ . + fi ## build mmdeploy examples cp -r ${MMDEPLOY_DIR}/demo/csrc ${MMDEPLOY_DIR}/build/example @@ -356,9 +357,6 @@ mmdeploy(){ mkdir build -p && cd build cmake -DMMDeploy_DIR=${INSTALL_PREFIX} .. make all - - # deactivate venv before exit - deactivate } all() { From ebe88ff55761f867b1dfd2e79136713a3eb461ed Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Thu, 19 May 2022 16:40:18 +0200 Subject: [PATCH 10/16] * Add support for unattended, interactive install. * Cleanup misc. comments. --- build_linux_nvidia.sh | 141 ++++++++++++++++++++++++++++++++---------- 1 file changed, 110 insertions(+), 31 deletions(-) diff --git a/build_linux_nvidia.sh b/build_linux_nvidia.sh index 617fb928fd..285d51c4de 100755 --- a/build_linux_nvidia.sh +++ b/build_linux_nvidia.sh @@ -2,8 +2,8 @@ # build_linux_nvidia.sh # Date: 08-03-2022, 24-04-2022 # -# Run this script to build MMDeploy SDK and necessary prerequisites. -# This script will also setup python venv +# Run this script to build MMDeploy SDK and install necessary prerequisites. +# This script will also setup python venv and generate prebuild binaries if requested. # ##### @@ -34,6 +34,8 @@ WITH_PYTHON=1 WITH_CLEAN=1 # WITH_PREBUILD: Generate prebuild archives WITH_PREBUILD=0 +# WITH_UNATTENDED: Unattended install, skip/use default options +WITH_UNATTENDED=0 ##### # Prefix: Set install prefix for ppl.cv, mmdeploy SDK depending on arch @@ -45,7 +47,7 @@ fi PYTHON_VENV_DIR=${WORKING_DIR}/venv-mmdeploy appargument1=$1 -#appargument2=$2 +appargument2=$2 echo_green() { if [ -n "$1" ]; then @@ -71,18 +73,46 @@ contains_element () { function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } +prompt_yesno() { + if [ -n "$1" ]; then + echo_blue "$1" + fi + if [[ $WITH_UNATTENDED -eq 1 ]] + then + echo_green "Unattended install, selecting default option" + return 2 + else + read -p "(y/n/q)" -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] + then + return 1 + elif [[ $REPLY =~ ^[Nn]$ ]] + then + return 0 + elif [[ $REPLY =~ ^[Qq]$ ]] + then + echo_green "Quitting!" + exit + else + echo_red "Invalid argument. Try again" + prompt_yesno + fi + fi +} prereqs() { # spdlog echo_green "Checking spdlog version..." - read -p "Install latest spdlog from source? (y/n)" -n 1 -r - echo - if [[ $REPLY =~ ^[Yy]$ ]] + prompt_yesno "Install latest spdlog from source? (Default:no)" + local res=$? + if [[ $res -eq 1 ]] # || [ $res -eq 2 ] then + echo_green "Building and installing latest spdlog from source" + # remove libspdlog, as it might be an old version sudo apt-get remove libspdlog-dev -y - echo "Building and installing latest spdlog from source" git clone https://github.com/gabime/spdlog.git spdlog cd spdlog git pull @@ -98,9 +128,9 @@ prereqs() { echo_green "Checking your cmake version..." CMAKE_DETECT_VER=$(cmake --version | grep -oP '(?<=version).*') if [ $(version $CMAKE_DETECT_VER) -ge $(version "3.14.0") ]; then - echo_green "Cmake version $CMAKE_DETECT_VER is up to date" + echo "Cmake version $CMAKE_DETECT_VER is up to date" else - echo_green "CMake too old, purging existing cmake and installing ${CMAKE_VER}..." + echo "CMake too old, purging existing cmake and installing ${CMAKE_VER}..." # purge existing sudo apt-get purge cmake sudo snap remove cmake @@ -114,10 +144,10 @@ prereqs() { echo_green "Checking your gcc version..." GCC_DETECT_VER=$(gcc --version | grep -oP '(?<=\)).*' -m1) if [ $(version $GCC_DETECT_VER) -ge $(version "7.0.0") ]; then - echo_green "GCC version $GCC_DETECT_VER is up to date" + echo "GCC version $GCC_DETECT_VER is up to date" else - echo_green "gcc version too old, installing ${CMAKE_VER}..." - echo_green "Purge existing cmake and install ${GCC_DETECT_VER}..." + echo "gcc version too old, installing ${CMAKE_VER}..." + echo "Purge existing cmake and install ${GCC_DETECT_VER}..." # Add repository if ubuntu < 18.04 sudo add-apt-repository ppa:ubuntu-toolchain-r/test sudo apt-get update @@ -130,11 +160,11 @@ prereqs() { echo_green "Check your TensorRT version:" ## Check if ${TENSORRT_DIR} env variable has been set if [ -d "${TENSORRT_DIR}" ]; then - echo_green "TENSORRT_DIR env. variable has been set ${TENSORRT_DIR}" + echo "TENSORRT_DIR env. variable has been set ${TENSORRT_DIR}" else echo_red "TENSORRT_DIR env. variable has NOT been set." if [[ "$ARCH" == aarch64 ]]; then - echo_green "Added TENSORRT_DIR, CUDNN_DIR to env." + echo "Added TENSORRT_DIR, CUDNN_DIR to env." echo 'export TENSORRT_DIR="/usr/include/'${ARCH}'-linux-gnu/"' >> ${HOME}/.bashrc echo 'export CUDNN_DIR="/usr/include/'${ARCH}'-linux-gnu/"' >> ${HOME}/.bashrc echo 'export LD_LIBRARY_PATH="/usr/lib/'${ARCH}'-linux-gnu/:$LD_LIBRARY_PATH"' >> ${HOME}/.bashrc @@ -156,18 +186,21 @@ prereqs() { else cat ${TENSORRT_DIR}/include/NvInferVersion.h | grep NV_TENSORRT fi - read -p "Is TensorRT >=8.0.1.6 installed? (Always installed on Jetson) (y/n)" -n 1 -r - echo - if [[ $REPLY =~ ^[Nn]$ ]] + prompt_yesno "Is TensorRT >=8.0.1.6 installed? (Always installed on Jetson) (Default:yes)" + local res=$? + if [[ $res -eq 1 ]] || [ $res -eq 2 ] then - echo "Error: You must install TensorRT before installing MMDeploy!" - exit + echo "TensorRT appears to be installed..." + else + echo_red "Error: You must install TensorRT before installing MMDeploy!" + exit fi - read -p "Install OpenCV? (Always installed on Jetson) (y/n)" -n 1 -r - echo - if [[ $REPLY =~ ^[Yy]$ ]] + prompt_yesno "Install OpenCV? (Always installed on Jetson) (Default:no)" + local res=$? + if [[ $res -eq 1 ]] # || [ $res -eq 2 ] then + echo "Installing libopencv-dev..." # opencv sudo apt-get install libopencv-dev fi @@ -194,9 +227,9 @@ py_venv() { pip3 install --upgrade setuptools wheel if [ -d "${PYTHON_VENV_DIR}" ]; then - read -p "Reinstall existing Python venv ${PYTHON_VENV_DIR}? (y/n)" -n 1 -r - echo - if [[ $REPLY =~ ^[Yy]$ ]] + prompt_yesno "Reinstall existing Python venv ${PYTHON_VENV_DIR}?" + local res=$? + if [[ $res -eq 1 ]] then rm -r ${PYTHON_VENV_DIR} python3 -m venv ${PYTHON_VENV_DIR} --system-site-packages #system site packages to keep trt from system installation @@ -217,7 +250,7 @@ py_venv() { then # protofbuf on jetson is quite old - must be upgraded pip3 install --upgrade protobuf - # Install numpy >1.19.4 might give "Illegal instruction (core dumped)" on Jetson + # Install numpy >1.19.4 might give "Illegal instruction (core dumped)" on Jetson/aarch64 # To solve it, we should set OPENBLAS_CORETYPE echo 'export OPENBLAS_CORETYPE=ARMV8' >> ~/.bashrc #source ${HOME}/.bashrc @@ -228,9 +261,9 @@ py_venv() { pip3 install opencv-python pip3 install matplotlib - read -p "Install PyTorch, Torchvision, mmcv? (y/n)" -n 1 -r - echo - if [[ $REPLY =~ ^[Yy]$ ]] + prompt_yesno "Install PyTorch, Torchvision, mmcv in the active venv? (Default:no)" + local res=$? + if [[ $res -eq 1 ]] then # pytorch, torchvision, torchaudio if [[ "$ARCH" == aarch64 ]] @@ -264,6 +297,9 @@ py_venv() { pip3 install mmcv-full==1.4.1 -f https://download.openmmlab.com/mmcv/dist/cu113/torch1.10.0/index.html fi fi + + # cleanup + rm get-pip.py } pplcv() { @@ -271,7 +307,7 @@ pplcv() { cd ${WORKING_DIR} echo_blue "checking out '${PPLCV_DIR}' pkg..." if [ -d "${PPLCV_DIR}" ]; then - echo_green "Already exists! Checking out the requested version..." + echo "Already exists! Checking out the requested version..." else git clone https://github.com/openppl-public/ppl.cv.git ${PPLCV_DIR} fi @@ -302,7 +338,7 @@ mmdeploy(){ ## mmdeploy SDK cd ${MMDEPLOY_DIR} - MMDEPLOY_DETECT_VER=$(cat MMDeploy/mmdeploy/version.py | grep -Eo '[0-9]\.[0-9].[0-9]+') + MMDEPLOY_DETECT_VER=$(cat mmdeploy/version.py | grep -Eo '[0-9]\.[0-9].[0-9]+') # reinit submodules git submodule update --init --recursive @@ -377,6 +413,49 @@ package_list=( "mmdeploy" ) +##### +# Unattended/auto install +if [[ $appargument2 == "auto" ]] +then + WITH_UNATTENDED=1 +fi + +##### +# Install dependencies in venv? +prompt_yesno "Install misc. dependencies in the active venv? (Default:${WITH_PYTHON})" +res=$? +if [[ $res -eq 1 ]] +then + WITH_PYTHON=1 +elif [[ $res -eq 0 ]] +then + WITH_PYTHON=0 +fi + +##### +# Clean previous build dirs? +prompt_yesno "Clean previous build dirs? (Default:${WITH_CLEAN})" +res=$? +if [[ $res -eq 1 ]] +then + WITH_CLEAN=1 +elif [[ $res -eq 0 ]] +then + WITH_CLEAN=0 +fi + +##### +# Generate prebuild dirs? +prompt_yesno "Generate prebuild dirs? (Default:${WITH_PREBUILD})" +res=$? +if [[ $res -eq 1 ]] +then + WITH_PREBUILD=1 +elif [[ $res -eq 0 ]] +then + WITH_PREBUILD=0 +fi + ##### # check input argument if contains_element "$appargument1" "${package_list[@]}"; then From 9a2885b68a247b5d348ad0e5bb7c0f3b68703b65 Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Fri, 20 May 2022 07:45:39 +0200 Subject: [PATCH 11/16] Check input arguments before continuing. --- build_linux_nvidia.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/build_linux_nvidia.sh b/build_linux_nvidia.sh index 285d51c4de..8115124426 100755 --- a/build_linux_nvidia.sh +++ b/build_linux_nvidia.sh @@ -413,6 +413,19 @@ package_list=( "mmdeploy" ) +##### +# check input argument +if contains_element "$appargument1" "${package_list[@]}"; then + echo_green "Build and install '$appargument1'..." +else + echo_red "Unsupported argument '$appargument1'. Use one of the following:" + for i in ${package_list[@]} + do + echo $i + done + exit +fi + ##### # Unattended/auto install if [[ $appargument2 == "auto" ]] @@ -456,19 +469,6 @@ then WITH_PREBUILD=0 fi -##### -# check input argument -if contains_element "$appargument1" "${package_list[@]}"; then - echo_green "Build and install '$appargument1'..." -else - echo_red "Unsupported argument '$appargument1'. Use one of the following:" - for i in ${package_list[@]} - do - echo $i - done - exit -fi - $appargument1 cd ${WORKING_DIR} From f6f11f0ee631afe0335d1ffc37918a7f37fa25c2 Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Fri, 20 May 2022 07:58:28 +0200 Subject: [PATCH 12/16] Added additional log messages, comments added --- build_linux_nvidia.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/build_linux_nvidia.sh b/build_linux_nvidia.sh index 8115124426..ec3004ecf1 100755 --- a/build_linux_nvidia.sh +++ b/build_linux_nvidia.sh @@ -49,6 +49,8 @@ PYTHON_VENV_DIR=${WORKING_DIR}/venv-mmdeploy appargument1=$1 appargument2=$2 +##### +# helper functions echo_green() { if [ -n "$1" ]; then echo "$(tput setaf 10)$1$(tput sgr 0)" @@ -102,6 +104,8 @@ prompt_yesno() { } prereqs() { + echo_green "Installing prerequisites..." + # spdlog echo_green "Checking spdlog version..." prompt_yesno "Install latest spdlog from source? (Default:no)" @@ -207,6 +211,9 @@ prereqs() { } py_venv() { + ## python venv + echo_green "Installing python venv..." + #check for python installed version pyv="$(python3 -V 2>&1)" pyv_old="Python 3.6" @@ -304,6 +311,8 @@ py_venv() { pplcv() { ## ppl.cv + echo_green "Building and installing ppl.cv..." + cd ${WORKING_DIR} echo_blue "checking out '${PPLCV_DIR}' pkg..." if [ -d "${PPLCV_DIR}" ]; then @@ -336,6 +345,8 @@ pplcv() { mmdeploy(){ ## mmdeploy SDK + echo_green "Building and installing mmdeploysdk..." + cd ${MMDEPLOY_DIR} MMDEPLOY_DETECT_VER=$(cat mmdeploy/version.py | grep -Eo '[0-9]\.[0-9].[0-9]+') From f467c6ed76f4e4e399fddd5574e29a193e83c05a Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Fri, 20 May 2022 10:28:59 +0200 Subject: [PATCH 13/16] Added default option when pressing [ENTER] --- build_linux_nvidia.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build_linux_nvidia.sh b/build_linux_nvidia.sh index ec3004ecf1..f0b0b04468 100755 --- a/build_linux_nvidia.sh +++ b/build_linux_nvidia.sh @@ -84,7 +84,8 @@ prompt_yesno() { echo_green "Unattended install, selecting default option" return 2 else - read -p "(y/n/q)" -n 1 -r + echo_blue "(y/n/q) or press [ENTER] to select default option" + read -p "?" -n 1 -r echo if [[ $REPLY =~ ^[Yy]$ ]] then @@ -92,6 +93,10 @@ prompt_yesno() { elif [[ $REPLY =~ ^[Nn]$ ]] then return 0 + elif [[ $REPLY = "" ]] + then + echo "Selecting default option..." + return 2 elif [[ $REPLY =~ ^[Qq]$ ]] then echo_green "Quitting!" @@ -127,6 +132,7 @@ prereqs() { sudo make install sudo ldconfig fi + exit # cmake check & install echo_green "Checking your cmake version..." From 06f8d7874d1853801ae2510802d6a73cd6d5582f Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Mon, 23 May 2022 12:59:12 +0200 Subject: [PATCH 14/16] * Rearranged cmake, gcc build/install order. * Mention default option when reinstalling venv * sudo make install instead of cmake --install --- build_linux_nvidia.sh | 49 +++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/build_linux_nvidia.sh b/build_linux_nvidia.sh index f0b0b04468..4919c5ee06 100755 --- a/build_linux_nvidia.sh +++ b/build_linux_nvidia.sh @@ -111,29 +111,6 @@ prompt_yesno() { prereqs() { echo_green "Installing prerequisites..." - # spdlog - echo_green "Checking spdlog version..." - prompt_yesno "Install latest spdlog from source? (Default:no)" - local res=$? - if [[ $res -eq 1 ]] # || [ $res -eq 2 ] - then - echo_green "Building and installing latest spdlog from source" - - # remove libspdlog, as it might be an old version - sudo apt-get remove libspdlog-dev -y - - git clone https://github.com/gabime/spdlog.git spdlog - cd spdlog - git pull - git checkout tags/v1.8.1 - mkdir build -p && cd build - # we must build spdlog with -fPIC enabled - cmake .. -DCMAKE_POSITION_INDEPENDENT_CODE=ON && make -j${PROC_NUM} - sudo make install - sudo ldconfig - fi - exit - # cmake check & install echo_green "Checking your cmake version..." CMAKE_DETECT_VER=$(cmake --version | grep -oP '(?<=version).*') @@ -166,6 +143,28 @@ prereqs() { GCC_COMPILER="g++-7" fi + # spdlog + echo_green "Checking spdlog version..." + prompt_yesno "Install latest spdlog from source? (Default:no)" + local res=$? + if [[ $res -eq 1 ]] # || [ $res -eq 2 ] + then + echo_green "Building and installing latest spdlog from source" + + # remove libspdlog, as it might be an old version + sudo apt-get remove libspdlog-dev -y + + git clone https://github.com/gabime/spdlog.git spdlog + cd spdlog + git pull + git checkout tags/v1.8.1 + mkdir build -p && cd build + # we must build spdlog with -fPIC enabled + cmake .. -DCMAKE_POSITION_INDEPENDENT_CODE=ON && make -j${PROC_NUM} + sudo make install + sudo ldconfig + fi + # tensorrt check echo_green "Check your TensorRT version:" ## Check if ${TENSORRT_DIR} env variable has been set @@ -240,7 +239,7 @@ py_venv() { pip3 install --upgrade setuptools wheel if [ -d "${PYTHON_VENV_DIR}" ]; then - prompt_yesno "Reinstall existing Python venv ${PYTHON_VENV_DIR}?" + prompt_yesno "Reinstall existing Python venv ${PYTHON_VENV_DIR}? (Default:no)" local res=$? if [[ $res -eq 1 ]] then @@ -393,7 +392,7 @@ mmdeploy(){ -DMMDEPLOY_BUILD_SDK_PYTHON_API=ON \ -DTENSORRT_DIR=${TENSORRT_DIR} \ -DCUDNN_DIR=${CUDNN_DIR} - cmake --build . -- -j${PROC_NUM} && sudo cmake --install . + cmake --build . -- -j${PROC_NUM} && sudo make install sudo ldconfig # generate prebuild and pack into .tar.gz From e15f0f7a8f6d21b29f6b2f9abe429c2540472693 Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Tue, 24 May 2022 07:17:31 +0200 Subject: [PATCH 15/16] Moved script to tools/scripts --- build_linux_nvidia.sh => tools/scripts/build_linux_nvidia.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build_linux_nvidia.sh => tools/scripts/build_linux_nvidia.sh (100%) diff --git a/build_linux_nvidia.sh b/tools/scripts/build_linux_nvidia.sh similarity index 100% rename from build_linux_nvidia.sh rename to tools/scripts/build_linux_nvidia.sh From ee3f7bd7d7dc27f616d45fac8a3cdfb65df3c206 Mon Sep 17 00:00:00 2001 From: Johannes Linde Date: Wed, 25 May 2022 08:45:47 +0200 Subject: [PATCH 16/16] Fixed relative path when generating WORKING_DIR --- tools/scripts/build_linux_nvidia.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/scripts/build_linux_nvidia.sh b/tools/scripts/build_linux_nvidia.sh index 4919c5ee06..082e88feda 100755 --- a/tools/scripts/build_linux_nvidia.sh +++ b/tools/scripts/build_linux_nvidia.sh @@ -16,8 +16,8 @@ GCC_COMPILER="g++" ##### # Directories -# WORKING_DIR must correspond to script dir, i.e. MMDeploy root -WORKING_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# WORKING_DIR must correspond to MMDeploy root dir +WORKING_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )" PPLCV_DIR=${WORKING_DIR}/ppl.cv MMDEPLOY_DIR=${WORKING_DIR}