From 41d15dad1d91b9f6edeb4ba581f5cd6309089ec8 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 22 May 2021 18:10:42 +0530 Subject: [PATCH 01/22] HADOOP-17727. Modularize docker images --- dev-support/docker/Dockerfile | 136 +-------- dev-support/docker/Dockerfile_centos_8 | 5 +- dev-support/docker/Dockerfile_pkg_test | 0 .../pkg-resolver/install-common-packages.sh | 81 +++++ dev-support/docker/pkg-resolver/packages.json | 287 ++++++++++++++++++ .../docker/pkg-resolver/platforms.json | 6 + dev-support/docker/pkg-resolver/resolve.py | 41 +++ 7 files changed, 428 insertions(+), 128 deletions(-) create mode 100644 dev-support/docker/Dockerfile_pkg_test create mode 100644 dev-support/docker/pkg-resolver/install-common-packages.sh create mode 100644 dev-support/docker/pkg-resolver/packages.json create mode 100644 dev-support/docker/pkg-resolver/platforms.json create mode 100644 dev-support/docker/pkg-resolver/resolve.py diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index ef2f1562613e8..3a43610dc1d05 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -32,57 +32,11 @@ RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true -# hadolint ignore=DL3008 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends \ - ant \ - apt-utils \ - bats \ - build-essential \ - bzip2 \ - clang \ - cmake \ - curl \ - doxygen \ - fuse \ - g++ \ - gcc \ - git \ - gnupg-agent \ - hugo \ - libbcprov-java \ - libbz2-dev \ - libcurl4-openssl-dev \ - libfuse-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libsasl2-dev \ - libsnappy-dev \ - libssl-dev \ - libtool \ - libzstd-dev \ - locales \ - make \ - maven \ - nodejs \ - node-yarn \ - npm \ - openjdk-11-jdk \ - openjdk-8-jdk \ - pinentry-curses \ - pkg-config \ - python3 \ - python3-pip \ - python3-pkg-resources \ - python3-setuptools \ - python3-wheel \ - rsync \ - shellcheck \ - software-properties-common \ - sudo \ - valgrind \ - zlib1g-dev \ - && apt-get clean \ + && apt-get -q install -y --no-install-recommends python3 +COPY pkg-resolver pkg-resolver +RUN apt-get -q install -y --no-install-recommends `pkg-resolver/resolve.py ubuntu:focal` +RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* RUN locale-gen en_US.UTF-8 @@ -97,89 +51,16 @@ ENV MAVEN_HOME /usr ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ####### -# Install SpotBugs 4.2.2 +# Set env vars for SpotBugs 4.2.2 ####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Install Boost 1.72 (1.71 ships with Focal) +# Set env vars for Google Protobuf 3.7.1 ####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 - -### -# Install hadolint -#### -RUN curl -L -s -S \ - https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ - -o /bin/hadolint \ - && chmod a+rx /bin/hadolint \ - && shasum -a 512 /bin/hadolint | \ - awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' - -###### -# Intel ISA-L 2.29.0 -###### -# hadolint ignore=DL3003,DL3008 -RUN mkdir -p /opt/isa-l-src \ - && apt-get -q update \ - && apt-get install -y --no-install-recommends automake yasm \ - && apt-get clean \ - && curl -L -s -S \ - https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ - -o /opt/isa-l.tar.gz \ - && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \ - && cd /opt/isa-l-src \ - && ./autogen.sh \ - && ./configure \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/isa-l-src - ### # Avoid out of memory errors in builds ### @@ -188,6 +69,11 @@ ENV MAVEN_OPTS -Xms256m -Xmx3072m # Skip gpg verification when downloading Yetus via yetus-wrapper ENV HADOOP_SKIP_YETUS_VERIFICATION true +#### +# Install common packages +#### +RUN pkg-resolver/install-common-packages.sh + ### # Everything past this point is either not needed for testing or breaks Yetus. # So tell Yetus not to read the rest of the file: diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 7f1f6b4ac2239..92ec720f2ec21 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -33,11 +33,10 @@ RUN yum update -y \ cyrus-sasl-devel \ dnf \ fuse \ - git \ - libcurl-devel \ - fuse \ fuse-libs \ fuse-devel \ + git \ + libcurl-devel \ libtool \ libtirpc-devel \ lz4-devel \ diff --git a/dev-support/docker/Dockerfile_pkg_test b/dev-support/docker/Dockerfile_pkg_test new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/dev-support/docker/pkg-resolver/install-common-packages.sh b/dev-support/docker/pkg-resolver/install-common-packages.sh new file mode 100644 index 0000000000000..1e3d139bba488 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-common-packages.sh @@ -0,0 +1,81 @@ +####### +# Install SpotBugs 4.2.2 +####### +mkdir -p /opt/spotbugs \ + && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ + -o /opt/spotbugs.tgz \ + && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ + && chmod +x /opt/spotbugs/bin/* + +####### +# Install Boost 1.72 (1.71 ships with Focal) +####### +# hadolint ignore=DL3003 +mkdir -p /opt/boost-library \ + && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ + && mv boost_1_72_0.tar.bz2 /opt/boost-library \ + && cd /opt/boost-library \ + && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ + && cd /opt/boost-library/boost_1_72_0 \ + && ./bootstrap.sh --prefix=/usr/ \ + && ./b2 --without-python install \ + && cd /root \ + && rm -rf /opt/boost-library + +###### +# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) +###### +# hadolint ignore=DL3003 +mkdir -p /opt/protobuf-src \ + && curl -L -s -S \ + https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ + -o /opt/protobuf.tar.gz \ + && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ + && cd /opt/protobuf-src \ + && ./configure --prefix=/opt/protobuf \ + && make "-j$(nproc)" \ + && make install \ + && cd /root \ + && rm -rf /opt/protobuf-src + +###### +# Install pylint and python-dateutil +###### +pip3 install pylint==2.6.0 python-dateutil==2.8.1 + +###### +# Install bower +###### +# hadolint ignore=DL3008 +npm install -g bower@1.8.8 + +###### +# Install hadolint +###### +curl -L -s -S \ + https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ + -o /bin/hadolint \ + && chmod a+rx /bin/hadolint \ + && shasum -a 512 /bin/hadolint | \ + awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' + +###### +# Intel ISA-L 2.29.0 +###### +# hadolint ignore=DL3003,DL3008 +mkdir -p /opt/isa-l-src \ + && apt-get -q update \ + && apt-get install -y --no-install-recommends automake yasm \ + && apt-get clean \ + && curl -L -s -S \ + https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ + -o /opt/isa-l.tar.gz \ + && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \ + && cd /opt/isa-l-src \ + && ./autogen.sh \ + && ./configure \ + && make "-j$(nproc)" \ + && make install \ + && cd /root \ + && rm -rf /opt/isa-l-src + diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json new file mode 100644 index 0000000000000..cf59c63992d32 --- /dev/null +++ b/dev-support/docker/pkg-resolver/packages.json @@ -0,0 +1,287 @@ +{ + "ant": { + "ubuntu:focal": "ant", + "ubuntu:focal::arch64": "ant", + "centos:7": "ant", + "centos:8": "ant" + }, + "apt-utils": { + "ubuntu:focal": "apt-utils", + "ubuntu:focal::arch64": "apt-utils" + }, + "bats": { + "ubuntu:focal": "bats", + "ubuntu:focal::arch64": "bats" + }, + "build-essential": { + "ubuntu:focal": "build-essential", + "ubuntu:focal::arch64": "build-essential", + "centos:7": "build-essential", + "centos:8": "build-essential" + }, + "bzip2": { + "ubuntu:focal": [ + "bzip2", + "libbz2-dev" + ], + "ubuntu:focal::arch64": [ + "bzip2", + "libbz2-dev" + ], + "centos:7": [ + "bzip2", + "bzip2-devel" + ], + "centos:8": [ + "bzip2", + "bzip2-devel" + ] + }, + "clang": { + "ubuntu:focal": "clang", + "ubuntu:focal::arch64": "clang", + "centos:7": "clang", + "centos:8": "clang" + }, + "cmake": { + "ubuntu:focal": "cmake", + "ubuntu:focal::arch64": "cmake" + }, + "curl": { + "ubuntu:focal": [ + "curl", + "libcurl4-openssl-dev" + ], + "ubuntu:focal::arch64": [ + "curl", + "libcurl4-openssl-dev" + ], + "centos:7": [ + "curl", + "libcurl-devel" + ], + "centos:8": [ + "curl", + "libcurl-devel" + ] + }, + "doxygen": { + "ubuntu:focal": "doxygen", + "ubuntu:focal::arch64": "doxygen", + "centos:7": "doxygen" + }, + "dnf": { + "centos:8": "dnf" + }, + "fuse": { + "ubuntu:focal": [ + "fuse", + "libfuse-dev" + ], + "ubuntu:focal::arch64": [ + "fuse", + "libfuse-dev" + ], + "centos:7": [ + "fuse", + "fuse-libs", + "fuse-devel" + ], + "centos:8": [ + "fuse", + "fuse-libs", + "fuse-devel" + ] + }, + "gcc": { + "ubuntu:focal": [ + "gcc", + "g++" + ], + "ubuntu:focal::arch64": [ + "gcc", + "g++" + ], + "centos:7": [ + "centos-release-scl", + "devtoolset-9" + ] + }, + "git": { + "ubuntu:focal": "git", + "ubuntu:focal::arch64": "git", + "centos:7": "git", + "centos:8": "git" + }, + "gnupg-agent": { + "ubuntu:focal": "gnupg-agent", + "ubuntu:focal::arch64": "gnupg-agent" + }, + "hugo": { + "ubuntu:focal": "hugo", + "ubuntu:focal::arch64": "hugo" + }, + "libbcprov-java": { + "ubuntu:focal": "libbcprov-java", + "ubuntu:focal::arch64": "libbcprov-java" + }, + "libtool": { + "ubuntu:focal": "libtool", + "ubuntu:focal::arch64": "libtool", + "centos:7": "libtool", + "centos:8": "libtool" + }, + "openssl": { + "ubuntu:focal": "libssl-dev", + "ubuntu:focal::arch64": "libssl-dev", + "centos:7": "openssl-devel", + "centos:8": "openssl-devel" + }, + "protocol-buffers": { + "ubuntu:focal": [ + "libprotobuf-dev", + "libprotoc-dev" + ], + "ubuntu:focal::arch64": [ + "libprotobuf-dev", + "libprotoc-dev" + ] + }, + "sasl": { + "ubuntu:focal": "libsasl2-dev", + "ubuntu:focal::arch64": "libsasl2-dev", + "centos:7": "cyrus-sasl-devel", + "centos:8": "cyrus-sasl-devel" + }, + "snappy": { + "ubuntu:focal": "libsnappy-dev", + "ubuntu:focal::arch64": "libsnappy-dev", + "centos:7": "snappy-devel" + }, + "zlib": { + "ubuntu:focal": [ + "libzstd-dev", + "zlib1g-dev" + ], + "ubuntu:focal::arch64": [ + "libzstd-dev", + "zlib1g-dev" + ], + "centos:7": [ + "zlib-devel", + "lz4-devel" + ], + "centos:8": [ + "zlib-devel", + "lz4-devel" + ] + }, + "locales": { + "ubuntu:focal": "locales", + "ubuntu:focal::arch64": "locales" + }, + "libtirpc-devel": { + "centos:7": "libtirpc-devel", + "centos:8": "libtirpc-devel" + }, + "libpmem": { + "centos:7": "libpmem-devel" + }, + "make": { + "ubuntu:focal": "make", + "ubuntu:focal::arch64": "make", + "centos:7": "make", + "centos:8": "make" + }, + "maven": { + "ubuntu:focal": "maven", + "ubuntu:focal::arch64": "maven" + }, + "node.js": { + "ubuntu:focal": [ + "nodejs", + "node-yarn", + "npm" + ], + "ubuntu:focal::arch64": [ + "nodejs", + "node-yarn", + "npm" + ] + }, + "java": { + "ubuntu:focal": [ + "openjdk-8-jdk", + "openjdk-11-jdk" + ], + "ubuntu:focal::arch64": [ + "openjdk-8-jdk", + "openjdk-11-jdk" + ] + }, + "pinentry-curses": { + "ubuntu:focal": "pinentry-curses", + "ubuntu:focal::arch64": "pinentry-curses", + "centos:7": "pinentry-curses", + "centos:8": "pinentry-curses" + }, + "pkg-config": { + "ubuntu:focal": "pkg-config", + "ubuntu:focal::arch64": "pkg-config", + "centos:8": "pkg-config" + }, + "python3": { + "ubuntu:focal": [ + "python3", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + "python3-wheel" + ], + "ubuntu:focal::arch64": [ + "python3", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + "python3-wheel" + ], + "centos:7": [ + "python3", + "python3-pip", + "python3-setuptools", + "python3-wheel" + ], + "centos:8": [ + "python3", + "python3-pip", + "python3-setuptools", + "python3-wheel" + ] + }, + "rsync": { + "ubuntu:focal": "rsync", + "ubuntu:focal::arch64": "rsync", + "centos:7": "rsync", + "centos:8": "rsync" + }, + "shellcheck": { + "ubuntu:focal": "shellcheck", + "ubuntu:focal::arch64": "shellcheck" + }, + "software-properties-common": { + "ubuntu:focal": "software-properties-common", + "ubuntu:focal::arch64": "software-properties-common" + }, + "sudo": { + "ubuntu:focal": "sudo", + "ubuntu:focal::arch64": "sudo", + "centos:7": "sudo", + "centos:8": "sudo" + }, + "valgrind": { + "ubuntu:focal": "valgrind", + "ubuntu:focal::arch64": "valgrind", + "centos:7": "valgrind", + "centos:8": "valgrind" + } +} \ No newline at end of file diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json new file mode 100644 index 0000000000000..4861c5dd55c00 --- /dev/null +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -0,0 +1,6 @@ +[ + "ubuntu:focal", + "ubuntu:focal::arch64", + "centos:7", + "centos:8" +] \ No newline at end of file diff --git a/dev-support/docker/pkg-resolver/resolve.py b/dev-support/docker/pkg-resolver/resolve.py new file mode 100644 index 0000000000000..7c73bb1d96314 --- /dev/null +++ b/dev-support/docker/pkg-resolver/resolve.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import json +import sys + + +def get_packages(platform): + with open('pkg-resolver/packages.json', encoding='utf-8', mode='r') as pkg_file: + pkgs = json.loads(pkg_file.read()) + packages = [] + for platforms in pkgs.values(): + if platforms.get(platform) is None: + continue + else: + if type(platforms.get(platform)) == list: + for p in platforms.get(platform): + packages.append(p) + else: + packages.append(platforms.get(platform)) + return packages + + +def get_platforms(): + with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file: + return json.loads(platforms_file.read()) + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1), file=sys.stderr) + sys.exit(1) + + supported_platforms = get_platforms() + platform = sys.argv[1] + if platform not in supported_platforms: + print('ERROR: The given platform {} is not supported. ' + 'Please refer to platforms.json for a list of supported platforms'.format(platform), file=sys.stderr) + sys.exit(1) + + packages = get_packages(platform) + print(' '.join(packages)) From f806b266f4bff12718830de710e32a42304e1a1e Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 22 May 2021 18:13:11 +0530 Subject: [PATCH 02/22] Add license headers --- .../pkg-resolver/install-common-packages.sh | 18 ++++++++++++++++++ dev-support/docker/pkg-resolver/resolve.py | 16 ++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/dev-support/docker/pkg-resolver/install-common-packages.sh b/dev-support/docker/pkg-resolver/install-common-packages.sh index 1e3d139bba488..00a5e8ee1cac1 100644 --- a/dev-support/docker/pkg-resolver/install-common-packages.sh +++ b/dev-support/docker/pkg-resolver/install-common-packages.sh @@ -1,3 +1,21 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ####### # Install SpotBugs 4.2.2 ####### diff --git a/dev-support/docker/pkg-resolver/resolve.py b/dev-support/docker/pkg-resolver/resolve.py index 7c73bb1d96314..56a9b7d0fb5fb 100644 --- a/dev-support/docker/pkg-resolver/resolve.py +++ b/dev-support/docker/pkg-resolver/resolve.py @@ -1,5 +1,21 @@ #!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import sys From 801c2e67f713dbbea97ca33a1f4858c487493e32 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 22 May 2021 20:22:06 +0530 Subject: [PATCH 03/22] Add the required permissions * Added the necessary permissions to the files in pkg-resolver. --- dev-support/docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 3a43610dc1d05..37ab911640ca2 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -35,6 +35,8 @@ ENV DEBCONF_TERSE true RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends python3 COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/install-common-packages.sh pkg-resolver/resolve.py \ + && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json RUN apt-get -q install -y --no-install-recommends `pkg-resolver/resolve.py ubuntu:focal` RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* From b4d99f4cb54ba148ee6f2668fd2d9848d21862dd Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 22 May 2021 20:25:41 +0530 Subject: [PATCH 04/22] Refactor install-common-pkgs * Formatted the file and renamed it. --- dev-support/docker/Dockerfile | 4 +- .../pkg-resolver/install-common-packages.sh | 99 ------------------- .../pkg-resolver/install-common-pkgs.sh | 98 ++++++++++++++++++ 3 files changed, 100 insertions(+), 101 deletions(-) delete mode 100644 dev-support/docker/pkg-resolver/install-common-packages.sh create mode 100644 dev-support/docker/pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 37ab911640ca2..415dd2fa27bb9 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -35,7 +35,7 @@ ENV DEBCONF_TERSE true RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends python3 COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/install-common-packages.sh pkg-resolver/resolve.py \ +RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json RUN apt-get -q install -y --no-install-recommends `pkg-resolver/resolve.py ubuntu:focal` RUN apt-get clean \ @@ -74,7 +74,7 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true #### # Install common packages #### -RUN pkg-resolver/install-common-packages.sh +RUN pkg-resolver/install-common-pkgs.sh ### # Everything past this point is either not needed for testing or breaks Yetus. diff --git a/dev-support/docker/pkg-resolver/install-common-packages.sh b/dev-support/docker/pkg-resolver/install-common-packages.sh deleted file mode 100644 index 00a5e8ee1cac1..0000000000000 --- a/dev-support/docker/pkg-resolver/install-common-packages.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -####### -# Install SpotBugs 4.2.2 -####### -mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* - -####### -# Install Boost 1.72 (1.71 ships with Focal) -####### -# hadolint ignore=DL3003 -mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src - -###### -# Install pylint and python-dateutil -###### -pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -###### -# Install bower -###### -# hadolint ignore=DL3008 -npm install -g bower@1.8.8 - -###### -# Install hadolint -###### -curl -L -s -S \ - https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ - -o /bin/hadolint \ - && chmod a+rx /bin/hadolint \ - && shasum -a 512 /bin/hadolint | \ - awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' - -###### -# Intel ISA-L 2.29.0 -###### -# hadolint ignore=DL3003,DL3008 -mkdir -p /opt/isa-l-src \ - && apt-get -q update \ - && apt-get install -y --no-install-recommends automake yasm \ - && apt-get clean \ - && curl -L -s -S \ - https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ - -o /opt/isa-l.tar.gz \ - && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \ - && cd /opt/isa-l-src \ - && ./autogen.sh \ - && ./configure \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/isa-l-src - diff --git a/dev-support/docker/pkg-resolver/install-common-pkgs.sh b/dev-support/docker/pkg-resolver/install-common-pkgs.sh new file mode 100644 index 0000000000000..62e8e7bd7f0e2 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-common-pkgs.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +####### +# Install SpotBugs 4.2.2 +####### +mkdir -p /opt/spotbugs && + curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ + -o /opt/spotbugs.tgz && + tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs && + chmod +x /opt/spotbugs/bin/* + +####### +# Install Boost 1.72 (1.71 ships with Focal) +####### +# hadolint ignore=DL3003 +mkdir -p /opt/boost-library && + curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download >boost_1_72_0.tar.bz2 && + mv boost_1_72_0.tar.bz2 /opt/boost-library && + cd /opt/boost-library && + tar --bzip2 -xf boost_1_72_0.tar.bz2 && + cd /opt/boost-library/boost_1_72_0 && + ./bootstrap.sh --prefix=/usr/ && + ./b2 --without-python install && + cd /root && + rm -rf /opt/boost-library + +###### +# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) +###### +# hadolint ignore=DL3003 +mkdir -p /opt/protobuf-src && + curl -L -s -S \ + https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ + -o /opt/protobuf.tar.gz && + tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src && + cd /opt/protobuf-src && + ./configure --prefix=/opt/protobuf && + make "-j$(nproc)" && + make install && + cd /root && + rm -rf /opt/protobuf-src + +###### +# Install pylint and python-dateutil +###### +pip3 install pylint==2.6.0 python-dateutil==2.8.1 + +###### +# Install bower +###### +# hadolint ignore=DL3008 +npm install -g bower@1.8.8 + +###### +# Install hadolint +###### +curl -L -s -S \ + https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ + -o /bin/hadolint && + chmod a+rx /bin/hadolint && + shasum -a 512 /bin/hadolint | + awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' + +###### +# Intel ISA-L 2.29.0 +###### +# hadolint ignore=DL3003,DL3008 +mkdir -p /opt/isa-l-src && + apt-get -q update && + apt-get install -y --no-install-recommends automake yasm && + apt-get clean && + curl -L -s -S \ + https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ + -o /opt/isa-l.tar.gz && + tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src && + cd /opt/isa-l-src && + ./autogen.sh && + ./configure && + make "-j$(nproc)" && + make install && + cd /root && + rm -rf /opt/isa-l-src From e259f161515a89278e822b6b55fd9981ed0fe7e6 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sun, 23 May 2021 17:57:40 +0530 Subject: [PATCH 05/22] Fix warnings reported in CI run * Added necessary warning suppressions. * Fixed warnings reported by pylint. --- dev-support/docker/Dockerfile | 13 +++++-- dev-support/docker/Dockerfile_pkg_test | 0 dev-support/docker/pkg-resolver/resolve.py | 43 ++++++++++++++-------- 3 files changed, 37 insertions(+), 19 deletions(-) delete mode 100644 dev-support/docker/Dockerfile_pkg_test diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 415dd2fa27bb9..1de50619383b9 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -32,13 +32,18 @@ RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true -RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends python3 COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json -RUN apt-get -q install -y --no-install-recommends `pkg-resolver/resolve.py ubuntu:focal` -RUN apt-get clean \ + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008 +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends python3 \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) + && apt-get clean \ && rm -rf /var/lib/apt/lists/* RUN locale-gen en_US.UTF-8 diff --git a/dev-support/docker/Dockerfile_pkg_test b/dev-support/docker/Dockerfile_pkg_test deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/dev-support/docker/pkg-resolver/resolve.py b/dev-support/docker/pkg-resolver/resolve.py index 56a9b7d0fb5fb..f78e5727ab66b 100644 --- a/dev-support/docker/pkg-resolver/resolve.py +++ b/dev-support/docker/pkg-resolver/resolve.py @@ -16,42 +16,55 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Platform package dependency resolver for building Apache Hadoop. +""" + import json import sys def get_packages(platform): + """ + Resolve and get the list of packages to install for the given platform. + + :param platform: The platform for which the packages needs to be resolved. + :return: A list of resolved packages to install. + """ with open('pkg-resolver/packages.json', encoding='utf-8', mode='r') as pkg_file: pkgs = json.loads(pkg_file.read()) packages = [] - for platforms in pkgs.values(): - if platforms.get(platform) is None: - continue + for platforms in filter(lambda x: x.get(platform) is not None, pkgs.values()): + if isinstance(platforms.get(platform), list): + packages.extend(platforms.get(platform)) else: - if type(platforms.get(platform)) == list: - for p in platforms.get(platform): - packages.append(p) - else: - packages.append(platforms.get(platform)) + packages.append(platforms.get(platform)) return packages def get_platforms(): + """ + :return: A list of the supported platforms managed by pkg-resolver. + """ + with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file: return json.loads(platforms_file.read()) if __name__ == '__main__': if len(sys.argv) < 2: - print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1), file=sys.stderr) + print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1), + file=sys.stderr) sys.exit(1) supported_platforms = get_platforms() - platform = sys.argv[1] - if platform not in supported_platforms: - print('ERROR: The given platform {} is not supported. ' - 'Please refer to platforms.json for a list of supported platforms'.format(platform), file=sys.stderr) + platform_arg = sys.argv[1] + if platform_arg not in supported_platforms: + print( + 'ERROR: The given platform {} is not supported. ' + 'Please refer to platforms.json for a list of supported platforms'.format( + platform_arg), file=sys.stderr) sys.exit(1) - packages = get_packages(platform) - print(' '.join(packages)) + packages_to_install = get_packages(platform_arg) + print(' '.join(packages_to_install)) From 17a1825f5087959c462351ba158f338badbce00a Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sun, 23 May 2021 18:05:33 +0530 Subject: [PATCH 06/22] Add trailing slash * Added a trailing / for continuation of the command. --- dev-support/docker/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 1de50619383b9..7be1c31efb975 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -32,6 +32,9 @@ RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true +###### +# Platform package dependency resolver +###### COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json @@ -42,7 +45,7 @@ RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ # hadolint ignore=DL3008 RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends python3 \ - && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From 01fc15759c755136d92e134b075da08be319fbbb Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sun, 23 May 2021 20:08:27 +0530 Subject: [PATCH 07/22] Add quotes to prevent word split * Added a quote as per the reported warning in CI run to prevent the splitting of word. --- dev-support/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 7be1c31efb975..9addd4b971e97 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -45,7 +45,7 @@ RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ # hadolint ignore=DL3008 RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends python3 \ - && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \ + && apt-get -q install -y --no-install-recommends "$(pkg-resolver/resolve.py ubuntu:focal)" \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From 656849e839ec234da73e83a196955f1d82db1011 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sun, 23 May 2021 22:21:50 +0530 Subject: [PATCH 08/22] Unquote and ignore warning * Quoting $(...) causes the command execution to fail. Thus, ignoring the warning instead. --- dev-support/docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 9addd4b971e97..19fad3aa7ab50 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -42,10 +42,10 @@ RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ ###### # Install packages from apt ###### -# hadolint ignore=DL3008 +# hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends python3 \ - && apt-get -q install -y --no-install-recommends "$(pkg-resolver/resolve.py ubuntu:focal)" \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From 38c337da7c871d2e0b072993f23612c82e0404f5 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 24 May 2021 20:41:10 +0530 Subject: [PATCH 09/22] Use pkg-resolver for other platforms * Using pkg-resolver for resolving the dependencies on Ubuntu Focal arch64, Centos 7 and Centos 8. --- dev-support/docker/Dockerfile_aarch64 | 64 ++--------- dev-support/docker/Dockerfile_centos_7 | 104 +++--------------- dev-support/docker/Dockerfile_centos_8 | 100 +++-------------- dev-support/docker/pkg-resolver/packages.json | 8 +- 4 files changed, 52 insertions(+), 224 deletions(-) diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 5c2613268a65b..7fd21fbdfbe0d 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -33,61 +33,19 @@ ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true ###### -# Install common dependencies from packages. Versions here are either -# sufficient or irrelevant. +# Platform package dependency resolver ###### -# hadolint ignore=DL3008 +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ + && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends \ - ant \ - apt-utils \ - bats \ - build-essential \ - bzip2 \ - clang \ - cmake \ - curl \ - doxygen \ - fuse \ - g++ \ - gcc \ - git \ - gnupg-agent \ - hugo \ - libbcprov-java \ - libbz2-dev \ - libcurl4-openssl-dev \ - libfuse-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libsasl2-dev \ - libsnappy-dev \ - libssl-dev \ - libtool \ - libzstd-dev \ - locales \ - make \ - maven \ - nodejs \ - node-yarn \ - npm \ - openjdk-11-jdk \ - openjdk-8-jdk \ - phantomjs \ - pinentry-curses \ - pkg-config \ - python2.7 \ - python3 \ - python3-pip \ - python3-pkg-resources \ - python3-setuptools \ - python3-wheel \ - rsync \ - shellcheck \ - software-properties-common \ - sudo \ - valgrind \ - zlib1g-dev \ + && apt-get -q install -y --no-install-recommends python3 \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index 319942e2f6d4a..061c29ad63e5d 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -23,39 +23,20 @@ WORKDIR /root SHELL ["/bin/bash", "-o", "pipefail", "-c"] +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ + && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json + +###### +# Install packages from yum +###### +# hadolint ignore=DL3008,SC2046 RUN yum update -y \ - && yum install -y centos-release-scl \ - && yum install -y devtoolset-9 \ - && yum install -y \ - ant \ - build-essential \ - bzip2 \ - bzip2-devel \ - clang \ - curl \ - cyrus-sasl-devel \ - doxygen \ - fuse \ - fuse-libs \ - fuse-devel \ - git \ - libcurl-devel \ - libtirpc-devel \ - libpmem-devel \ - libtool \ - lz4-devel \ - make \ - openssl-devel \ - pinentry-curses \ - python3 \ - python3-pip \ - python3-setuptools \ - python3-wheel \ - rsync \ - snappy-devel \ - sudo \ - valgrind \ - zlib-devel + && yum install -y python3 \ + && yum install -y $(pkg-resolver/resolve.py centos:7) # Set GCC 9 as the default C/C++ compiler RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc @@ -97,6 +78,11 @@ RUN locale-gen en_US.UTF-8 ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' ENV PYTHONIOENCODING=utf-8 +###### +# Install common packages +###### +RUN pkg-resolver/install-common-pkgs.sh + ###### # Set env vars required to build Hadoop ###### @@ -105,49 +91,6 @@ ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 -####### -# Install SpotBugs 4.2.2 -####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* -ENV SPOTBUGS_HOME /opt/spotbugs - -####### -# Install Boost 1.72 (1.71 ships with Focal) -####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" - #### # Install Node.js #### @@ -160,14 +103,3 @@ RUN mkdir -p /tmp/node \ && make "-j$(nproc)" \ && make install \ && cd /root || exit - -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 92ec720f2ec21..5b0f8868565e8 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -23,35 +23,20 @@ WORKDIR /root SHELL ["/bin/bash", "-o", "pipefail", "-c"] +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ + && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json + +###### +# Install packages from yum +###### +# hadolint ignore=DL3008,SC2046 RUN yum update -y \ - && yum install -y \ - ant \ - bzip2 \ - bzip2-devel \ - clang \ - curl \ - cyrus-sasl-devel \ - dnf \ - fuse \ - fuse-libs \ - fuse-devel \ - git \ - libcurl-devel \ - libtool \ - libtirpc-devel \ - lz4-devel \ - make \ - openssl-devel \ - pinentry-curses \ - pkg-config \ - python3 \ - python3-pip \ - python3-setuptools \ - python3-wheel \ - rsync \ - sudo \ - valgrind \ - zlib-devel + && yum install -y python3 \ + && yum install -y $(pkg-resolver/resolve.py centos:8) #### # Install EPEL @@ -113,6 +98,11 @@ RUN locale-gen en_US.UTF-8 ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' ENV PYTHONIOENCODING=utf-8 +###### +# Install common packages +###### +RUN pkg-resolver/install-common-pkgs.sh + ###### # Set env vars required to build Hadoop ###### @@ -120,57 +110,3 @@ ENV MAVEN_HOME /opt/maven/apache-maven-3.6.3 ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 - -####### -# Install SpotBugs 4.2.2 -####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* -ENV SPOTBUGS_HOME /opt/spotbugs - -####### -# Install Boost 1.72 -####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" - -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index cf59c63992d32..a5ce5402f3e64 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -197,7 +197,7 @@ "ubuntu:focal": "maven", "ubuntu:focal::arch64": "maven" }, - "node.js": { + "javascript": { "ubuntu:focal": [ "nodejs", "node-yarn", @@ -206,7 +206,8 @@ "ubuntu:focal::arch64": [ "nodejs", "node-yarn", - "npm" + "npm", + "phantomjs" ] }, "java": { @@ -230,7 +231,7 @@ "ubuntu:focal::arch64": "pkg-config", "centos:8": "pkg-config" }, - "python3": { + "python": { "ubuntu:focal": [ "python3", "python3-pip", @@ -239,6 +240,7 @@ "python3-wheel" ], "ubuntu:focal::arch64": [ + "python2.7", "python3", "python3-pip", "python3-pkg-resources", From 9b0b3328053a6cfa6dbcb430a5597dd64b86915a Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 24 May 2021 23:30:07 +0530 Subject: [PATCH 10/22] Install centos-release-scl as prerequisite * The centos-release-scl package must be installed as a prerequisite before installing devtoolset-9 for gcc. --- dev-support/docker/Dockerfile_centos_7 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index 061c29ad63e5d..9e4f741be624f 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -35,7 +35,9 @@ RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ ###### # hadolint ignore=DL3008,SC2046 RUN yum update -y \ - && yum install -y python3 \ + && yum install -y \ + centos-release-scl \ + python3 \ && yum install -y $(pkg-resolver/resolve.py centos:7) # Set GCC 9 as the default C/C++ compiler From 8923e3aae2be7899dfe4899fea7d72f538f590e9 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 29 May 2021 13:40:39 +0530 Subject: [PATCH 11/22] Fix installation of Intel ISA-L library * Intel ISA-L library has dependencies on yasm, shasum which needs to be installed prior to installing it. --- dev-support/docker/Dockerfile_centos_7 | 16 ++++++++++++---- dev-support/docker/Dockerfile_centos_8 | 9 +++++---- .../pkg-resolver/install-common-pkgs.sh | 3 --- dev-support/docker/pkg-resolver/packages.json | 19 +++++++++++++++++-- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index 9e4f741be624f..44d6f7b0ac981 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -76,9 +76,17 @@ RUN mkdir -p /opt/zstd /tmp/zstd \ && make install \ && cd /root || exit -RUN locale-gen en_US.UTF-8 -ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -ENV PYTHONIOENCODING=utf-8 +# RUN locale-gen en_US.UTF-8 +# ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +# ENV PYTHONIOENCODING=utf-8 + +###### +# Install yasm +###### +RUN mkdir -p /tmp/yasm \ + && curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \ + -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm \ + && rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm ###### # Install common packages @@ -104,4 +112,4 @@ RUN mkdir -p /tmp/node \ && ./configure \ && make "-j$(nproc)" \ && make install \ - && cd /root || exit + && cd /root || exit \ No newline at end of file diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 5b0f8868565e8..8fbb2910aea8b 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -48,7 +48,8 @@ RUN mkdir -p /tmp/epel \ RUN dnf --enablerepo=powertools install -y \ doxygen \ - snappy-devel + snappy-devel \ + yasm RUN dnf install -y \ bouncycastle \ @@ -94,9 +95,9 @@ RUN mkdir -p /opt/zstd /tmp/zstd \ && make install \ && cd /root || exit -RUN locale-gen en_US.UTF-8 -ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -ENV PYTHONIOENCODING=utf-8 +# RUN locale-gen en_US.UTF-8 +# ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +# ENV PYTHONIOENCODING=utf-8 ###### # Install common packages diff --git a/dev-support/docker/pkg-resolver/install-common-pkgs.sh b/dev-support/docker/pkg-resolver/install-common-pkgs.sh index 62e8e7bd7f0e2..9e4028f4572eb 100644 --- a/dev-support/docker/pkg-resolver/install-common-pkgs.sh +++ b/dev-support/docker/pkg-resolver/install-common-pkgs.sh @@ -82,9 +82,6 @@ curl -L -s -S \ ###### # hadolint ignore=DL3003,DL3008 mkdir -p /opt/isa-l-src && - apt-get -q update && - apt-get install -y --no-install-recommends automake yasm && - apt-get clean && curl -L -s -S \ https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ -o /opt/isa-l.tar.gz && diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index a5ce5402f3e64..e6cb625f502a7 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -9,6 +9,12 @@ "ubuntu:focal": "apt-utils", "ubuntu:focal::arch64": "apt-utils" }, + "automake": { + "ubuntu:focal": "automake", + "ubuntu:focal::arch64": "automake", + "centos:7": "automake", + "centos:8": "automake" + }, "bats": { "ubuntu:focal": "bats", "ubuntu:focal::arch64": "bats" @@ -16,8 +22,7 @@ "build-essential": { "ubuntu:focal": "build-essential", "ubuntu:focal::arch64": "build-essential", - "centos:7": "build-essential", - "centos:8": "build-essential" + "centos:7": "build-essential" }, "bzip2": { "ubuntu:focal": [ @@ -270,6 +275,12 @@ "ubuntu:focal": "shellcheck", "ubuntu:focal::arch64": "shellcheck" }, + "shasum": { + "ubuntu:focal": "shasum", + "ubuntu:focal::arch64": "shasum", + "centos:7": "perl-Digest-SHA", + "centos:8": "perl-Digest-SHA" + }, "software-properties-common": { "ubuntu:focal": "software-properties-common", "ubuntu:focal::arch64": "software-properties-common" @@ -285,5 +296,9 @@ "ubuntu:focal::arch64": "valgrind", "centos:7": "valgrind", "centos:8": "valgrind" + }, + "yasm": { + "ubuntu:focal": "yasm", + "ubuntu:focal::arch64": "yasm" } } \ No newline at end of file From 175deb762f42943bba5d7e0ddd3193a2cc6d9cbb Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 29 May 2021 16:18:04 +0530 Subject: [PATCH 12/22] Add checks for package installation * Moved the build and installation of separate packages into individual scripts. --- dev-support/docker/Dockerfile | 11 ++- .../docker/pkg-resolver/check_platform.py | 50 ++++++++++++++ .../docker/pkg-resolver/install-boost.sh | 56 ++++++++++++++++ .../pkg-resolver/install-common-pkgs.sh | 67 ------------------- .../docker/pkg-resolver/install-hadolint.sh | 35 ++++++++++ .../pkg-resolver/install-intel-isa-l.sh | 58 ++++++++++++++++ .../docker/pkg-resolver/install-protobuf.sh | 57 ++++++++++++++++ .../docker/pkg-resolver/install-spotbugs.sh | 50 ++++++++++++++ dev-support/docker/pkg-resolver/resolve.py | 13 +--- 9 files changed, 316 insertions(+), 81 deletions(-) create mode 100644 dev-support/docker/pkg-resolver/check_platform.py create mode 100644 dev-support/docker/pkg-resolver/install-boost.sh create mode 100644 dev-support/docker/pkg-resolver/install-hadolint.sh create mode 100644 dev-support/docker/pkg-resolver/install-intel-isa-l.sh create mode 100644 dev-support/docker/pkg-resolver/install-protobuf.sh create mode 100644 dev-support/docker/pkg-resolver/install-spotbugs.sh diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 19fad3aa7ab50..fac364bbd4363 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -36,8 +36,8 @@ ENV DEBCONF_TERSE true # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ - && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json ###### # Install packages from apt @@ -80,9 +80,14 @@ ENV MAVEN_OPTS -Xms256m -Xmx3072m ENV HADOOP_SKIP_YETUS_VERIFICATION true #### -# Install common packages +# Install packages #### RUN pkg-resolver/install-common-pkgs.sh +RUN pkg-resolver/install-spotbugs.sh ubuntu:focal +RUN pkg-resolver/install-boost.sh ubuntu:focal +RUN pkg-resolver/install-protobuf.sh ubuntu:focal +RUN pkg-resolver/install-hadolint.sh ubuntu:focal +RUN pkg-resolver/install-intel-isa-l.sh ubuntu:focal ### # Everything past this point is either not needed for testing or breaks Yetus. diff --git a/dev-support/docker/pkg-resolver/check_platform.py b/dev-support/docker/pkg-resolver/check_platform.py new file mode 100644 index 0000000000000..fa5529a58be20 --- /dev/null +++ b/dev-support/docker/pkg-resolver/check_platform.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks whether the given platform is supported for building Apache Hadoop +""" + +import json +import sys + + +def get_platforms(): + """ + :return: A list of the supported platforms managed by pkg-resolver. + """ + + with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file: + return json.loads(platforms_file.read()) + + +def is_supported_platform(platform): + """ + :param platform: The name of the platform + :return: Whether the platform is supported + """ + return platform in get_platforms() + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print('ERROR: Expecting 1 argument, {} were provided'.format(len(sys.argv) - 1), + file=sys.stderr) + sys.exit(1) + + sys.exit(0 if is_supported_platform(sys.argv[1]) else 1) diff --git a/dev-support/docker/pkg-resolver/install-boost.sh b/dev-support/docker/pkg-resolver/install-boost.sh new file mode 100644 index 0000000000000..eaca09effa2c0 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-boost.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="1.72.0" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "1.72.0" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "1.72.0" ]; then + # hadolint ignore=DL3003 + mkdir -p /opt/boost-library && + curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download >boost_1_72_0.tar.bz2 && + mv boost_1_72_0.tar.bz2 /opt/boost-library && + cd /opt/boost-library && + tar --bzip2 -xf boost_1_72_0.tar.bz2 && + cd /opt/boost-library/boost_1_72_0 && + ./bootstrap.sh --prefix=/usr/ && + ./b2 --without-python install && + cd /root && + rm -rf /opt/boost-library +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-common-pkgs.sh b/dev-support/docker/pkg-resolver/install-common-pkgs.sh index 9e4028f4572eb..d6756eea73d79 100644 --- a/dev-support/docker/pkg-resolver/install-common-pkgs.sh +++ b/dev-support/docker/pkg-resolver/install-common-pkgs.sh @@ -16,46 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -####### -# Install SpotBugs 4.2.2 -####### -mkdir -p /opt/spotbugs && - curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz && - tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs && - chmod +x /opt/spotbugs/bin/* - -####### -# Install Boost 1.72 (1.71 ships with Focal) -####### -# hadolint ignore=DL3003 -mkdir -p /opt/boost-library && - curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download >boost_1_72_0.tar.bz2 && - mv boost_1_72_0.tar.bz2 /opt/boost-library && - cd /opt/boost-library && - tar --bzip2 -xf boost_1_72_0.tar.bz2 && - cd /opt/boost-library/boost_1_72_0 && - ./bootstrap.sh --prefix=/usr/ && - ./b2 --without-python install && - cd /root && - rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -mkdir -p /opt/protobuf-src && - curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz && - tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src && - cd /opt/protobuf-src && - ./configure --prefix=/opt/protobuf && - make "-j$(nproc)" && - make install && - cd /root && - rm -rf /opt/protobuf-src - ###### # Install pylint and python-dateutil ###### @@ -66,30 +26,3 @@ pip3 install pylint==2.6.0 python-dateutil==2.8.1 ###### # hadolint ignore=DL3008 npm install -g bower@1.8.8 - -###### -# Install hadolint -###### -curl -L -s -S \ - https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ - -o /bin/hadolint && - chmod a+rx /bin/hadolint && - shasum -a 512 /bin/hadolint | - awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' - -###### -# Intel ISA-L 2.29.0 -###### -# hadolint ignore=DL3003,DL3008 -mkdir -p /opt/isa-l-src && - curl -L -s -S \ - https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ - -o /opt/isa-l.tar.gz && - tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src && - cd /opt/isa-l-src && - ./autogen.sh && - ./configure && - make "-j$(nproc)" && - make install && - cd /root && - rm -rf /opt/isa-l-src diff --git a/dev-support/docker/pkg-resolver/install-hadolint.sh b/dev-support/docker/pkg-resolver/install-hadolint.sh new file mode 100644 index 0000000000000..1e2081f38c403 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-hadolint.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +curl -L -s -S \ + https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ + -o /bin/hadolint && + chmod a+rx /bin/hadolint && + shasum -a 512 /bin/hadolint | + awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' diff --git a/dev-support/docker/pkg-resolver/install-intel-isa-l.sh b/dev-support/docker/pkg-resolver/install-intel-isa-l.sh new file mode 100644 index 0000000000000..c6b4de782282e --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-intel-isa-l.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="2.29.0" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "2.29.0" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "2.29.0" ]; then + # hadolint ignore=DL3003,DL3008 + mkdir -p /opt/isa-l-src && + curl -L -s -S \ + https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ + -o /opt/isa-l.tar.gz && + tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src && + cd /opt/isa-l-src && + ./autogen.sh && + ./configure && + make "-j$(nproc)" && + make install && + cd /root && + rm -rf /opt/isa-l-src +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-protobuf.sh b/dev-support/docker/pkg-resolver/install-protobuf.sh new file mode 100644 index 0000000000000..7303b4048226a --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-protobuf.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="3.7.1" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "3.7.1" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "3.7.1" ]; then + # hadolint ignore=DL3003 + mkdir -p /opt/protobuf-src && + curl -L -s -S \ + https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ + -o /opt/protobuf.tar.gz && + tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src && + cd /opt/protobuf-src && + ./configure --prefix=/opt/protobuf && + make "-j$(nproc)" && + make install && + cd /root && + rm -rf /opt/protobuf-src +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-spotbugs.sh b/dev-support/docker/pkg-resolver/install-spotbugs.sh new file mode 100644 index 0000000000000..65a8f2e692418 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-spotbugs.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="4.2.2" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "4.2.2" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "4.2.2" ]; then + mkdir -p /opt/spotbugs && + curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ + -o /opt/spotbugs.tgz && + tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs && + chmod +x /opt/spotbugs/bin/* +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/resolve.py b/dev-support/docker/pkg-resolver/resolve.py index f78e5727ab66b..3a8bb46b02747 100644 --- a/dev-support/docker/pkg-resolver/resolve.py +++ b/dev-support/docker/pkg-resolver/resolve.py @@ -22,6 +22,7 @@ import json import sys +from check_platform import is_supported_platform def get_packages(platform): @@ -42,24 +43,14 @@ def get_packages(platform): return packages -def get_platforms(): - """ - :return: A list of the supported platforms managed by pkg-resolver. - """ - - with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file: - return json.loads(platforms_file.read()) - - if __name__ == '__main__': if len(sys.argv) < 2: print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1), file=sys.stderr) sys.exit(1) - supported_platforms = get_platforms() platform_arg = sys.argv[1] - if platform_arg not in supported_platforms: + if is_supported_platform(platform_arg): print( 'ERROR: The given platform {} is not supported. ' 'Please refer to platforms.json for a list of supported platforms'.format( From f95bf840c227f56256e688984c348cf659273ac6 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 29 May 2021 17:12:28 +0530 Subject: [PATCH 13/22] Fix negation bug * Forgot to negate the platform check which caused a bug. --- dev-support/docker/pkg-resolver/resolve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/pkg-resolver/resolve.py b/dev-support/docker/pkg-resolver/resolve.py index 3a8bb46b02747..5d62edd323e78 100644 --- a/dev-support/docker/pkg-resolver/resolve.py +++ b/dev-support/docker/pkg-resolver/resolve.py @@ -50,7 +50,7 @@ def get_packages(platform): sys.exit(1) platform_arg = sys.argv[1] - if is_supported_platform(platform_arg): + if not is_supported_platform(platform_arg): print( 'ERROR: The given platform {} is not supported. ' 'Please refer to platforms.json for a list of supported platforms'.format( From 08a482b250a1fc33588a1ad30c810b1d408d576a Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 29 May 2021 17:29:23 +0530 Subject: [PATCH 14/22] Remove shasum for Ubuntu * shasum must be installed through other deps. --- dev-support/docker/pkg-resolver/packages.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index e6cb625f502a7..4ebbc900fe4de 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -276,8 +276,6 @@ "ubuntu:focal::arch64": "shellcheck" }, "shasum": { - "ubuntu:focal": "shasum", - "ubuntu:focal::arch64": "shasum", "centos:7": "perl-Digest-SHA", "centos:8": "perl-Digest-SHA" }, From 0d104ecc31e489c4274f6d1ca5be233c9e67d68f Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 31 May 2021 20:21:09 +0530 Subject: [PATCH 15/22] Modularize Dockerfile for Centos 7 * Moved the packages' installation to separate scripts. --- dev-support/docker/Dockerfile_centos_7 | 75 ++++--------------- .../docker/pkg-resolver/install-cmake.sh | 52 +++++++++++++ .../docker/pkg-resolver/install-maven.sh | 49 ++++++++++++ .../docker/pkg-resolver/install-nodejs.sh | 53 +++++++++++++ .../docker/pkg-resolver/install-yasm.sh | 49 ++++++++++++ .../docker/pkg-resolver/install-zstandard.sh | 52 +++++++++++++ 6 files changed, 268 insertions(+), 62 deletions(-) create mode 100644 dev-support/docker/pkg-resolver/install-cmake.sh create mode 100644 dev-support/docker/pkg-resolver/install-maven.sh create mode 100644 dev-support/docker/pkg-resolver/install-nodejs.sh create mode 100644 dev-support/docker/pkg-resolver/install-yasm.sh create mode 100644 dev-support/docker/pkg-resolver/install-zstandard.sh diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index 44d6f7b0ac981..840de00a275c2 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -27,8 +27,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ - && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json ###### # Install packages from yum @@ -44,54 +44,7 @@ RUN yum update -y \ RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc SHELL ["/bin/bash", "--login", "-c"] -#### -# Install Maven 3.6.3 -#### -RUN mkdir -p /opt/maven /tmp/maven \ - && curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \ - -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \ - && tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven - -#### -# Install CMake 3.19 -#### -# hadolint ignore=DL3003 -RUN mkdir -p /tmp/cmake /opt/cmake \ - && curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \ - && tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \ - && cd /opt/cmake || exit && ./bootstrap \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -#### -# Install zstandard -#### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/zstd /tmp/zstd \ - && curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \ - && tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \ - && cd /opt/zstd || exit \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -# RUN locale-gen en_US.UTF-8 -# ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -# ENV PYTHONIOENCODING=utf-8 - -###### -# Install yasm -###### -RUN mkdir -p /tmp/yasm \ - && curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \ - -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm \ - && rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm - -###### -# Install common packages -###### -RUN pkg-resolver/install-common-pkgs.sh +# TODO: Set locale ###### # Set env vars required to build Hadoop @@ -101,15 +54,13 @@ ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 -#### -# Install Node.js -#### -# hadolint ignore=DL3003 -RUN mkdir -p /tmp/node \ - && curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz \ - && tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node \ - && cd /tmp/node || exit \ - && ./configure \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit \ No newline at end of file + +###### +# Install packages +###### +RUN pkg-resolver/install-maven.sh centos:7 +RUN pkg-resolver/install-cmake.sh centos:7 +RUN pkg-resolver/install-zstandard.sh centos:7 +RUN pkg-resolver/install-yasm.sh centos:7 +RUN pkg-resolver/install-nodejs.sh centos:7 +RUN pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/pkg-resolver/install-cmake.sh b/dev-support/docker/pkg-resolver/install-cmake.sh new file mode 100644 index 0000000000000..a6a2f549acc50 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-cmake.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="3.19.0" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "3.19.0" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "3.19.0" ]; then + mkdir -p /tmp/cmake /opt/cmake && + curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz && + tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake && + cd /opt/cmake || exit && ./bootstrap && + make "-j$(nproc)" && + make install && + cd /root || exit +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-maven.sh b/dev-support/docker/pkg-resolver/install-maven.sh new file mode 100644 index 0000000000000..f9ff961a190f9 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-maven.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="3.6.3" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "3.6.3" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "3.6.3" ]; then + mkdir -p /opt/maven /tmp/maven && + curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \ + -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz && + tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-nodejs.sh b/dev-support/docker/pkg-resolver/install-nodejs.sh new file mode 100644 index 0000000000000..7975d3e112407 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-nodejs.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="14.16.1" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "14.16.1" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "14.16.1" ]; then + mkdir -p /tmp/node && + curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz && + tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node && + cd /tmp/node || exit && + ./configure && + make "-j$(nproc)" && + make install && + cd /root || exit +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-yasm.sh b/dev-support/docker/pkg-resolver/install-yasm.sh new file mode 100644 index 0000000000000..a5f6162bc38d7 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-yasm.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="1.2.0-4" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "1.2.0-4" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "1.2.0-4" ]; then + mkdir -p /tmp/yasm && + curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \ + -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm && + rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-zstandard.sh b/dev-support/docker/pkg-resolver/install-zstandard.sh new file mode 100644 index 0000000000000..e4b1efa08b65f --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-zstandard.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="1.4.9" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "1.4.9" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "1.4.9" ]; then + mkdir -p /opt/zstd /tmp/zstd && + curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz && + tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd && + cd /opt/zstd || exit && + make "-j$(nproc)" && + make install && + cd /root || exit +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi From 88657e66939704b0fb60c3203c6b5af4e2cd7fbd Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 31 May 2021 22:40:47 +0530 Subject: [PATCH 16/22] Modularize Dockerfile for Centos 8 * Replaced explicit installation of packages with script invocations. --- dev-support/docker/Dockerfile_centos_8 | 58 ++++--------------- .../docker/pkg-resolver/install-cmake.sh | 1 + .../docker/pkg-resolver/install-epel.sh | 49 ++++++++++++++++ .../docker/pkg-resolver/install-nodejs.sh | 1 + .../docker/pkg-resolver/install-zstandard.sh | 1 + 5 files changed, 64 insertions(+), 46 deletions(-) create mode 100644 dev-support/docker/pkg-resolver/install-epel.sh diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 8fbb2910aea8b..809bb2b1e8548 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -27,8 +27,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ - && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json ###### # Install packages from yum @@ -41,10 +41,7 @@ RUN yum update -y \ #### # Install EPEL #### -RUN mkdir -p /tmp/epel \ - && curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ - -o /tmp/epel/epel-release-latest-8.noarch.rpm \ - && rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm +RUN pkg-resolver/install-epel.sh centos:8 RUN dnf --enablerepo=powertools install -y \ doxygen \ @@ -63,46 +60,7 @@ RUN dnf install -y \ RUN echo "source /opt/rh/gcc-toolset-9/enable" >> /etc/bashrc SHELL ["/bin/bash", "--login", "-c"] -#### -# Install Maven 3.6.3 -#### -RUN mkdir -p /opt/maven /tmp/maven \ - && curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \ - -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \ - && tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven - -#### -# Install CMake 3.19 -#### -# hadolint ignore=DL3003 -RUN mkdir -p /tmp/cmake /opt/cmake \ - && curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \ - && tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \ - && cd /opt/cmake || exit && ./bootstrap \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -#### -# Install zstandard -#### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/zstd /tmp/zstd \ - && curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \ - && tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \ - && cd /opt/zstd || exit \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -# RUN locale-gen en_US.UTF-8 -# ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -# ENV PYTHONIOENCODING=utf-8 - -###### -# Install common packages -###### -RUN pkg-resolver/install-common-pkgs.sh +# TODO: Set locale ###### # Set env vars required to build Hadoop @@ -111,3 +69,11 @@ ENV MAVEN_HOME /opt/maven/apache-maven-3.6.3 ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 + +###### +# Install packages +###### +RUN pkg-resolver/install-maven.sh centos:8 +RUN pkg-resolver/install-cmake.sh centos:8 +RUN pkg-resolver/install-zstandard.sh centos:8 +RUN pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/pkg-resolver/install-cmake.sh b/dev-support/docker/pkg-resolver/install-cmake.sh index a6a2f549acc50..29e2733e70196 100644 --- a/dev-support/docker/pkg-resolver/install-cmake.sh +++ b/dev-support/docker/pkg-resolver/install-cmake.sh @@ -39,6 +39,7 @@ if [ "$version_to_install" != "3.19.0" ]; then fi if [ "$version_to_install" == "3.19.0" ]; then + # hadolint ignore=DL3003 mkdir -p /tmp/cmake /opt/cmake && curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz && tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake && diff --git a/dev-support/docker/pkg-resolver/install-epel.sh b/dev-support/docker/pkg-resolver/install-epel.sh new file mode 100644 index 0000000000000..875dce3a9ae85 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-epel.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="8" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "8" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "8" ]; then + mkdir -p /tmp/epel && + curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ + -o /tmp/epel/epel-release-latest-8.noarch.rpm && + rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-nodejs.sh b/dev-support/docker/pkg-resolver/install-nodejs.sh index 7975d3e112407..5ba1c22808640 100644 --- a/dev-support/docker/pkg-resolver/install-nodejs.sh +++ b/dev-support/docker/pkg-resolver/install-nodejs.sh @@ -39,6 +39,7 @@ if [ "$version_to_install" != "14.16.1" ]; then fi if [ "$version_to_install" == "14.16.1" ]; then + # hadolint ignore=DL3003 mkdir -p /tmp/node && curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz && tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node && diff --git a/dev-support/docker/pkg-resolver/install-zstandard.sh b/dev-support/docker/pkg-resolver/install-zstandard.sh index e4b1efa08b65f..3aafd469d2be3 100644 --- a/dev-support/docker/pkg-resolver/install-zstandard.sh +++ b/dev-support/docker/pkg-resolver/install-zstandard.sh @@ -39,6 +39,7 @@ if [ "$version_to_install" != "1.4.9" ]; then fi if [ "$version_to_install" == "1.4.9" ]; then + # hadolint ignore=DL3003 mkdir -p /opt/zstd /tmp/zstd && curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz && tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd && From 21067891aa1bfb6da15b1bde0cccc0d0cf9d650e Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 31 May 2021 22:54:06 +0530 Subject: [PATCH 17/22] Modularize Dockerfile for aarch64 * Replaced explicit installations of packages with script invocations. --- dev-support/docker/Dockerfile_aarch64 | 59 ++++++--------------------- 1 file changed, 12 insertions(+), 47 deletions(-) diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 7fd21fbdfbe0d..dd0348961f464 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -36,8 +36,8 @@ ENV DEBCONF_TERSE true # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ - && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json ###### # Install packages from apt @@ -61,59 +61,16 @@ ENV MAVEN_HOME /usr ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64 ####### -# Install SpotBugs 4.2.2 +# Set env vars for SpotBugs 4.2.2 ####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Install Boost 1.72 (1.71 ships with Focal) +# Set env vars for Google Protobuf 3.7.1 ####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 - ### # Avoid out of memory errors in builds ### @@ -125,6 +82,14 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true # Force PhantomJS to be in 'headless' mode, do not connect to Xwindow ENV QT_QPA_PLATFORM offscreen +#### +# Install packages +#### +RUN pkg-resolver/install-common-pkgs.sh +RUN pkg-resolver/install-spotbugs.sh ubuntu:focal::arch64 +RUN pkg-resolver/install-boost.sh ubuntu:focal::arch64 +RUN pkg-resolver/install-protobuf.sh ubuntu:focal::arch64 + ### # Everything past this point is either not needed for testing or breaks Yetus. # So tell Yetus not to read the rest of the file: From 75a9f184c2342724dcdf90fd0069c5c8bbd23554 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 31 May 2021 23:37:53 +0530 Subject: [PATCH 18/22] Add README.md * Added a readme file for describing the package management module. --- dev-support/docker/README.md | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 dev-support/docker/README.md diff --git a/dev-support/docker/README.md b/dev-support/docker/README.md new file mode 100644 index 0000000000000..4abc65fd01ac3 --- /dev/null +++ b/dev-support/docker/README.md @@ -0,0 +1,57 @@ +# Docker images for building Hadoop + +This folder contains the Dockerfiles for building Hadoop on various platforms. + +# Dependency management + +The mode of installation of the dependencies needed for building Hadoop varies from one platform to +the other. Different platforms have different toolchains. Some packages tend to be polymorphic +across platforms and most commonly, a package that's readily available in one platform's toolchain +isn't available on another. We thus, resort to building and installing the package from source, +causing duplication of code since this needs to be done for all the Dockerfiles pertaining to all +the platforms. We need a system to track a dependency - for a package - for a platform. Thus, +there's a lot of diversity that needs to be handled for managing package dependencies and +`pkg-resolver` caters to that. + +## Supported platforms + +`pkg-resolver/platforms.json` contains a list of the supported platforms for dependency management. + +## Package dependencies + +`pkg-resolver/packages.json` maps a dependency to a given platform. Here's the schema of this JSON. + +```json +{ + "dependency_1": { + "platform_1": "package_1", + "platform_2": [ + "package_1", + "package_2" + ] + }, + "dependency_2": { + "platform_1": [ + "package_1", + "package_2", + "package_3" + ] + } +} +``` + +The root JSON element contains unique _dependency_ children. This in turn contains the name of the _ +platforms_ and the list of _packages_ to be installed for that platform. Just to give an example of +how to interpret the above JSON - + +1. For `dependency_1`, `package_1` needs to be installed for `platform_1`. +2. For `dependency_2`, `package_1` and `package_2` needs to be installed for `platform_2`. +3. For `dependency_2`, `package_1`, `package_3` and `package_3` needs to be installed for + `platform_1`. + +## Standalone packages + +Most commonly, some packages are not available across the toolchains in various platforms. Thus, we +would need to build and install them. Since we need to do this across all the Dockerfiles for all +the platforms, it could lead to code duplication and managing them becomes a hassle. Thus, we put +the build steps in a `pkg-resolver/install-.sh` and invoke this in all the Dockerfiles. \ No newline at end of file From a6528c3a382d4bc5cc4692e40ab0a1153b461f65 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Wed, 2 Jun 2021 16:12:26 +0530 Subject: [PATCH 19/22] Fix build issues * Added the necessary environment variables and install scripts. --- dev-support/docker/Dockerfile_centos_7 | 14 ++++++++++++++ dev-support/docker/Dockerfile_centos_8 | 15 ++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index 840de00a275c2..c74230e758cf6 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -54,6 +54,17 @@ ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 +####### +# Set env vars for SpotBugs +####### +ENV SPOTBUGS_HOME /opt/spotbugs + +####### +# Set env vars for Google Protobuf +####### +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" + ###### # Install packages @@ -62,5 +73,8 @@ RUN pkg-resolver/install-maven.sh centos:7 RUN pkg-resolver/install-cmake.sh centos:7 RUN pkg-resolver/install-zstandard.sh centos:7 RUN pkg-resolver/install-yasm.sh centos:7 +RUN pkg-resolver/install-protobuf.sh centos:7 +RUN pkg-resolver/install-boost.sh centos:7 +RUN pkg-resolver/install-spotbugs.sh centos:7 RUN pkg-resolver/install-nodejs.sh centos:7 RUN pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 809bb2b1e8548..5ddbec1482ef8 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -65,15 +65,28 @@ SHELL ["/bin/bash", "--login", "-c"] ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME /opt/maven/apache-maven-3.6.3 +ENV MAVEN_HOME /opt/maven ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 +####### +# Set env vars for SpotBugs +####### +ENV SPOTBUGS_HOME /opt/spotbugs + +####### +# Set env vars for Google Protobuf +#######ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" + ###### # Install packages ###### RUN pkg-resolver/install-maven.sh centos:8 RUN pkg-resolver/install-cmake.sh centos:8 +RUN pkg-resolver/install-boost.sh centos:8 +RUN pkg-resolver/install-spotbugs.sh centos:8 +RUN pkg-resolver/install-protobuf.sh centos:8 RUN pkg-resolver/install-zstandard.sh centos:8 RUN pkg-resolver/install-common-pkgs.sh From 1dc29dadcea7de995d1eae32ef8ca40382e89a52 Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Thu, 3 Jun 2021 10:25:04 +0530 Subject: [PATCH 20/22] Set env vars for protobuf * PROTOBUF_HOME was commented out. Fixed it. --- dev-support/docker/Dockerfile_centos_8 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 5ddbec1482ef8..b4109d825e47a 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -77,7 +77,8 @@ ENV SPOTBUGS_HOME /opt/spotbugs ####### # Set env vars for Google Protobuf -#######ENV PROTOBUF_HOME /opt/protobuf +####### +ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" ###### From f6433cc0a41761903241760fbde5798f0a6dcf5c Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 7 Jun 2021 11:00:05 +0530 Subject: [PATCH 21/22] Add ASF license header for README.md * Added ASF license header for README.md in dev-support/docker. --- dev-support/docker/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dev-support/docker/README.md b/dev-support/docker/README.md index 4abc65fd01ac3..32269c0929a02 100644 --- a/dev-support/docker/README.md +++ b/dev-support/docker/README.md @@ -1,3 +1,20 @@ + + # Docker images for building Hadoop This folder contains the Dockerfiles for building Hadoop on various platforms. From 263666dd7c100f3451f2e6d74b0ded80fb7ff2b7 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 7 Jun 2021 23:26:16 +0530 Subject: [PATCH 22/22] Fix create-release * create-release script was failing since it couldn't find pkg-resolver directory. * Fixed it by specifying the path to the build context. --- dev-support/bin/create-release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release index 39a5d0d319837..31ae6ee1b0659 100755 --- a/dev-support/bin/create-release +++ b/dev-support/bin/create-release @@ -514,7 +514,7 @@ function dockermode echo "USER ${user_name}" printf "\n\n" - ) | docker build -t "${imgname}" - + ) | docker build -t "${imgname}" -f - "${BASEDIR}"/dev-support/docker/ run docker run -i -t \ --privileged \