sonic_debian_extension.j2

#!/bin/bash
## This script is to automate loading of vendor specific docker images
## and instalation of configuration files and vendor specific packages
## to debian file system.
##
## USAGE:
##   ./sonic_debian_extension.sh FILESYSTEM_ROOT PLATFORM_DIR
## PARAMETERS:
##   FILESYSTEM_ROOT
##          Path to debian file system root directory

FILESYSTEM_ROOT=$1
[ -n "$FILESYSTEM_ROOT" ] || {
    echo "Error: no or empty FILESYSTEM_ROOT argument"
    exit 1
}

PLATFORM_DIR=$2
[ -n "$PLATFORM_DIR" ] || {
    echo "Error: no or empty PLATFORM_DIR argument"
    exit 1
}

## Enable debug output for script
set -x -e

. functions.sh
BUILD_TEMPLATES=files/build_templates
IMAGE_CONFIGS=files/image_config
SCRIPTS_DIR=files/scripts

# Define target fold macro
FILESYSTEM_ROOT_USR="$FILESYSTEM_ROOT/usr"
FILESYSTEM_ROOT_USR_SHARE="$FILESYSTEM_ROOT_USR/share"
FILESYSTEM_ROOT_USR_SHARE_SONIC="$FILESYSTEM_ROOT_USR_SHARE/sonic"
FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES="$FILESYSTEM_ROOT_USR_SHARE_SONIC/templates"

clean_sys() {
    sudo umount $FILESYSTEM_ROOT/sys/fs/cgroup/*            \
                $FILESYSTEM_ROOT/sys/fs/cgroup              \
                $FILESYSTEM_ROOT/sys || true
}
trap_push clean_sys
sudo LANG=C chroot $FILESYSTEM_ROOT mount sysfs /sys -t sysfs

sudo bash -c "echo \"DOCKER_OPTS=\"--storage-driver=overlay2\"\" >> $FILESYSTEM_ROOT/etc/default/docker"
sudo cp files/docker/docker $FILESYSTEM_ROOT/etc/init.d/
sudo chroot $FILESYSTEM_ROOT service docker start

# Apply apt configuration files
sudo cp $IMAGE_CONFIGS/apt/sources.list $FILESYSTEM_ROOT/etc/apt/
sudo cp -R $IMAGE_CONFIGS/apt/sources.list.d/ $FILESYSTEM_ROOT/etc/apt/
cat $IMAGE_CONFIGS/apt/sonic-dev.gpg.key | sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add -

# Update apt's snapshot of its repos
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get update

# Apply environtment configuration files
sudo cp $IMAGE_CONFIGS/environment/environment $FILESYSTEM_ROOT/etc/
sudo cp $IMAGE_CONFIGS/environment/motd $FILESYSTEM_ROOT/etc/

# Create all needed directories
sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic/
sudo mkdir -p $FILESYSTEM_ROOT/etc/modprobe.d/
sudo mkdir -p $FILESYSTEM_ROOT/var/cache/sonic/
sudo mkdir -p $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/

# Install dependencies for SONiC config engine
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install \
    python-dev     \
    python-lxml    \
    python-yaml    \
    python-bitarray

# Install SONiC config engine Python package
CONFIG_ENGINE_WHEEL_NAME=$(basename {{config_engine_wheel_path}})
sudo cp {{config_engine_wheel_path}} $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $CONFIG_ENGINE_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_WHEEL_NAME

# Install Python client for Redis
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install "redis==2.10.6"

# Install redis-dump-load Python 2 package
REDIS_DUMP_LOAD_PY2_WHEEL_NAME=$(basename {{redis_dump_load_py2_wheel_path}})
sudo cp {{redis_dump_load_py2_wheel_path}} $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $REDIS_DUMP_LOAD_PY2_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$REDIS_DUMP_LOAD_PY2_WHEEL_NAME

# Install Python module for ipaddress
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install ipaddress

# Install SwSS SDK Python 2 package
SWSSSDK_PY2_WHEEL_NAME=$(basename {{swsssdk_py2_wheel_path}})
sudo cp {{swsssdk_py2_wheel_path}} $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SWSSSDK_PY2_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SWSSSDK_PY2_WHEEL_NAME

# Install sonic-platform-common Python 2 package
PLATFORM_COMMON_PY2_WHEEL_NAME=$(basename {{platform_common_py2_wheel_path}})
sudo cp {{platform_common_py2_wheel_path}} $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY2_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $PLATFORM_COMMON_PY2_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY2_WHEEL_NAME

# Install built Python Click package (and its dependencies via 'apt-get -y install -f')
# Do this before installing sonic-utilities so that it doesn't attempt to install
# an older version as part of its dependencies
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python-click*_all.deb || \
    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f

# Install python pexpect used by sonic-utilities consutil
# using pip install instead to get a more recent version than is available through debian
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install pexpect
# Install python click-default-group by sonic-utilities
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install click-default-group==1.2

# Install tabulate >= 0.8.1 via pip in order to support multi-line row output for sonic-utilities
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install tabulate==0.8.2

# Install SONiC Utilities (and its dependencies via 'apt-get -y install -f')
sudo dpkg --root=$FILESYSTEM_ROOT -i $python_debs_path/python-sonic-utilities_*.deb || \
    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f

# SONiC utilities installs bash-completion as a dependency. However, it is disabled by default
# in bash.bashrc, so we copy a version of the file with it enabled here.
sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/

# Install SONiC Device Data  (and its dependencies via 'apt-get -y install -f')
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-device-data_*.deb || \
    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f

# Install pam-tacplus and nss-tacplus
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libtac2_*.deb || \
    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libpam-tacplus_*.deb || \
    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libnss-tacplus_*.deb || \
    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
# Disable tacplus by default
sudo LANG=C chroot $FILESYSTEM_ROOT pam-auth-update --remove tacplus
sudo sed -i -e '/^passwd/s/ tacplus//' $FILESYSTEM_ROOT/etc/nsswitch.conf

# Copy crontabs
sudo cp -f $IMAGE_CONFIGS/cron.d/* $FILESYSTEM_ROOT/etc/cron.d/

# Copy NTP configuration files and templates
sudo cp $IMAGE_CONFIGS/ntp/ntp-config.service $FILESYSTEM_ROOT/etc/systemd/system/
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable ntp-config.service
sudo cp $IMAGE_CONFIGS/ntp/ntp-config.sh $FILESYSTEM_ROOT/usr/bin/
sudo cp $IMAGE_CONFIGS/ntp/ntp.conf.j2 $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/

# Copy warmboot-finalizer files
sudo LANG=C cp $IMAGE_CONFIGS/warmboot-finalizer/finalize-warmboot.sh $FILESYSTEM_ROOT/usr/local/bin/finalize-warmboot.sh
sudo LANG=C cp $IMAGE_CONFIGS/warmboot-finalizer/warmboot-finalizer.service $FILESYSTEM_ROOT/etc/systemd/system/
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable warmboot-finalizer.service

# Copy rsyslog configuration files and templates
sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog-config.service  $FILESYSTEM_ROOT/etc/systemd/system/
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable rsyslog-config.service
sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog-config.sh $FILESYSTEM_ROOT/usr/bin/
sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.conf.j2 $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.d/* $FILESYSTEM_ROOT/etc/rsyslog.d/

# Copy logrotate.d configuration files
sudo cp -f $IMAGE_CONFIGS/logrotate/logrotate.d/* $FILESYSTEM_ROOT/etc/logrotate.d/

# Copy systemd-journald configuration files
sudo cp -f $IMAGE_CONFIGS/systemd/journald.conf $FILESYSTEM_ROOT/etc/systemd/

# Copy interfaces configuration files and templates
sudo cp $IMAGE_CONFIGS/interfaces/interfaces-config.service  $FILESYSTEM_ROOT/etc/systemd/system/
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable interfaces-config.service
sudo cp $IMAGE_CONFIGS/interfaces/interfaces-config.sh $FILESYSTEM_ROOT/usr/bin/
sudo cp $IMAGE_CONFIGS/interfaces/*.j2 $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/

# Copy initial interfaces configuration file, will be overwritten on first boot
sudo cp $IMAGE_CONFIGS/interfaces/init_interfaces $FILESYSTEM_ROOT/etc/network/interfaces
sudo mkdir -p $FILESYSTEM_ROOT/etc/network/interfaces.d

# Copy hostcfgd files
sudo cp $IMAGE_CONFIGS/hostcfgd/hostcfgd.service $FILESYSTEM_ROOT/etc/systemd/system/
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable hostcfgd.service
sudo cp $IMAGE_CONFIGS/hostcfgd/hostcfgd $FILESYSTEM_ROOT/usr/bin/
sudo cp $IMAGE_CONFIGS/hostcfgd/*.j2 $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/

# Copy the buffer configuration template
sudo cp $BUILD_TEMPLATES/buffers_config.j2 $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/

# Copy the qos configuration template
sudo cp $BUILD_TEMPLATES/qos_config.j2 $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/

# Copy hostname configuration scripts
sudo cp $IMAGE_CONFIGS/hostname/hostname-config.service  $FILESYSTEM_ROOT/etc/systemd/system/
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable hostname-config.service
sudo cp $IMAGE_CONFIGS/hostname/hostname-config.sh $FILESYSTEM_ROOT/usr/bin/

# Copy updategraph script and service file
j2 files/build_templates/updategraph.service.j2 | sudo tee $FILESYSTEM_ROOT/etc/systemd/system/updategraph.service
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable updategraph.service
sudo cp $IMAGE_CONFIGS/updategraph/updategraph $FILESYSTEM_ROOT/usr/bin/
{% if enable_dhcp_graph_service == "y" %}
sudo bash -c "echo enabled=true > $FILESYSTEM_ROOT/etc/sonic/updategraph.conf"
sudo bash -c "echo src=dhcp >> $FILESYSTEM_ROOT/etc/sonic/updategraph.conf"
sudo bash -c "echo dhcp_as_static=true >> $FILESYSTEM_ROOT/etc/sonic/updategraph.conf"
{% else %}
sudo bash -c "echo enabled=false > $FILESYSTEM_ROOT/etc/sonic/updategraph.conf"
{% endif %}
sudo bash -c "echo '{ \"DEVICE_METADATA\": { \"localhost\": { \"default_bgp_status\": {% if shutdown_bgp_on_start == "y" %}\"down\"{% else %}\"up\"{% endif %}, \"default_pfcwd_status\": {% if enable_pfcwd_on_start == "y" %}\"enable\"{% else %}\"disable\"{% endif %}  } },
{%- print ' \\"CRM\\": { \\"Config\\": { \\"polling_interval\\": \\"300\\", ' %}
{%- for crm_res in ["ipv4_route", "ipv6_route", "ipv4_nexthop", "ipv6_nexthop", "ipv4_neighbor", "ipv6_neighbor", "nexthop_group_member", "nexthop_group", "acl_table", "acl_group", "acl_entry", "acl_counter", "fdb_entry"] -%}
\"{{crm_res}}_threshold_type\": \"percentage\", \"{{crm_res}}_low_threshold\": \"70\", \"{{crm_res}}_high_threshold\": \"85\"{% if not loop.last %}, {% endif %}
{%- endfor %} } } }' >> $FILESYSTEM_ROOT/etc/sonic/init_cfg.json"

# Copy SNMP configuration files
sudo cp $IMAGE_CONFIGS/snmp/snmp.yml $FILESYSTEM_ROOT/etc/sonic/

# Copy ASN configuration files
sudo cp $IMAGE_CONFIGS/asn/deployment_id_asn_map.yml $FILESYSTEM_ROOT/etc/sonic/

# Copy sudoers configuration file
sudo cp $IMAGE_CONFIGS/sudoers/sudoers $FILESYSTEM_ROOT/etc/
sudo cp $IMAGE_CONFIGS/sudoers/sudoers.lecture $FILESYSTEM_ROOT/etc/

# Copy control plane ACL management daemon files
sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd.service  $FILESYSTEM_ROOT/etc/systemd/system/
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable caclmgrd.service
sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd $FILESYSTEM_ROOT/usr/bin/

## Install package without starting service
## ref: https://wiki.debian.org/chroot
sudo tee -a $FILESYSTEM_ROOT/usr/sbin/policy-rc.d > /dev/null <<EOF
#!/bin/sh
exit 101
EOF
sudo chmod a+x $FILESYSTEM_ROOT/usr/sbin/policy-rc.d

{% if installer_debs.strip() -%}
{% for deb in installer_debs.strip().split(' ') -%}
{% if sonic_asic_platform == "mellanox" %}
sudo dpkg --extract {{deb}} $FILESYSTEM_ROOT
{% else %}
sudo dpkg --root=$FILESYSTEM_ROOT -i {{deb}} || sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
{% endif %}
{% endfor %}
{% endif %}

## Run depmod command for target kernel modules
sudo LANG=C chroot $FILESYSTEM_ROOT depmod -a {{kversion}}

## download all dependency packages for platform debian packages
{% if lazy_installer_debs.strip() -%}
{% for file in lazy_installer_debs.strip().split(' ') -%}

{% set dev = file.split('@')[0] -%}
{% set deb = file.split('@')[1] -%}
{% set debfilename = deb.split('/')|last -%}
{% set debname = debfilename.split('_')|first -%}

sudo dpkg --root=$FILESYSTEM_ROOT -i {{deb}} || sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f --download-only

sudo mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR/{{dev}}
sudo cp {{ deb }} $FILESYSTEM_ROOT/$PLATFORM_DIR/{{dev}}/
for f in $(find $FILESYSTEM_ROOT/var/cache/apt/archives -name "*.deb"); do
    sudo mv $f $FILESYSTEM_ROOT/$PLATFORM_DIR/{{dev}}/
done

sudo dpkg --root=$FILESYSTEM_ROOT -P {{ debname }}

{% endfor %}
{% endif %}

sudo rm -f $FILESYSTEM_ROOT/usr/sbin/policy-rc.d

# Copy fstrim service and timer file, enable fstrim timer
sudo cp $IMAGE_CONFIGS/fstrim/* $FILESYSTEM_ROOT/etc/systemd/system/
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable fstrim.timer

## copy platform rc.local
sudo cp $IMAGE_CONFIGS/platform/rc.local $FILESYSTEM_ROOT/etc/

## copy blacklist file
sudo cp $IMAGE_CONFIGS/platform/linux_kernel_bde.conf $FILESYSTEM_ROOT/etc/modprobe.d/

{% if installer_images.strip() -%}
sudo chroot $FILESYSTEM_ROOT docker info
{% for image in installer_images.strip().split(' ') -%}
{% set imagefilename = image.split('/')|last -%}
{% set imagename = imagefilename.split('.')|first -%}
sudo LANG=C chroot $FILESYSTEM_ROOT docker load < {{image}}
sudo LANG=C chroot $FILESYSTEM_ROOT docker tag {{imagename}}:latest {{imagename}}:$(sonic_get_version)
{% if imagename.endswith('-dbg') %}
{% set imagebasename = imagename.replace('-dbg', '') -%}
sudo LANG=C chroot $FILESYSTEM_ROOT docker tag {{imagename}}:latest {{imagebasename}}:$(sonic_get_version)
sudo LANG=C chroot $FILESYSTEM_ROOT docker tag {{imagename}}:latest {{imagebasename}}:latest
{% endif %}
{% endfor %}
sudo chroot $FILESYSTEM_ROOT service docker stop
sudo rm $FILESYSTEM_ROOT/etc/init.d/docker
{% for script in installer_start_scripts.split(' ') -%}
sudo cp {{script}} $FILESYSTEM_ROOT/usr/bin/
{% endfor %}
{% for service in installer_services.split(' ') -%}
if [ -f {{service}} ]; then
    sudo cp {{service}} $FILESYSTEM_ROOT/etc/systemd/system/
    sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable {{service}}
fi
{% endfor %}
sudo LANG=C chroot $FILESYSTEM_ROOT fuser -km /sys || true
sudo LANG=C chroot $FILESYSTEM_ROOT umount -lf /sys
{% endif %}

# Copy swss and syncd service script
sudo LANG=C cp $SCRIPTS_DIR/swss.sh $FILESYSTEM_ROOT/usr/local/bin/swss.sh
sudo LANG=C cp $SCRIPTS_DIR/syncd.sh $FILESYSTEM_ROOT/usr/local/bin/syncd.sh

# Copy systemd timer configuration
# It implements delayed start of services
sudo cp $BUILD_TEMPLATES/snmp.timer $FILESYSTEM_ROOT/etc/systemd/system/
sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable snmp.timer

sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get purge -y python-dev
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get clean -y
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get autoremove -y

{% for file in installer_extra_files.split(' ') -%}
{% if file.strip() -%}
{% set src = file.split(':')[0] -%}
{% set dst = file.split(':')[1] -%}
sudo cp {{src}} $FILESYSTEM_ROOT/{{dst}}
{% endif -%}
{% endfor -%}

{% if sonic_asic_platform == "mellanox" %}
sudo mkdir -p $FILESYSTEM_ROOT/etc/mlnx/
sudo cp $files_path/$MLNX_SPC_FW_FILE $FILESYSTEM_ROOT/etc/mlnx/fw-SPC.mfa
sudo cp $files_path/$MLNX_SPC2_FW_FILE $FILESYSTEM_ROOT/etc/mlnx/fw-SPC2.mfa
sudo cp $files_path/$ISSU_VERSION_FILE $FILESYSTEM_ROOT/etc/mlnx/issu-version
sudo cp $files_path/$MLNX_FFB_SCRIPT $FILESYSTEM_ROOT/usr/bin/mlnx-ffb.sh
j2 platform/mellanox/mlnx-fw-upgrade.j2 | sudo tee $FILESYSTEM_ROOT/usr/bin/mlnx-fw-upgrade.sh
sudo chmod 755 $FILESYSTEM_ROOT/usr/bin/mlnx-fw-upgrade.sh
{% endif %}

{%- if SONIC_ROUTING_STACK == "frr" %}
sudo mkdir $FILESYSTEM_ROOT/etc/sonic/frr
sudo touch $FILESYSTEM_ROOT/etc/sonic/frr/frr.conf
sudo touch $FILESYSTEM_ROOT/etc/sonic/frr/vtysh.conf
sudo chown -R $FRR_USER_UID:$FRR_USER_GID $FILESYSTEM_ROOT/etc/sonic/frr
sudo chmod -R 640 $FILESYSTEM_ROOT/etc/sonic/frr/
sudo chmod 750 $FILESYSTEM_ROOT/etc/sonic/frr
{%- endif %}