diff --git a/Makefile.include b/Makefile.include index 1d7b1c9febbe..38fc55b3c441 100644 --- a/Makefile.include +++ b/Makefile.include @@ -352,7 +352,7 @@ ifeq (,$(and $(DOWNLOAD_TO_STDOUT),$(DOWNLOAD_TO_FILE))) DOWNLOAD_TO_STDOUT ?= $(if $(CURL),$(CURL) -s,$(WGET) -q -O-) endif ifeq (,$(DOWNLOAD_TO_FILE)) - DOWNLOAD_TO_FILE ?= $(if $(WGET),$(WGET) -nv -c -O,$(CURL) -s -o) + DOWNLOAD_TO_FILE ?= $(DLCACHE) endif endif diff --git a/dist/tools/dlcache/README.md b/dist/tools/dlcache/README.md index 059f19212000..b88d1f45b594 100644 --- a/dist/tools/dlcache/README.md +++ b/dist/tools/dlcache/README.md @@ -1,11 +1,24 @@ # Introduction -This script aims to cache http(s)-downloads +This script aims to cache http(s)-downloads of (mainly) zip archives. ## How it works -- if a file with the right name and md5 exists, nothing happens -- if a file with the right name and md5 is in cache, use that -- if a file with the right name but wrong md5 is in cache, redownload +- if a file with the right name and SHA512 exists, nothing happens +- if a file with the right name and SHA512 is in cache, use that +- if a file with the right name but wrong SHA512 is in cache, redownload download to cache -- after download, check md5, then copy to target +- after download, check SHA512, then copy to target +- if no SHA512 checksum is given, always redownload + +## Usage + +In order to stay compatible with the `$(DOWNLOAD_TO_FILE)` command, +the command usage was changed for Release 2025.10. + +If no SHA512 checksum was given, the file will *always* be downloaded, +regardless of whether it is in the cache or not. + +```sh +dist/tools/dlcache/dlcache.sh [SHA512 checksum] +``` diff --git a/dist/tools/dlcache/dlcache.sh b/dist/tools/dlcache/dlcache.sh index 1585f9751b11..a7ab43c3c78c 100755 --- a/dist/tools/dlcache/dlcache.sh +++ b/dist/tools/dlcache/dlcache.sh @@ -13,13 +13,13 @@ if [ "$(uname)" = Darwin ]; then local lockfile="$1" shift - while ! shlock -p $$ -f $lockfile; do + while ! shlock -p "$$" -f "$lockfile"; do sleep 0.2 done - $* + "$@" - rm $lockfile + rm "$lockfile" } else _locked() { @@ -28,31 +28,28 @@ else ( flock -w 600 9 || exit 1 - $* + "$@" ) 9>"$lockfile" } fi -if [ "$(uname)" = Darwin ]; then - MD5="md5 -r" -else - MD5=md5sum -fi +# shasum is supported on Linux and Darwin +SHA512="shasum -a 512" -calcmd5() { +calcsha512() { local file="$1" - local md5="$2" + local sha512="$2" + local file_sha512 + file_sha512=$(${SHA512} "$file" | cut -d\ -f1) - local file_md5=$(${MD5} "$file" | cut -d\ -f1) - - test "$md5" = "$file_md5" + test "$sha512" = "$file_sha512" } downloader() { if [ -n "$(command -v wget)" ]; then - wget -nv "$1" -O $2 + wget -nv "$1" -O "$2" elif [ -n "$(command -v curl)" ]; then - curl -L $1 -o $2 + curl -L "$1" -o "$2" else _echo "$0: neither wget nor curl available!" return 1 @@ -60,26 +57,28 @@ downloader() { } download() { - local url="$1" - local _md5="$2" - local basename_url=$(basename ${url}) - local target="${3:-${basename_url}}" - - [ -f "$target" ] && { - # if our target file exists, check it's md5. - calcmd5 "$target" "$_md5" && { - _echo "$0: target exists, md5 matches." + local url="$2" + local _sha512="$3" + local basename_url + basename_url="$(basename "${url}")" + local target="${1:-"${basename_url}"}" + + if [ -f "$target" ] && [ -n "$_sha512" ]; then + # if our target file exists and an SHA512 sum was given, check it's SHA512 + calcsha512 "$target" "$_sha512" && { + _echo "$0: target exists, SHA512 matches." exit 0 } - } + fi - local filename="$(basename $url)" + local filename + filename="$(basename "$url")" [ -f "$DLCACHE_DIR/$filename" ] && { - # if the file exists in cache, check it's md5 and possibly remove it. - if calcmd5 "$DLCACHE_DIR/$filename" "$_md5"; then + # if the file exists in cache, check it's SHA512 and possibly remove it. + if calcsha512 "$DLCACHE_DIR/$filename" "$_sha512"; then _echo "$0: getting \"$url\" from cache" else - _echo "$0: \"$DLCACHE_DIR/$filename\" has wrong checksum, re-downloading" + _echo "$0: \"$DLCACHE_DIR/$filename\" has wrong or no checksum, re-downloading" rm "$DLCACHE_DIR/$filename" fi } @@ -93,10 +92,13 @@ download() { _echo "$0: done downloading \"$url\"" } - calcmd5 "$DLCACHE_DIR/$filename" "$_md5" || { - _echo "$0: checksum mismatch!" - exit 1 - } + # only try to calculate the checksum if a checksum was given + if [ -n "$_sha512" ]; then + calcsha512 "$DLCACHE_DIR/$filename" "$_sha512" || { + _echo "$0: checksum mismatch!" + exit 1 + } + fi if [ "$target" = "-" ]; then cat "$DLCACHE_DIR/$filename" @@ -105,4 +107,4 @@ download() { fi } -_locked "$DLCACHE_DIR/$(basename $1).locked" download "$@" +_locked "$DLCACHE_DIR/$(basename "$1").locked" download "$@" diff --git a/makefiles/mcuboot.mk b/makefiles/mcuboot.mk index 611c6c75672f..8bb4e47cdea4 100644 --- a/makefiles/mcuboot.mk +++ b/makefiles/mcuboot.mk @@ -8,7 +8,7 @@ SIGN_BINFILE = $(BINDIR)/signed-$(APPLICATION).bin MCUBOOT_KEYFILE ?= $(BINDIR)/key.pem MCUBOOT_BIN ?= $(BINDIR)/mcuboot.bin MCUBOOT_BIN_URL ?= http://download.riot-os.org/mynewt.mcuboot.bin -MCUBOOT_BIN_MD5 ?= 0c71a0589bd3709fc2d90f07a0035ce7 +MCUBOOT_BIN_SHA512 ?= export IMAGE_HDR_SIZE ?= 512 @@ -37,7 +37,7 @@ mcuboot: mcuboot-create-key link @$(COLOR_ECHO) $(MCUBOOT_BIN): - $(Q)$(DLCACHE) $(MCUBOOT_BIN_URL) $(MCUBOOT_BIN_MD5) $@ + $(Q)$(DLCACHE) $@ $(MCUBOOT_BIN_URL) $(MCUBOOT_BIN_SHA512) .PHONY: mcuboot-flash-bootloader mcuboot-flash diff --git a/pkg/Makefile.http b/pkg/Makefile.http index 15d1ae398665..23999abb49dd 100644 --- a/pkg/Makefile.http +++ b/pkg/Makefile.http @@ -3,6 +3,7 @@ PKG_URL = http://example.com/downloads # source url of the package e.g. a gi PKG_VERSION = v1.2.3 # version of the package to use e.g. a git commit/ref PKG_EXT = zip # extension of this package PKG_LICENSE = MIT # license of the package +PKG_SHA512 = 5bcc4310f09ea247f5b65d63afae872fb92bb5c39029f336d94503fe04feccfcd22aa42b377b0927ab7f0a19111fd5a0a6842ecab147761c8f218f7f115e0da5 # SHA512 checksum of the package ifneq ($(RIOTBASE),) include $(RIOTBASE)/Makefile.base @@ -30,7 +31,9 @@ $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION)/: $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION).$(PK $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT): # Get PKG_VERSION of package from PKG_URL - $(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT) + $(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT) $(PKG_SHA512) + # Note: Since Release 2025.10, DOWNLOAD_TO_FILE uses dlcache and + # requires an SHA512 checksum for caching! clean:: # Reset package to checkout state. diff --git a/pkg/c25519/Makefile b/pkg/c25519/Makefile index 06dd824850e6..ed486b97d86e 100644 --- a/pkg/c25519/Makefile +++ b/pkg/c25519/Makefile @@ -26,7 +26,7 @@ $(PKG_SOURCE_DIR)/: $(PKG_ZIPFILE) $(PKG_ZIPFILE): $(QQ)mkdir -p $(PKGDIRBASE) - $(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_ZIP_URL) + $(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_ZIP_URL) $(PKG_SHA512) clean:: # Reset package to checkout state. diff --git a/pkg/driver_cryptocell_310/Makefile b/pkg/driver_cryptocell_310/Makefile index 0871087a0201..0c2c34afbae4 100644 --- a/pkg/driver_cryptocell_310/Makefile +++ b/pkg/driver_cryptocell_310/Makefile @@ -15,7 +15,7 @@ PKG_ZIP_URL = $(PKG_URL)/$(PKG_DIR_NAME)_$(PKG_VERSION).$(PKG_EXT) NRF_CC310_PATH = external/nrf_cc310 ifneq ($(RIOTBASE),) -include $(RIOTBASE)/Makefile.base + include $(RIOTBASE)/Makefile.base endif .PHONY: all clean distclean @@ -34,7 +34,7 @@ $(PKG_SOURCE_DIR)/: $(PKG_ZIPFILE) $(PKG_ZIPFILE): $(QQ)mkdir -p $(PKGDIRBASE) - $(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_ZIP_URL) + $(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_ZIP_URL) $(PKG_SHA512) clean:: rm -rf $(PKG_SOURCE_DIR)