pkg: use esptool.py v4.9.0 as package

cpu/esp32/Makefile.dep

@@ -2,6 +2,7 @@
 
 include $(RIOTCPU)/esp_common/Makefile.dep
 
+USEPKG += esptool
 USEPKG += esp32_sdk
 
 USEMODULE += esp_idf_common

cpu/esp32/Makefile.include

@@ -92,7 +92,10 @@ endif
 
 FLASHFILE_POS ?= 0x10000
 
-ESPTOOL ?= $(RIOTTOOLS)/esptools/esptool_v3.2.py
+ifeq (,$(ESPTOOL))
+  ESPTOOL ?= $(PKGDIRBASE)/esptool/venv/bin/esptool.py
+  ESPTOOL_FLASH ?= $(PKGDIRBASE)/esptool/venv_flash/bin/esptool.py
+endif
 
 include $(RIOTCPU)/esp_common/Makefile.include
 

cpu/esp32/bootloader/Makefile

@@ -418,7 +418,7 @@ $(ESP_SDK_BOOTLOADER_ELF): \
 	$(Q)$(CC) -o $@ $(LINKFLAGS) -Wl,-Map=$(@:%.elf=%.map)
 
 FLASH_CHIP = $(CPU_FAM)
-ESPTOOL ?= $(RIOTTOOLS)/esptools/esptool_v3.2.py
+ESPTOOL ?= $(PKGDIRBASE)/esptool/venv/bin/esptool.py
 # TODO: These should be exported/configurable from the app side. That would
 # require to export these values.
 FLASH_MODE ?= dio   # ESP-IDF uses dio as default flash mode for the bootloader

cpu/esp32/doc.txt

@@ -439,30 +439,7 @@ The building process using Docker comprises two steps:
 Both steps can also be performed with a single command on the host system
 by setting the `BUILD_IN_DOCKER` variable:
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-$ BUILD_IN_DOCKER=1 DOCKER="sudo docker" \
-  make flash BOARD=...
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-@note
-During the migration phase from the ESP32 toolchain with GCC 5.2.0, which was
-specially compiled for RIOT, to Espressif's precompiled ESP32 vendor toolchain
-with GCC 8.4.0, the RIOT Docker build image
-[schorcht/riotbuild_esp32_espressif_gcc_8.4.0](https://hub.docker.com/repository/docker/schorcht/riotbuild_esp32_espressif_gcc_8.4.0)
-has to be used instead of `riot/riotbuild` as this already contains the
-precompiled ESP32 vendor toolchain from Espressif while `riot/riotbuild`
-does not.
-Therefore, the RIOT Docker build image has to be pulled with command:
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-$ sudo docker pull schorcht/riotbuild_esp32_espressif_gcc_8.4.0
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-and the RIOT Docker build image in step 1 has to be started with command:
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-$ sudo docker run --rm -i -t -u $UID -v $(pwd):/data/riotbuild schorcht/riotbuild_esp32_espressif_gcc_8.4.0
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-The single step build command on the host system has then to be:
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-$ BUILD_IN_DOCKER=1 DOCKER="sudo docker" DOCKER_IMAGE=schorcht/riotbuild_esp32_espressif_gcc_8.4.0 \
-  make flash BOARD=...
+$ BUILD_IN_DOCKER=1 make flash BOARD=...
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 [Back to table of contents](#esp32_toc)
@@ -547,25 +524,17 @@ install it separately.
 
 ### Installation of esptool.py (ESP flash programmer tool) {#esp32_installation_of_esptool}
 
-The RIOT port does not work with the `esptool.py` ESP flasher program
-available on [GitHub](https://github.com/espressif/esptool) or
-as a package for your OS. Instead, a modified version included in
-ESP-IDF SDK is required.
+The RIOT port uses the ESP flasher programme `esptool.py` as a package directly
+from [GitHub](https://github.com/espressif/esptool) within a virtual Python
+environment. The `esptool.py` version that is available as a Python package
+for your operating system does not normally work.
 
-To avoid the installation of the complete ESP-IDF SDK, for example, because
-RIOT Docker build image is used for compilation, `esptool.py` has been
-extracted from the SDK and placed in RIOT's directory `dist/tools/esptool`.
-For convenience, the build system uses always the version from this directory.
+The RIOT port uses the ESP flasher program `esptool.py` as package directly from [GitHub](https://github.com/espressif/esptool) within an virtual Python environment. The version of `esptool.py` available as Python package for your OS does not usually work.
 
-Therefore, it is **not necessary to install** `esptool.py` explicitly. However
-`esptool.py` depends on `pySerial` which can be installed either
-using `pip`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-$ sudo pip3 install pyserial
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-or the package manager of your OS, for example on Debian/Ubuntu systems:
+It is also possible to use your own version of `esptool.py` by overriding
+the make variable `ESPTOOL` in the command line, for example:
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-$ apt install python3-serial
+$ ESPTOOL=/path/to/esptool.py make flash BOARD=...
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 For more information on `esptool.py`, please refer to the

makefiles/tools/esptool.inc.mk

@@ -5,6 +5,7 @@ ifneq (,$(filter qout qio,$(FLASH_MODE)))
 endif
 
 ESPTOOL ?= $(RIOTTOOLS)/esptools/esptool.py
+ESPTOOL_FLASH ?= $(ESPTOOL)
 
 # flasher configuration
 ifneq (,$(filter esp_qemu,$(USEMODULE)))
@@ -13,17 +14,18 @@ ifneq (,$(filter esp_qemu,$(USEMODULE)))
   FLASHDEPS += esp-qemu
 else
   PROGRAMMER_SPEED ?= 460800
-  FLASHER = $(ESPTOOL)
+  FLASHER = $(ESPTOOL_FLASH)
   FFLAGS += --chip $(FLASH_CHIP) --port $(PROG_DEV) --baud $(PROGRAMMER_SPEED)
   FFLAGS += --before default_reset write_flash -z
   FFLAGS += --flash_mode $(FLASH_MODE) --flash_freq $(FLASH_FREQ)
   FFLAGS += --flash_size detect
   FFLAGS += $(BOOTLOADER_POS) $(BOOTLOADER_BIN)
   FFLAGS += 0x8000 $(BINDIR)/partitions.bin
   FFLAGS += $(FLASHFILE_POS) $(FLASHFILE)
+  FLASHDEPS += esptool_flash
 endif
 
-.PHONY: esp-qemu
+.PHONY: esp-qemu esptool_flash clean
 
 esp-qemu: $(FLASHFILE)
 ifeq (esp32,$(CPU))
@@ -51,6 +53,12 @@ else
 		head -c $(FLASH_SIZE)MB > $(BINDIR)/$(CPU)flash.bin && rm tmp.bin
 endif
 
+esptool_flash: $(PKGDIRBASE)/esptool/venv_flash/bin/esptool.py
+
+$(PKGDIRBASE)/esptool/venv_flash/bin/esptool.py:
+	python3 -m venv $(PKGDIRBASE)/esptool/venv_flash
+	$(PKGDIRBASE)/esptool/venv_flash/bin/pip install esptool
+
 # reset tool configuration
 RESET ?= $(RIOTTOOLS)/esptools/espreset.py
 RESET_FLAGS ?= --port $(PROG_DEV)

pkg/esptool/Makefile

@@ -0,0 +1,16 @@
+PKG_NAME=esptool
+PKG_URL=https://github.com/espressif/esptool
+# esptool v4.9.0
+PKG_VERSION=8c3e3bf5d4292f1451744a838b2e60f50aa02b3c
+PKG_LICENSE=GPLv2
+
+include $(RIOTBASE)/pkg/pkg.mk
+
+all: $(PKG_SOURCE_DIR)/venv/bin/esptool.py
+
+clean::
+	rm -rf $(PKG_SOURCE_DIR)/venv*
+
+$(PKG_SOURCE_DIR)/venv/bin/esptool.py:
+	python3 -m venv $(PKG_SOURCE_DIR)/venv
+	$(PKG_SOURCE_DIR)/venv/bin/pip install esptool

pkg/esptool/Makefile.include

@@ -0,0 +1 @@
+PSEUDOMODULES += esptool