diff --git a/docs/en/get-started/windows-setup.rst b/docs/en/get-started/windows-setup.rst index 9dccad8dc..a8efd5639 100644 --- a/docs/en/get-started/windows-setup.rst +++ b/docs/en/get-started/windows-setup.rst @@ -24,7 +24,7 @@ ESP-IDF Tools Installer The easiest way to install ESP-IDF's prerequisites is to download the ESP-IDF Tools installer from this URL: -https://dl.espressif.com/dl/esp-idf-tools-setup-2.0.exe +https://dl.espressif.com/dl/esp-idf-tools-setup-2.1.exe The installer includes the cross-compilers, OpenOCD, cmake_ and Ninja_ build tool, and a configuration tool called mconf-idf_. The installer can also download and run installers for Python_ 3.7 and `Git For Windows`_ if they are not already installed on the computer. diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index c5dab206d..0d756d532 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -345,19 +345,5 @@ build_installer: - build_cmdlinerunner before_script: [] script: - - mkdir idf_tools_tmp - - export IDF_TOOLS_PATH=$PWD/idf_tools_tmp - - tools/idf_tools.py --non-interactive download --platform Windows-x86_64 all - - tools/idf_tools.py --tools-json tools/windows/tool_setup/tools_fallback.json --non-interactive download --platform Windows-x86_64 all - - mkdir tools/windows/tool_setup/dist - - mv idf_tools_tmp/dist/* tools/windows/tool_setup/dist/ - - cd tools/windows/tool_setup/ - - mkdir unzip - - cd unzip - - wget --no-verbose https://www.7-zip.org/a/7z1900-extra.7z - - 7zr e -y 7z1900-extra.7z - - cd .. - - - wget --no-verbose https://dl.espressif.com/dl/esp-idf/idf_versions.txt - - iscc idf_tool_setup.iss + - ./build_installer.sh diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 06a9ff318..d3d559c7e 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -81,3 +81,4 @@ tools/test_idf_tools/test_idf_tools.py tools/unit-test-app/unit_test.py tools/windows/eclipse_make.sh tools/windows/tool_setup/build_installer.sh +tools/windows/tool_setup/sign_installer.sh diff --git a/tools/idf_tools.py b/tools/idf_tools.py index 3d2341237..a04cc1c9c 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -61,6 +61,12 @@ try: except ImportError: from urllib import urlretrieve +try: + from exceptions import WindowsError +except ImportError: + class WindowsError(OSError): + pass + TOOLS_FILE = 'tools/tools.json' TOOLS_SCHEMA_FILE = 'tools/tools_schema.json' @@ -254,13 +260,34 @@ def unpack(filename, destination): archive_obj.extractall(destination) +# Sometimes renaming a directory on Windows (randomly?) causes a PermissionError. +# This is confirmed to be a workaround: +# https://github.com/espressif/esp-idf/issues/3819#issuecomment-515167118 +# https://github.com/espressif/esp-idf/issues/4063#issuecomment-531490140 +# https://stackoverflow.com/a/43046729 +def rename_with_retry(path_from, path_to): + if sys.platform.startswith('win'): + retry_count = 100 + else: + retry_count = 1 + + for retry in range(retry_count): + try: + os.rename(path_from, path_to) + return + except (OSError, WindowsError): # WindowsError until Python 3.3, then OSError + if retry == retry_count - 1: + raise + warn('Rename {} to {} failed, retrying...'.format(path_from, path_to)) + + def strip_container_dirs(path, levels): assert levels > 0 # move the original directory out of the way (add a .tmp suffix) tmp_path = path + '.tmp' if os.path.exists(tmp_path): shutil.rmtree(tmp_path) - os.rename(path, tmp_path) + rename_with_retry(path, tmp_path) os.mkdir(path) base_path = tmp_path # walk given number of levels down @@ -276,7 +303,7 @@ def strip_container_dirs(path, levels): for name in contents: move_from = os.path.join(base_path, name) move_to = os.path.join(path, name) - os.rename(move_from, move_to) + rename_with_retry(move_from, move_to) shutil.rmtree(tmp_path) @@ -544,7 +571,7 @@ class IDFTool(object): if not self.check_download_file(download_obj, local_temp_path): warn('Failed to download file {}'.format(local_temp_path)) continue - os.rename(local_temp_path, local_path) + rename_with_retry(local_temp_path, local_path) downloaded = True break if not downloaded: @@ -810,7 +837,7 @@ def get_python_env_path(): with open(version_file_path, "r") as version_file: idf_version_str = version_file.read() else: - idf_version_str = subprocess.check_output(['git', '-C', global_idf_path, 'describe', '--tags'], cwd=global_idf_path, env=os.environ).decode() + idf_version_str = subprocess.check_output(['git', '--work-tree=' + global_idf_path, 'describe', '--tags'], cwd=global_idf_path, env=os.environ).decode() match = re.match(r'^v([0-9]+\.[0-9]+).*', idf_version_str) idf_version = match.group(1) diff --git a/tools/windows/tool_setup/README.md b/tools/windows/tool_setup/README.md index f9e71e9b4..d7118f695 100644 --- a/tools/windows/tool_setup/README.md +++ b/tools/windows/tool_setup/README.md @@ -14,7 +14,17 @@ Some functionality of the installer depends on additional programs: * [cmdlinerunner](cmdlinerunner/cmdlinerunner.c) — a helper DLL used to run external command line programs from the installer, capture live console output, and get the exit code. -## Steps required to build the installer +## Building the installer + +### In Docker + +This uses `wine-innosetup` Docker image and `build_installer.sh` script. This is how the installer is built in CI. + +``` +docker run --rm -v $IDF_PATH:/idf -w /idf/tools/windows/tool_setup -it $CI_DOCKER_REGISTRY/wine-innosetup:1 /bin/bash build_installer.sh +``` + +### Manually, step by step * Build cmdlinerunner DLL. - On Linux/Mac, install mingw-w64 toolchain (`i686-w64-mingw32-gcc`). Then build the DLL using CMake: @@ -35,5 +45,10 @@ Some functionality of the installer depends on additional programs: * Build the installer using Inno Setup Compiler: `ISCC.exe idf_tools_setup.iss`. -* Obtain the signing keys, then sign `Output/esp-idf-tools-setup-unsigned.exe`. +## Signing the installer +* Obtain the signing key (e.g `key.pem`) and the certificate chain (e.g. `certchain.pem`). Set the environment variables to point to these files: + - `export KEYFILE=key.pem` + - `export CERTCHAIN=certchain.pem` + +* Run `sign_installer.sh` script. This will ask for the `key.pem` password, and produce the signed installer in the Output directory. If you plan to run the script multiple times, you may also set `KEYPASSWORD` environment variable to the `key.pem` password, to avoid the prompt. diff --git a/tools/windows/tool_setup/build_installer.sh b/tools/windows/tool_setup/build_installer.sh index 270fa8efd..b3e089cc1 100755 --- a/tools/windows/tool_setup/build_installer.sh +++ b/tools/windows/tool_setup/build_installer.sh @@ -1,67 +1,43 @@ #!/bin/bash # -# Setup script to build Windows tool installer with Inno Setup +# Script to build the IDF Tools installer for Windows with Inno Setup. +# This script should be executed inside wine-innosetup docker image. # -# Designed to be run on Linux (with wine) but could be adapted to run under MSYS2 on Windows -# pretty easily... -# -# - Downloads (if necessary) all tools to install to the "dl/" directory -# - Deletes the "input" directory contains and copies everything under there +# - Downloads all tools to install into the "dist/" directory +# - Downloads 7z and idf_versions.txt # - Runs ISCC under wine to compile the installer itself -set -e -if [ -z "${KEYPASSWORD}" ]; then - echo "KEYPASSWORD should be set" +set -e +set -u + +iscc_path=$(which iscc) +if [[ -z "$iscc_path" ]]; then + echo "Inno setup compiler (iscc) not found. Are you running wine-innosetup Docker image?" exit 1 fi -if [ "$1" != "--no-download" ]; then - - mkdir -p dl input - - cd `dirname $0` - pushd dl - wget --continue "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip" - wget --continue "https://github.com/espressif/binutils-esp32ulp/releases/download/v2.28.51-esp32ulp-20180809/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip" - wget --continue "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20180920/openocd-esp32-win32-0.10.0-esp32-20180920.zip" - wget --continue "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20180525/mconf-v4.6.0.0-idf-20180525-win32.zip" - wget --continue "https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip" - popd - - rm -rf input/* - pushd input - unzip ../dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip - unzip ../dl/mconf-v4.6.0.0-idf-20180525-win32.zip - unzip ../dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip - unzip ../dl/openocd-esp32-win32-0.10.0-esp32-20180920.zip - unzip ../dl/ninja-win.zip - popd +if [[ -z "${IDF_PATH:-}" ]]; then + export IDF_PATH=$(cd ../../../; pwd) + echo "Assuming IDF_PATH: ${IDF_PATH}" fi -wine "C:\Program Files\Inno Setup 5\ISCC.exe" "`winepath -w ./idf_tool_setup.iss`" +echo "Downloading IDF Tools..." +mkdir -p idf_tools_tmp +export IDF_TOOLS_PATH=$PWD/idf_tools_tmp +$IDF_PATH/tools/idf_tools.py --non-interactive download --platform Windows-x86_64 all +$IDF_PATH/tools/idf_tools.py --tools-json tools_fallback.json --non-interactive download --platform Windows-x86_64 all +mkdir -p dist +cp idf_tools_tmp/dist/* dist/ -# sign the installer with osslsigncode, parsing the version number out of the -# installer config +echo "Downloading 7z..." +mkdir -p unzip +pushd unzip +wget --no-verbose -O 7z1900-extra.7z https://www.7-zip.org/a/7z1900-extra.7z +7zr e -y 7z1900-extra.7z +popd -VERSION=`grep "^AppVersion=" idf_tool_setup.iss | cut -d'=' -f2` +echo "Downloading idf_versions.txt..." +wget --no-verbose -O idf_versions.txt https://dl.espressif.com/dl/esp-idf/idf_versions.txt -echo "Signing installer..." - -# Note: The cert chain passed to -certs needs to contain the intermediate -# cert(s) as well, appended after the code signing cert, or Windows may see -# it as "Unknown Publisher" -# -# See https://stackoverflow.com/a/52637050 for full details -# -umask 770 # for the process substitution FIFO - -osslsigncode -certs ./keys/certchain.pem -key ./keys/key.pem \ - -readpass <(echo "$KEYPASSWORD") \ - -in Output/esp-idf-tools-setup-unsigned.exe \ - -out Output/esp-idf-tools-setup-${VERSION}.exe \ - -h sha256 \ - -n "Espressif Systems (Shanghai) Pte. Ltd." \ - -i "https://www.espressif.com/" \ - -ts http://timestamp.digicert.com - -chmod 644 Output/esp-idf-tools-setup-${VERSION}.exe # make up for the umask +echo "Running ISCC..." +iscc idf_tool_setup.iss diff --git a/tools/windows/tool_setup/idf_tool_setup.iss b/tools/windows/tool_setup/idf_tool_setup.iss index 13869feae..be4dff87a 100644 --- a/tools/windows/tool_setup/idf_tool_setup.iss +++ b/tools/windows/tool_setup/idf_tool_setup.iss @@ -5,7 +5,7 @@ #include #define MyAppName "ESP-IDF Tools" -#define MyAppVersion "2.0" +#define MyAppVersion "2.1" #define MyAppPublisher "Espressif Systems (Shanghai) Co. Ltd." #define MyAppURL "https://github.com/espressif/esp-idf" diff --git a/tools/windows/tool_setup/sign_installer.sh b/tools/windows/tool_setup/sign_installer.sh new file mode 100755 index 000000000..59298edf3 --- /dev/null +++ b/tools/windows/tool_setup/sign_installer.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Script to sign the IDF Tools installer for Windows, built with build_installer.sh. +# + +set -e +set -u + +if [[ -z "${KEYFILE:-}" || -z "${CERTCHAIN:-}" ]]; then + echo "To sign the installer, set the following environment variables:" + echo " KEYFILE - private key file" + echo " KEYPASSWORD - password for the private key file (optional, will prompt for password if not set)" + echo " CERTCHAIN - certificate chain file" + exit 1 +fi + +umask 770 # for the process substitution FIFO + +VERSION=`grep "#define MyAppVersion " idf_tool_setup.iss | cut -d ' ' -f3 | tr -d '"'` +echo "Installer version ${VERSION}" + +IN_FILE="Output/esp-idf-tools-setup-unsigned.exe" +OUT_FILE="Output/esp-idf-tools-setup-${VERSION}.exe" + +if [[ -n "${KEYPASSWORD:-}" ]]; then + PASSARG="-readpass <(echo \"$KEYPASSWORD\")" +else + PASSARG="-askpass" +fi + +echo "Signing the installer (${IN_FILE})..." +# Note: The cert chain passed to -certs needs to contain the intermediate +# cert(s) as well, appended after the code signing cert, or Windows may see +# it as "Unknown Publisher" +# +# See https://stackoverflow.com/a/52637050 for full details +# +osslsigncode -certs ${CERTCHAIN} -key ${KEYFILE} \ + ${PASSARG} \ + -in ${IN_FILE} \ + -out ${OUT_FILE} \ + -h sha256 \ + -n "Espressif Systems (Shanghai) Co., Ltd." \ + -i "https://www.espressif.com/" \ + -ts http://timestamp.digicert.com + +chmod 644 ${OUT_FILE} # make up for the umask + +echo "Generated ${OUT_FILE}"