From 291735c7dd0f63a8d7022bccc6f9525bd9893324 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 4 Feb 2020 17:36:23 +1100 Subject: [PATCH] docs: Move to Python 3.5+ and Sphinx 2.3 --- docs/README.md | 16 ++++------ docs/build_docs.py | 2 +- docs/conf_common.py | 4 +-- docs/en/contribute/documenting-code.rst | 30 ++++++++++++++----- docs/idf_extensions/format_idf_target.py | 2 +- docs/idf_extensions/gen_idf_tools_links.py | 2 +- .../gen_version_specific_includes.py | 10 +++---- docs/idf_extensions/link_roles.py | 8 ++--- docs/requirements.txt | 17 ++++++----- tools/ci/config/build.yml | 4 +-- tools/ci/config/post_deploy.yml | 4 +-- tools/ci/config/pre_check.yml | 2 +- 12 files changed, 57 insertions(+), 44 deletions(-) diff --git a/docs/README.md b/docs/README.md index 89a0db0a9..12202ca5b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,16 +16,12 @@ The above URLs are all for the master branch latest version. Click the drop-down # Building Documentation -* Install `make` and `doxygen` for your platform (`make` may already be installed as an ESP-IDF prerequisite). -* Change to either the docs/en or docs/zh_CN subdirectory and run `make html` -* `make` will probably prompt you to run a python pip install step to get some other Python-related prerequisites. Run the command as shown, then re-run `make html` to build the docs. +* Documentation build requres Python 3 and will not work with Python 2 +* Install dependencies for ESP-IDF as per the Getting Started guide +* Install documentation Python depdendencies, ie `pip install -r $IDF_PATH/docs/requirements.txt` +* Run `./build_docs.py build` to build docs for all supported Language & Target combinations, or `./build_docs.py -t esp32 -l en build` to build docs for a single supported language & target combination only. -## For MSYS2 MINGW32 on Windows +See [Documenting Code](https://docs.espressif.com/projects/esp-idf/en/latest/contribute/documenting-code.rst) for more information. -If using Windows and the MSYS2 MINGW32 terminal, run this command before running "make html" the first time: +Available languages are `en` and `zh_CN`, targets are any target supported by ESP-IDF - for example `esp32` or `esp32s2`. -``` -pacman -S doxygen mingw-w64-i686-python2-pillow -``` - -Note: Currently it is not possible to build docs on Windows without using a Unix-on-Windows layer such as MSYS2 MINGW32. diff --git a/docs/build_docs.py b/docs/build_docs.py index 1c36615ff..cf6ef6ec3 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding=utf-8 # # Top-level docs builder diff --git a/docs/conf_common.py b/docs/conf_common.py index 36fd57dc3..bc9fc63f5 100644 --- a/docs/conf_common.py +++ b/docs/conf_common.py @@ -107,7 +107,7 @@ master_doc = 'index' # This is supposed to be "the short X.Y version", but it's the only version # visible when you open index.html. # Display full version to make things less confusing. -version = subprocess.check_output(['git', 'describe']).strip() +version = subprocess.check_output(['git', 'describe']).strip().decode('utf-8') # The full version, including alpha/beta/rc tags. # If needed, nearest tag is returned by 'git describe --abbrev=0'. release = version @@ -121,7 +121,7 @@ print('Version: {0} Release: {1}'.format(version, release)) # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['**/inc/**', '_static'] +exclude_patterns = ['**/inc/**', '_static', '**/_build'] # Add target-specific excludes based on tags (for the IDF_TARGET). Haven't found any better way to do this yet diff --git a/docs/en/contribute/documenting-code.rst b/docs/en/contribute/documenting-code.rst index 20d3cea46..677e50777 100644 --- a/docs/en/contribute/documenting-code.rst +++ b/docs/en/contribute/documenting-code.rst @@ -320,6 +320,9 @@ OK, but I am new to Sphinx! Setup for building documentation locally ---------------------------------------- +Install Dependencies +"""""""""""""""""""" + You can setup environment to build documentation locally on your PC by installing: 1. Doxygen - https://www.stack.nl/~dimitri/doxygen/ @@ -334,6 +337,8 @@ The package "sphinx_rtd_theme" is added to have the same "look and feel" of `ESP Do not worry about being confronted with several packages to install. Besides Doxygen, all remaining packages are written in Python. Therefore installation of all of them is combined into one simple step. +.. important:: Docs building now supports Python 3 only. Python 2 installations will not work. + Installation of Doxygen is OS dependent: **Linux** @@ -356,7 +361,7 @@ Installation of Doxygen is OS dependent: .. note:: - If you are installing on Windows system (Linux and MacOS users should skip this note), **before** going further, execute two extra steps below. These steps are required to install dependencies of "blockdiag" discussed under :ref:`add-illustrations`. + If you are installing on Windows MSYS2 system (Linux and MacOS users should skip this note, Windows users who don't use MSYS2 will need to find other alternatives), **before** going further, execute two extra steps below. These steps are required to install dependencies of "blockdiag" discussed under :ref:`add-illustrations`. 1. Update all the system packages: @@ -370,9 +375,9 @@ Installation of Doxygen is OS dependent: :: - $ pacman -S mingw32/mingw-w64-i686-python2-pillow + $ pacman -S mingw32/mingw-w64-i686-python-pillow - Check the log on the screen that ``mingw-w64-i686-python2-pillow-4.3.0-1`` is installed. Previous versions of *pillow* will not work. + Check the log on the screen that ``mingw-w64-i686-python-pillow-4.3.0-1`` or newer is installed. Previous versions of *pillow* will not work. A downside of Windows installation is that fonts of the `blockdiag pictures ` do not render correctly, you will see some random characters instead. Until this issue is fixed, you can use the `interactive shell`_ to see how the complete picture looks like. @@ -387,15 +392,26 @@ All remaining applications are `Python `_ packages and Installation steps assume that ESP-IDF is placed in ``~/esp/esp-idf`` directory, that is default location of ESP-IDF used in documentation. -Change to directory with files for specific language:: +Building Documentation +"""""""""""""""""""""" - cd en +:: + + cd ~/esp/esp-idf/docs Now you should be ready to build documentation by invoking:: - make html + ./build_docs.py build -This may take couple of minutes. After completion, documentation will be placed in ``~/esp/esp-idf/docs/en/_build/html`` folder. To see it, open ``index.html`` in a web browser. +This will build docs for all supported ESP-IDF languages & targets. This can take some time, although jobs will run in parallel up to the number of CPU cores you have (can modify this with the ``--sphinx-parallel-builds`` option, see ``./build_docs.py --help`` for details). + +To build for a single language and target combination only:: + + ./build_docs.py -l en -t esp32 build + +Choices for language (``-l``) are ``en`` and ``zh_CN``. Choices for target (``-t``) are any supported ESP-IDF build system target (for example ``esp32`` and ``esp32s2``). + +Build documentation will be placed in ``_build///html`` folder. To see it, open the ``index.html`` inside this directory in a web browser. Wrap up diff --git a/docs/idf_extensions/format_idf_target.py b/docs/idf_extensions/format_idf_target.py index da342841f..15c0db68f 100644 --- a/docs/idf_extensions/format_idf_target.py +++ b/docs/idf_extensions/format_idf_target.py @@ -75,7 +75,7 @@ class StringSubstituter: # Add any new local tags that matches the reg.ex. sub_defs = re.findall(self.RE_PATTERN, source[0]) - if len(sub_defs) is not 0: + if len(sub_defs) != 0: self.add_local_subs(sub_defs) # Remove the tag defines diff --git a/docs/idf_extensions/gen_idf_tools_links.py b/docs/idf_extensions/gen_idf_tools_links.py index 4a02b8126..126efe63c 100644 --- a/docs/idf_extensions/gen_idf_tools_links.py +++ b/docs/idf_extensions/gen_idf_tools_links.py @@ -1,7 +1,7 @@ # Generate toolchain download links from toolchain info makefile from __future__ import print_function import os.path -from util import copy_if_modified, call_with_python +from .util import copy_if_modified, call_with_python def setup(app): diff --git a/docs/idf_extensions/gen_version_specific_includes.py b/docs/idf_extensions/gen_version_specific_includes.py index fae92512b..b93d31e6a 100755 --- a/docs/idf_extensions/gen_version_specific_includes.py +++ b/docs/idf_extensions/gen_version_specific_includes.py @@ -201,21 +201,21 @@ def get_version(): # Otherwise, use git to look for a tag try: - tag = subprocess.check_output(["git", "describe", "--tags", "--exact-match"]).strip() + tag = subprocess.check_output(["git", "describe", "--tags", "--exact-match"]).strip().decode('utf-8') is_stable = re.match(r"v[0-9\.]+$", tag) is not None return (tag, "tag", is_stable) except subprocess.CalledProcessError: pass # No tag, look for a branch - refs = subprocess.check_output(["git", "for-each-ref", "--points-at", "HEAD", "--format", "%(refname)"]) + refs = subprocess.check_output(["git", "for-each-ref", "--points-at", "HEAD", "--format", "%(refname)"]).decode('utf-8') print("refs:\n%s" % refs) - refs = refs.split(b"\n") + refs = refs.split("\n") # Note: this looks for branches in 'origin' because GitLab CI doesn't check out a local branch - branches = [r.replace(b"refs/remotes/origin/",b"").strip() for r in refs if r.startswith(b"refs/remotes/origin/")] + branches = [r.replace("refs/remotes/origin/","").strip() for r in refs if r.startswith("refs/remotes/origin/")] if len(branches) == 0: # last resort, return the commit (may happen on Gitlab CI sometimes, unclear why) - return (subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).strip(), "commit", False) + return (subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).strip().decode('utf-8'), "commit", False) if "master" in branches: return ("master", "branch", False) else: diff --git a/docs/idf_extensions/link_roles.py b/docs/idf_extensions/link_roles.py index ee1a6adc5..5d8728f65 100644 --- a/docs/idf_extensions/link_roles.py +++ b/docs/idf_extensions/link_roles.py @@ -9,9 +9,9 @@ from docutils import nodes def get_github_rev(): - path = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip() + path = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip().decode('utf-8') try: - tag = subprocess.check_output(['git', 'describe', '--exact-match']).strip() + tag = subprocess.check_output(['git', 'describe', '--exact-match']).strip().decode('utf-8') except subprocess.CalledProcessError: tag = None print('Git commit ID: ', path) @@ -41,11 +41,11 @@ def setup(app): if on_rtd: # provide RTD specific commit identification to be included in the link tag_rev = 'latest' - if (subprocess.check_output(['git','rev-parse', '--short', 'HEAD']).strip() != rev): + if (subprocess.check_output(['git','rev-parse', '--short', 'HEAD']).decode('utf-8').strip() != rev): tag_rev = rev else: # if not on the RTD then provide generic identification - tag_rev = subprocess.check_output(['git', 'describe', '--always']).strip() + tag_rev = subprocess.check_output(['git', 'describe', '--always']).decode('utf-8').strip() app.add_role('link_to_translation', crosslink('%s../../%s/{}/%s.html'.format(tag_rev))) diff --git a/docs/requirements.txt b/docs/requirements.txt index c217a4004..52424f7b8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,15 +1,16 @@ # This is a list of python packages used to generate documentation. This file is used with pip: # pip install --user -r requirements.txt # -sphinx>=1.8.4 -breathe==4.11.1 +sphinx==2.3.1 +breathe==4.14.1 sphinx-rtd-theme sphinx-notfound-page -sphinxcontrib-blockdiag>=1.5.5, <2.0.0 -sphinxcontrib-seqdiag>=0.8.5, <2.0.0 -sphinxcontrib-actdiag>=0.8.5, <2.0.0 -sphinxcontrib-nwdiag>=0.9.5, <2.0.0 -nwdiag==1.0.4 +sphinxcontrib-blockdiag==2.0.0 +sphinxcontrib-seqdiag==2.0.0 +sphinxcontrib-actdiag==2.0.0 +sphinxcontrib-nwdiag==2.0.0 +sphinxcontrib-wavedrom==2.0.0 +nwdiag==2.0.0 recommonmark future>=0.16.0 # for ../tools/gen_esp_err_to_name.py -sphinx_selective_exclude>=1.0.3 +sphinx_selective_exclude==1.0.3 diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index b6af246e0..cf682c8a4 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -188,7 +188,7 @@ build_examples_cmake_esp32s2: .build_docs_template: &build_docs_template stage: build - image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:esp-docs-20200204-ea18dcbd + image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:v2-d4034371 tags: - build_docs artifacts: @@ -206,7 +206,7 @@ build_examples_cmake_esp32s2: dependencies: [] script: - cd docs - - ./build_docs.py -l $DOCLANG -t $DOCTGT build + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -l $DOCLANG -t $DOCTGT build build_docs_en_esp32: extends: .build_docs_template diff --git a/tools/ci/config/post_deploy.yml b/tools/ci/config/post_deploy.yml index 024b0c5d2..8cc46821a 100644 --- a/tools/ci/config/post_deploy.yml +++ b/tools/ci/config/post_deploy.yml @@ -1,6 +1,6 @@ .check_doc_links_template: &check_doc_links_template stage: post_deploy - image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:esp-docs-20200204-ea18dcbd + image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:v2-d4034371 tags: [ "build", "amd64", "internet" ] only: - master @@ -15,7 +15,7 @@ dependencies: [] script: - cd docs - - ./build_docs.py -l $DOCLANG -t $DOCTGT linkcheck + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -l $DOCLANG -t $DOCTGT linkcheck check_doc_links_en_esp32: extends: .check_doc_links_template diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index 6e23e0cfc..6439cdda7 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -24,7 +24,7 @@ check_docs_gh_links: SUBMODULES_TO_FETCH: "none" script: - cd docs - - ./build_docs.py gh-linkcheck + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck check_version: extends: .check_job_template