diff --git a/docs/Doxyfile b/docs/Doxyfile
index e62b4604e..397540511 100644
--- a/docs/Doxyfile
+++ b/docs/Doxyfile
@@ -302,7 +302,7 @@ EXPAND_ONLY_PREDEF = YES
PREDEFINED = \
$(ENV_DOXYGEN_DEFINES) \
__attribute__(x)= \
- _Static_assert(x, y) = \
+ _Static_assert(x) = \
IDF_DEPRECATED(X)= \
IRAM_ATTR= \
configSUPPORT_DYNAMIC_ALLOCATION=1 \
diff --git a/docs/build_docs.py b/docs/build_docs.py
index 4e3932e39..be354fa4c 100755
--- a/docs/build_docs.py
+++ b/docs/build_docs.py
@@ -17,6 +17,7 @@ import sys
LANGUAGES = ["en", "zh_CN"]
TARGETS = ["esp32", "esp32s2"]
+
def main():
# check Python dependencies for docs
try:
@@ -43,13 +44,13 @@ def main():
print("Building all languages")
languages = LANGUAGES
else:
- languages = [ args.language ]
+ languages = [args.language]
if args.target is None:
print("Building all targets")
targets = TARGETS
else:
- targets = [ args.target ]
+ targets = [args.target]
for language in languages:
for target in targets:
@@ -76,9 +77,9 @@ def build_docs(language, target, build_dir):
"-w", "sphinx-warning.log",
"-t", target,
"-D", "idf_target={}".format(target),
- os.path.join(os.path.abspath(os.path.dirname(__file__)), language), # srcdir for this language
+ os.path.join(os.path.abspath(os.path.dirname(__file__)), language), # srcdir for this language
os.path.join(build_dir, "html") # build directory
- ]
+ ]
cwd = build_dir # also run sphinx in the build directory
print("Running '{}'".format(" ".join(args)))
subprocess.check_call(args, cwd=cwd, env=environ)
diff --git a/docs/conf_common.py b/docs/conf_common.py
index c30c5e7a7..2deb903a9 100644
--- a/docs/conf_common.py
+++ b/docs/conf_common.py
@@ -18,65 +18,13 @@ from __future__ import print_function
from __future__ import unicode_literals
import sys
import os
+import os.path
import re
import subprocess
-# Note: If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute
-
-from local_util import run_cmd_get_output, copy_if_modified, call_with_python
-
# build_docs on the CI server sometimes fails under Python3. This is a workaround:
sys.setrecursionlimit(3500)
-try:
- build_dir = os.environ['BUILDDIR']
-except KeyError:
- build_dir = '_build'
-
-build_dir = os.path.abspath(build_dir)
-
-# Fill in a default IDF_PATH if it's missing (ie when Read The Docs is building the docs)
-try:
- idf_path = os.environ['IDF_PATH']
-except KeyError:
- idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
-
-docs_root = os.path.join(idf_path, "docs")
-
-try:
- os.mkdir(build_dir)
-except OSError:
- pass
-
-try:
- os.mkdir(os.path.join(build_dir, 'inc'))
-except OSError:
- pass
-
-# Generate version-related includes
-#
-# (Note: this is in a function as it needs to access configuration to get the language)
-def generate_version_specific_includes(app):
- print("Generating version-specific includes...")
- version_tmpdir = '{}/version_inc'.format(build_dir)
- call_with_python('{}/gen-version-specific-includes.py {} {}'.format(docs_root, app.config.language, version_tmpdir))
- copy_if_modified(version_tmpdir, '{}/inc'.format(build_dir))
-
-# Generate toolchain download links
-print("Generating toolchain download links")
-base_url = 'https://dl.espressif.com/dl/'
-toolchain_tmpdir = '{}/toolchain_inc'.format(build_dir)
-call_with_python('{}/gen-toolchain-links.py ../../tools/toolchain_versions.mk {} {}'.format(docs_root, base_url, toolchain_tmpdir))
-copy_if_modified(toolchain_tmpdir, '{}/inc'.format(build_dir))
-
-print("Generating IDF Tools list")
-os.environ["IDF_MAINTAINER"] = "1"
-tools_rst = os.path.join(builddir, 'idf-tools-inc.rst')
-tools_rst_tmp = os.path.join(builddir, 'inc', 'idf-tools-inc.rst')
-call_with_python("{}/tools/idf_tools.py gen-doc --output {}".format(idf_path, tools_rst_tmp))
-copy_if_modified(tools_rst_tmp, tools_rst)
# http://stackoverflow.com/questions/12772927/specifying-an-online-image-in-sphinx-restructuredtext-format
#
@@ -84,7 +32,6 @@ suppress_warnings = ['image.nonlocal_uri']
# -- General configuration ------------------------------------------------
-
# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'
@@ -92,24 +39,33 @@ suppress_warnings = ['image.nonlocal_uri']
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['breathe',
- 'link-roles',
+
+ 'sphinx.ext.todo',
+
'sphinxcontrib.blockdiag',
'sphinxcontrib.seqdiag',
'sphinxcontrib.actdiag',
'sphinxcontrib.nwdiag',
'sphinxcontrib.rackdiag',
'sphinxcontrib.packetdiag',
- 'html_redirects',
- 'idf_build_system',
- 'kconfig_reference',
- 'doxygen_idf',
- 'sphinx.ext.todo',
- 'include_build_file',
- 'toctree_filter',
+
+ 'extensions.html_redirects',
+ 'extensions.toctree_filter',
+
+ 'idf_extensions.include_build_file',
+ 'idf_extensions.link_roles',
+ 'idf_extensions.build_system',
+ 'idf_extensions.esp_err_definitions',
+ 'idf_extensions.gen_toolchain_links',
+ 'idf_extensions.gen_version_specific_includes',
+ 'idf_extensions.kconfig_reference',
+ 'idf_extensions.run_doxygen',
+ 'idf_extensions.gen_idf_tools_links',
+
# from https://github.com/pfalcon/sphinx_selective_exclude
'sphinx_selective_exclude.eager_only',
- #'sphinx_selective_exclude.search_auto_exclude',
- #'sphinx_selective_exclude.modindex_exclude',
+ # TODO: determine if we need search_auto_exclude
+ # 'sphinx_selective_exclude.search_auto_exclude',
]
# sphinx.ext.todo extension parameters
@@ -120,13 +76,6 @@ todo_include_todos = False
# Enabling this fixes cropping of blockdiag edge labels
seqdiag_antialias = True
-# Breathe extension variables
-
-# Doxygen regenerates files in 'xml/' directory every time,
-# but we copy files to 'xml_in/' only when they change, to speed up
-# incremental builds.
-breathe_projects = {"esp32-idf": os.path.join(build_dir, "xml_in/")}
-breathe_default_project = "esp32-idf"
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@@ -155,7 +104,7 @@ master_doc = 'index'
# This is supposed to be "the short X.Y version", but it's the only version
# visible when you open index.html.
# Display full version to make things less confusing.
-version = run_cmd_get_output('git describe')
+version = subprocess.check_output(['git', 'describe']).strip()
# The full version, including alpha/beta/rc tags.
# If needed, nearest tag is returned by 'git describe --abbrev=0'.
release = version
@@ -171,7 +120,8 @@ print('Version: {0} Release: {1}'.format(version, release))
# directories to ignore when looking for source files.
exclude_patterns = ['**/inc/**']
-# Add target-specific excludes based on tags. Haven't found any better way to do this yet
+
+# Add target-specific excludes based on tags (for the IDF_TARGET). Haven't found any better way to do this yet
def update_exclude_patterns(tags):
if "esp32" not in tags:
# Exclude ESP32-only document pages so they aren't found in the initial search for .rst files
@@ -186,6 +136,7 @@ def update_exclude_patterns(tags):
'gnu-make-legacy.rst']:
exclude_patterns.append(e)
+
# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None
@@ -389,6 +340,9 @@ texinfo_documents = [
# Override RTD CSS theme to introduce the theme corrections
# https://github.com/rtfd/sphinx_rtd_theme/pull/432
def setup(app):
- app.config.build_dir = build_dir
app.add_stylesheet('theme_overrides.css')
- generate_version_specific_includes(app)
+
+ # Breathe extension variables (depend on build_dir)
+ # note: we generate into xml_in and then copy_if_modified to xml dir
+ app.config.breathe_projects = {"esp32-idf": os.path.join(app.config.build_dir, "xml_in/")}
+ app.config.breathe_default_project = "esp32-idf"
diff --git a/docs/doxygen_idf.py b/docs/doxygen_idf.py
deleted file mode 100644
index af7fb633a..000000000
--- a/docs/doxygen_idf.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# Extension to generate Doxygen XML include files, with IDF config & soc macros included
-import glob
-import os.path
-import re
-import sys
-import subprocess
-gen_dxd = __import__("gen-dxd")
-
-from local_util import copy_if_modified, call_with_python
-
-def setup(app):
- # The idf_build_system extension will emit this event once it
- app.connect('idf-info', generate_doxygen)
-
- return { 'parallel_read_safe' : True, 'parallel_write_safe': True, 'version': '0.1' }
-
-
-def _parse_defines(header_path):
- defines = {}
- # Note: we run C preprocessor here without any -I arguments, so assumption is
- # that these headers are all self-contained and don't include any other headers
- # not in the same directory
- print("Reading macros from %s..." % (header_path))
- processed_output = subprocess.check_output(["xtensa-esp32-elf-gcc", "-dM", "-E", header_path]).decode()
- for line in processed_output.split("\n"):
- line = line.strip()
- m = re.search("#define ([^ ]+) ?(.*)", line)
- if m and not m.group(1).startswith("_"):
- defines[m.group(1)] = m.group(2)
-
- return defines
-
-
-def generate_doxygen(app, project_description):
- build_dir = os.path.dirname(app.doctreedir.rstrip(os.sep))
-
- # Parse kconfig macros to pass into doxygen
- #
- # TODO: this should use the set of "config which can't be changed" eventually,
- # not the header
- defines = _parse_defines(os.path.join(project_description["build_dir"],
- "config", "sdkconfig.h"))
-
- # Add all SOC _caps.h headers to the defines
- #
- # kind of a hack, be nicer to add a component info dict in project_description.json
- soc_path = [p for p in project_description["build_component_paths"] if p.endswith("/soc")][0]
- for soc_header in glob.glob(os.path.join(soc_path, project_description["target"],
- "include", "soc", "*_caps.h")):
- defines.update(_parse_defines(soc_header))
-
- # Call Doxygen to get XML files from the header files
- print("Calling Doxygen to generate latest XML files")
- doxy_env = {
- "ENV_DOXYGEN_DEFINES": " ".join(defines),
- "IDF_PATH": app.config.idf_path,
- }
- doxyfile = os.path.join(app.config.docs_root, "Doxyfile")
- # note: run Doxygen in the build directory, so the xml & xml_in files end up in there
- subprocess.check_call(["doxygen", doxyfile], env=doxy_env, cwd=build_dir)
-
- # Doxygen has generated XML files in 'xml' directory.
- # Copy them to 'xml_in', only touching the files which have changed.
- copy_if_modified(os.path.join(build_dir, 'xml/'), os.path.join(build_dir, 'xml_in/'))
-
- # Generate 'api_name.inc' files using the XML files by Doxygen
- gen_dxd.builddir = build_dir
- gen_dxd.doxyfile_path = doxyfile
- gen_dxd.header_file_path_prefix = "components/"
- gen_dxd.xml_directory_path = "{}/xml".format(build_dir)
- gen_dxd.inc_directory_path = "{}/inc".format(build_dir)
- gen_dxd.generate_api_inc_files()
-
- # Generate 'esp_err_defs.inc' file with ESP_ERR_ error code definitions from inc file
- esp_err_inc_path = '{}/inc/esp_err_defs.inc'.format(build_dir)
- call_with_python('{}/tools/gen_esp_err_to_name.py --rst_output {}.in'.format(app.config.idf_path, esp_err_inc_path))
- copy_if_modified(esp_err_inc_path + '.in', esp_err_inc_path)
-
diff --git a/docs/en/api-guides/external-ram.rst b/docs/en/api-guides/external-ram.rst
index 6aa14019a..8fa7569c9 100644
--- a/docs/en/api-guides/external-ram.rst
+++ b/docs/en/api-guides/external-ram.rst
@@ -122,6 +122,7 @@ External RAM use has the following restrictions:
* External RAM cannot be used as task stack memory. Due to this, :cpp:func:`xTaskCreate` and similar functions will always allocate internal memory for stack and task TCBs, and functions such as :cpp:func:`xTaskCreateStatic` will check if the buffers passed are internal.
* By default, failure to initialize external RAM will cause the ESP-IDF startup to abort. This can be disabled by enabling the config item :ref:`CONFIG_SPIRAM_IGNORE_NOTFOUND`. If :ref:`CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY` is enabled, the option to ignore failure is not available as the linker will have assigned symbols to external memory addresses at link time.
+
.. only:: esp32
.. include:: inc/external-ram-esp32-notes.inc
diff --git a/docs/en/api-guides/tools/idf-tools.rst b/docs/en/api-guides/tools/idf-tools.rst
index 628364f45..6ea511a3f 100644
--- a/docs/en/api-guides/tools/idf-tools.rst
+++ b/docs/en/api-guides/tools/idf-tools.rst
@@ -138,4 +138,4 @@ Although the methods above are recommended for ESP-IDF users, they are not a mus
List of IDF Tools
-----------------
-.. include:: /_build/inc/idf-tools-inc.rst
+.. include-build-file:: idf-tools-inc.rst
diff --git a/docs/en/conf.py b/docs/en/conf.py
index 68bcf8766..ec996f62f 100644
--- a/docs/en/conf.py
+++ b/docs/en/conf.py
@@ -10,7 +10,7 @@ import sys
import os
sys.path.insert(0, os.path.abspath('..'))
from conf_common import * # noqa: F401, F403 - need to make available everything from common
-from local_util import download_file_if_missing # noqa: E402 - need to import from common folder
+from idf_extensions.util import download_file_if_missing # noqa: E402 - need to import from common folder
# General information about the project.
project = u'ESP-IDF Programming Guide'
@@ -32,5 +32,4 @@ nwdiag_fontpath = '../_static/DejaVuSans.ttf'
rackdiag_fontpath = '../_static/DejaVuSans.ttf'
packetdiag_fontpath = '../_static/DejaVuSans.ttf'
-update_exclude_patterns(tags)
-
+update_exclude_patterns(tags) # noqa: F405, need to import * from conf_common
diff --git a/docs/en/contribute/add-ons-reference.rst b/docs/en/contribute/add-ons-reference.rst
index b1942b46e..30fedb8ed 100644
--- a/docs/en/contribute/add-ons-reference.rst
+++ b/docs/en/contribute/add-ons-reference.rst
@@ -5,6 +5,8 @@ This documentation is created using `Sphinx `_ appli
Besides Sphinx there are several other applications that help to provide nicely formatted and easy to navigate documentation. These applications are listed in section :ref:`setup-for-building-documentation` with the installed version numbers provided in file :idf_file:`docs/requirements.txt`.
+We build ESP-IDF documentation for two languages (English, Simplified Chinese) and for multiple chips. Therefore we don't run ``sphinx`` directly, there is a wrapper Python program ``build_docs.py`` that runs Sphinx.
+
On top of that we have created a couple of custom add-ons and extensions to help integrate documentation with underlining `ESP-IDF`_ repository and further improve navigation as well as maintenance of documentation.
The purpose of this section is to provide a quick reference to the add-ons and the extensions.
@@ -17,48 +19,43 @@ Documentation Folder Structure
* The ``docs`` folder contains localized documentation in :idf:`docs/en` (English) and :idf:`docs/zh_CN` (simplified Chinese) subfolders.
* Graphics files and fonts common to localized documentation are contained in :idf:`docs/_static` subfolder
* Remaining files in the root of ``docs`` as well as ``docs/en`` and ``docs/zh_CN`` provide configuration and scripts used to automate documentation processing including the add-ons and extensions.
-* Several folders and files are generated dynamically during documentations build and placed primarily in ``docs/[lang]/_build`` folders. These folders are temporary and not visible in `ESP-IDF`_ repository,
+* Sphinx extensions are provided in two directories, ``extensions`` and ``idf_extensions``
+* A ``_build`` directory is created in the ``docs`` folder by ``build_docs.py``. This directory is not added to the `ESP-IDF`_ repository.
Add-ons and Extensions Reference
--------------------------------
+Config Files
+^^^^^^^^^^^^
+
:idf_file:`docs/conf_common.py`
This file contains configuration common to each localized documentation (e.g. English, Chinese). The contents of this file is imported to standard Sphinx configuration file ``conf.py`` located in respective language folders (e.g. ``docs/en``, ``docs/zh_CN``) during build for each language.
-:idf_file:`docs/check_doc_warnings.sh`
- If there are any warnings reported during documentation build, then the build is failed. The warnings should be resolved before merging any documentation updates. This script is doing check for warnings in respective log file to fail the build. See also description of ``sphinx-known-warnings.txt`` below.
+:idf_file:`docs/sphinx-known-warnings.txt`
+ There are couple of spurious Sphinx warnings that cannot be resolved without doing update to the Sphinx source code itself. For such specific cases respective warnings are documented in ``sphinx-known-warnings.txt`` file, that is checked during documentation build, to ignore the spurious warnings.
+
+
+Scripts
+^^^^^^^
+
+:idf_file:`docs/build_docs.py`
+
+ Top-level executable program which runs a Sphinx build for one or more language/target combinations. Run ``build_docs.py --help`` for full command line options.
+
+ When ``build_docs.py`` runs Sphinx it sets the ``idf_target`` configuration variable, sets a Sphinx tag with the same name as the configuration variable, and uses some environment variables to communicate paths to :ref:`IDF-Specific Extensions`.
:idf_file:`docs/check_lang_folder_sync.sh`
To reduce potential discrepancies when maintaining concurrent language version, the structure and filenames of language folders ``docs/en`` and ``docs/zh_CN`` folders should be kept identical. The script ``check_lang_folder_sync.sh`` is run on each documentation build to verify if this condition is met.
- .. note::
+.. note::
If a new content is provided in e.g. English, and there is no any translation yet, then the corresponding file in ``zh_CN`` folder should contain an ``.. include::`` directive pointing to the source file in English. This will automatically include the English version visible to Chinese readers. For example if a file ``docs/zh_CN/contribute/documenting-code.rst`` does not have a Chinese translation, then it should contain ``.. include:: ../../en/contribute/documenting-code.rst`` instead.
-:idf_file:`docs/docs_common.mk`
- It contains the common code which is included into the language-specific ``Makefiles``. Note that this file contains couple of customizations comparing to what is provided within standard Sphinx installation, e.g. ``gh-linkcheck`` command has been added.
+Non-Docs Scripts
+^^^^^^^^^^^^^^^^
-:idf_file:`docs/gen-dxd.py`
- A Python script that generates API reference files based on Doxygen ``xml`` output. The files have an ``inc`` extension and are located in ``docs/[lang]/_build/inc`` directory created dynamically when documentation is build. Please refer to :doc:`documenting-code` and :doc:`../api-reference/template`, section **API Reference** for additional details on this process.
-
-:idf_file:`docs/gen-toolchain-links.py`
- There couple of places in documentation that provide links to download the toolchain. To provide one source of this information and reduce effort to manually update several files, this script generates toolchain download links and toolchain unpacking code snippets based on information found in :idf_file:`tools/toolchain_versions.mk`.
-
-:idf_file:`docs/gen-version-specific-includes.py`
- Another Python script to automatically generate reStructuredText Text ``.inc`` snippets with version-based content for this ESP-IDF version.
-
-:idf_file:`docs/html_redirects.py`
- During documentation lifetime some source files are moved between folders or renamed. This Python script is adding a mechanism to redirect documentation pages that have changed URL by generating in the Sphinx output static HTML redirect pages. The script is used together with a redirection list ``html_redirect_pages`` defined in file :idf_file:`docs/conf_common.py`.
-
-:idf_file:`docs/link-roles.py`
- This is an implementation of a custom `Sphinx Roles `_ to help linking from documentation to specific files and folders in `ESP-IDF`_. For description of implemented roles please see :ref:`link-custom-roles` and :ref:`link-language-versions`.
-
-:idf_file:`docs/local_util.py`
- A collection of utility functions useful primarily when building documentation locally (see :ref:`setup-for-building-documentation`) to reduce the time to generate documentation on a second and subsequent builds. The utility functions check what Doxygen ``xml`` input files have been changed and copy these files to destination folders, so only the changed files are used during build process.
-
-:idf_file:`docs/sphinx-known-warnings.txt`
- There are couple of spurious Sphinx warnings that cannot be resolved without doing update to the Sphinx source code itself. For such specific cases respective warnings are documented in ``sphinx-known-warnings.txt`` file, that is checked during documentation build, to ignore the spurious warnings.
+These scripts are used to build docs but also used for other purposes:
:idf_file:`tools/gen_esp_err_to_name.py`
This script is traversing the `ESP-IDF`_ directory structure looking for error codes and messages in source code header files to generate an ``.inc`` file to include in documentation under :doc:`../api-reference/error-codes`.
@@ -66,7 +63,89 @@ Add-ons and Extensions Reference
:idf_file:`tools/kconfig_new/confgen.py`
Options to configure ESP-IDF's :idf:`components` are contained in ``Kconfig`` files located inside directories of individual components, e.g. :idf_file:`components/bt/Kconfig`. This script is traversing the ``component`` directories to collect configuration options and generate an ``.inc`` file to include in documentation under :ref:`configuration-options-reference`.
+Generic Extensions
+^^^^^^^^^^^^^^^^^^
+These are Sphinx extensions developed for IDF that don't rely on any IDF-docs-specific behaviour or configuration:
+
+:idf_file:`docs/extensions/toctree_filter.py`
+ Sphinx extensions overrides the ``:toctree:`` directive to allow filtering entries based on whether a tag is set, as ``:tagname: toctree_entry``. See the Python file for a more complete description.
+
+:idf_file:`docs/extensions/html_redirects.py`
+ During documentation lifetime some source files are moved between folders or renamed. This Sphinx extension adds a mechanism to redirect documentation pages that have changed URL by generating in the Sphinx output static HTML redirect pages. The script is used together with a redirection list ``html_redirect_pages``. ``conf_common.py`` builds this list from :idf_file:`docs/page_redirects.txt`
+
+
+Third Party Extensions
+^^^^^^^^^^^^^^^^^^^^^^
+
+- ``sphinxcontrib`` extensions for blockdiag, seqdiag, actdiag, nwdiag, rackdiag & packetdiag diagrams.
+- `Sphinx selective exclude`_ ``eager_only`` extension
+
+.. _idf-specific extensions:
+
+IDF-Specific Extensions
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Build System Integration
+########################
+
+:idf:`docs/idf_extensions/build_system/`
+
+Python package implementing a Sphinx extension to pull IDF build system information into the docs build
+
+* Creates a dummy CMake IDF project and runs CMake to generate metadata
+* Registers some new configuration variables and emits a new Sphinx event, both for use by other extensions.
+
+Configuration Variables
+@@@@@@@@@@@@@@@@@@@@@@@
+
+* ``docs_root`` - The absolute path of the $IDF_PATH/docs directory
+* ``idf_path`` - The value of IDF_PATH variable, or the absolute path of IDF_PATH if environment unset
+* ``build_dir`` - The build directory passed in by ``build_docs.py``, default will be like ``_build//``
+* ``idf_target`` - The IDF_TARGET value. Expected that ``build_docs.py`` set this on the Sphinx command line
+
+New Event
+@@@@@@@@@
+
+``idf-info`` event is emitted early in the build, after the dummy project CMake run is complete.
+
+Arguments are ``(app, project_description)`` where ``project_description`` is a dict containing the values parsed from ``project_description.json`` in the CMake build directory.
+
+Other IDF-specific extensions subscribe to this event and use it to set up some docs parameters based on build system info.
+
+Other Extensions
+################
+
+:idf_file:`docs/idf_extensions/include_build_file.py`
+ The ``include-build-file`` directive is like the built-in ``include-file`` directive, but file path is evaluated relative to ``build_dir``.
+
+
+:idf_file:`docs/idf_extensions/kconfig_reference.py`
+ Subscribes to ``idf-info`` event and uses confgen to generate ``kconfig.inc`` from the components included in the default project build. This file is then included into :doc:`/api-reference/kconfig`.
+
+:idf_file:`docs/idf_extensions/link_roles.py`
+ This is an implementation of a custom `Sphinx Roles `_ to help linking from documentation to specific files and folders in `ESP-IDF`_. For description of implemented roles please see :ref:`link-custom-roles` and :ref:`link-language-versions`.
+
+:idf_file:`docs/idf_extensions/run_doxygen.py`
+ Subscribes to ``idf-info`` event and runs Doxygen (:idf_file:`docs/Doxyfile`) to generate XML files describing key headers, and then runs Breathe to convert these to ``.inc`` files which can be included directly into API reference pages.
+
+ Pushes a number of target-specific custom environment variables into Doxygen, including all macros defined in the project's default ``sdkconfig.h`` file and all macros defined in all ``soc`` component ``xxx_caps.h`` headers. This means that public API headers can depend on target-specific configuration options or ``soc`` capabilities headers options as ``#ifdef`` & ``#if`` preprocessor selections in the header.
+
+ This means we can generate different Doxygen files, depending on the target we are building docs for.
+
+ Please refer to :doc:`documenting-code` and :doc:`../api-reference/template`, section **API Reference** for additional details on this process.
+
+:idf_file:`docs/idf_extensions/esp_err_definitions.py`
+ Small wrapper extension that calls ``gen_esp_err_to_name.py`` and updates the included .rst file if it has changed.
+
+:idf_file:`docs/idf_extensions/gen_toolchain_links.py`
+ There couple of places in documentation that provide links to download the toolchain. To provide one source of this information and reduce effort to manually update several files, this script generates toolchain download links and toolchain unpacking code snippets based on information found in :idf_file:`tools/toolchain_versions.mk`.
+
+:idf_file:`docs/idf_extensions/gen_version_specific_includes.py`
+ Another extension to automatically generate reStructuredText Text ``.inc`` snippets with version-based content for this ESP-IDF version.
+
+:idf_file:`docs/idf_extensions/util.py`
+ A collection of utility functions useful primarily when building documentation locally (see :ref:`setup-for-building-documentation`) to reduce the time to generate documentation on a second and subsequent builds.
Related Documents
-----------------
@@ -75,3 +154,4 @@ Related Documents
.. _ESP-IDF: https://github.com/espressif/esp-idf/
+.. _Sphinx selective exclude: https://github.com/pfalcon/sphinx_selective_exclude
diff --git a/docs/extensions/README.md b/docs/extensions/README.md
new file mode 100644
index 000000000..e32b49af0
--- /dev/null
+++ b/docs/extensions/README.md
@@ -0,0 +1 @@
+See docs/en/contribute/add-ons-reference.rst (or in the IDF docs) for details.
diff --git a/docs/extensions/__init__.py b/docs/extensions/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/docs/html_redirects.py b/docs/extensions/html_redirects.py
similarity index 94%
rename from docs/html_redirects.py
rename to docs/extensions/html_redirects.py
index 45a28cac3..5019e84a6 100644
--- a/docs/html_redirects.py
+++ b/docs/extensions/html_redirects.py
@@ -17,7 +17,7 @@
# Mechanism to generate static HTML redirect pages in the output
#
# Uses redirect_template.html and the list of pages given in
-# conf.html_redirect_pages
+# the file conf.html_redirect_pages
#
# Adapted from ideas in https://tech.signavio.com/2017/managing-sphinx-redirects
import os.path
@@ -45,7 +45,7 @@ def setup(app):
# to create HTML redirects
app.connect('html-collect-pages', create_redirect_pages)
- return { 'parallel_read_safe' : True, 'parallel_write_safe': True, 'version': '0.1' }
+ return {'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '0.1'}
def create_redirect_pages(app):
diff --git a/docs/toctree_filter.py b/docs/extensions/toctree_filter.py
similarity index 79%
rename from docs/toctree_filter.py
rename to docs/extensions/toctree_filter.py
index 7082fad7b..2f537a5fe 100644
--- a/docs/toctree_filter.py
+++ b/docs/extensions/toctree_filter.py
@@ -1,13 +1,13 @@
# Based on https://stackoverflow.com/a/46600038 with some modifications
import re
from sphinx.directives.other import TocTree
-from sphinx.util.nodes import explicit_title_re
-from sphinx.util import docname_join
+
def setup(app):
app.add_directive('toctree', TocTreeFilt, override=True)
- return { 'parallel_read_safe' : True, 'parallel_write_safe': True, 'version': '0.1' }
+ return {'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '0.1'}
+
class TocTreeFilt(TocTree):
"""
@@ -24,16 +24,14 @@ class TocTreeFilt(TocTree):
when it scan the src/ directory, so it's also necessary to make sure that the files
are covered by the exclude_patterns list in conf.py
"""
- RE_PATTERN = re.compile('^\s*:(.+):\s*(.+)$')
-
+ RE_PATTERN = re.compile(r'^\s*:(.+):\s*(.+)$')
def run(self):
# Remove all TOC entries that should not be on display
env = self.state.document.settings.env
- self.content = [ self.filter_entry(env, e) for e in self.content if e is not None ]
+ self.content = [self.filter_entry(env, e) for e in self.content if e is not None]
return super(TocTreeFilt, self).run()
-
def filter_entry(self, env, entry):
m = self.RE_PATTERN.match(entry)
if m is not None:
@@ -41,4 +39,3 @@ class TocTreeFilt(TocTree):
if not env.app.builder.tags.eval_condition(tag_filter):
return None
return entry
-
diff --git a/docs/idf_extensions/README.md b/docs/idf_extensions/README.md
new file mode 100644
index 000000000..0d7cb9d91
--- /dev/null
+++ b/docs/idf_extensions/README.md
@@ -0,0 +1,2 @@
+See docs/en/contribute/add-ons-reference.rst (or in the IDF docs) for details.
+
diff --git a/docs/idf_extensions/__init__.py b/docs/idf_extensions/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/docs/idf_build_system/CMakeLists.txt b/docs/idf_extensions/build_system/CMakeLists.txt
similarity index 100%
rename from docs/idf_build_system/CMakeLists.txt
rename to docs/idf_extensions/build_system/CMakeLists.txt
diff --git a/docs/idf_build_system/__init__.py b/docs/idf_extensions/build_system/__init__.py
similarity index 60%
rename from docs/idf_build_system/__init__.py
rename to docs/idf_extensions/build_system/__init__.py
index 9f3e662ed..f1e2e0913 100644
--- a/docs/idf_build_system/__init__.py
+++ b/docs/idf_extensions/build_system/__init__.py
@@ -14,19 +14,43 @@ import json
# this directory also contains the dummy IDF project
project_path = os.path.abspath(os.path.dirname(__file__))
+
def setup(app):
- builddir = os.path.dirname(app.doctreedir.rstrip(os.sep))
- app.add_config_value('docs_root', "", 'env')
- app.add_config_value('idf_path', os.environ.get("IDF_PATH", ""), 'env')
+ # Setup some common paths
+
+ try:
+ build_dir = os.environ["BUILDDIR"] # TODO see if we can remove this
+ except KeyError:
+ build_dir = os.path.dirname(app.doctreedir.rstrip(os.sep))
+
+ try:
+ os.mkdir(build_dir)
+ except OSError:
+ pass
+
+ try:
+ os.mkdir(os.path.join(build_dir, 'inc'))
+ except OSError:
+ pass
+
+ # Fill in a default IDF_PATH if it's missing (ie when Read The Docs is building the docs)
+ try:
+ idf_path = os.environ['IDF_PATH']
+ except KeyError:
+ idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
+
+ app.add_config_value('docs_root', os.path.join(idf_path, "docs"), 'env')
+ app.add_config_value('idf_path', idf_path, 'env')
app.add_config_value('idf_target', 'esp32', 'env')
- app.add_config_value('build_dir', os.environ.get("BUILDDIR", ""), 'env') # not actually an IDF thing
+ app.add_config_value('build_dir', build_dir, 'env') # not actually an IDF thing
app.add_event('idf-info')
# Attaching the generate event to env-get-outdated is a bit of a hack,
# we want this to run early in the docs build but unclear exactly when
app.connect('env-get-outdated', generate_idf_info)
- return { 'parallel_read_safe' : True, 'parallel_write_safe': True, 'version': '0.1' }
+ return {'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '0.1'}
+
def generate_idf_info(app, env, added, changed, removed):
print("Running CMake on dummy project to get build info...")
@@ -43,17 +67,20 @@ def generate_idf_info(app, env, added, changed, removed):
if not os.path.exists(os.path.join(cmake_build_dir, "CMakeCache.txt")):
# if build directory not created yet, run a reconfigure pass over it
print("Starting new dummy IDF project...")
- subprocess.check_call(idf_py + [ "set-target", app.config.idf_target])
+ subprocess.check_call(idf_py + ["set-target", app.config.idf_target])
else:
print("Re-running CMake on the existing IDF project in {}".format(cmake_build_dir))
- subprocess.check_call(idf_py + [ "reconfigure"])
+ subprocess.check_call(idf_py + ["reconfigure"])
with open(os.path.join(cmake_build_dir, "project_description.json")) as f:
project_description = json.load(f)
if project_description["target"] != app.config.idf_target:
# this shouldn't really happen unless someone has been moving around directories inside _build, as
# the cmake_build_dir path should be target-specific
- raise RuntimeError("Error configuring the dummy IDF project for {}. Target in project description is {}. Is _build directory contents corrupt?".format(app.config.idf_target, project_description["target"]))
+ raise RuntimeError(("Error configuring the dummy IDF project for {}. " +
+ "Target in project description is {}. " +
+ "Is build directory contents corrupt?")
+ .format(app.config.idf_target, project_description["target"]))
app.emit('idf-info', project_description)
return []
diff --git a/docs/idf_extensions/esp_err_definitions.py b/docs/idf_extensions/esp_err_definitions.py
new file mode 100644
index 000000000..c77ff5bb7
--- /dev/null
+++ b/docs/idf_extensions/esp_err_definitions.py
@@ -0,0 +1,14 @@
+# Extension to generate esp_err definition as .rst
+from util import copy_if_modified, call_with_python
+
+
+def setup(app):
+ app.connect('idf-info', generate_err_defs)
+ return {'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '0.1'}
+
+
+def generate_err_defs(app, project_description):
+ # Generate 'esp_err_defs.inc' file with ESP_ERR_ error code definitions from inc file
+ esp_err_inc_path = '{}/inc/esp_err_defs.inc'.format(app.config.build_dir)
+ call_with_python('{}/tools/gen_esp_err_to_name.py --rst_output {}.in'.format(app.config.idf_path, esp_err_inc_path))
+ copy_if_modified(esp_err_inc_path + '.in', esp_err_inc_path)
diff --git a/docs/idf_extensions/gen_idf_tools_links.py b/docs/idf_extensions/gen_idf_tools_links.py
new file mode 100644
index 000000000..abba61daa
--- /dev/null
+++ b/docs/idf_extensions/gen_idf_tools_links.py
@@ -0,0 +1,19 @@
+# Generate toolchain download links from toolchain info makefile
+from __future__ import print_function
+import os.path
+from util import copy_if_modified, call_with_python
+
+def setup(app):
+ # we don't actually need idf-info, just a convenient event to trigger this on
+ app.connect('idf-info', generate_idf_tools_links)
+
+ return {'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '0.1'}
+
+
+def generate_idf_tools_links(app, project_description):
+ print("Generating IDF Tools list")
+ os.environ["IDF_MAINTAINER"] = "1"
+ tools_rst = os.path.join(app.config.build_dir, 'inc', 'idf-tools-inc.rst')
+ tools_rst_tmp = os.path.join(app.config.build_dir, 'idf-tools-inc.rst')
+ call_with_python("{}/tools/idf_tools.py gen-doc --output {}".format(app.config.idf_path, tools_rst_tmp))
+ copy_if_modified(tools_rst_tmp, tools_rst)
diff --git a/docs/gen-toolchain-links.py b/docs/idf_extensions/gen_toolchain_links.py
similarity index 67%
rename from docs/gen-toolchain-links.py
rename to docs/idf_extensions/gen_toolchain_links.py
index b98a68e2c..183e39f81 100644
--- a/docs/gen-toolchain-links.py
+++ b/docs/idf_extensions/gen_toolchain_links.py
@@ -1,15 +1,10 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# This script generates toolchain download links and toolchain unpacking
-# code snippets based on information found in $IDF_PATH/tools/toolchain_versions.mk
-#
-
+# Generate toolchain download links from toolchain info makefile
from __future__ import print_function
-
-import sys
-import os
+import os.path
from collections import namedtuple
+from util import copy_if_modified
+
+BASE_URL = 'https://dl.espressif.com/dl/'
PlatformInfo = namedtuple("PlatformInfo", [
"platform_name",
@@ -20,19 +15,22 @@ PlatformInfo = namedtuple("PlatformInfo", [
])
-def main():
- if len(sys.argv) != 4:
- print("Usage: gen-toolchain-links.py