doc: add generation of tags from sdkconfig and x_caps.h headers

Also updates the way we handle exclude_patterns to use the new tags

Closes IDF-1484
This commit is contained in:
Marius Vikhammer 2020-04-30 16:55:12 +08:00 committed by bot
parent 695f075a13
commit fbb54184ef
15 changed files with 328 additions and 227 deletions

View file

@ -67,6 +67,8 @@ extensions = ['breathe',
'idf_extensions.gen_idf_tools_links',
'idf_extensions.format_idf_target',
'idf_extensions.latex_builder',
'idf_extensions.gen_defines',
'idf_extensions.exclude_docs',
# from https://github.com/pfalcon/sphinx_selective_exclude
'sphinx_selective_exclude.eager_only',
@ -124,47 +126,49 @@ print('Version: {0} Release: {1}'.format(version, release))
exclude_patterns = ['**/inc/**', '_static', '**/_build']
# Add target-specific excludes based on tags (for the IDF_TARGET). Haven't found any better way to do this yet
def update_exclude_patterns(tags):
if "esp32" not in tags:
# Exclude ESP32-only document pages so they aren't found in the initial search for .rst files
# note: in toctrees, these also need to be marked with a :esp32: filter
for e in ['api-guides/blufi.rst',
'api-guides/build-system-legacy.rst',
'api-guides/esp-ble-mesh/**',
'api-guides/RF_calibration.rst', # temporary until support re-added in esp_wifi
'api-guides/ulp-legacy.rst',
'api-guides/unit-tests-legacy.rst',
'api-guides/ulp_instruction_set.rst',
'api-guides/jtag-debugging/configure-wrover.rst',
'api-reference/system/himem.rst',
'api-reference/bluetooth/**',
'api-reference/peripherals/sdio_slave.rst',
'api-reference/peripherals/esp_slave_protocol.rst',
'api-reference/peripherals/mcpwm.rst',
'api-reference/peripherals/sd_pullup_requirements.rst',
'api-reference/peripherals/sdmmc_host.rst',
'api-reference/protocols/esp_serial_slave_link.rst',
'api-reference/system/ipc.rst',
'get-started-legacy/**',
'security/secure-boot-v1.rst',
'security/secure-boot-v2.rst',
'gnu-make-legacy.rst',
'hw-reference/esp32/**',
]:
exclude_patterns.append(e)
BT_DOCS = ['api-guides/blufi.rst',
'api-guides/esp-ble-mesh/**',
'api-reference/bluetooth/**']
if "esp32s2" not in tags:
# Exclude ESP32-S2-only document pages so they aren't found in the initial search for .rst files
# note: in toctrees, these also need to be marked with a :esp32: filter
for e in ['esp32s2.rst',
'hw-reference/esp32s2/**',
'api-guides/dfu.rst',
'api-guides/ulps2_instruction_set.rst',
'api-reference/peripherals/hmac.rst',
'api-reference/peripherals/temp_sensor.rst']:
exclude_patterns.append(e)
SDMMC_DOCS = ['api-reference/peripherals/sdmmc_host.rst',
'api-reference/peripherals/sd_pullup_requirements.rst']
SDIO_SLAVE_DOCS = ['api-reference/peripherals/sdio_slave.rst',
'api-reference/peripherals/esp_slave_protocol.rst',
'api-reference/protocols/esp_serial_slave_link.rst']
MCPWM_DOCS = ['api-reference/peripherals/mcpwm.rst']
LEGACY_DOCS = ['api-guides/build-system-legacy.rst',
'gnu-make-legacy.rst',
'api-guides/ulp-legacy.rst',
'api-guides/unit-tests-legacy.rst',
'get-started-legacy/**']
ESP32_DOCS = ['api-guides/ulp_instruction_set.rst',
'api-guides/jtag-debugging/configure-wrover.rst',
'api-reference/system/himem.rst',
'api-guides/RF_calibration.rst',
'api-reference/system/ipc.rst',
'security/secure-boot-v1.rst',
'security/secure-boot-v2.rst',
'hw-reference/esp32/**'] + LEGACY_DOCS
ESP32S2_DOCS = ['esp32s2.rst',
'hw-reference/esp32s2/**',
'api-guides/ulps2_instruction_set.rst',
'api-guides/dfu.rst',
'api-reference/peripherals/hmac.rst',
'api-reference/peripherals/temp_sensor.rst'
'']
# format: {tag needed to include: documents to included}, tags are parsed from sdkconfig and peripheral_caps.h headers
conditional_include_dict = {'SOC_BT_SUPPORTED':BT_DOCS,
'SOC_SDMMC_HOST_SUPPORTED':SDMMC_DOCS,
'SOC_SDIO_SLAVE_SUPPORTED':SDIO_SLAVE_DOCS,
'SOC_MCPWM_SUPPORTED':MCPWM_DOCS,
'esp32':ESP32_DOCS,
'esp32s2':ESP32S2_DOCS}
# The reST default role (used for this markup: `text`) to use for all
# documents.
@ -378,6 +382,8 @@ def setup(app):
app.add_config_value('idf_target', None, 'env')
app.add_config_value('idf_targets', None, 'env')
app.add_config_value('conditional_include_dict', None, 'env')
# Breathe extension variables (depend on build_dir)
# note: we generate into xml_in and then copy_if_modified to xml dir
app.config.breathe_projects = {"esp32-idf": os.path.join(app.config.build_dir, "xml_in/")}

View file

@ -4,7 +4,7 @@ ESP-IDF FreeRTOS SMP Changes
Overview
--------
.. only:: esp32
.. only:: not CONFIG_FREERTOS_UNICORE
The vanilla FreeRTOS is designed to run on a single core. However the ESP32 is
dual core containing a Protocol CPU (known as **CPU 0** or **PRO_CPU**) and an
@ -25,7 +25,7 @@ see :doc:`ESP-IDF FreeRTOS Additions<../api-reference/system/freertos_additions>
port of FreeRTOS v8.2.0, a number of FreeRTOS v9.0.0 features have been backported
to ESP-IDF.
.. only:: esp32
.. only:: not CONFIG_FREERTOS_UNICORE
:ref:`tasks-and-task-creation`: Use :cpp:func:`xTaskCreatePinnedToCore` or
:cpp:func:`xTaskCreateStaticPinnedToCore` to create tasks in ESP-IDF FreeRTOS. The
@ -355,7 +355,7 @@ context switches and servicing of ISRs during a critical section. Therefore,
critical sections are used as a valid protection method against simultaneous
access in vanilla FreeRTOS.
.. only:: esp32
.. only:: not CONFIG_FREERTOS_UNICORE
On the other hand, the ESP32 has no hardware method for cores to disable each
others interrupts. Calling ``portDISABLE_INTERRUPTS()`` will have no effect on
@ -364,7 +364,7 @@ access in vanilla FreeRTOS.
leaves the other core free to access the data even if the current core has
disabled its own interrupts.
.. only:: esp32s2
.. only:: CONFIG_FREERTOS_UNICORE
ESP-IDF contains some modifications to work with dual core concurrency,
and the dual core API is used even on a single core only chip.
@ -407,7 +407,7 @@ spinlock is provided upon entering and exiting, the type of call should not
matter.
.. only:: esp32
.. only:: not CONFIG_FREERTOS_UNICORE
.. _floating-points:
@ -495,7 +495,7 @@ The ESP-IDF FreeRTOS can be configured in the project configuration menu
highlights some of the ESP-IDF FreeRTOS configuration options. For a full list of
ESP-IDF FreeRTOS configurations, see :doc:`FreeRTOS <../api-reference/kconfig>`
.. only:: esp32
.. only:: not CONFIG_FREERTOS_UNICORE
:ref:`CONFIG_FREERTOS_UNICORE` will run ESP-IDF FreeRTOS only
on **PRO_CPU**. Note that this is **not equivalent to running vanilla
@ -504,9 +504,9 @@ ESP-IDF FreeRTOS configurations, see :doc:`FreeRTOS <../api-reference/kconfig>`
effects of running ESP-IDF FreeRTOS on a single core, search for
occurences of ``CONFIG_FREERTOS_UNICORE`` in the ESP-IDF components.
.. only:: esp32s2
.. only:: CONFIG_FREERTOS_UNICORE
As ESP32-S2 is a single core SoC, the config item :ref:`CONFIG_FREERTOS_UNICORE` is
As {IDF_TARGET_NAME} is a single core SoC, the config item :ref:`CONFIG_FREERTOS_UNICORE` is
always set. This means ESP-IDF only runs on the single CPU. Note that this is **not
equivalent to running vanilla FreeRTOS**. Behaviors of multiple components in ESP-IDF
will be modified. For more details regarding the effects of running ESP-IDF FreeRTOS

View file

@ -6,14 +6,14 @@ API Guides
:maxdepth: 1
Application Level Tracing <app_trace>
:esp32: BluFi <blufi>
:SOC_BT_SUPPORTED: BluFi <blufi>
Bootloader <bootloader>
Build System <build-system>
:esp32: Build System (Legacy GNU Make) <build-system-legacy>
Deep Sleep Wake Stubs <deep-sleep-stub>
:esp32s2: Device Firmware Upgrade through USB <dfu>
Error Handling <error-handling>
:esp32: ESP-BLE-MESH <esp-ble-mesh/ble-mesh-index>
:SOC_BT_SUPPORTED: ESP-BLE-MESH <esp-ble-mesh/ble-mesh-index>
ESP-MESH (Wi-Fi) <mesh>
Core Dump <core_dump>
Event Handling <event-handling>

View file

@ -6,7 +6,7 @@ API Reference
.. toctree::
:maxdepth: 2
:esp32: Bluetooth <bluetooth/index>
:SOC_BT_SUPPORTED: Bluetooth <bluetooth/index>
Networking <network/index>
Peripherals <peripherals/index>
Protocols <protocols/index>

View file

@ -15,13 +15,13 @@ Peripherals API
I2C <i2c>
I2S <i2s>
LED Control <ledc>
:esp32: MCPWM <mcpwm>
:SOC_MCPWM_SUPPORTED: MCPWM <mcpwm>
Pulse Counter <pcnt>
Remote Control <rmt>
:esp32: SD Pull-up Requirements <sd_pullup_requirements>
:esp32: SDMMC Host <sdmmc_host>
:SOC_SDMMC_HOST_SUPPORTED: SDMMC Host <sdmmc_host>
SD SPI Host <sdspi_host>
:esp32: SDIO Slave <sdio_slave>
:SOC_SDIO_SLAVE_SUPPORTED: SDIO Slave <sdio_slave>
Sigma-delta Modulation <sigmadelta>
SPI Master <spi_master>
SPI Slave <spi_slave>

View file

@ -18,5 +18,3 @@ copyright = u'2016 - 2020, Espressif Systems (Shanghai) CO., LTD'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
language = 'en'
update_exclude_patterns(tags) # noqa: F405, need to import * from conf_common

View file

@ -128,15 +128,6 @@ Other Extensions
:idf_file:`docs/idf_extensions/link_roles.py`
This is an implementation of a custom `Sphinx Roles <https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html>`_ to help linking from documentation to specific files and folders in `ESP-IDF`_. For description of implemented roles please see :ref:`link-custom-roles` and :ref:`link-language-versions`.
:idf_file:`docs/idf_extensions/run_doxygen.py`
Subscribes to ``idf-info`` event and runs Doxygen (:idf_file:`docs/Doxyfile`) to generate XML files describing key headers, and then runs Breathe to convert these to ``.inc`` files which can be included directly into API reference pages.
Pushes a number of target-specific custom environment variables into Doxygen, including all macros defined in the project's default ``sdkconfig.h`` file and all macros defined in all ``soc`` component ``xxx_caps.h`` headers. This means that public API headers can depend on target-specific configuration options or ``soc`` capabilities headers options as ``#ifdef`` & ``#if`` preprocessor selections in the header.
This means we can generate different Doxygen files, depending on the target we are building docs for.
Please refer to :doc:`documenting-code` and :doc:`../api-reference/template`, section **API Reference** for additional details on this process.
:idf_file:`docs/idf_extensions/esp_err_definitions.py`
Small wrapper extension that calls ``gen_esp_err_to_name.py`` and updates the included .rst file if it has changed.
@ -166,6 +157,27 @@ Other Extensions
Creates and adds the espidf.sty latex package to the output directory, which contains some macros for run-time variables such as IDF-Target.
:idf_file:`docs/idf_extensions/gen_defines.py`
Sphinx extension to integrate defines from IDF into the Sphinx build, runs after the IDF dummy project has been built.
Parses defines and adds them as sphinx tags.
Emits the new 'idf-defines-generated' event which has a dictionary of raw text define values that other extensions can use to generate relevant data.
:idf_file:`docs/idf_extensions/exclude_docs.py`
Sphinx extension that updates the excluded documents according to the conditional_include_dict {tag:documents}. If the tag is set, then the list of documents will be included.
Subscribes to ``idf-defines-generated`` as it relies on the sphinx tags to determine which documents to exclude
:idf_file:`docs/idf_extensions/run_doxygen.py`
Subscribes to ``idf-defines-generated`` event and runs Doxygen (:idf_file:`docs/Doxyfile`) to generate XML files describing key headers, and then runs Breathe to convert these to ``.inc`` files which can be included directly into API reference pages.
Pushes a number of target-specific custom environment variables into Doxygen, including all macros defined in the project's default ``sdkconfig.h`` file and all macros defined in all ``soc`` component ``xxx_caps.h`` headers. This means that public API headers can depend on target-specific configuration options or ``soc`` capabilities headers options as ``#ifdef`` & ``#if`` preprocessor selections in the header.
This means we can generate different Doxygen files, depending on the target we are building docs for.
Please refer to :doc:`documenting-code` and :doc:`../api-reference/template`, section **API Reference** for additional details on this process.
Related Documents
-----------------

View file

@ -252,11 +252,22 @@ The documentation for all of Espressif's chips is built from the same files. To
Exclusion of content based on chip-target
"""""""""""""""""""""""""""""""""""""""""
Occasionally there will be content that is only relevant for one of targets. When this is the case, you can exclude that content by using the ''.. only:: TARGET'' directive, where you replace 'TARGET' with one of the chip names. As of now the following targets are available:
Occasionally there will be content that is only relevant for one of targets. When this is the case, you can exclude that content by using the ''.. only:: TAG'' directive, where you replace 'TAG' with one of the following names:
Chip name:
* esp32
* esp32s2
Define identifiers from 'sdkconfig.h', generated by the default menuconfig settings for the target, e.g:
* CONFIG_FREERTOS_UNICORE
Define identifiers from the soc '*_caps' headers, e.g:
* SOC_BT_SUPPORTED
* SOC_CAN_SUPPORTED
Example:
.. code-block:: none
@ -265,6 +276,14 @@ Example:
ESP32 specific content.
This directive also supports the boolean operators 'and', 'or' and 'not'. Example:
.. code-block:: none
.. only:: SOC_BT_SUPPORTED and CONFIG_FREERTOS_UNICORE
BT specific content only relevant for single-core targets.
This functionality is provided by the `Sphinx selective exclude <https://github.com/pfalcon/sphinx_selective_exclude>`_ extension.
A weakness in this extension is that it does not correctly handle the case were you exclude a section, and that is directly followed by a labeled new section. In these cases everything will render correctly, but the label will not correctly link to the section that follows. A temporary work-around for the cases were this can't be avoided is the following:
@ -290,7 +309,7 @@ A weakness in this extension is that it does not correctly handle the case were
^^^^^^^^^
Section 2 content
The :TARGET: role is used for excluding content from a table of content tree. For example:
The :TAG: role is used for excluding content from a table of content tree. For example:
.. code-block:: none
@ -302,16 +321,20 @@ The :TARGET: role is used for excluding content from a table of content tree. Fo
When building the documents, Sphinx will use the above mentioned directive and role to include or exclude content based on the target tag it was called with.
.. note:: If excluding an entire document from the toctree based on targets, it's necessary to also update the ``exclude_patterns`` list in :idf_file:`docs/conf_common.py` to exclude the file for other targets, or a Sphinx warning "WARNING: document isn't included in any toctree" will be generated..
.. note::
If you need to exclude content inside a list or bullet points then this should be done by using the '':TARGET:'' role inside the ''.. list:: '' directive.
If excluding an entire document from the toctree based on targets, it's necessary to also update the ``exclude_patterns`` list in :idf_file:`docs/conf_common.py` to exclude the file for other targets, or a Sphinx warning "WARNING: document isn't included in any toctree" will be generated..
The recommended way of doing it is adding the document to one of the list that gets included in ``conditional_include_dict`` in :idf_file:`docs/conf_common.py`, e.g. a document which should only be shown for BT capable targets should be added to ``BT_DOCS``. :idf_file:`docs/idf_extensions/exclude_docs.py` will then take care of adding it to ``exclude_patterns`` if the corresponding tag is not set.
If you need to exclude content inside a list or bullet points then this should be done by using the '':TAG:'' role inside the ''.. list:: '' directive.
.. code-block:: none
.. list::
:esp32: - ESP32 specific content
:esp32s2: - ESP32 S2 specific content
:SOC_BT_SUPPORTED: - BT specific content
- Common bullet point
- Also common bullet point

View file

@ -20,8 +20,8 @@ Introduction
.. list::
* Wi-Fi (2.4 GHz band)
:esp32: * Bluetooth
:esp32: * Dual high performance cores
:SOC_BT_SUPPORTED: * Bluetooth
:CONFIG_FREERTOS_UNICORE: * Dual high performance cores
* Ultra Low Power co-processor
* Multiple peripherals
:esp32s2: * Built-in security hardware

View file

@ -0,0 +1,12 @@
# Updates the excluded documents according to the conditional_include_dict {tag:documents}
def update_exclude_patterns(app, config):
for tag, docs in config.conditional_include_dict.items():
if not app.tags.has(tag):
app.config.exclude_patterns.extend(docs)
def setup(app):
# Tags are generated together with defines
app.connect('config-inited', update_exclude_patterns)
return {'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '0.1'}

View file

@ -0,0 +1,72 @@
# Sphinx extension to integrate defines into the Sphinx Build
#
# Runs after the IDF dummy project has been built
#
# Then emits the new 'idf-defines-generated' event which has a dictionary of raw text define values
# that other extensions can use to generate relevant data.
import glob
import os
import subprocess
import re
def generate_defines(app, project_description):
sdk_config_path = os.path.join(project_description["build_dir"], "config")
# Parse kconfig macros to pass into doxygen
#
# TODO: this should use the set of "config which can't be changed" eventually,
# not the header
defines = get_defines(os.path.join(project_description["build_dir"],
"config", "sdkconfig.h"), sdk_config_path)
# Add all SOC _caps.h headers and kconfig macros to the defines
#
# kind of a hack, be nicer to add a component info dict in project_description.json
soc_path = [p for p in project_description["build_component_paths"] if p.endswith("/soc")][0]
soc_headers = glob.glob(os.path.join(soc_path, "soc", project_description["target"],
"include", "soc", "*_caps.h"))
assert len(soc_headers) > 0
for soc_header in soc_headers:
defines.update(get_defines(soc_header, sdk_config_path))
add_tags(app, defines)
app.emit('idf-defines-generated', defines)
def get_defines(header_path, sdk_config_path):
defines = {}
# Note: we run C preprocessor here without any -I arguments (except "sdkconfig.h"), so assumption is
# that these headers are all self-contained and don't include any other headers
# not in the same directory
print("Reading macros from %s..." % (header_path))
processed_output = subprocess.check_output(["xtensa-esp32-elf-gcc", "-I", sdk_config_path,
"-dM", "-E", header_path]).decode()
for line in processed_output.split("\n"):
line = line.strip()
m = re.search("#define ([^ ]+) ?(.*)", line)
if m and not m.group(1).startswith("_"):
defines[m.group(1)] = m.group(2)
return defines
def add_tags(app, defines):
# try to parse define values as ints and add to tags
for name, value in defines.items():
try:
define_value = int(value.strip("()"))
if define_value > 0:
app.tags.add(name)
except ValueError:
continue
def setup(app):
app.connect('idf-info', generate_defines)
app.add_event('idf-defines-generated')
return {'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '0.1'}

View file

@ -2,7 +2,6 @@
from __future__ import print_function
from __future__ import unicode_literals
from io import open
import glob
import os
import os.path
import re
@ -45,28 +44,9 @@ def _parse_defines(header_path, sdk_config_path):
return defines
def generate_doxygen(app, project_description):
def generate_doxygen(app, defines):
build_dir = os.path.dirname(app.doctreedir.rstrip(os.sep))
sdk_config_path = os.path.join(project_description["build_dir"], "config")
# Parse kconfig macros to pass into doxygen
#
# TODO: this should use the set of "config which can't be changed" eventually,
# not the header
defines = _parse_defines(os.path.join(project_description["build_dir"],
"config", "sdkconfig.h"), sdk_config_path)
# Add all SOC _caps.h headers to the defines
#
# kind of a hack, be nicer to add a component info dict in project_description.json
soc_path = [p for p in project_description["build_component_paths"] if p.endswith("/soc")][0]
soc_headers = glob.glob(os.path.join(soc_path, "soc", project_description["target"],
"include", "soc", "*_caps.h"))
assert len(soc_headers) > 0
for soc_header in soc_headers:
defines.update(_parse_defines(soc_header, sdk_config_path))
# Call Doxygen to get XML files from the header files
print("Calling Doxygen to generate latest XML files")
doxy_env = os.environ

View file

@ -31,9 +31,9 @@ API 指南
ROM debug console <romconsole>
:esp32: RF Calibration <RF_calibration>
WiFi Driver <wifi>
:esp32: ESP-BLE-MESH <esp-ble-mesh/ble-mesh-index>
:SOC_BT_SUPPORTED: ESP-BLE-MESH <esp-ble-mesh/ble-mesh-index>
ESP-MESH (Wi-Fi) <mesh>
:esp32: BluFi <blufi>
:SOC_BT_SUPPORTED: BluFi <blufi>
External SPI-connected RAM <external-ram>
链接脚本生成机制 <linker-script-generation>
LwIP <lwip>

View file

@ -14,12 +14,12 @@
I2C <i2c>
I2S <i2s>
LED Control <ledc>
:esp32: MCPWM <mcpwm>
:SOC_MCPWM_SUPPORTED: MCPWM <mcpwm>
Pulse Counter <pcnt>
Remote Control <rmt>
:esp32: SDMMC Host <sdmmc_host>
:SOC_SDMMC_HOST_SUPPORTED: SDMMC Host <sdmmc_host>
SD SPI Host <sdspi_host>
:esp32: SDIO Slave <sdio_slave>
:SOC_SDIO_SLAVE_SUPPORTED: SDIO Slave <sdio_slave>
Sigma-delta Modulation <sigmadelta>
SPI Master <spi_master>
SPI Slave <spi_slave>

View file

@ -18,5 +18,3 @@ copyright = u'2016 - 2020 乐鑫信息科技(上海)股份有限公司'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
language = 'zh_CN'
update_exclude_patterns(tags) # noqa: F405, need to import * from conf_common