Merge branch 'bugfix/py3_error_code_generator_from_docs' into 'master'

Docs: Check Python packages

See merge request idf/esp-idf!3135

docs/conf_common.py

@@ -30,6 +30,11 @@ builddir = builddir
 if 'BUILDDIR' in os.environ:
     builddir = os.environ['BUILDDIR']
 
+def call_with_python(cmd):
+    # using sys.executable ensures that the scripts are called with the same Python interpreter
+    if os.system('{} {}'.format(sys.executable, cmd)) != 0:
+        raise RuntimeError('{} failed'.format(cmd))
+
 # Call Doxygen to get XML files from the header files
 print("Calling Doxygen to generate latest XML files")
 if os.system("doxygen ../Doxyfile") != 0:
@@ -40,20 +45,17 @@ if os.system("doxygen ../Doxyfile") != 0:
 copy_if_modified('xml/', 'xml_in/')
 
 # Generate 'api_name.inc' files using the XML files by Doxygen
-if os.system('python ../gen-dxd.py') != 0:
+call_with_python('../gen-dxd.py')
-    raise RuntimeError('gen-dxd.py failed')
 
 # Generate 'kconfig.inc' file from components' Kconfig files
 kconfig_inc_path = '{}/inc/kconfig.inc'.format(builddir)
-if os.system('python ../gen-kconfig-doc.py > ' + kconfig_inc_path + '.in') != 0:
+call_with_python('../gen-kconfig-doc.py > ' + kconfig_inc_path + '.in')
-    raise RuntimeError('gen-kconfig-doc.py failed')
 
 copy_if_modified(kconfig_inc_path + '.in', kconfig_inc_path)
 
 # Generate 'esp_err_defs.inc' file with ESP_ERR_ error code definitions
 esp_err_inc_path = '{}/inc/esp_err_defs.inc'.format(builddir)
-if os.system('python ../../tools/gen_esp_err_to_name.py --rst_output ' + esp_err_inc_path + '.in') != 0:
+call_with_python('../../tools/gen_esp_err_to_name.py --rst_output ' + esp_err_inc_path + '.in')
-    raise RuntimeError('gen_esp_err_to_name.py failed')
 copy_if_modified(esp_err_inc_path + '.in', esp_err_inc_path)
 
 # Generate version-related includes
@@ -62,8 +64,7 @@ copy_if_modified(esp_err_inc_path + '.in', esp_err_inc_path)
 def generate_version_specific_includes(app):
     print("Generating version-specific includes...")
     version_tmpdir = '{}/version_inc'.format(builddir)
-    if os.system('python ../gen-version-specific-includes.py {} {}'.format(app.config.language, version_tmpdir)):
+    call_with_python('../gen-version-specific-includes.py {} {}'.format(app.config.language, version_tmpdir))
-        raise RuntimeError('gen-version-specific-includes.py failed')
     copy_if_modified(version_tmpdir, '{}/inc'.format(builddir))
 
 

docs/docs_common.mk

@@ -28,7 +28,7 @@ ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) -w
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext dependencies version-specific-includes
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext dependencies version-specific-includes check_python_packages
 
 help:
 	@echo "Please use \`make <target>\' where <target> is one of"
@@ -58,38 +58,44 @@ help:
 clean:
 	rm -rf $(BUILDDIR)/*
 
-html:
+# Notify users when some of the required python packages are not installed.
+# Note: This is intended to help developers who generate the documentation on their local machine. Read The Docs uses
+# the requirements.txt file directly and calls sphinx also directly without the use of the makefile!
+check_python_packages:
+	$(IDF_PATH)/tools/check_python_dependencies.py -r $(IDF_PATH)/docs/requirements.txt
+
+html: | check_python_packages
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
-dirhtml:
+dirhtml: | check_python_packages
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 
-singlehtml:
+singlehtml: | check_python_packages
 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 	@echo
 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 
-pickle:
+pickle: | check_python_packages
 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 	@echo
 	@echo "Build finished; now you can process the pickle files."
 
-json:
+json: | check_python_packages
 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 	@echo
 	@echo "Build finished; now you can process the JSON files."
 
-htmlhelp:
+htmlhelp: | check_python_packages
 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 	@echo
 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 
-qthelp:
+qthelp: | check_python_packages
 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 	@echo
 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
@@ -98,7 +104,7 @@ qthelp:
 	@echo "To view the help file:"
 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ReadtheDocsTemplate.qhc"
 
-devhelp:
+devhelp: | check_python_packages
 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 	@echo
 	@echo "Build finished."
@@ -107,70 +113,70 @@ devhelp:
 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ReadtheDocsTemplate"
 	@echo "# devhelp"
 
-epub:
+epub: | check_python_packages
 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
 	@echo
 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
 
-latex:
+latex: | check_python_packages
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
 	      "(use \`make latexpdf' here to do that automatically)."
 
-latexpdf:
+latexpdf: | check_python_packages
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through pdflatex..."
 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
-latexpdfja:
+latexpdfja: | check_python_packages
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through platex and dvipdfmx..."
 	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
 
-text:
+text: | check_python_packages
 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
 	@echo
 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
 
-man:
+man: | check_python_packages
 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
 	@echo
 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
 
-texinfo:
+texinfo: | check_python_packages
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
 	@echo
 	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
 	@echo "Run \`make' in that directory to run these through makeinfo" \
 	      "(use \`make info' here to do that automatically)."
 
-info:
+info: | check_python_packages
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
 	@echo "Running Texinfo files through makeinfo..."
 	make -C $(BUILDDIR)/texinfo info
 	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
 
-gettext:
+gettext: | check_python_packages
 	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
 	@echo
 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
 
-changes:
+changes: | check_python_packages
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 	@echo
 	@echo "The overview file is in $(BUILDDIR)/changes."
 
-linkcheck:
+linkcheck: | check_python_packages
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
 	@echo
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in $(BUILDDIR)/linkcheck/output.txt."
 
-gh-linkcheck:
+gh-linkcheck: | check_python_packages
 	@echo "Checking for hardcoded GitHub links"
 	@if (find ../ -name '*.rst' | xargs grep \
 		'https://github.com/espressif/esp-idf/tree\|https://github.com/espressif/esp-idf/blob\|https://github.com/espressif/esp-idf/raw'\
@@ -194,17 +200,17 @@ gh-linkcheck:
 	fi
 	@echo "No hardcoded links found"
 
-doctest:
+doctest: | check_python_packages
 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
 	@echo "Testing of doctests in the sources finished, look at the " \
 	      "results in $(BUILDDIR)/doctest/output.txt."
 
-xml:
+xml: | check_python_packages
 	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
 	@echo
 	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
 
-pseudoxml:
+pseudoxml: | check_python_packages
 	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
 	@echo
 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."

docs/gen-dxd.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+#
 # gen-dxd.py - Generate Doxygen Directives
 #
 # This code is in the Public Domain (or CC0 licensed, at your option.)

docs/requirements.txt

@@ -4,8 +4,9 @@
 sphinx==1.6.5
 sphinx-rtd-theme
 breathe==4.7.3
-sphinxcontrib.blockdiag==1.5.3
+sphinxcontrib-blockdiag==1.5.3
-sphinxcontrib.seqdiag==0.8.5
+sphinxcontrib-seqdiag==0.8.5
-sphinxcontrib.actdiag==0.8.5
+sphinxcontrib-actdiag==0.8.5
-sphinxcontrib.nwdiag==0.9.5
+sphinxcontrib-nwdiag==0.9.5
 recommonmark
+future>=0.16.0 # for ../tools/gen_esp_err_to_name.py

tools/check_python_dependencies.py

@@ -16,6 +16,7 @@
 
 import os
 import sys
+import argparse
 try:
     import pkg_resources
 except:
@@ -24,11 +25,17 @@ except:
            'setting up the required packages.')
     sys.exit(1)
 
-req_file = '{}/requirements.txt'.format(os.getenv("IDF_PATH"))
-
 if __name__ == "__main__":
+    idf_path = os.getenv("IDF_PATH")
+
+    parser = argparse.ArgumentParser(description='ESP32 Python package dependency checker')
+    parser.add_argument('--requirements', '-r',
+            help='Path to the requrements file',
+            default=idf_path + '/requirements.txt')
+    args = parser.parse_args()
+
     not_satisfied = []
-    with open(req_file) as f:
+    with open(args.requirements) as f:
         for line in f:
             line = line.strip()
             try:
@@ -40,7 +47,7 @@ if __name__ == "__main__":
         print('The following Python requirements are not satisfied:')
         for requirement in not_satisfied:
             print(requirement)
-        print('Please run "{} -m pip install -r {}" for resolving the issue.'.format(sys.executable, req_file))
+        print('Please run "{} -m pip install -r {}" for resolving the issue.'.format(sys.executable, args.requirements))
         sys.exit(1)
 
-    print('Python requirements are satisfied.')
+    print('Python requirements from {} are satisfied.'.format(args.requirements))

tools/ci/executable-list.txt

@@ -41,3 +41,5 @@ tools/test_idf_monitor/run_test_idf_monitor.py
 tools/mass_mfg/mfg_gen.py
 tools/unit-test-app/unit_test.py
 tools/test_idf_size/test.sh
+tools/check_python_dependencies.py
+docs/gen-dxd.py

tools/gen_esp_err_to_name.py

@@ -16,9 +16,20 @@
 
 from __future__ import print_function
 from __future__ import unicode_literals
-from builtins import str
+import sys
-from builtins import range
+try:
-from builtins import object
+    from builtins import str
+    from builtins import range
+    from builtins import object
+except ImportError:
+    # This should not happen because the Python packages are checked before invoking this script. However, here is
+    # some output which should help if we missed something.
+    print('Import has failed probably because of the missing "future" package. Please install all the packages for '
+          'interpreter {} from the requirements.txt file.'.format(sys.executable))
+    # The path to requirements.txt is not provided because this script could be invoked from an IDF project (then the
+    # requirements.txt from the IDF_PATH should be used) or from the documentation project (then the requirements.txt
+    # for the documentation directory should be used).
+    sys.exit(1)
 from io import open
 import os
 import argparse