doc: add latex and PDF generation to build_docs

Adds options for generating tex-files and PDFs when building documentation

Closes IDF-1217
Closes IDF-1464

docs/build_docs.py

@@ -78,6 +78,8 @@ def main():
     parser.add_argument("--language", "-l", choices=LANGUAGES, required=False)
     parser.add_argument("--target", "-t", choices=TARGETS, required=False)
     parser.add_argument("--build-dir", "-b", type=str, default="_build")
+    parser.add_argument("--builders", "-bs", nargs='+', type=str, default=["html"],
+                        help="List of builders for Sphinx, e.g. html or latex, for latex a PDF is also generated")
     parser.add_argument("--sphinx-parallel-builds", "-p", choices=["auto"] + [str(x) for x in range(8)],
                         help="Parallel Sphinx builds - number of independent Sphinx builds to run", default="auto")
     parser.add_argument("--sphinx-parallel-jobs", "-j", choices=["auto"] + [str(x) for x in range(8)],
@@ -151,7 +153,7 @@ def parallel_call(args, callback):
     for target in targets:
         for language in languages:
             build_dir = os.path.realpath(os.path.join(args.build_dir, language, target))
-            entries.append((language, target, build_dir, args.sphinx_parallel_jobs))
+            entries.append((language, target, build_dir, args.sphinx_parallel_jobs, args.builders))
 
     print(entries)
     errcodes = pool.map(callback, entries)
@@ -251,12 +253,73 @@ def action_build(args):
                               log_file=os.path.join(build_dir, SPHINX_WARN_LOG),
                               known_warnings_file=SPHINX_KNOWN_WARNINGS,
                               out_sanitized_log_file=os.path.join(build_dir, SPHINX_SANITIZED_LOG))
+
     if ret != 0:
         return ret
 
 
 def call_build_docs(entry):
-    return sphinx_call(*entry, buildername="html")
+    (language, target, build_dir, sphinx_parallel_jobs, builders) = entry
+    for buildername in builders:
+        ret = sphinx_call(language, target, build_dir, sphinx_parallel_jobs, buildername)
+        if ret != 0:
+            return ret
+
+    # Build PDF from tex
+    if 'latex' in builders:
+        latex_dir = os.path.join(build_dir, "latex")
+        ret = build_pdf(language, target, latex_dir)
+
+    return ret
+
+
+def build_pdf(language, target, latex_dir):
+    # Note: because this runs in a multiprocessing Process, everything which happens here should be isolated to a single process
+
+    # wrap stdout & stderr in a way that lets us see which build_docs instance they come from
+    #
+    # this doesn't apply to subprocesses, they write to OS stdout & stderr so no prefix appears
+    prefix = "%s/%s: " % (language, target)
+
+    print("Building PDF in latex_dir: %s" % (latex_dir))
+
+    saved_cwd = os.getcwd()
+    os.chdir(latex_dir)
+
+    # Based on read the docs PDFBuilder
+    rcfile = 'latexmkrc'
+
+    cmd = [
+        'latexmk',
+        '-r',
+        rcfile,
+        '-pdf',
+        # When ``-f`` is used, latexmk will continue building if it
+        # encounters errors. We still receive a failure exit code in this
+        # case, but the correct steps should run.
+        '-f',
+        '-dvi-',    # dont generate dvi
+        '-ps-',     # dont generate ps
+        '-interaction=nonstopmode',
+        '-quiet',
+        '-outdir=build',
+    ]
+
+    try:
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        for c in iter(lambda: p.stdout.readline(), b''):
+            sys.stdout.write(prefix)
+            sys.stdout.write(c.decode('utf-8'))
+        ret = p.wait()
+        assert (ret is not None)
+        sys.stdout.flush()
+    except KeyboardInterrupt:  # this seems to be the only way to get Ctrl-C to kill everything?
+        p.kill()
+        os.chdir(saved_cwd)
+        return 130  # FIXME It doesn't return this errorcode, why? Just prints stacktrace
+    os.chdir(saved_cwd)
+
+    return ret
 
 
 SANITIZE_FILENAME_REGEX = re.compile("[^:]*/([^/:]*)(:.*)")
@@ -331,7 +394,8 @@ def action_linkcheck(args):
 
 
 def call_linkcheck(entry):
-    return sphinx_call(*entry, buildername="linkcheck")
+    # Remove the last entry which the buildername, since the linkcheck builder is not supplied through the builder list argument
+    return sphinx_call(*entry[:4], buildername="linkcheck")
 
 
 # https://github.com/espressif/esp-idf/tree/

docs/conf_common.py

@@ -66,6 +66,7 @@ extensions = ['breathe',
               'idf_extensions.run_doxygen',
               'idf_extensions.gen_idf_tools_links',
               'idf_extensions.format_idf_target',
+              'idf_extensions.latex_builder',
 
               # from https://github.com/pfalcon/sphinx_selective_exclude
               'sphinx_selective_exclude.eager_only',
@@ -293,48 +294,40 @@ html_static_path = ['../_static']
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'ReadtheDocsTemplatedoc'
 
-
 # -- Options for LaTeX output ---------------------------------------------
 
+latex_template_dir = os.path.join(config_dir, 'latex_templates')
+
+preamble = ''
+with open(os.path.join(latex_template_dir, 'preamble.tex')) as f:
+    preamble = f.read()
+
+titlepage = ''
+with open(os.path.join(latex_template_dir, 'titlepage.tex')) as f:
+    titlepage = f.read()
+
+
 latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
+    'papersize': 'a4paper',
-    # 'papersize': 'letterpaper',
+
-    #
+    # Latex figure (float) alignment
-    # The font size ('10pt', '11pt' or '12pt').
+    'figure_align':'htbp',
-    # 'pointsize': '10pt',
+
-    #
+    'pointsize': '10pt',
+
     # Additional stuff for the LaTeX preamble.
-    # 'preamble': '',
+    'fncychap': '\\usepackage[Sonny]{fncychap}',
+
+    'preamble': preamble,
+
+    'maketitle': titlepage,
 }
 
-# Grouping the document tree into LaTeX files. List of tuples
+# The name of an image file (relative to this directory) to place at the bottom of
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    ('index', 'ReadtheDocsTemplate.tex', u'Read the Docs Template Documentation',
-     u'Read the Docs', 'manual'),
-]
-
-# The name of an image file (relative to this directory) to place at the top of
 # the title page.
-# latex_logo = None
+latex_logo = "../_static/espressif2.pdf"
-
+latex_engine = 'xelatex'
-# For "manual" documents, if this is true, then toplevel headings are parts,
+latex_use_xindy = False
-# not chapters.
-# latex_use_parts = False
-
-# If true, show page references after internal links.
-# latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-# latex_show_urls = False
-
-# Documents to append as an appendix to all manuals.
-# latex_appendices = []
-
-# If false, no module index is generated.
-# latex_domain_indices = True
-
 
 # -- Options for manual page output ---------------------------------------
 
@@ -390,6 +383,22 @@ def setup(app):
 
     setup_diag_font(app)
 
+    # Config values pushed by -D using the cmdline is not available when setup is called
+    app.connect('config-inited',  setup_config_values)
+
+
+def setup_config_values(app, config):
+    # Sets up global config values needed by other extensions
+    idf_target_title_dict = {
+        'esp32': 'ESP32',
+        'esp32s2': 'ESP32-S2'
+    }
+
+    app.add_config_value('idf_target_title_dict', idf_target_title_dict, 'env')
+
+    pdf_name = "esp-idf-{}-{}-{}".format(app.config.language, app.config.version, app.config.idf_target)
+    app.add_config_value('pdf_file', pdf_name, 'env')
+
 
 def setup_diag_font(app):
     # blockdiag and other tools require a font which supports their character set

docs/en/contribute/add-ons-reference.rst

@@ -161,6 +161,10 @@ Other Extensions
 
     These replacements cannot be used inside markup that rely on alignment of characters, e.g. tables.
 
+:idf_file:`docs/idf_extensions/latex_builder.py`
+    An extension for adding ESP-IDF specific functionality to the latex builder. Overrides the default Sphinx latex builder.
+
+    Creates and adds the espidf.sty latex package to the output directory, which contains some macros for run-time variables such as IDF-Target.
 
 Related Documents
 -----------------

docs/en/contribute/documenting-code.rst

@@ -489,6 +489,32 @@ Choices for language (``-l``) are ``en`` and ``zh_CN``. Choices for target (``-t
 Build documentation will be placed in ``_build/<language>/<target>/html`` folder. To see it, open the ``index.html`` inside this directory in a web browser.
 
 
+Building PDF
+""""""""""""
+
+It is also possible to build latex files and a PDF of the documentation using ``build_docs.py``. To do this the following Latex packages are required to be installed:
+
+ * latexmk
+ * texlive-latex-recommended
+ * texlive-fonts-recommended
+ * texlive-xetex
+
+The following fonts are also required to be installed:
+
+ * Freefont Serif, Sans and Mono OpenType fonts, available as the package ``fonts-freefont-otf`` on Ubuntu
+ * Lmodern, available as the package ``fonts-lmodern`` on Ubuntu
+ * Fandol, can be downloaded from `here <https://ctan.org/tex-archive/fonts/fandol>`_
+
+Now you can build the PDF for a target by invoking::
+
+    ./build_docs.py -bs latex -l en -t esp32 build
+
+Or alternatively build both html and PDF::
+
+    ./build_docs.py -bs html latex -l en -t esp32 build
+
+Latex files and the PDF will be placed in ``_build/<language>/<target>/latex`` folder.
+
 Wrap up
 -------
 

docs/idf_extensions/latex_builder.py

@@ -0,0 +1,55 @@
+from sphinx.builders.latex import LaTeXBuilder
+import os
+
+
+# Overrides the default Sphinx latex build
+class IdfLatexBuilder(LaTeXBuilder):
+
+    def __init__(self, app):
+
+        # Sets up the latex_documents config value, done here instead of conf.py since it depends on the runtime value 'idf_target'
+        self.init_latex_documents(app)
+
+        super().__init__(app)
+
+    def init_latex_documents(self, app):
+
+        file_name = app.config.pdf_file + '.tex'
+
+        if app.config.language == 'zh_CN':
+            latex_documents = [('index', file_name, u'ESP-IDF 编程指南', u'乐鑫信息科技', 'manual')]
+        else:
+            # Default to english naming
+            latex_documents = [('index', file_name, u'ESP-IDF Programming Guide', u'Espressif Systems', 'manual')]
+
+        app.config.latex_documents = latex_documents
+
+    def prepare_latex_macros(self, package_path, config):
+
+        PACKAGE_NAME = "espidf.sty"
+        latex_package = ''
+        with open(package_path, 'r') as template:
+
+            latex_package = template.read()
+
+        idf_target_title = config.idf_target_title_dict[config.idf_target]
+        latex_package = latex_package.replace('<idf_target_title>', idf_target_title)
+
+        # Release name for the PDF front page, remove '_' as this is used for subscript in Latex
+        idf_release_name = "Release {}".format(config.version.replace('_', '-'))
+        latex_package = latex_package.replace('<idf_release_name>', idf_release_name)
+
+        with open(os.path.join(self.outdir, PACKAGE_NAME), 'w') as package_file:
+            package_file.write(latex_package)
+
+    def finish(self):
+        super().finish()
+
+        TEMPLATE_PATH = "../latex_templates/espidf.sty"
+        self.prepare_latex_macros(os.path.join(self.confdir,TEMPLATE_PATH), self.config)
+
+
+def setup(app):
+    app.add_builder(IdfLatexBuilder, override=True)
+
+    return {'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '0.1'}

docs/latex_templates/espidf.sty

@@ -0,0 +1,7 @@
+\NeedsTeXFormat{LaTeX2e}[1995/12/01]
+\ProvidesPackage{espidf}[2020/03/25 v0.1.0 LaTeX package (ESP-IDF markup)]
+
+\newcommand{\idfTarget}{<idf_target_title>}
+\newcommand{\idfReleaseName}{<idf_release_name>}
+
+\endinput

docs/latex_templates/preamble.tex

@@ -0,0 +1,129 @@
+% package with esp-idf specific macros
+\usepackage{espidf}
+
+\setcounter{secnumdepth}{2}
+\setcounter{tocdepth}{2}
+
+\usepackage{amsmath,amsfonts,amssymb,amsthm}
+\usepackage{graphicx}
+%%% reduce spaces for Table of contents, figures and tables
+%%% it is used "\addtocontents{toc}{\vskip -1.2cm}" etc. in the document
+\usepackage[notlot,nottoc,notlof]{}
+
+\usepackage{color}
+\usepackage{transparent}
+\usepackage{eso-pic}
+\usepackage{lipsum}
+
+%%% Needed for displaying Chinese in English documentation
+\usepackage{xeCJK}
+
+\usepackage{footnotebackref} %%link at the footnote to go to the place of footnote in the text
+
+%% spacing between line
+\usepackage{setspace}
+\singlespacing
+
+
+\definecolor{myred}{RGB}{229, 32, 26}
+\definecolor{mygrayy}{RGB}{127, 127, 127}
+\definecolor{myblack}{RGB}{64, 64, 64}
+
+
+%%%%%%%%%%% datetime
+\usepackage{datetime}
+
+\newdateformat{MonthYearFormat}{%
+    \monthname[\THEMONTH], \THEYEAR}
+
+
+%% RO, LE will not work for 'oneside' layout.
+%% Change oneside to twoside in document class
+\usepackage{fancyhdr}
+\pagestyle{fancy}
+\fancyhf{}
+
+% Header and footer
+\makeatletter
+  \fancypagestyle{normal}{
+    \fancyhf{}
+    \fancyhead[L]{\nouppercase{\leftmark}}
+    \fancyfoot[C]{\py@HeaderFamily\thepage \\ \href{https://www.espressif.com/en/company/documents/documentation_feedback?docId=4287&sections=&version=\idfReleaseName}{Submit Document Feedback}}
+    \fancyfoot[L]{Espressif Systems}
+    \fancyfoot[R]{\idfReleaseName}
+    \renewcommand{\headrulewidth}{0.4pt}
+    \renewcommand{\footrulewidth}{0.4pt}
+  }
+\makeatother
+
+\renewcommand{\headrulewidth}{0.5pt}
+\renewcommand{\footrulewidth}{0.5pt}
+
+
+% Define a spacing for section, subsection and subsubsection
+% http://tex.stackexchange.com/questions/108684/spacing-before-and-after-section-titles
+
+\titlespacing*{\section}{0pt}{6pt plus 0pt minus 0pt}{6pt plus 0pt minus 0pt}
+\titlespacing*{\subsection}{0pt}{18pt plus 64pt minus 0pt}{0pt}
+\titlespacing*{\subsubsection}{0pt}{12pt plus 0pt minus 0pt}{0pt}
+\titlespacing*{\paragraph}    {0pt}{3.25ex plus 1ex minus .2ex}{1.5ex plus .2ex}
+\titlespacing*{\subparagraph} {0pt}{3.25ex plus 1ex minus .2ex}{1.5ex plus .2ex}
+
+% Define the colors of table of contents
+% This is helpful to understand http://tex.stackexchange.com/questions/110253/what-the-first-argument-for-lsubsection-actually-is
+\definecolor{LochmaraColor}{HTML}{1020A0}
+
+% Hyperlinks
+\hypersetup{
+    colorlinks = true,
+    allcolors = {LochmaraColor},
+}
+
+
+\RequirePackage{tocbibind} %%% comment this to remove page number for following
+\addto\captionsenglish{\renewcommand{\contentsname}{Table of contents}}
+\addto\captionsenglish{\renewcommand{\listfigurename}{List of figures}}
+\addto\captionsenglish{\renewcommand{\listtablename}{List of tables}}
+% \addto\captionsenglish{\renewcommand{\chaptername}{Chapter}}
+
+
+
+
+%%reduce spacing for itemize
+\usepackage{enumitem}
+\setlist{nosep}
+
+%%%%%%%%%%% Quote Styles at the top of chapter
+\usepackage{epigraph}
+\setlength{\epigraphwidth}{0.8\columnwidth}
+\newcommand{\chapterquote}[2]{\epigraphhead[60]{\epigraph{\textit{#1}}{\textbf {\textit{--#2}}}}}
+%%%%%%%%%%% Quote for all places except Chapter
+\newcommand{\sectionquote}[2]{{\quote{\textit{``#1''}}{\textbf {\textit{--#2}}}}}
+
+% Insert 22pt white space before roc title. \titlespacing at line 65 changes it by -22 later on.
+\renewcommand*\contentsname{\hspace{0pt}Contents}
+
+
+% Define section, subsection and subsubsection font size and color
+\usepackage{sectsty}
+\definecolor{AllportsColor}{HTML}{A02010}
+\allsectionsfont{\color{AllportsColor}}
+
+\usepackage{titlesec}
+\titleformat{\section}
+{\color{AllportsColor}\LARGE\bfseries}{\thesection.}{1em}{}
+
+\titleformat{\subsection}
+{\color{AllportsColor}\Large\bfseries}{\thesubsection.}{1em}{}
+
+\titleformat{\subsubsection}
+{\color{AllportsColor}\large\bfseries}{\thesubsubsection.}{1em}{}
+
+\titleformat{\paragraph}
+{\color{AllportsColor}\large\bfseries}{\theparagraph}{1em}{}
+
+\titleformat{\subparagraph}
+  {\normalfont\normalsize\bfseries}{\thesubparagraph}{1em}{}
+
+\titleformat{\subsubparagraph}
+  {\normalfont\normalsize\bfseries}{\thesubsubparagraph}{1em}{}

docs/latex_templates/titlepage.tex

@@ -0,0 +1,39 @@
+\makeatletter
+\newgeometry{left=0cm,right=0cm,bottom=2cm}
+
+
+\cfoot{www.espressif.com}
+
+\renewcommand{\headrulewidth}{0pt}
+
+{\color{myred}\rule{30pt}{2.1cm}}
+    \hspace{0.2cm}
+    \begin{minipage}[b]{18cm}
+    {\fontsize{36pt}{48pt}\textbf{\idfTarget}}\\
+
+    {\fontsize{28pt}{18pt}\textbf{\color{mygrayy}\@title}}
+    \end{minipage}
+    \hspace{\stretch{1}}
+
+\vspace{48em}
+
+
+\begin{flushright}
+ \setlength\parindent{8em}
+    \begin{minipage}[b]{2cm}
+    \sphinxlogo
+    \end{minipage}
+    \hspace{0.2cm}
+    \rule{3pt}{1.9cm}
+    \hspace{0.2cm}
+    \begin{minipage}[b]{7cm}
+       {\large{\idfReleaseName}}\smallskip\newline
+        {\large{\@author}}\smallskip\newline
+         {\large{\@date}}\smallskip
+    \end{minipage}
+    {\color{myred}\rule{30pt}{1.9cm}}
+\end{flushright}
+
+
+\restoregeometry
+\makeatother

tools/ci/config/build.yml

@@ -258,7 +258,7 @@ build_test_apps_esp32s2:
   script:
     - cd docs
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -l $DOCLANG -t $DOCTGT build
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs html latex -l $DOCLANG -t $DOCTGT build
 
 build_docs_en_esp32:
   extends: .build_docs_template

tools/ci/deploy_docs.py

@@ -75,7 +75,7 @@ def main():
 
     print("DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}".format(docs_server, docs_path))
 
-    tarball_path, version_urls = build_doc_tarball(version, build_dir)
+    tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
 
     deploy(version, tarball_path, docs_path, docs_server)
 
@@ -90,7 +90,7 @@ def main():
     # process but call the version 'stable' this time
     if is_stable_version(version):
         print("Deploying again as stable version...")
-        tarball_path, version_urls = build_doc_tarball("stable", build_dir)
+        tarball_path, version_urls = build_doc_tarball("stable", git_ver, build_dir)
         deploy("stable", tarball_path, docs_path, docs_server)
 
 
@@ -118,7 +118,7 @@ def deploy(version, tarball_path, docs_path, docs_server):
     # another thing made much more complex by the directory structure putting language before version...
 
 
-def build_doc_tarball(version, build_dir):
+def build_doc_tarball(version, git_ver, build_dir):
     """ Make a tar.gz archive of the docs, in the directory structure used to deploy as
         the given version """
     version_paths = []
@@ -128,6 +128,12 @@ def build_doc_tarball(version, build_dir):
     html_dirs = glob.glob("{}/**/html/".format(build_dir), recursive=True)
     print("Found %d html directories" % len(html_dirs))
 
+    pdfs = glob.glob("{}/**/latex/build/*.pdf".format(build_dir), recursive=True)
+    print("Found %d PDFs in latex directories" % len(pdfs))
+
+    # add symlink for stable and latest and adds them to PDF blob
+    symlinks = create_and_add_symlinks(version, git_ver, pdfs)
+
     def not_sources_dir(ti):
         """ Filter the _sources directories out of the tarballs """
         if ti.name.endswith("/_sources"):
@@ -154,9 +160,41 @@ def build_doc_tarball(version, build_dir):
             tarball.add(html_dir, archive_path, filter=not_sources_dir)
             version_paths.append(archive_path)
 
+        for pdf_path in pdfs:
+            # pdf_path has the form '<ignored>/<language>/<target>/latex/build'
+            latex_dirname = os.path.dirname(pdf_path)
+            pdf_filename = os.path.basename(pdf_path)
+            target_dirname = os.path.dirname(os.path.dirname(latex_dirname))
+            target = os.path.basename(target_dirname)
+            language = os.path.basename(os.path.dirname(target_dirname))
+
+            # when deploying, we want the layout 'language/version/target/pdf'
+            archive_path = "{}/{}/{}/{}".format(language, version, target, pdf_filename)
+            print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
+            tarball.add(pdf_path, archive_path)
+
+    for symlink in symlinks:
+        os.unlink(symlink)
+
     return (os.path.abspath(tarball_path), version_paths)
 
 
+def create_and_add_symlinks(version, git_ver, pdfs):
+    """ Create symbolic links for PDFs for 'latest' and 'stable' releases """
+
+    symlinks = []
+    if 'stable' in version or 'latest' in version:
+        for pdf_path in pdfs:
+            symlink_path = pdf_path.replace(git_ver, version)
+            os.symlink(pdf_path, symlink_path)
+            symlinks.append(symlink_path)
+
+        pdfs.extend(symlinks)
+        print("Found %d PDFs in latex directories after adding symlink" % len(pdfs))
+
+    return symlinks
+
+
 def is_stable_version(version):
     """ Heuristic for whether this is the latest stable release """
     if not version.startswith("v"):