docs: Run build_docs script in ci

2019-11-22 18:58:32 +11:00 · 2019-11-22 18:58:32 +11:00 · ee03e04947
parent 35db219be2
commit ee03e04947
6 changed files with 125 additions and 62 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -6,7 +6,7 @@ stages:
  - target_test
  - post_check
  - deploy
-  - post_check
+  - post_deploy

 variables:
 # System environment
@ -76,6 +76,9 @@ variables:
  rm -rf "$CUSTOM_TOOLCHAIN_PATH"

 .setup_tools_unless_target_test: &setup_tools_unless_target_test |
+  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
+  export IDF_MIRROR_PREFIX_MAP=
+  fi
  if [[ "$SETUP_TOOLS" == "1" || "$CI_JOB_STAGE" != "target_test" ]]; then
  tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
  fi
@ -142,7 +145,7 @@ after_script:
  tags:
    - host_test
  dependencies: []
-  extends: .before_script_lesser_nofilter
+  extends: .before_script_lesser

 .macos_build_template:
  stage: build
@ -163,3 +166,4 @@ include:
  - '/tools/ci/config/target-test.yml'
  - '/tools/ci/config/post_check.yml'
  - '/tools/ci/config/deploy.yml'
+  - '/tools/ci/config/post_deploy.yml'
--- a/tools/ci/config/build.yml
+++ b/tools/ci/config/build.yml
@ -186,25 +186,16 @@ build_examples_cmake_esp32s2:

 # If you want to add new build example jobs, please add it into dependencies of `.example_test_template`

-build_docs:
+.build_docs_template: &build_docs_template
  stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:esp-docs-20200204-ea18dcbd
  tags:
    - build_docs
  artifacts:
    when: always
    paths:
-      # English version of documentation
-      - docs/en/doxygen-warning-log.txt
-      - docs/en/sphinx-warning-log.txt
-      - docs/en/sphinx-warning-log-sanitized.txt
-      - docs/en/_build/html
-      - docs/sphinx-err-*
-      # Chinese version of documentation
-      - docs/zh_CN/doxygen-warning-log.txt
-      - docs/zh_CN/sphinx-warning-log.txt
-      - docs/zh_CN/sphinx-warning-log-sanitized.txt
-      - docs/zh_CN/_build/html
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
    expire_in: 4 days
  only:
    variables:
@ -212,17 +203,34 @@ build_docs:
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_BUILD_DOCS
      - $BOT_LABEL_REGULAR_TEST
+  dependencies: []
  script:
    - cd docs
-    - ./check_lang_folder_sync.sh
-    - cd en
-    - make gh-linkcheck
-    - make html
-    - ../check_doc_warnings.sh
-    - cd ../zh_CN
-    - make gh-linkcheck
-    - make html
-    - ../check_doc_warnings.sh
+    - ./build_docs.py -l $DOCLANG -t $DOCTGT build
+
+build_docs_en_esp32:
+  extends: .build_docs_template
+  variables:
+    DOCLANG: "en"
+    DOCTGT: "esp32"
+
+build_docs_en_esp32s2:
+  extends: .build_docs_template
+  variables:
+    DOCLANG: "en"
+    DOCTGT: "esp32s2"
+
+build_docs_zh_CN_esp32:
+  extends: .build_docs_template
+  variables:
+    DOCLANG: "zh_CN"
+    DOCTGT: "esp32"
+
+build_docs_zh_CN_esp32s2:
+  extends: .build_docs_template
+  variables:
+    DOCLANG: "zh_CN"
+    DOCTGT: "esp32s2"

 test_build_system:
  extends: .build_template
--- a/tools/ci/config/deploy.yml
+++ b/tools/ci/config/deploy.yml
@ -70,6 +70,14 @@ push_to_github:
    - git remote add github git@github.com:espressif/esp-idf.git
    - tools/ci/push_to_github.sh

+.upload_doc_archive: &upload_doc_archive |
+  cd docs/_build/$DOCLANG/$DOCTGT
+  mv html $GIT_VER
+  tar czf $GIT_VER.tar.gz $GIT_VER
+  scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/$DOCLANG/$DOCTGT
+  ssh $DOCS_SERVER -x "cd $DOCS_PATH/$DOCLANG/$DOCTGT && tar xzf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+  cd -
+
 deploy_docs:
  stage: deploy
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
@ -86,7 +94,10 @@ deploy_docs:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD_DOCS
  dependencies:
-    - build_docs
+    - build_docs_en_esp32
+    - build_docs_en_esp32s2
+    - build_docs_zh_CN_esp32
+    - build_docs_zh_CN_esp32s2
  extends: .before_script_lesser
  script:
    - mkdir -p ~/.ssh
@ -96,19 +107,24 @@ deploy_docs:
    - chmod 600 ~/.ssh/id_rsa
    - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
    - export GIT_VER=$(git describe --always)
-    - cd docs/en/_build/
-    - mv html $GIT_VER
-    - tar czvf $GIT_VER.tar.gz $GIT_VER
-    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/en
-    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/en && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
-    - cd ../../zh_CN/_build/
-    - mv html $GIT_VER
-    - tar czvf $GIT_VER.tar.gz $GIT_VER
-    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/zh_CN
-    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/zh_CN && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+
+    - DOCLANG=en; DOCTGT=esp32
+    - *upload_doc_archive
+
+    - DOCLANG=en; DOCTGT=esp32s2
+    - *upload_doc_archive
+
+    - DOCLANG=zh_CN; DOCTGT=esp32
+    - *upload_doc_archive
+
+    - DOCLANG=zh_CN; DOCTGT=esp32s2
+    - *upload_doc_archive
+
    # add link to preview doc
-    - echo "[document preview][en] $CI_DOCKER_REGISTRY/docs/esp-idf/en/${GIT_VER}/index.html"
-    - echo "[document preview][zh_CN] $CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/${GIT_VER}/index.html"
+    - echo "[document preview][en][esp32] https://$CI_DOCKER_REGISTRY/docs/esp-idf/en/esp32/${GIT_VER}/index.html"
+    - echo "[document preview][en][esp32s2] https://$CI_DOCKER_REGISTRY/docs/esp-idf/en/esp32s2/${GIT_VER}/index.html"
+    - echo "[document preview][zh_CN][esp32] https://$CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/esp32/${GIT_VER}/index.html"
+    - echo "[document preview][zh_CN][esp32s2] https://$CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/esp32s2/${GIT_VER}/index.html"

 deploy_test_result:
  stage: deploy
--- a/tools/ci/config/host-test.yml
+++ b/tools/ci/config/host-test.yml
@ -290,28 +290,3 @@ test_sysviewtrace_proc:
  script:
    - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
-
-check_doc_links:
-  stage: host_test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
-  tags:
-    - check_doc_links
-  only:
-    refs:
-      # can only be triggered
-      - triggers
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD_DOCS
-  artifacts:
-    paths:
-      - docs/_build/linkcheck
-    expire_in: 1 week
-  dependencies: []
-  script:
-    # must be triggered with CHECK_LINKS=Yes, otherwise exit without test
-    - test "$CHECK_LINKS" = "Yes" || exit 0
-    # can only run on master branch (otherwise the commit is not on Github yet)
-    - test "${CI_COMMIT_REF_NAME}" = "master" || exit 0
-    - cd docs
-    - make linkcheck
--- a/tools/ci/config/post_deploy.yml
+++ b/tools/ci/config/post_deploy.yml
@ -0,0 +1,42 @@
+.check_doc_links_template: &check_doc_links_template
+  stage: post_deploy
+  image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:esp-docs-20200204-ea18dcbd
+  tags: [ "build", "amd64", "internet" ]
+  only:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+    expire_in: 1 week
+  allow_failure: true
+  dependencies: []
+  script:
+    - cd docs
+    - ./build_docs.py -l $DOCLANG -t $DOCTGT linkcheck
+
+check_doc_links_en_esp32:
+  extends: .check_doc_links_template
+  variables:
+    DOCLANG: "en"
+    DOCTGT: "esp32"
+
+check_doc_links_en_esp32s2:
+  extends: .check_doc_links_template
+  variables:
+    DOCLANG: "en"
+    DOCTGT: "esp32s2"
+
+check_doc_links_zh_CN_esp32:
+  extends: .check_doc_links_template
+  variables:
+    DOCLANG: "zh_CN"
+    DOCTGT: "esp32"
+
+check_doc_links_zh_CN_esp32s2:
+  extends: .check_doc_links_template
+  variables:
+    DOCLANG: "zh_CN"
+    DOCTGT: "esp32s2"
--- a/tools/ci/config/pre_check.yml
+++ b/tools/ci/config/pre_check.yml
@ -8,6 +8,24 @@ check_permissions:
  script:
    - tools/ci/check-executable.sh

+check_docs_lang_sync:
+  extends: .check_job_template
+  stage: pre_check
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+  script:
+    - cd docs
+    - ./check_lang_folder_sync.sh
+
+check_docs_gh_links:
+  extends: .build_docs_template
+  stage: pre_check
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+  script:
+    - cd docs
+    - ./build_docs.py gh-linkcheck
+
 check_version:
  extends: .check_job_template
  # Don't run this for feature/bugfix branches, so that it is possible to modify