Merge branch 'test/workaround_ci_failures_v3.0' into 'release/v3.0'

CI: erase nvs partition before test (backport v3.0) See merge request idf/esp-idf!2870
2018-08-01 17:02:50 +08:00 · 2018-08-01 17:02:50 +08:00 · 054e82b4b3
commit 054e82b4b3
parent 7d4a9db191 8f81157432
3 changed files with 88 additions and 107 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -1,9 +1,9 @@
 stages:
  - build
  - assign_test
+  - host_test
  - unit_test
-  - test
-  - test_report
+  - integration_test
  - deploy

 variables:
@ -133,6 +133,8 @@ build_ssc_01:
 build_ssc_02:
  <<: *build_ssc_template

+# If you want to add new build ssc jobs, please add it into dependencies of `assign_test` and `.test_template`
+
 build_esp_idf_tests:
  <<: *build_template
  artifacts:
@ -140,7 +142,7 @@ build_esp_idf_tests:
      - tools/unit-test-app/output
      - components/idf_test/unit_test/TestCaseAll.yml
      - components/idf_test/unit_test/CIConfigs/*.yml
-    expire_in: 6 mos
+    expire_in: 1 mos
  script:
    - cd tools/unit-test-app
    - make help # make sure kconfig tools are built in single process
@ -199,6 +201,7 @@ build_examples_06:
 build_examples_07:
  <<: *build_examples_template

+# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`

 build_docs:
  stage: build
@ -222,54 +225,44 @@ build_docs:
    - make html
    - ./check_doc_warnings.sh

-test_nvs_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+.host_test_template: &host_test_template
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
-    - nvs_host_test
+    - host_test
  dependencies: []
+    
+test_nvs_on_host:
+  <<: *host_test_template
  script:
    - cd components/nvs_flash/test_nvs_host
    - make test

 test_partition_table_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  <<: *host_test_template
  tags:
    - build
-  dependencies: []
  script:
    - cd components/partition_table/test_gen_esp32part_host
    - ./gen_esp32part_tests.py

 test_wl_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - wl_host_test
+  <<: *host_test_template
  artifacts:
    paths:
      - components/wear_levelling/test_wl_host/coverage_report.zip
-  dependencies: []
  script:
    - cd components/wear_levelling/test_wl_host
    - make test

 test_multi_heap_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - wl_host_test
+  <<: *host_test_template
  script:
    - cd components/heap/test_multi_heap_host
    - ./test_all_configs.sh

 test_build_system:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - build_test
-  dependencies: []
+  <<: *host_test_template
  script:
    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
    - rm -rf test_build_system
@ -277,60 +270,6 @@ test_build_system:
    - cd test_build_system
    - ${IDF_PATH}/tools/ci/test_build_system.sh

-test_report:
-  stage: test_report
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - report
-  only:
-    - master
-    - triggers
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-  variables:
-    LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
-    TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test"
-    REPORT_PATH: "$CI_PROJECT_DIR/CI_Test_Report"
-    MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/tools/unit-test-app/tools/ModuleDefinition.yml"
-  #dependencies:
-  #We need all UT* and IT* artifacts except for only a few other
-  artifacts:
-    when: always
-    paths:
-      - $REPORT_PATH
-      - $LOG_PATH
-    expire_in: 12 mos
-  script:
-    # calc log path
-    - VER_NUM=`git rev-list HEAD | wc -l | awk '{print $1}'`
-    - SHA_ID=`echo $CI_COMMIT_SHA | cut -c 1-7`
-    - REVISION="${VER_NUM}_${SHA_ID}"
-    # replace / to _ in branch name
-    - ESCAPED_BRANCH_NAME=`echo $CI_COMMIT_REF_NAME | sed 's/\//___/g'`
-    # result path and artifacts path
-    - RESULT_PATH="$CI_PROJECT_NAME/$ESCAPED_BRANCH_NAME/$REVISION"
-    - ARTIFACTS_PATH="$GITLAB_HTTP_SERVER/idf/esp-idf/builds/$CI_JOB_ID/artifacts/browse/$CI_COMMIT_SHA"
-    # clone test bench
-    - git clone $GITLAB_SSH_SERVER/yinling/auto_test_script.git
-    - cd auto_test_script
-    - python $CHECKOUT_REF_SCRIPT auto_test_script
-    # generate report
-    - TEST_RESULT=Pass
-    - python CITestReport.py -l $LOG_PATH -t $TEST_CASE_FILE_PATH -p $REPORT_PATH -r $RESULT_PATH -a $ARTIFACTS_PATH -m $MODULE_UPDATE_FILE || TEST_RESULT=Fail
-    # commit to CI-test-result project
-    - git clone $GITLAB_SSH_SERVER/qa/CI-test-result.git
-    - rm -rf "CI-test-result/RawData/$RESULT_PATH"
-    - cp -R $CI_PROJECT_NAME CI-test-result/RawData
-    - cd CI-test-result
-    # config git user
-    - git config --global user.email "ci-test-result@espressif.com"
-    - git config --global user.name "ci-test-result"
-    # commit test result
-    - git add .
-    - git commit . -m "update test result for $CI_PROJECT_NAME/$CI_COMMIT_REF_NAME/$CI_COMMIT_SHA, pipeline ID $CI_PIPELINE_ID" || exit 0
-    - git push origin master
-    - test "${TEST_RESULT}" = "Pass" || exit 1
-
 push_master_to_github:
  stage: deploy
  image: $CI_DOCKER_REGISTRY/esp32-ci-env
@ -362,8 +301,8 @@ push_master_to_github:


 deploy_docs:
-  stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - deploy
  only:
@ -389,8 +328,8 @@ deploy_docs:
    - ssh $DOCS_SERVER -x "cd $DOCS_PATH && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"

 check_doc_links:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - check_doc_links
  only:
@ -478,21 +417,28 @@ assign_test:
    - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/SSC/ssc_bin

 .example_test_template: &example_test_template
-  stage: test
+  stage: integration_test
  when: on_success
  only:
    - master
    - /^release\/v/
    - /^v\d+\.\d+(\.\d+)?($|-)/
    - triggers
-  # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
-  # we have a lot build example jobs and the binaries them exceed the limitation of artifacts.
-  # we can't artifact them in one job. For example test jobs, download all artifacts from previous stages.
+  dependencies:
+    - assign_test
+    - build_examples_00
+    - build_examples_01
+    - build_examples_02
+    - build_examples_03
+    - build_examples_04
+    - build_examples_05
+    - build_examples_06
+    - build_examples_07
  artifacts:
    when: always
    paths:
      - $LOG_PATH
-    expire_in: 6 mos
+    expire_in: 1 mos
  variables:
    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
    TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
@ -506,14 +452,13 @@ assign_test:
    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE

 .test_template: &test_template
-  stage: test
+  stage: integration_test
  when: on_success
  only:
    - master
    - /^release\/v/
    - /^v\d+\.\d+(\.\d+)?($|-)/
    - triggers
-  allow_failure: true
  dependencies:
    - assign_test
    - build_ssc_00
@ -523,7 +468,7 @@ assign_test:
    when: always
    paths:
      - $LOG_PATH
-    expire_in: 6 mos
+    expire_in: 1 mos
  variables:
    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
    LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
@ -564,7 +509,7 @@ nvs_compatible_test:
    paths:
      - $LOG_PATH
      - nvs_wifi.bin
-    expire_in: 6 mos
+    expire_in: 1 mos
  tags:
    - ESP32_IDF
    - NVS_Compatible
--- a/tools/tiny-test-fw/IDF/IDFDUT.py
+++ b/tools/tiny-test-fw/IDF/IDFDUT.py
@ -17,6 +17,8 @@ import os
 import re
 import subprocess
 import functools
+import random
+import tempfile

 import DUT

@ -40,6 +42,8 @@ class IDFDUT(DUT.SerialDUT):
    """ IDF DUT, extends serial with ESPTool methods """

    CHIP_TYPE_PATTERN = re.compile(r"Detecting chip type[.:\s]+(.+)")
+    # if need to erase NVS partition in start app
+    ERASE_NVS = True

    def __init__(self, name, port, log_file, app, **kwargs):
        self.download_config, self.partition_table = app.process_app_info()
@ -68,24 +72,39 @@ class IDFDUT(DUT.SerialDUT):
        return cls.get_chip(app, port) is not None

    @_tool_method
-    def start_app(self):
+    def start_app(self, erase_nvs=ERASE_NVS):
        """
        download and start app.

+        :param: erase_nvs: whether erase NVS partition during flash
        :return: None
        """
+        if erase_nvs:
+            address = self.partition_table["nvs"]["offset"]
+            size = self.partition_table["nvs"]["size"]
+            nvs_file = tempfile.NamedTemporaryFile()
+            nvs_file.write(chr(0xFF) * size)
+            nvs_file.flush()
+            download_config = self.download_config + [address, nvs_file.name]
+        else:
+            download_config = self.download_config
+
        retry_baud_rates = ["921600", "115200"]
        error = IDFToolError()
-        for baud_rate in retry_baud_rates:
-            try:
-                subprocess.check_output(["python", self.app.esptool,
-                                         "--port", self.port, "--baud", baud_rate]
-                                        + self.download_config)
-                break
-            except subprocess.CalledProcessError as error:
-                continue
-        else:
-            raise error
+        try:
+            for baud_rate in retry_baud_rates:
+                try:
+                    subprocess.check_output(["python", self.app.esptool,
+                                             "--port", self.port, "--baud", baud_rate]
+                                            + download_config)
+                    break
+                except subprocess.CalledProcessError as error:
+                    continue
+            else:
+                raise error
+        finally:
+            if erase_nvs:
+                nvs_file.close()

    @_tool_method
    def reset(self):
@ -96,6 +115,17 @@ class IDFDUT(DUT.SerialDUT):
        """
        subprocess.check_output(["python", self.app.esptool, "--port", self.port, "run"])

+    @_tool_method
+    def erase_partition(self, partition):
+        """
+        :param partition: partition name to erase
+        :return: None
+        """
+        address = self.partition_table[partition]["offset"]
+        size = self.partition_table[partition]["size"]
+        with open(".erase_partition.tmp", "wb") as f:
+            f.write(chr(0xFF) * size)
+
    @_tool_method
    def dump_flush(self, output_file, **kwargs):
        """
--- a/tools/tiny-test-fw/IDF/init.py
+++ b/tools/tiny-test-fw/IDF/init.py
@ -20,9 +20,8 @@ from IDF.IDFApp import IDFApp, Example, UT
 from IDF.IDFDUT import IDFDUT


-def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32",
-                     module="examples", execution_time=1,
-                     **kwargs):
+def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", execution_time=1,
+                     level="example", erase_nvs=True, **kwargs):
    """
    decorator for testing idf examples (with default values for some keyword args).

@ -31,12 +30,19 @@ def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32",
    :param chip: chip supported, string or tuple
    :param module: module, string
    :param execution_time: execution time in minutes, int
+    :param level: test level, could be used to filter test cases, string
+    :param erase_nvs: if need to erase_nvs in DUT.start_app()
    :param kwargs: other keyword args
    :return: test method
    """
-    # not use partial function as define as function support auto generating document
+    try:
+        # try to config the default behavior of erase nvs
+        dut.ERASE_NVS = erase_nvs
+    except AttributeError:
+        pass
+
    return TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
-                              execution_time=execution_time, **kwargs)
+                              execution_time=execution_time, level=level, **kwargs)


 def log_performance(item, value):