From 9e7a69be88cbc46e2bb3a6f165895918a822d57e Mon Sep 17 00:00:00 2001
From: He Yin Ling <heyinling@espressif.com>
Date: Mon, 8 Jan 2018 20:33:09 +0800
Subject: [PATCH] example test: integrate wifi throughput test to CI

---
 .gitlab-ci.yml                             | 23 ++++++++++++++++++++--
 tools/tiny-test-fw/TinyFW.py               |  1 +
 tools/tiny-test-fw/Utility/CIAssignTest.py | 14 +++++++++----
 3 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d5491a64d..f7bff45e3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -493,10 +493,17 @@ check_submodule_sync:
     - git submodule update --init --recursive
 
 assign_test:
-  <<: *build_template
+  tags:
+    - assign_test
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
   stage: assign_test
   # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
   # we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage.
+  dependencies:
+    - build_ssc_00
+    - build_ssc_01
+    - build_ssc_02
+    - build_esp_idf_tests
   variables:
     UT_BIN_PATH: "tools/unit-test-app/output"
     OUTPUT_BIN_PATH: "test_bins/ESP32_IDF"
@@ -548,12 +555,17 @@ assign_test:
     TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
     CONFIG_FILE: "$CI_PROJECT_DIR/examples/test_configs/$CI_JOB_NAME.yml"
     LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
   script:
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0
+    # clone test env configs
+    - git clone $TEST_ENV_CONFIG_REPOSITORY
+    - cd ci-test-runner-configs
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs
     - cd $TEST_FW_PATH
     # run test
-    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE
+    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
 
 .unit_test_template: &unit_test_template
   <<: *example_test_template
@@ -565,6 +577,7 @@ assign_test:
     TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
     CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/unit_test/CIConfigs/$CI_JOB_NAME.yml"
     LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
 
 .test_template: &test_template
   stage: test
@@ -638,6 +651,12 @@ example_test_001_01:
     - ESP32
     - Example_WIFI
 
+example_test_002_01:
+  <<: *example_test_template
+  tags:
+    - ESP32
+    - Example_ShieldBox
+
 UT_001_01:
   <<: *unit_test_template
   tags:
diff --git a/tools/tiny-test-fw/TinyFW.py b/tools/tiny-test-fw/TinyFW.py
index b227cf7aa..34463b285 100644
--- a/tools/tiny-test-fw/TinyFW.py
+++ b/tools/tiny-test-fw/TinyFW.py
@@ -106,6 +106,7 @@ get_passed_cases = TestResult.get_passed_cases
 MANDATORY_INFO = {
     "execution_time": 1,
     "env_tag": "default",
+    "category": "function",
 }
 
 
diff --git a/tools/tiny-test-fw/Utility/CIAssignTest.py b/tools/tiny-test-fw/Utility/CIAssignTest.py
index 04e43f076..3e570cc31 100644
--- a/tools/tiny-test-fw/Utility/CIAssignTest.py
+++ b/tools/tiny-test-fw/Utility/CIAssignTest.py
@@ -122,6 +122,10 @@ class AssignTest(object):
     """
     # subclass need to rewrite CI test job pattern, to filter all test jobs
     CI_TEST_JOB_PATTERN = re.compile(r"^test_.+")
+    # by default we only run function in CI, as other tests could take long time
+    DEFAULT_FILTER = {
+        "category": "function",
+    }
 
     def __init__(self, test_case_path, ci_config_file, case_group=Group):
         self.test_case_path = test_case_path
@@ -140,15 +144,17 @@ class AssignTest(object):
                 job_list.append(GitlabCIJob.Job(ci_config[job_name], job_name))
         return job_list
 
-    @staticmethod
-    def _search_cases(test_case_path, case_filter=None):
+    def _search_cases(self, test_case_path, case_filter=None):
         """
         :param test_case_path: path contains test case folder
-        :param case_filter: filter for test cases
+        :param case_filter: filter for test cases. the filter to use is default filter updated with case_filter param.
         :return: filtered test case list
         """
+        _case_filter = self.DEFAULT_FILTER.copy()
+        if case_filter:
+            _case_filter.update(case_filter)
         test_methods = SearchCases.Search.search_test_cases(test_case_path)
-        return CaseConfig.filter_test_cases(test_methods, case_filter if case_filter else dict())
+        return CaseConfig.filter_test_cases(test_methods, _case_filter)
 
     def _group_cases(self):
         """