Merge branch 'ci/workaround_multiple_ci_failures_v4.1' into 'release/v4.1'

CI: workaround multiple ci failures (v4.1)

See merge request espressif/esp-idf!9685

components/idf_test/integration_test/KnownIssues

@@ -1,88 +1,7 @@
 
-# NOT SUPPORT
-
-# ICMP send Ping not supported
-ESP32.TCPIP_ICMP_0101
-ESP32.TCPIP_ICMP_0101_01
-
-# Bug or not stable cases
-
-# Wifi scan issue
-ESP32.WIFI_SCAN_0303
-ESP32.WIFI_SCAN_0303_01
-ESP32.WIFI_CONN_0302
-ESP32.WIFI_CONN_0302_01
-ESP32.WIFI_MODE_0102
-ESP32.WIFI_MODE_0103
-ESP32.WIFI_ADDR_0102
-ESP32.WIFI_ADDR_0102_01
-
-# IGMP
-ESP32.TCPIP_IGMP_0201
-ESP32.TCPIP_IGMP_0201_01
-ESP32.TCPIP_IGMP_0202
-ESP32.TCPIP_IGMP_0202_01
-ESP32.TCPIP_IGMP_0204
-ESP32.TCPIP_IGMP_0204_01
-ESP32.TCPIP_IGMP_0301
-ESP32.TCPIP_IGMP_0301_01
-ESP32.TCPIP_IGMP_0303
-ESP32.TCPIP_IGMP_0303_01
-ESP32.TCPIP_IGMP_0402
-ESP32.TCPIP_IGMP_0402_01
-ESP32.TCPIP_IGMP_0406
-ESP32.TCPIP_IGMP_0406_01
-
-# TCP
-ESP32.TCPIP_TCP_0401_01_02
-
-# UDP
-ESP32.TCPIP_UDP_0103
-ESP32.TCPIP_UDP_0103_01
-ESP32.TCPIP_UDP_0103_02
-ESP32.TCPIP_UDP_0103_01_02
-
-# BLE
-
 # BT heap size issue
 ESP32.BTSTK_MISC_0301
 
-# SMP
-ESP32.BTSTK_SMP_05001
-ESP32.BTSTK_SMP_05002
-ESP32.BTSTK_SMP_05003
-ESP32.BTSTK_SMP_06004
-ESP32.BTSTK_SMP_07007
-ESP32.BTSTK_SMP_07006
-
-# multi connect
-ESP32.BTSTK_GAP_04002
-ESP32.BTSTK_GAP_04003
-ESP32.BTSTK_GAP_10001
-ESP32.BTSTK_GAP_10002
-ESP32.BTSTK_GAP_10003
-ESP32.BTSTK_GAP_10004
-ESP32.BTSTK_GAP_11001
-ESP32.BTSTK_GAP_11002
-ESP32.BTSTK_GAP_12001
-ESP32.BTSTK_GAP_12002
-ESP32.BTSTK_GATT_30001
-ESP32.BTSTK_GATT_30002
-ESP32.BTSTK_GATT_30003
-ESP32.BTSTK_GATT_31001
-ESP32.BTSTK_GATT_31002
-ESP32.BTSTK_GATT_31003
-ESP32.BTSTK_GATT_32001
-ESP32.BTSTK_GATT_32002
-ESP32.BTSTK_GATT_32003
-ESP32.BTSTK_GATT_32004
-ESP32.BTSTK_GATT_33001
-ESP32.BTSTK_GATT_33002
-ESP32.BTSTK_GATT_33003
-ESP32.BTSTK_GATT_34001
-ESP32.BTSTK_GATT_34002
-ESP32.BTSTK_GATT_34003
-
 # GATT read multiple
 ESP32.BTSTK_GATT_27002
 ESP32.BTSTK_GATT_27003

components/idf_test/integration_test/TC_IT_BLUEDROID_SMP.yml

@@ -477,6 +477,7 @@ test cases:
     - ['P SSC1 RE "\+BLESMP:RemoveBond,Success,%%s"%%(<dut2_bt_mac>)']
 - ID: BLUEDROID_SMP_08003
   <<: *SMP_CASE
+  CI ready: 'No'
   test point 2: BLE SMP bond item management test
   summary: BLE SMP get bond list and number when pairing and after reboot for BLUEDROID
   steps: |

components/idf_test/integration_test/TC_IT_BTSTK_GAP.yml

@@ -1420,6 +1420,7 @@ test cases:
       - 'R SSC2 C Complete'
 - ID: BTSTK_GAP_10001
   <<: *GAP_CASE
+  CI ready: 'No'
   test point 2: BLE GAP master multi connection test
   summary: master connect to 3 slaves with different app and master do disconnect
   steps: |
@@ -1447,6 +1448,7 @@ test cases:
     - ['P SSC1 C +BLECONN:GapDisconnect,OK', 'P SSC{%d} C +BLECONN:GapDisconnect,OK']
 - ID: BTSTK_GAP_10002
   <<: *GAP_CASE
+  CI ready: 'No'
   test point 2: BLE GAP master multi connection test
   summary: master connect to 3 slaves with different app and slave do disconnect
   steps: |
@@ -1474,6 +1476,7 @@ test cases:
     - ['P SSC1 C +BLECONN:GapDisconnect,OK,001[0-2]', 'P SSC{%d} C +BLECONN:GapDisconnect,OK']
 - ID: BTSTK_GAP_10003
   <<: *GAP_CASE
+  CI ready: 'No'
   test point 2: BLE GAP master multi connection test
   summary: master connect to 4 slaves with same app and master do disconnect
   steps: |
@@ -1501,6 +1504,7 @@ test cases:
     - ['P SSC1 C +BLECONN:GapDisconnect,OK', 'P SSC{%d} C +BLECONN:GapDisconnect,OK']
 - ID: BTSTK_GAP_10004
   <<: *GAP_CASE
+  CI ready: 'No'
   test point 2: BLE GAP master multi connection test
   summary: master connect to 4 slaves with same app and slave do disconnect
   steps: |
@@ -1528,6 +1532,7 @@ test cases:
     - ['P SSC1 C +BLECONN:GapDisconnect,OK', 'P SSC{%d} C +BLECONN:GapDisconnect,OK']
 - ID: BTSTK_GAP_11001
   <<: *GAP_CASE
+  CI ready: 'No'
   test point 2: BLE GAP slave multi connection test
   summary: slave connected by 4 master and master do disconnect
   steps: |
@@ -1557,6 +1562,7 @@ test cases:
     - ['P SSC{%d} C +BLECONN:GapDisconnect,OK', 'P SSC1 C +BLECONN:GapDisconnect,OK']
 - ID: BTSTK_GAP_11002
   <<: *GAP_CASE
+  CI ready: 'No'
   test point 2: BLE GAP slave multi connection test
   summary: slave connected by 4 master and slave do disconnect
   steps: |
@@ -1587,6 +1593,7 @@ test cases:
     - ['P SSC{%d} C +BLECONN:GapDisconnect,OK', 'P SSC1 C +BLECONN:GapDisconnect,OK']
 - ID: BTSTK_GAP_12001
   <<: *GAP_CASE
+  CI ready: 'No'
   test point 2: BLE GAP multi connection as both master and slave role test
   summary:  connected by 2 masters and connect to 2 slaves and do disconnect
   steps: |
@@ -1623,6 +1630,7 @@ test cases:
     - ["P SSC[2-5] C Disconnect"]
 - ID: BTSTK_GAP_12002
   <<: *GAP_CASE
+  CI ready: 'No'
   test point 2: BLE GAP multi connection as both master and slave role test
   summary:  connected by 2 masters and connect to 2 slaves and be disconnected
   steps: |

components/idf_test/integration_test/TC_IT_BTSTK_GATT.yml

@@ -1118,6 +1118,7 @@ test cases:
     - ["R SSC1 C +GATTC:OK","P SSC[1-2] C ConfigMTU,OK,261"]
 - ID: BTSTK_GATT_30001
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection service discovery test
   summary: do primary service discovery when DUT1 role is master, connected with 4 slaves
   steps: |
@@ -1135,6 +1136,7 @@ test cases:
     - ["R SSC1 C +GATTC:Discover,OK"]
 - ID: BTSTK_GATT_30002
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection service discovery test
   summary: do primary service discovery when DUT1 is slave, connected by 4 masters
   steps: |
@@ -1150,6 +1152,7 @@ test cases:
     - ["R SSC[2-5] C +GATTC:Discover,OK"]
 - ID: BTSTK_GATT_30003
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection service discovery test
   summary: do primary service discovery when DUT1 connect to DUT[2-3] and connected by DUT[4-5]
   steps: |
@@ -1171,6 +1174,7 @@ test cases:
     - ["R SSC[4-5] C +GATTC:Discover,OK"]
 - ID: BTSTK_GATT_31001
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection read test
   summary: do read when DUT1 role is master, connected with 4 slaves
   steps: |
@@ -1196,6 +1200,7 @@ test cases:
     - ["R SSC1 C +GATTC:ReadOnce,0010,A002,C301,256"]
 - ID: BTSTK_GATT_31002
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection read test
   summary: do read when DUT1 is slave, connected by 4 masters
   steps: |
@@ -1215,6 +1220,7 @@ test cases:
     - ["R SSC[2-5] C +GATTC:ReadOnce,0010,A002,C301,256"]
 - ID: BTSTK_GATT_31003
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection read test
   summary: do read when DUT1 connect to DUT[2-3] and connected by DUT[4-5]
   steps: |
@@ -1248,6 +1254,7 @@ test cases:
     - ["R SSC[4-5] C +GATTC:ReadOnce,0010,A002,C301,256"]
 - ID: BTSTK_GATT_32001
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection write test
   summary: do write when DUT1 role is master, connected with 4 slaves
   steps: |
@@ -1273,6 +1280,7 @@ test cases:
     - ["R SSC1 C +GATTC:Write,OK,0010,A002,C304"]
 - ID: BTSTK_GATT_32002
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection write test
   summary: do write when DUT1 is slave, connected by 4 masters
   steps: |
@@ -1292,6 +1300,7 @@ test cases:
     - ["R SSC[2-5] C +GATTC:Write,OK,0010,A002,C304"]
 - ID: BTSTK_GATT_32003
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection write test
   summary: do write when DUT1 connect to DUT[2-3] and connected by DUT[4-5]
   steps: |
@@ -1325,6 +1334,7 @@ test cases:
     - ["R SSC[4-5] C +GATTC:Write,OK,0010,A002,C304"]
 - ID: BTSTK_GATT_33001
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection notify test
   summary: do notify when DUT1 role is master, connected with 4 slaves
   steps: |
@@ -1350,6 +1360,7 @@ test cases:
     - ['P SSC1 RE "\+GATTC:Notification,0010,A002,C305,1,%%s"%%(<dut{%d}_bt_mac>)']
 - ID: BTSTK_GATT_33002
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection notify test
   summary: do notify when DUT1 is slave, connected by 4 masters
   steps: |
@@ -1375,6 +1386,7 @@ test cases:
     - ['P SSC{%d} RE "\+GATTC:Notification,0010,A002,C305,1,%%s"%%(<dut1_bt_mac>)']
 - ID: BTSTK_GATT_33003
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection notify test
   summary: do notify when DUT1 connect to DUT[2-3] and connected by DUT[4-5]
   steps: |
@@ -1414,6 +1426,7 @@ test cases:
     - ['P SSC{%d} RE "\+GATTC:Notification,0010,A002,C305,1,%%s"%%(<dut1_bt_mac>)']
 - ID: BTSTK_GATT_34001
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection indicate test
   summary: do indicate when DUT1 role is master, connected with 4 slaves
   steps: |
@@ -1439,6 +1452,7 @@ test cases:
     - ['P SSC1 RE "\+GATTC:Indication,0010,A002,C306,1,%%s"%%(<dut[2-5]_bt_mac>)']
 - ID: BTSTK_GATT_34002
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection indicate test
   summary: do indicate when DUT1 is slave, connected by 4 masters
   steps: |
@@ -1464,6 +1478,7 @@ test cases:
     - ['P SSC{%d} RE "\+GATTC:Indication,0010,A002,C306,1,%%s"%%(<dut1_bt_mac>)']
 - ID: BTSTK_GATT_34003
   <<: *GATT_CASE
+  CI ready: 'No'
   test point 2: BLE GATT multi connection indicate test
   summary: do indicate when DUT1 connect to DUT[2-3] and connected by DUT[4-5]
   steps: |

components/idf_test/integration_test/TC_IT_BTSTK_SMP.yml

@@ -1651,6 +1651,7 @@ test cases:
   - *check_connection
 - ID: BTSTK_SMP_05003
   <<: *SMP_CASE
+  CI ready: 'No'
   test point 2: BLE SMP unsuccessful pair reply test
   summary: BLE SMP unsuccessful passkey entry on both side test
   steps: |
@@ -1766,6 +1767,7 @@ test cases:
     - ['P SSC[1-2] C +BLESMP:AuthComplete,Success,0']
 - ID: BTSTK_SMP_06004
   <<: *SMP_CASE
+  CI ready: 'No'
   test point 2: BLE SMP use API in abnormal state
   summary: BLE SMP master disconnect during pairing and repairing
   steps: |

components/idf_test/integration_test/TC_IT_TCPIP_ICMP.yml

@@ -1,5 +1,5 @@
 test cases:
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_ICMP_0101
   SDK: |-
     8266_NonOS

components/idf_test/integration_test/TC_IT_TCPIP_IGMP.yml

@@ -545,7 +545,7 @@ test cases:
   test point 1: basic function
   test point 2: IGMP API parameter check
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_IGMP_0201
   SDK: |-
     8266_RTOS
@@ -593,7 +593,7 @@ test cases:
   test point 1: basic function
   test point 2: IGMP send/recv test
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_IGMP_0202
   SDK: |-
     8266_RTOS
@@ -703,7 +703,7 @@ test cases:
   test point 1: basic function
   test point 2: IGMP send/recv test
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_IGMP_0204
   SDK: |-
     8266_RTOS
@@ -759,7 +759,7 @@ test cases:
   test point 1: basic function
   test point 2: IGMP send/recv test
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_IGMP_0301
   SDK: |-
     8266_RTOS
@@ -845,7 +845,7 @@ test cases:
   test point 1: basic function
   test point 2: IGMP API parameter check
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_IGMP_0303
   SDK: |-
     8266_RTOS
@@ -1151,7 +1151,7 @@ test cases:
   test point 1: basic function
   test point 2: IGMP send/recv test
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_IGMP_0402
   SDK: |-
     8266_RTOS
@@ -1377,7 +1377,7 @@ test cases:
   test point 1: basic function
   test point 2: IGMP send/recv test
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_IGMP_0406
   SDK: |-
     8266_RTOS

components/idf_test/integration_test/TC_IT_TCPIP_TCP.yml

@@ -2167,7 +2167,7 @@ test cases:
   test point 1: basic function
   test point 2: socket option test
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_TCP_0401
   SDK: |-
     8266_NonOS

components/idf_test/integration_test/TC_IT_TCPIP_UDP.yml

@@ -85,7 +85,7 @@ test cases:
   test point 1: basic function
   test point 2: use UDP SAP (socket/espconn API) with different parameter
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_UDP_0103
   SDK: |-
     8266_NonOS
@@ -136,7 +136,7 @@ test cases:
   test point 1: basic function
   test point 2: use UDP SAP (socket/espconn API) with different parameter
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_UDP_0103
   SDK: ESP32_IDF
   Test App: SSC
@@ -180,7 +180,7 @@ test cases:
   test point 1: basic function
   test point 2: use UDP SAP (socket/espconn API) with different parameter
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_UDP_0104
   SDK: |-
     8266_NonOS
@@ -233,7 +233,7 @@ test cases:
   test point 1: basic function
   test point 2: use UDP SAP (socket/espconn API) with different parameter
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_UDP_0104_02
   SDK: |-
     8266_NonOS
@@ -286,7 +286,7 @@ test cases:
   test point 1: basic function
   test point 2: use UDP SAP (socket/espconn API) with different parameter
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: TCPIP_UDP_0104_01_02
   SDK: |-
     8266_NonOS

components/idf_test/integration_test/TC_IT_WIFI_ADDR.yml

@@ -43,7 +43,7 @@ test cases:
   test point 1: basic function
   test point 2: mac address function test
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: WIFI_ADDR_0102
   SDK: |-
     8266_NonOS

components/idf_test/integration_test/TC_IT_WIFI_CONN.yml

@@ -418,7 +418,7 @@ test cases:
   test point 1: interaction
   test point 2: Conn interact with other WiFi operation
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: WIFI_CONN_0302
   SDK: |-
     ESP32_IDF

components/idf_test/integration_test/TC_IT_WIFI_MODE.yml

@@ -47,7 +47,7 @@ test cases:
   test point 1: basic function
   test point 2: wifi mode fucntion
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: WIFI_MODE_0102
   SDK: |-
     8266_NonOS
@@ -88,7 +88,7 @@ test cases:
   test point 1: basic function
   test point 2: wifi mode fucntion
   version: v1 (2016-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: WIFI_MODE_0103
   SDK: |-
     8266_NonOS

components/idf_test/integration_test/TC_IT_WIFI_SCAN.yml

@@ -587,7 +587,7 @@ test cases:
   test point 1: interaction
   test point 2: Scan interact with other WiFi operation
   version: v1 (2015-8-15)
-- CI ready: 'Yes'
+- CI ready: 'No'
   ID: WIFI_SCAN_0303
   SDK: ESP32_IDF
   Test App: SSC

tools/ci/config/target-test.yml

@@ -548,14 +548,13 @@ IT_004:
 
 IT_005:
   extends: .test_template
-  parallel: 2
   tags:
     - ESP32_IDF
     - SSC_T1_5
 
 IT_006:
   extends: .test_template
-  parallel: 8
+  parallel: 7
   tags:
     - ESP32_IDF
     - SSC_T1_6
@@ -579,13 +578,6 @@ IT_009:
     - ESP32_IDF
     - SSC_T1_3
 
-IT_010:
-  extends: .test_template
-  parallel: 4
-  tags:
-    - ESP32_IDF
-    - SSC_T5_1
-
 IT_011:
   extends: .test_template
   tags:
@@ -619,6 +611,7 @@ IT_015:
 
 IT_016:
   extends: .test_template
+  allow_failure: true
   tags:
     - ESP32_IDF
     - SSC_T50_MESH1
@@ -631,7 +624,6 @@ IT_017:
 
 IT_018:
   extends: .test_template
-  parallel: 2
   tags:
     - ESP32_IDF
     - SSC_T1_9