Merge branch 'bugfix/py_imp_deprecated' into 'master'

Fix Python3 deprecation warning for the imp module & run multi-device tests only under Python 2

See merge request idf/esp-idf!3972

examples/protocols/http_server/advanced_tests/http_server_advanced_test.py

@@ -17,7 +17,6 @@
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-import imp
 import re
 import os
 import sys
@@ -41,7 +40,7 @@ import Utility
 
 # Import client module
 expath = os.path.dirname(os.path.realpath(__file__))
-client = imp.load_source("client", expath + "/scripts/test.py")
+client = Utility.load_source("client", expath + "/scripts/test.py")
 
 
 # Due to connectivity issues (between runner host and DUT) in the runner environment,

examples/protocols/http_server/persistent_sockets/http_server_persistence_test.py

@@ -19,7 +19,6 @@ from __future__ import print_function
 from __future__ import unicode_literals
 from builtins import str
 from builtins import range
-import imp
 import re
 import os
 import sys
@@ -43,7 +42,7 @@ import Utility
 
 # Import client module
 expath = os.path.dirname(os.path.realpath(__file__))
-client = imp.load_source("client", expath + "/scripts/adder.py")
+client = Utility.load_source("client", expath + "/scripts/adder.py")
 
 
 @IDF.idf_example_test(env_tag="Example_WIFI")

examples/protocols/http_server/simple/http_server_simple_test.py

@@ -18,7 +18,6 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 from builtins import range
-import imp
 import re
 import os
 import sys
@@ -44,7 +43,7 @@ import Utility
 
 # Import client module
 expath = os.path.dirname(os.path.realpath(__file__))
-client = imp.load_source("client", expath + "/scripts/client.py")
+client = Utility.load_source("client", expath + "/scripts/client.py")
 
 
 @IDF.idf_example_test(env_tag="Example_WIFI")

tools/ci/setup_python.sh

@@ -2,8 +2,10 @@
 
 # Regexp for matching job names which are incompatible with Python 3
 # - assign_test, nvs_compatible_test, IT - auto_test_script causes the incompatibility
-# - UT_009_ - RS485 multi-device test is not started properly
+# - UT_009_ - multi-device tests are not compatible
-py3_incomp='assign_test|nvs_compatible_test|IT|UT_009_'
+# - UT_014_ - multi-device tests are not compatible
+# - UT_017_ - multi-device tests are not compatible
+py3_incomp='assign_test|nvs_compatible_test|IT|UT_009_|UT_014_|UT_017_'
 
 if [ -z ${PYTHON_VER+x} ] || [[ $CI_JOB_NAME =~ $py3_incomp ]]; then
     # Use this version of the Python interpreter if it was not defined before or

tools/esp_prov/proto/__init__.py

@@ -13,21 +13,31 @@
 # limitations under the License.
 #
 
-import imp
 import os
 
+
+def _load_source(name, path):
+    try:
+        from importlib.machinery import SourceFileLoader
+        return SourceFileLoader(name, path).load_module()
+    except ImportError:
+        # importlib.machinery doesn't exists in Python 2 so we will use imp (deprecated in Python 3)
+        import imp
+        return imp.load_source(name, path)
+
+
 idf_path = os.environ['IDF_PATH']
 
 # protocomm component related python files generated from .proto files
-constants_pb2 = imp.load_source("constants_pb2", idf_path + "/components/protocomm/python/constants_pb2.py")
+constants_pb2 = _load_source("constants_pb2", idf_path + "/components/protocomm/python/constants_pb2.py")
-sec0_pb2      = imp.load_source("sec0_pb2",      idf_path + "/components/protocomm/python/sec0_pb2.py")
+sec0_pb2      = _load_source("sec0_pb2",      idf_path + "/components/protocomm/python/sec0_pb2.py")
-sec1_pb2      = imp.load_source("sec1_pb2",      idf_path + "/components/protocomm/python/sec1_pb2.py")
+sec1_pb2      = _load_source("sec1_pb2",      idf_path + "/components/protocomm/python/sec1_pb2.py")
-session_pb2   = imp.load_source("session_pb2",   idf_path + "/components/protocomm/python/session_pb2.py")
+session_pb2   = _load_source("session_pb2",   idf_path + "/components/protocomm/python/session_pb2.py")
 
 # wifi_provisioning component related python files generated from .proto files
-wifi_constants_pb2 = imp.load_source("wifi_constants_pb2", idf_path + "/components/wifi_provisioning/python/wifi_constants_pb2.py")
+wifi_constants_pb2 = _load_source("wifi_constants_pb2", idf_path + "/components/wifi_provisioning/python/wifi_constants_pb2.py")
-wifi_config_pb2    = imp.load_source("wifi_config_pb2",    idf_path + "/components/wifi_provisioning/python/wifi_config_pb2.py")
+wifi_config_pb2    = _load_source("wifi_config_pb2",    idf_path + "/components/wifi_provisioning/python/wifi_config_pb2.py")
 
 # custom_provisioning component related python files generated from .proto files
-custom_config_pb2  = imp.load_source("custom_config_pb2", idf_path +
+custom_config_pb2  = _load_source("custom_config_pb2",  idf_path +
-                                     "/examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py")
+                                  "/examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py")

tools/tiny-test-fw/Utility/CaseConfig.py

@@ -43,13 +43,12 @@ Template Config File::
       - name: xxx
 """
 
-# TODO: add a function to use suitable import lib for python2 and python3
-import imp
-
 import yaml
 
 import TestCase
 
+from Utility import load_source
+
 
 def _convert_to_lower_case_bytes(item):
     """
@@ -169,8 +168,7 @@ class Parser(object):
         output = dict()
         for key in overwrite:
             _path = overwrite[key]["path"]
-            # TODO: add a function to use suitable import lib for python2 and python3
+            _module = load_source(str(hash(_path)), overwrite[key]["path"])
-            _module = imp.load_source(str(hash(_path)), overwrite[key]["path"])
             output[key] = _module.__getattribute__(overwrite[key]["class"])
         return output
 

tools/tiny-test-fw/Utility/SearchCases.py

@@ -17,8 +17,7 @@ import os
 import fnmatch
 import types
 import copy
-# TODO: add a function to use suitable import lib for python2 and python3
+from Utility import load_source
-import imp
 
 
 class Search(object):
@@ -31,8 +30,7 @@ class Search(object):
         print("Try to get cases from: " + file_name)
         test_functions = []
         try:
-            # TODO: add a function to use suitable import lib for python2 and python3
+            mod = load_source(str(hash(file_name)), file_name)
-            mod = imp.load_source(str(hash(file_name)), file_name)
             for func in [mod.__getattribute__(x) for x in dir(mod)
                          if isinstance(mod.__getattribute__(x), types.FunctionType)]:
                 try:

tools/tiny-test-fw/Utility/__init__.py

@@ -36,3 +36,13 @@ def console_log(data, color="white", end="\n"):
         # reset color to white for later logs
         print(_COLOR_CODES["white"] + u"\r")
     sys.stdout.flush()
+
+
+def load_source(name, path):
+    try:
+        from importlib.machinery import SourceFileLoader
+        return SourceFileLoader(name, path).load_module()
+    except ImportError:
+        # importlib.machinery doesn't exists in Python 2 so we will use imp (deprecated in Python 3)
+        import imp
+        return imp.load_source(name, path)