diff --git a/components/mbedtls/CMakeLists.txt b/components/mbedtls/CMakeLists.txt
index b7e048a1f..eac4096f1 100644
--- a/components/mbedtls/CMakeLists.txt
+++ b/components/mbedtls/CMakeLists.txt
@@ -81,20 +81,35 @@ endif()
 # Add port files to mbedtls targets
 target_sources(mbedtls PRIVATE ${mbedtls_target_sources})
 
-
-
 target_sources(mbedcrypto PRIVATE "${COMPONENT_DIR}/port/esp_hardware.c"
-                                "${COMPONENT_DIR}/port/esp_mem.c"
-                                "${COMPONENT_DIR}/port/esp_timing.c"
-                                "${COMPONENT_DIR}/port/esp_sha.c"
-				"${COMPONENT_DIR}/port/esp_bignum.c"
-                                "${COMPONENT_DIR}/port/esp_aes_xts.c"
-                                "${COMPONENT_DIR}/port/${idf_target}/bignum.c"
-                                "${COMPONENT_DIR}/port/${idf_target}/aes.c"
-                                "${COMPONENT_DIR}/port/${idf_target}/sha.c"
-                                "${COMPONENT_DIR}/port/${idf_target}/esp_sha1.c"
-                                "${COMPONENT_DIR}/port/${idf_target}/esp_sha256.c"
-                                "${COMPONENT_DIR}/port/${idf_target}/esp_sha512.c")
+                                  "${COMPONENT_DIR}/port/esp_mem.c"
+                                  "${COMPONENT_DIR}/port/esp_timing.c"
+                                  "${COMPONENT_DIR}/port/esp_sha.c"
+                                  "${COMPONENT_DIR}/port/esp_aes_xts.c"
+                                  "${COMPONENT_DIR}/port/${idf_target}/aes.c"
+                                  "${COMPONENT_DIR}/port/${idf_target}/sha.c"
+)
+
+# Note: some mbedTLS hardware acceleration can be enabled/disabled by config.
+#
+# We don't need to filter aes.c as this uses a different prefix (esp_aes_x) and the
+# config option only changes the prefixes in the header so mbedtls_aes_x compiles to esp_aes_x
+#
+# The other port-specific files don't override internal mbedTLS functions, they just add new functions.
+
+if(CONFIG_MBEDTLS_HARDWARE_MPI)
+    target_sources(mbedcrypto PRIVATE  "${COMPONENT_DIR}/port/esp_bignum.c"
+                                       "${COMPONENT_DIR}/port/${idf_target}/bignum.c"
+    )
+endif()
+
+if(CONFIG_MBEDTLS_HARDWARE_SHA)
+    target_sources(mbedcrypto PRIVATE  "${COMPONENT_DIR}/port/${idf_target}/esp_sha1.c"
+                                       "${COMPONENT_DIR}/port/${idf_target}/esp_sha256.c"
+                                       "${COMPONENT_DIR}/port/${idf_target}/esp_sha512.c"
+    )
+endif()
+
 
 foreach(target ${mbedtls_targets})
     target_compile_definitions(${target} PUBLIC -DMBEDTLS_CONFIG_FILE="mbedtls/esp_config.h")
diff --git a/components/mbedtls/component.mk b/components/mbedtls/component.mk
index fc8f1d434..196370dff 100644
--- a/components/mbedtls/component.mk
+++ b/components/mbedtls/component.mk
@@ -10,6 +10,20 @@ COMPONENT_OBJEXCLUDE := mbedtls/library/net_sockets.o
 
 COMPONENT_SUBMODULES += mbedtls
 
+# Note: some mbedTLS hardware acceleration can be enabled/disabled by config.
+#
+# We don't need to exclude aes.o as these functions use a different prefix (esp_aes_x) and the
+# config option only changes the prefixes in the header so mbedtls_aes_x compiles to esp_aes_x
+#
+# The other port-specific files don't override internal mbedTLS functions, they just add new functions.
+
+ifndef CONFIG_MBEDTLS_HARDWARE_MPI
+    COMPONENT_OBJEXCLUDE += port/esp_bignum.o port/$(IDF_TARGET)/bignum.o
+endif
+
+ifndef CONFIG_MBEDTLS_HARDWARE_SHA
+    COMPONENT_OBJEXCLUDE += port/$(IDF_TARGET)/esp_sha1.o port/$(IDF_TARGET)/esp_sha256.o port/$(IDF_TARGET)/esp_sha512.o
+endif
 
 ifdef CONFIG_MBEDTLS_CERTIFICATE_BUNDLE