diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 616848225..dc9c5520f 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -46,6 +46,7 @@ tools/ci/get_supported_examples.sh tools/ci/mirror-submodule-update.sh tools/ci/multirun_with_pyenv.sh tools/ci/push_to_github.sh +tools/ci/retry_failed.sh tools/ci/test_build_system.sh tools/ci/test_build_system_cmake.sh tools/ci/test_configure_ci_environment.sh diff --git a/tools/ci/retry_failed.sh b/tools/ci/retry_failed.sh new file mode 100755 index 000000000..2da480b47 --- /dev/null +++ b/tools/ci/retry_failed.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# +# Retries a command RETRY_ATTEMPTS times in case of failure +# +# Inspired by https://stackoverflow.com/a/8351489 +# + +max_attempts=${RETRY_ATTEMPTS-3} +RETRY_TIMEWAIT=${RETRY_TIMEWAIT-1} +attempt=1 +exitCode=0 +whole_start=$(date +%s) +attempt_start=whole_start + +while true; do + if "$@" ; then + exitCode=0 + break + else + exitCode=$? + fi + + if (( $attempt >= $max_attempts )) ; then + break + fi + + echo "Failed! ("$@") Spent time $(( $(date '+%s') - ${attempt_start} )) sec. Retrying in ${RETRY_TIMEWAIT}..." 1>&2 + sleep $RETRY_TIMEWAIT + attempt=$(( attempt + 1 )) + RETRY_TIMEWAIT=$(( RETRY_TIMEWAIT * 2 )) + attempt_start=$(date +%s) +done + +if [[ $exitCode != 0 ]] ; then + echo -n "Totally failed! ("$@")" 1>&2 +else + echo -n "Done ("$@")" 1>&2 +fi + +echo " Spent time $(( $(date '+%s') - ${whole_start} )) sec in total" 1>&2 + +exit $exitCode