Merge branch 'feature/retry_download_if_CRC_check_failed' into 'master'
ci: retry download if catched IOError/EOFError See merge request espressif/esp-idf!9134
This commit is contained in:
commit
d1bf7a8743
1 changed files with 29 additions and 0 deletions
|
@ -4,6 +4,7 @@ import argparse
|
||||||
import tempfile
|
import tempfile
|
||||||
import tarfile
|
import tarfile
|
||||||
import zipfile
|
import zipfile
|
||||||
|
from functools import wraps
|
||||||
|
|
||||||
import gitlab
|
import gitlab
|
||||||
|
|
||||||
|
@ -11,6 +12,8 @@ import gitlab
|
||||||
class Gitlab(object):
|
class Gitlab(object):
|
||||||
JOB_NAME_PATTERN = re.compile(r"(\w+)(\s+(\d+)/(\d+))?")
|
JOB_NAME_PATTERN = re.compile(r"(\w+)(\s+(\d+)/(\d+))?")
|
||||||
|
|
||||||
|
DOWNLOAD_ERROR_MAX_RETRIES = 3
|
||||||
|
|
||||||
def __init__(self, project_id=None):
|
def __init__(self, project_id=None):
|
||||||
config_data_from_env = os.getenv("PYTHON_GITLAB_CONFIG")
|
config_data_from_env = os.getenv("PYTHON_GITLAB_CONFIG")
|
||||||
if config_data_from_env:
|
if config_data_from_env:
|
||||||
|
@ -64,6 +67,31 @@ class Gitlab(object):
|
||||||
with zipfile.ZipFile(temp_file.name, "r") as archive_file:
|
with zipfile.ZipFile(temp_file.name, "r") as archive_file:
|
||||||
archive_file.extractall(destination)
|
archive_file.extractall(destination)
|
||||||
|
|
||||||
|
def retry_download(func):
|
||||||
|
"""
|
||||||
|
This wrapper will only catch IOError and retry the whole function.
|
||||||
|
|
||||||
|
So only use it with download functions, read() inside and atomic
|
||||||
|
functions
|
||||||
|
"""
|
||||||
|
@wraps(func)
|
||||||
|
def wrapper(self, *args, **kwargs):
|
||||||
|
retried = 0
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
res = func(self, *args, **kwargs)
|
||||||
|
except (IOError, EOFError) as e:
|
||||||
|
retried += 1
|
||||||
|
if retried > self.DOWNLOAD_ERROR_MAX_RETRIES:
|
||||||
|
raise e # get out of the loop
|
||||||
|
else:
|
||||||
|
print('Retried for the {} time'.format(retried))
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
return res
|
||||||
|
return wrapper
|
||||||
|
|
||||||
def download_artifact(self, job_id, artifact_path, destination=None):
|
def download_artifact(self, job_id, artifact_path, destination=None):
|
||||||
"""
|
"""
|
||||||
download specific path of job artifacts and extract to destination.
|
download specific path of job artifacts and extract to destination.
|
||||||
|
@ -118,6 +146,7 @@ class Gitlab(object):
|
||||||
job_id_list.append({"id": job.id, "parallel_num": match.group(3)})
|
job_id_list.append({"id": job.id, "parallel_num": match.group(3)})
|
||||||
return job_id_list
|
return job_id_list
|
||||||
|
|
||||||
|
@retry_download
|
||||||
def download_archive(self, ref, destination, project_id=None):
|
def download_archive(self, ref, destination, project_id=None):
|
||||||
"""
|
"""
|
||||||
Download archive of certain commit of a repository and extract to destination path
|
Download archive of certain commit of a repository and extract to destination path
|
||||||
|
|
Loading…
Reference in a new issue