Skip to content

Commit 2a1616a

Browse files
jamesfrascajamesfrasca
committed
Migrate worker container retries to RetryHelper
1 parent ff636ea commit 2a1616a

2 files changed

Lines changed: 86 additions & 80 deletions

File tree

‎src/Runner.Worker/ActionManager.cs‎

Lines changed: 42 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -767,29 +767,30 @@ private async Task PullActionContainerAsync(IExecutionContext executionContext,
767767

768768
// Pull down docker image with retry up to 3 times
769769
var dockerManager = HostContext.GetService<IDockerCommandManager>();
770-
int retryCount = 0;
771770
int pullExitCode = 0;
772-
while (retryCount < 3)
771+
var pullRetryHelper = new RetryHelper(Trace, new RetryStrategy
773772
{
774-
pullExitCode = await dockerManager.DockerPull(executionContext, setupInfo.Container.Image);
775-
if (pullExitCode == 0)
773+
MaxAttempts = 3,
774+
GetBackoff = (_, _, _) => BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)),
775+
OnRetry = (_, _, backoff) =>
776776
{
777-
break;
778-
}
779-
else
777+
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backoff.TotalSeconds} seconds before retry.");
778+
},
779+
});
780+
781+
await pullRetryHelper.ExecuteAsync(
782+
operationName: nameof(PullActionContainerAsync),
783+
operation: async () =>
780784
{
781-
retryCount++;
782-
if (retryCount < 3)
783-
{
784-
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
785-
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
786-
await Task.Delay(backOff);
787-
}
788-
}
789-
}
785+
pullExitCode = await dockerManager.DockerPull(executionContext, setupInfo.Container.Image);
786+
return pullExitCode == 0
787+
? OperationOutcome.Success(true)
788+
: OperationOutcome.TransientFailure<bool>($"Docker pull failed with exit code {pullExitCode}");
789+
},
790+
cancellationToken: executionContext.CancellationToken);
790791
executionContext.Output("##[endgroup]");
791792

792-
if (retryCount == 3 && pullExitCode != 0)
793+
if (pullExitCode != 0)
793794
{
794795
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
795796
}
@@ -811,35 +812,37 @@ private async Task BuildActionContainerAsync(IExecutionContext executionContext,
811812

812813
// Build docker image with retry up to 3 times
813814
var dockerManager = HostContext.GetService<IDockerCommandManager>();
814-
int retryCount = 0;
815815
int buildExitCode = 0;
816816
var imageName = $"{dockerManager.DockerInstanceLabel}:{Guid.NewGuid().ToString("N")}";
817-
while (retryCount < 3)
817+
var buildRetryHelper = new RetryHelper(Trace, new RetryStrategy
818818
{
819-
buildExitCode = await dockerManager.DockerBuild(
820-
executionContext,
821-
setupInfo.Container.WorkingDirectory,
822-
setupInfo.Container.Dockerfile,
823-
Directory.GetParent(setupInfo.Container.Dockerfile).FullName,
824-
imageName);
825-
if (buildExitCode == 0)
819+
MaxAttempts = 3,
820+
GetBackoff = (_, _, _) => BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)),
821+
OnRetry = (_, _, backoff) =>
826822
{
827-
break;
828-
}
829-
else
823+
executionContext.Warning($"Docker build failed with exit code {buildExitCode}, back off {backoff.TotalSeconds} seconds before retry.");
824+
},
825+
});
826+
827+
await buildRetryHelper.ExecuteAsync(
828+
operationName: nameof(BuildActionContainerAsync),
829+
operation: async () =>
830830
{
831-
retryCount++;
832-
if (retryCount < 3)
833-
{
834-
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
835-
executionContext.Warning($"Docker build failed with exit code {buildExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
836-
await Task.Delay(backOff);
837-
}
838-
}
839-
}
831+
buildExitCode = await dockerManager.DockerBuild(
832+
executionContext,
833+
setupInfo.Container.WorkingDirectory,
834+
setupInfo.Container.Dockerfile,
835+
Directory.GetParent(setupInfo.Container.Dockerfile).FullName,
836+
imageName);
837+
838+
return buildExitCode == 0
839+
? OperationOutcome.Success(true)
840+
: OperationOutcome.TransientFailure<bool>($"Docker build failed with exit code {buildExitCode}");
841+
},
842+
cancellationToken: executionContext.CancellationToken);
840843
executionContext.Output("##[endgroup]");
841844

842-
if (retryCount == 3 && buildExitCode != 0)
845+
if (buildExitCode != 0)
843846
{
844847
throw new InvalidOperationException($"Docker build failed with exit code {buildExitCode}");
845848
}

‎src/Runner.Worker/ContainerOperationProvider.cs‎

Lines changed: 44 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -196,31 +196,32 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta
196196
var configLocation = await ContainerRegistryLogin(executionContext, container);
197197

198198
// Pull down docker image with retry up to 3 times
199-
int retryCount = 0;
200199
int pullExitCode = 0;
201-
while (retryCount < 3)
200+
var pullRetryHelper = new RetryHelper(Trace, new RetryStrategy
202201
{
203-
pullExitCode = await _dockerManager.DockerPull(executionContext, container.ContainerImage, configLocation);
204-
if (pullExitCode == 0)
202+
MaxAttempts = 3,
203+
GetBackoff = (_, _, _) => BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)),
204+
OnRetry = (_, _, backoff) =>
205205
{
206-
break;
207-
}
208-
else
206+
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backoff.TotalSeconds} seconds before retry.");
207+
},
208+
});
209+
210+
await pullRetryHelper.ExecuteAsync(
211+
operationName: nameof(StartContainerAsync),
212+
operation: async () =>
209213
{
210-
retryCount++;
211-
if (retryCount < 3)
212-
{
213-
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
214-
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
215-
await Task.Delay(backOff);
216-
}
217-
}
218-
}
214+
pullExitCode = await _dockerManager.DockerPull(executionContext, container.ContainerImage, configLocation);
215+
return pullExitCode == 0
216+
? OperationOutcome.Success(true)
217+
: OperationOutcome.TransientFailure<bool>($"Docker pull failed with exit code {pullExitCode}");
218+
},
219+
cancellationToken: executionContext.CancellationToken);
219220

220221
// Remove credentials after pulling
221222
ContainerRegistryLogout(configLocation);
222223

223-
if (retryCount == 3 && pullExitCode != 0)
224+
if (pullExitCode != 0)
224225
{
225226
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
226227
}
@@ -468,33 +469,35 @@ private async Task<string> ContainerRegistryLogin(IExecutionContext executionCon
468469
}
469470

470471
// Login docker with retry up to 3 times
471-
int retryCount = 0;
472472
int loginExitCode = 0;
473-
while (retryCount < 3)
474-
{
475-
loginExitCode = await _dockerManager.DockerLogin(
476-
executionContext,
477-
configLocation,
478-
container.RegistryServer,
479-
container.RegistryAuthUsername,
480-
container.RegistryAuthPassword);
481-
if (loginExitCode == 0)
482-
{
483-
break;
484-
}
485-
else
473+
var loginRetryHelper = new RetryHelper(Trace, new RetryStrategy
474+
{
475+
MaxAttempts = 3,
476+
GetBackoff = (_, _, _) => BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)),
477+
OnRetry = (_, _, backoff) =>
486478
{
487-
retryCount++;
488-
if (retryCount < 3)
489-
{
490-
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
491-
executionContext.Warning($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
492-
await Task.Delay(backOff);
493-
}
494-
}
495-
}
479+
executionContext.Warning($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}, back off {backoff.TotalSeconds} seconds before retry.");
480+
},
481+
});
496482

497-
if (retryCount == 3 && loginExitCode != 0)
483+
await loginRetryHelper.ExecuteAsync(
484+
operationName: nameof(ContainerRegistryLogin),
485+
operation: async () =>
486+
{
487+
loginExitCode = await _dockerManager.DockerLogin(
488+
executionContext,
489+
configLocation,
490+
container.RegistryServer,
491+
container.RegistryAuthUsername,
492+
container.RegistryAuthPassword);
493+
494+
return loginExitCode == 0
495+
? OperationOutcome.Success(true)
496+
: OperationOutcome.TransientFailure<bool>($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}");
497+
},
498+
cancellationToken: executionContext.CancellationToken);
499+
500+
if (loginExitCode != 0)
498501
{
499502
throw new InvalidOperationException($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}");
500503
}

0 commit comments

Comments
 (0)