Don't continue uploading when server is down

Hotsapi.Uploader.Common.Test/MockUploader.cs

@@ -11,6 +11,9 @@ private class MockUploader : IUploader
             public bool UploadToHotslogs { get; set; }
 
             private Func<ReplayFile, Task> UploadCallback = _ => Task.CompletedTask;
+
+            public event Action<DateTime> NotifyServerDownUntil;
+
             public void SetUploadCallback(Func<ReplayFile, Task> onUpload)
             {
                 var old = UploadCallback;

Hotsapi.Uploader.Common/IUploader.cs

@@ -1,14 +1,17 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 
 namespace Hotsapi.Uploader.Common
 {
     public interface IUploader
     {
+        event Action<DateTime> NotifyServerDownUntil;
         bool UploadToHotslogs { get; set; }
         Task CheckDuplicate(IEnumerable<ReplayFile> replays);
         Task<int> GetMinimumBuild();
         Task Upload(ReplayFile file);
         Task<UploadStatus> Upload(string file);
     }
+
 }

Hotsapi.Uploader.Common/Manager.cs

@@ -101,6 +101,11 @@ public async void Start(IMonitor monitor, IAnalyzer analyzer, IUploader uploader
             _analyzer = analyzer;
             _monitor = monitor;
 
+            uploader.NotifyServerDownUntil += dt => {
+                ServerDownUntil = dt;
+                RefreshStatusAndAggregates();
+            };
+
             var replays = ScanReplays();
             Files.AddRange(replays);
             replays.Where(x => x.UploadStatus == UploadStatus.None).Reverse().Map(x => processingQueue.Add(x));
@@ -126,19 +131,41 @@ public void Stop()
             processingQueue.CompleteAdding();
         }
 
+        private DateTime ServerDownUntil { get; set; }
+
+        private async Task WhenUploaderAvailable()
+        {
+            var maxDelay = new TimeSpan(0, 10, 0);
+            var calculated = ServerDownUntil - DateTime.Now;
+            var delay = calculated > maxDelay ? maxDelay : calculated;
+            if (delay.TotalMilliseconds >= 0) {
+                await Task.Delay(delay);
+                await WhenUploaderAvailable();
+            }
+        }
+
+        private bool IsUploaderAvailable => ServerDownUntil <= DateTime.Now;
+
         private async Task UploadLoop()
         {
             while (await processingQueue.OutputAvailableAsync()) {
+                await WhenUploaderAvailable();
                 try {
                     var file = await processingQueue.TakeAsync();
+                    await WhenUploaderAvailable();
 
                     file.UploadStatus = UploadStatus.InProgress;
 
                     // test if replay is eligible for upload (not AI, PTR, Custom, etc)
                     var replay = _analyzer.Analyze(file);
+                    await WhenUploaderAvailable();
                     if (file.UploadStatus == UploadStatus.InProgress) {
                         // if it is, upload it
                         await _uploader.Upload(file);
+                        await Task.Yield();
+                        if (!IsUploaderAvailable) {
+                            await processingQueue.AddAsync(file);
+                        }
                     }
                     SaveReplayList();
                     if (ShouldDelete(file, replay)) {
@@ -153,7 +180,9 @@ private async Task UploadLoop()
 
         private void RefreshStatusAndAggregates()
         {
-            _status = Files.Any(x => x.UploadStatus == UploadStatus.InProgress) ? "Uploading..." : "Idle";
+            _status = !IsUploaderAvailable ? "Server unavailable" :
+                      Files.Any(x => x.UploadStatus == UploadStatus.InProgress) ? "Uploading..." :
+                      "Idle";
             _aggregates = Files.GroupBy(x => x.UploadStatus).ToDictionary(x => x.Key, x => x.Count());
             PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Status)));
             PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Aggregates)));

Hotsapi.Uploader.Common/Uploader.cs

@@ -20,6 +20,8 @@ public class Uploader : IUploader
 #endif
 
         public bool UploadToHotslogs { get; set; }
+        public event Action<DateTime> NotifyServerDownUntil;
+        public DateTime AssumeServerDownUntil {get; private set;} = DateTime.Now;
 
         /// <summary>
         /// New instance of replay uploader
@@ -72,7 +74,14 @@ public async Task<UploadStatus> Upload(string file)
                 }
             }
             catch (WebException ex) {
-                if (await CheckApiThrottling(ex.Response)) {
+                if(ex.Response is HttpWebResponse httpWebResponse && httpWebResponse.StatusCode == HttpStatusCode.ServiceUnavailable) {
+                    _log.Warn($"Service unavaiable, response is {httpWebResponse}");
+                    if(AssumeServerDownUntil < DateTime.Now) {
+                        AssumeServerDownUntil = DateTime.Now.AddMinutes(10);
+                        NotifyServerDownUntil(AssumeServerDownUntil);
+                    }
+                }
+                else if (await CheckApiThrottling(ex.Response)) {
                     return await Upload(file);
                 }
                 _log.Warn(ex, $"Error uploading file '{file}'");