Notice when Team Collection monitoring stops working, and go Disconnected (BL-16729)

DistFiles/localization/en/BloomMediumPriority.xlf

@@ -1579,6 +1579,21 @@
         <note>ID: ImageLibrary.ThisPage</note>
         <note>Short link text used in setup instructions (e.g., as in "go to this page"). Should be lowercase as it appears mid-sentence.</note>
       </trans-unit>
+      <trans-unit id="TeamCollection.LostContactWithRepo" translate="no">
+        <source xml:lang="en">Bloom can no longer watch the Team Collection folder at "{0}", so it will not see changes made by your teammates. Usually this means the folder, or the drive or network it is on, is no longer available.</source>
+        <note>ID: TeamCollection.LostContactWithRepo</note>
+        <note>Shown in the Team Collection status dialog when Bloom loses its ability to watch the shared Team Collection folder part way through a session, for example because the network share or Dropbox folder disappears. {0} is replaced with the full path of the shared folder. "Bloom" is a product name and must not be translated.</note>
+      </trans-unit>
+      <trans-unit id="TeamCollection.NoLongerSeeingChanges" translate="no">
+        <source xml:lang="en">Bloom can no longer see the Team Collection folder, so you will not see changes made by your teammates.</source>
+        <note>ID: TeamCollection.NoLongerSeeingChanges</note>
+        <note>A short pop-up notification (toast) in the bottom-right corner of the window, shown the moment Bloom notices part way through a session that it has lost contact with the shared Team Collection folder. Clicking it opens the Team Collection dialog, which shows the longer TeamCollection.LostContactWithRepo message. Keep it short; it has to fit in a small pop-up. "Bloom" is a product name and must not be translated.</note>
+      </trans-unit>
+      <trans-unit id="TeamCollection.MayHaveMissedChanges" translate="no">
+        <source xml:lang="en">Bloom may have missed some changes your teammates made. Please click "Reload Collection" to be sure you have the latest.</source>
+        <note>ID: TeamCollection.MayHaveMissedChanges</note>
+        <note>Used both as a pop-up notification (toast) and in the Team Collection status dialog, when so many files changed at once that Bloom's file-change notifications overflowed and some were lost. "Reload Collection" is the label of a button in that dialog and should match its translation there. "Bloom" is a product name and must not be translated.</note>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/BloomBrowserUI/collectionsTab/collectionsTabBookPane/CollectionsTabBookPane.tsx

@@ -79,7 +79,9 @@ export const CollectionsTabBookPane: React.FunctionComponent<{
 
                 setBookTeamCollectionStatus((prevBookStatus) => ({
                     ...prevBookStatus,
-                    disconnected: true,
+                    // The field is isDisconnected; setting "disconnected" left the panel
+                    // rendering the book as available for checkout. See BL-16729.
+                    isDisconnected: true,
                     error: errorMessage,
                 }));
             },

src/BloomExe/TeamCollection/ConnectionFailureTracker.cs

@@ -0,0 +1,56 @@
+namespace Bloom.TeamCollection
+{
+    /// <summary>
+    /// Decides when a run of failed connection checks has gone on long enough to believe.
+    /// Kept as a separate, pure class (no IO, no timers, no threads) so the policy can be
+    /// unit tested exhaustively; ConnectionHeartbeat supplies the timing. See BL-16729.
+    ///
+    /// The point of waiting for a second failure is that the things CheckConnection looks at
+    /// can lie: one dropped packet fails the probe to dropbox.com, a Wi-Fi roam briefly makes
+    /// NetworkInterface.GetIsNetworkAvailable() false, a flaky SMB share can answer "no such
+    /// folder" for a moment. Wrongly disconnecting a collection that is working is the worst
+    /// outcome available to us, because there is no automatic way back: the user has to
+    /// Reload Collection. Waiting fifteen seconds to be sure is cheap by comparison.
+    /// </summary>
+    internal class ConnectionFailureTracker
+    {
+        /// <summary>
+        /// How many checks in a row must report the same problem before we act on it.
+        /// </summary>
+        internal const int kRequiredConsecutiveFailures = 2;
+
+        private string _lastFailureL10nId;
+        private int _consecutiveFailures;
+
+        /// <summary>
+        /// Feed in the result of one connection check. Returns true when we have now seen
+        /// enough consecutive failures of the same kind to conclude we really are disconnected.
+        /// </summary>
+        /// <param name="problemOrNull">What CheckConnection returned: null means all is well.</param>
+        public bool RecordResult(TeamCollectionMessage problemOrNull)
+        {
+            if (problemOrNull == null)
+            {
+                Reset();
+                return false;
+            }
+            if (problemOrNull.L10NId != _lastFailureL10nId)
+            {
+                // A different problem from last time. "No network" followed by "repo missing"
+                // is two transients, not one sustained outage, so start counting again.
+                _lastFailureL10nId = problemOrNull.L10NId;
+                _consecutiveFailures = 0;
+            }
+            return ++_consecutiveFailures >= kRequiredConsecutiveFailures;
+        }
+
+        /// <summary>
+        /// Forget any run of failures, e.g. because we stopped checking for a while.
+        /// </summary>
+        public void Reset()
+        {
+            _consecutiveFailures = 0;
+            _lastFailureL10nId = null;
+        }
+    }
+}

src/BloomExe/TeamCollection/ConnectionHeartbeat.cs

@@ -0,0 +1,166 @@
+using System;
+using System.Threading;
+
+namespace Bloom.TeamCollection
+{
+    /// <summary>
+    /// Periodically re-checks that we can still reach the Team Collection repo, and disconnects
+    /// if we can't. See BL-16729.
+    ///
+    /// The file system watchers tell us at once when the shared folder is yanked away, but they
+    /// cannot tell us that Dropbox has stopped syncing: the folder is still there, we simply
+    /// stop receiving other people's work. Before this, CheckConnection ran only when the user
+    /// did something (check out, check in, delete), so somebody who was just reading and editing
+    /// their own checked-out book could go the whole session without finding out.
+    ///
+    /// Owned by TeamCollection.StartMonitoring/StopMonitoring, which gets the lifecycle right
+    /// for free: no heartbeat during SyncAtStartup (monitoring is deliberately off then), none
+    /// on a DisconnectedTeamCollection (whose Start/StopMonitoring are no-ops), and it stops
+    /// when we disconnect or dispose.
+    /// </summary>
+    internal sealed class ConnectionHeartbeat : IDisposable
+    {
+        /// <summary>
+        /// How long between checks when everything is fine. Long enough that the cost (which for
+        /// a Dropbox repo includes an HTTP HEAD to dropbox.com) is negligible, short enough that
+        /// the user finds out reasonably soon that they have stopped seeing their teammates' work.
+        /// Not const, and internal, so tests can shorten it.
+        /// </summary>
+        internal static int IntervalMs = 60 * 1000;
+
+        /// <summary>
+        /// How soon we look again after a check fails, to see whether it was just a blip.
+        /// Must be longer than DropboxUtils' 10-second cache of the dropbox.com probe, or the
+        /// second look would just return the first one's answer and confirm nothing.
+        /// </summary>
+        internal const int kConfirmIntervalMs = 15 * 1000;
+
+        private readonly TeamCollection _teamCollection;
+        private readonly ConnectionFailureTracker _tracker = new ConnectionFailureTracker();
+        private Timer _timer;
+        private volatile bool _disposed;
+
+        public ConnectionHeartbeat(TeamCollection teamCollection)
+        {
+            _teamCollection = teamCollection;
+        }
+
+        /// <summary>
+        /// Begin checking. Does nothing under unit tests, which must not be left with live
+        /// threadpool timers checking real folders and the real network.
+        /// </summary>
+        internal void Start()
+        {
+            if (Program.RunningUnitTests)
+                return;
+            // A one-shot timer that re-arms itself at the end of each tick (rather than a
+            // repeating one) makes overlapping ticks structurally impossible, so a probe that
+            // blocks for forty seconds on a dead share cannot pile up behind itself.
+            _timer = new Timer(Tick, null, IntervalMs, Timeout.Infinite);
+        }
+
+        /// <summary>
+        /// Runs on a threadpool thread. Internal so tests can drive the policy directly, with
+        /// no timer involved.
+        /// </summary>
+        internal void Tick(object unused)
+        {
+            var delayUntilNextTick = IntervalMs;
+            try
+            {
+                if (!OkToCheckNow())
+                {
+                    // Not the live collection, not watching just now (e.g. during a sync), or
+                    // busy writing to the repo. Anything we noticed before such a gap is no
+                    // longer part of a "consecutive" run, and an in-flight write reports its
+                    // own failures, so disconnecting out from under it would be disruptive.
+                    _tracker.Reset();
+                }
+                else
+                {
+                    // A joiner's Books folder can arrive minutes after Bloom starts, once
+                    // Dropbox delivers it. If we could not watch it then, this is where we
+                    // notice it has turned up. See BL-16729.
+                    _teamCollection.RetryDeferredWatching();
+
+                    // Deliberately the quiet overload: History messages are not de-duplicated,
+                    // so a probe that wrote them would fill log.txt and raise a status-changed
+                    // event on every tick of a perfectly healthy session.
+                    var problem = _teamCollection.CheckConnection(writeHistoryMessages: false);
+                    // The probe can block for several seconds (DropboxUtils allows 5 for its
+                    // request to dropbox.com), which is long enough for a check-in or a sync to
+                    // have started meanwhile. Re-ask before acting on what we found, or we
+                    // would disconnect in the middle of one.
+                    if (!OkToCheckNow())
+                    {
+                        _tracker.Reset();
+                    }
+                    else if (_tracker.RecordResult(problem))
+                    {
+                        // Logged as well as acted on: without this, someone testing a real
+                        // outage has no way to tell "the check ran and decided" from "the
+                        // check never ran".
+                        SIL.Reporting.Logger.WriteEvent(
+                            $"Team Collection periodic check confirmed a problem ({problem.L10NId}); disconnecting."
+                        );
+                        _teamCollection.ReportConnectionProblem(problem);
+                    }
+                    else if (problem != null)
+                    {
+                        SIL.Reporting.Logger.WriteEvent(
+                            $"Team Collection periodic check found a problem ({problem.L10NId}); "
+                                + $"looking again in {kConfirmIntervalMs / 1000}s before believing it."
+                        );
+                        delayUntilNextTick = kConfirmIntervalMs; // suspicious; confirm sooner
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                // An exception here is not evidence that the repo is gone, so we don't
+                // disconnect over it. This matches TeamCollectionManager.CheckConnection.
+                NonFatalProblem.ReportSentryOnly(ex);
+                // It is also not evidence that the repo is FINE, so this tick tells us nothing
+                // either way -- which means it breaks the run. Without this reset, a failure,
+                // then a throwing probe, then another failure would count as two consecutive
+                // failures and disconnect a collection that was never shown to be unreachable
+                // twice in a row.
+                _tracker.Reset();
+            }
+            finally
+            {
+                if (!_disposed)
+                {
+                    try
+                    {
+                        _timer?.Change(delayUntilNextTick, Timeout.Infinite);
+                    }
+                    catch (ObjectDisposedException)
+                    {
+                        // Disposed while we were checking. Nothing to re-arm.
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Whether this is a sensible moment to check the connection at all. Checked both
+        /// before and after the probe, because the probe can block long enough for the answer
+        /// to change.
+        /// </summary>
+        private bool OkToCheckNow()
+        {
+            return !_disposed
+                && _teamCollection.IsMonitoring
+                && _teamCollection.IsLiveCollection
+                && !_teamCollection.IsWritingToRepo;
+        }
+
+        public void Dispose()
+        {
+            _disposed = true;
+            _timer?.Dispose();
+            _timer = null;
+        }
+    }
+}