Merge branch 'feature/startup-healthreport'

ATxCommon/ATxCommon.csproj

@@ -46,6 +46,7 @@
     <Compile Include="ActiveDirectory.cs" />
     <Compile Include="BuildDetails.cs" />
     <Compile Include="Conv.cs" />
+    <Compile Include="DirectoryDetails.cs" />
     <Compile Include="FsUtils.cs" />
     <Compile Include="Monitoring\Cpu.cs" />
     <Compile Include="Monitoring\MonitorBase.cs" />
@@ -55,6 +56,7 @@
     <Compile Include="Serializables\ServiceConfig.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Serializables\ServiceStatus.cs" />
+    <Compile Include="StorageStatus.cs" />
     <Compile Include="SystemChecks.cs" />
     <Compile Include="TimeUtils.cs" />
   </ItemGroup>

ATxCommon/DirectoryDetails.cs

+using System;
+using System.IO;
+using NLog;
+
+namespace ATxCommon
+{
+    public class DirectoryDetails
+    {
+        private static readonly Logger Log = LogManager.GetCurrentClassLogger();
+
+        private long _size = -1;
+        private int _age = -1;
+
+        /// <summary>
+        /// Initialize the DirectoryDetails object from the given DirectoryInfo.
+        /// </summary>
+        /// <param name="dirInfo">The DirectoryInfo object of the directory to investigate.</param>
+        public DirectoryDetails(DirectoryInfo dirInfo) {
+            Dir = dirInfo;
+        }
+
+        /// <summary>
+        /// The underlying DirectoryInfo object.
+        /// </summary>
+        public DirectoryInfo Dir { get; }
+
+        /// <summary>
+        /// The age defined by the directory's NAME in days.
+        /// </summary>
+        public int AgeFromName {
+            get {
+                if (_age < 0)
+                    _age = FsUtils.DirNameToAge(Dir, DateTime.Now);
+
+                return _age;
+            }
+        }
+
+        /// <summary>
+        /// The full size of the directory tree in bytes.
+        /// </summary>
+        public long Size {
+            get {
+                if (_size >= 0)
+                    return _size;
+
+                try {
+                    _size = FsUtils.GetDirectorySize(Dir.FullName);
+                }
+                catch (Exception ex) {
+                    Log.Error("ERROR getting directory size of [{0}]: {1}",
+                        Dir.FullName, ex.Message);
+                }
+
+                return _size;
+            }
+        }
+
+        /// <summary>
+        /// Human friendly description of the directory's age, derived from its NAME.
+        /// </summary>
+        public string HumanAgeFromName => TimeUtils.DaysToHuman(AgeFromName, false);
+
+        /// <summary>
+        /// Human friendly description of the directory tree size (recursively).
+        /// </summary>
+        public string HumanSize => Conv.BytesToString(Size);
+    }
+}

ATxCommon/FsUtils.cs

 using System;
-using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
 using System.Linq;
@@ -52,11 +51,11 @@ namespace ATxCommon
                 Log.Error("ERROR: CheckForDirectory() parameter must not be empty!");
                 return false;
             }
-            return FsUtils.CreateNewDirectory(path, false) == path;
+            return CreateNewDirectory(path, false) == path;
         }
 
         /// <summary>
-        /// Recursively sum up size of all files under a given path.
+        /// Recursively sum up size (in bytes) of all files under a given path.
         /// </summary>
         /// <param name="path">Full path of the directory.</param>
         /// <returns>The total size in bytes.</returns>
@@ -88,67 +87,6 @@ namespace ATxCommon
             return (baseTime - dirTimestamp).Days;
         }
 
-        /// <summary>
-        /// Assemble a dictionary with information about expired directories.
-        /// </summary>
-        /// <param name="baseDir">The base directory to scan for subdirectories.</param>
-        /// <param name="thresh">The number of days used as expiration threshold.</param>
-        /// <returns>A dictionary having usernames as keys (of those users that actually do have
-        /// expired directories), where the values are lists of tuples with the DirInfo objects,
-        /// size (in bytes) and age (in days) of the expired directories.</returns>
-        public static Dictionary<string, List<Tuple<DirectoryInfo, long, int>>>
-            ExpiredDirs(DirectoryInfo baseDir,int thresh) {
-
-            var collection = new Dictionary<string, List<Tuple<DirectoryInfo, long, int>>>();
-            var now = DateTime.Now;
-            foreach (var userdir in baseDir.GetDirectories()) {
-                var expired = new List<Tuple<DirectoryInfo, long, int>>();
-                foreach (var subdir in userdir.GetDirectories()) {
-                    var age = DirNameToAge(subdir, now);
-                    if (age < thresh)
-                        continue;
-                    long size = -1;
-                    try {
-                        size = GetDirectorySize(subdir.FullName);
-                    }
-                    catch (Exception ex) {
-                        Log.Error("ERROR getting directory size of [{0}]: {1}",
-                            subdir.FullName, ex.Message);
-                    }
-                    expired.Add(new Tuple<DirectoryInfo, long, int>(subdir, size, age));
-                }
-                if (expired.Count > 0)
-                    collection.Add(userdir.Name, expired);
-            }
-            return collection;
-        }
-
-        /// <summary>
-        /// Generate a report on expired folders in the grace location.
-        /// 
-        /// Check all user-directories in the grace location for subdirectories whose timestamp
-        /// (the directory name) exceeds the configured grace period and generate a summary
-        /// containing the age and size of those directories.
-        /// </summary>
-        /// <param name="graceLocation">The location to scan for expired folders.</param>
-        /// <param name="threshold">The number of days used as expiration threshold.</param>
-        public static string GraceLocationSummary(DirectoryInfo graceLocation, int threshold) {
-            var expired = ExpiredDirs(graceLocation, threshold);
-            var report = "";
-            foreach (var userdir in expired.Keys) {
-                report += "\n - user '" + userdir + "'\n";
-                foreach (var subdir in expired[userdir]) {
-                    report += string.Format("   - {0} [age: {2}, size: {1}]\n",
-                        subdir.Item1, Conv.BytesToString(subdir.Item2),
-                        TimeUtils.DaysToHuman(subdir.Item3));
-                }
-            }
-            if (string.IsNullOrEmpty(report))
-                return "";
-
-            return "Expired folders in grace location:\n" + report;
-        }
-
         /// <summary>
         /// Check if a given directory is empty. If a marker file is set in the config a
         /// file with this name will be created inside the given directory and will be

ATxCommon/Serializables/DriveToCheck.cs

-namespace ATxCommon.Serializables
+using System.Xml.Serialization;
+
+namespace ATxCommon.Serializables
 {
     /// <summary>
     /// Helper class for the nested SpaceMonitoring sections.
@@ -14,5 +16,19 @@
         /// Limit (in GB) of free space, lower values will trigger a notification.
         /// </summary>
         public long SpaceThreshold { get; set; }
+
+        /// <summary>
+        /// Free space of a drive in bytes, set to -1 if unknown or check resulted in an error.
+        /// </summary>
+        [XmlIgnore]
+        public long FreeSpace { get; set; } = -1;
+
+        /// <summary>
+        /// Check if this drive's free space is below its threshold.
+        /// </summary>
+        /// <returns>True if free space is below the threshold, false otherwise.</returns>
+        public bool DiskSpaceLow() {
+            return FreeSpace < SpaceThreshold * Conv.GigaBytes;
+        }
     }
 }
\ No newline at end of file

ATxCommon/Serializables/ServiceConfig.cs

@@ -224,6 +224,12 @@ namespace ATxCommon.Serializables
         /// Minimum time in minutes between two low-space notifications. Default: 720 (12h).
         /// </summary>
         public int StorageNotificationDelta { get; set; } = 720;
+        
+        /// <summary>
+        /// Minimum time in minutes between two startup system health notifications.
+        /// Default: 2880 (2d).
+        /// </summary>
+        public int StartupNotificationDelta { get; set; } = 2880;
 
         #endregion
 
@@ -291,6 +297,11 @@ namespace ATxCommon.Serializables
         [XmlIgnore]
         public static string ValidatorWarnings { get; set; }
 
+        /// <summary>
+        /// Convenience property converting the grace period to a human-friendly format.
+        /// </summary>
+        [XmlIgnore]
+        public string HumanGracePeriod => TimeUtils.DaysToHuman(GracePeriod, false);
 
         #endregion
 
@@ -472,6 +483,7 @@ namespace ATxCommon.Serializables
             WarnOnHighValue(c.AdminNotificationDelta, nameof(c.AdminNotificationDelta), 1440);
             WarnOnHighValue(c.GraceNotificationDelta, nameof(c.GraceNotificationDelta), 10080);
             WarnOnHighValue(c.StorageNotificationDelta, nameof(c.StorageNotificationDelta), 10080);
+            WarnOnHighValue(c.StartupNotificationDelta, nameof(c.StartupNotificationDelta), 40320);
             WarnOnHighValue(c.GracePeriod, nameof(c.GracePeriod), 100);
 
             if (!c.DestinationDirectory.StartsWith(@"\\"))
@@ -557,6 +569,8 @@ namespace ATxCommon.Serializables
                     TimeUtils.MinutesToHuman(GraceNotificationDelta, false) + ")\n" +
                     $"StorageNotificationDelta: {StorageNotificationDelta} min (" +
                     TimeUtils.MinutesToHuman(StorageNotificationDelta, false) + ")\n" +
+                    $"StartupNotificationDelta: {StartupNotificationDelta} min (" +
+                    TimeUtils.MinutesToHuman(StartupNotificationDelta, false) + ")\n" +
                     "";
             }
             return msg;

ATxCommon/Serializables/ServiceStatus.cs

@@ -19,6 +19,7 @@ namespace ATxCommon.Serializables
         private DateTime _lastStorageNotification;
         private DateTime _lastAdminNotification;
         private DateTime _lastGraceNotification;
+        private DateTime _lastStartupNotification;
 
         private string _statusDescription;
         private string _currentTransferSrc;
@@ -159,6 +160,18 @@ namespace ATxCommon.Serializables
             }
         }
 
+        /// <summary>
+        /// Timestamp indicating when the last startup system health notification has been sent.
+        /// </summary>
+        [XmlElement("LastStartupNotification", DataType = "dateTime")]
+        public DateTime LastStartupNotification {
+            get => _lastStartupNotification;
+            set {
+                _lastStartupNotification = value;
+                Serialize();
+            }
+        }
+
         /// <summary>
         /// String indicating why the service is currently suspended (empty if not suspended).
         /// </summary>
@@ -374,7 +387,10 @@ namespace ATxCommon.Serializables
                 $"LastAdminNotification: {LastAdminNotification:yyyy-MM-dd HH:mm:ss}" +
                 $" ({TimeUtils.HumanSince(LastAdminNotification)})\n" +
                 $"LastGraceNotification: {LastGraceNotification:yyyy-MM-dd HH:mm:ss}" +
-                $" ({TimeUtils.HumanSince(LastGraceNotification)})\n";
+                $" ({TimeUtils.HumanSince(LastGraceNotification)})\n" +
+                $"LastStartupNotification: {LastStartupNotification:yyyy-MM-dd HH:mm:ss}" +
+                $" ({TimeUtils.HumanSince(LastStartupNotification)})\n" +
+                "";
         }
 
         #endregion validate and report

ATxCommon/StorageStatus.cs

+using System;
+using System.Collections.Generic;
+using System.IO;
+using ATxCommon.Serializables;
+using NLog;
+
+namespace ATxCommon
+{
+    public class StorageStatus
+    {
+        /// <summary>
+        /// By default the statuses will only be updated if more than UpdateDelta seconds have
+        /// elapsed since the last update to prevent too many updates causing high system load.
+        /// </summary>
+        public int UpdateDelta = 20;
+
+        private DateTime _lastUpdateFreeSpace = DateTime.MinValue;
+        private DateTime _lastUpdateGraceLocation = DateTime.MinValue;
+
+        private static readonly Logger Log = LogManager.GetCurrentClassLogger();
+
+        private readonly Dictionary<string, List<DirectoryDetails>> _expiredDirs;
+
+        private readonly List<DriveToCheck> _drives;
+        private readonly int _gracePeriod;
+        private readonly string _gracePeriodHuman;
+        private readonly DirectoryInfo _graceLocation;
+
+        /// <summary>
+        /// Initialize the StorageStatus object from the given ServiceConfig.
+        /// </summary>
+        /// <param name="config">The service configuration object.</param>
+        public StorageStatus(ServiceConfig config) {
+            _drives = config.SpaceMonitoring;
+            _gracePeriod = config.GracePeriod;
+            _gracePeriodHuman = config.HumanGracePeriod;
+            _graceLocation = new DirectoryInfo(config.DonePath);
+            _expiredDirs = new Dictionary<string, List<DirectoryDetails>>();
+            Log.Debug("StorageStatus initialization complete, updating status...");
+            Update(true);
+        }
+
+        /// <summary>
+        /// Number of expired directories in the grace location.
+        /// </summary>
+        public int ExpiredDirsCount {
+            get {
+                UpdateGraceLocation();
+                return _expiredDirs.Count;
+            }
+        }
+
+        /// <summary>
+        /// Check if free space on all configured drives is above their threshold.
+        /// </summary>
+        /// <returns>False if any of the drives is below its threshold, true otherwise.</returns>
+        public bool AllDrivesAboveThreshold() {
+            UpdateFreeSpace();
+            foreach (var drive in _drives) {
+                if (drive.DiskSpaceLow()) {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Get a dictionary of expired directories from the grace location.
+        /// </summary>
+        public Dictionary<string, List<DirectoryDetails>> ExpiredDirs {
+            get {
+                UpdateGraceLocation();
+                return _expiredDirs;
+            }
+        }
+
+        /// <summary>
+        /// Human-friendly summary of expired directories in the grace location.
+        /// </summary>
+        /// <returns>A human-readable (i.e. formatted) string with details on the grace location
+        /// and all expired directories, grouped by the topmost level (i.e. user dirs).</returns>
+        public string GraceLocationSummary() {
+            UpdateGraceLocation();
+            var summary = "------ Grace location status, " +
+                          $"threshold: {_gracePeriod} days ({_gracePeriodHuman}) ------\n\n" +
+                          $" - location: [{_graceLocation}]\n";
+
+            if (_expiredDirs.Count == 0)
+                return summary + " -- NO EXPIRED folders in grace location! --";
+
+            foreach (var dir in _expiredDirs.Keys) {
+                summary += "\n - directory '" + dir + "'\n";
+                foreach (var subdir in _expiredDirs[dir]) {
+                    summary += $"    - {subdir.Dir.Name} " +
+                               $"[age: {subdir.HumanAgeFromName}, " +
+                               $"size: {subdir.HumanSize}]\n";
+                }
+            }
+
+            return summary;
+        }
+
+        /// <summary>
+        /// Human-friendly summary of free disk space on all configured drives.
+        /// </summary>
+        /// <param name="onlyLowSpace"></param>
+        /// <returns>A human-readable (i.e. formatted) string with details on the free space on all
+        /// configured drives. If <paramref name="onlyLowSpace"/> is set to "true", space will only
+        /// be reported for drives that are below their corresponding threshold.</returns>
+        public string SpaceSummary(bool onlyLowSpace = false) {
+            UpdateFreeSpace();
+            var summary = "------ Storage space status ------\n\n";
+            foreach (var drive in _drives) {
+                var msg = $" - drive [{drive.DriveName}] " +
+                          $"free space: {Conv.BytesToString(drive.FreeSpace)} " +
+                          $"(threshold: {Conv.GigabytesToString(drive.SpaceThreshold)})\n";
+
+                if (onlyLowSpace && !drive.DiskSpaceLow()) {
+                    Log.Trace(msg);
+                    continue;
+                }
+
+                summary += msg;
+            }
+
+            return summary;
+        }
+
+        /// <summary>
+        /// Update the storage status of free drive space if it's older than its threshold.
+        /// </summary>
+        /// <param name="force">Update, independently of the last update timestamp.</param>
+        public void UpdateFreeSpace(bool force = false) {
+            if (force)
+                _lastUpdateFreeSpace = DateTime.MinValue;
+
+            if (TimeUtils.SecondsSince(_lastUpdateFreeSpace) < UpdateDelta)
+                return;
+
+            Log.Trace("Updating storage status: checking free disk space...");
+            foreach (var drive in _drives) {
+                try {
+                    drive.FreeSpace = new DriveInfo(drive.DriveName).TotalFreeSpace;
+                }
+                catch (Exception ex) {
+                    // log this as an error which then also gets sent via email (if configured) and
+                    // let the rate-limiter take care of not flooding the admin with mails:
+                    Log.Error("Error in GetFreeDriveSpace({0}): {1}", drive.DriveName, ex.Message);
+                }
+            }
+
+            _lastUpdateFreeSpace = DateTime.Now;
+        }
+
+        /// <summary>
+        /// Update the storage status of the grace location if it's older than its threshold.
+        /// </summary>
+        /// <param name="force">Update, independently of the last update timestamp.</param>
+        public void UpdateGraceLocation(bool force = false) {
+            if (force)
+                _lastUpdateGraceLocation = DateTime.MinValue;
+
+            if (TimeUtils.SecondsSince(_lastUpdateGraceLocation) < UpdateDelta)
+                return;
+
+            Log.Debug("Updating storage status: checking grace location...");
+            _expiredDirs.Clear();
+            foreach (var userdir in _graceLocation.GetDirectories()) {
+                Log.Trace("Scanning directory [{0}]", userdir.Name);
+                var expired = new List<DirectoryDetails>();
+                foreach (var subdir in userdir.GetDirectories()) {
+                    var dirDetails = new DirectoryDetails(subdir);
+                    Log.Trace("Checking directory [{0}]: {1}",
+                        dirDetails.Dir.Name, dirDetails.HumanAgeFromName);
+                    if (dirDetails.AgeFromName < _gracePeriod)
+                        continue;
+
+                    Log.Trace("Found expired directory [{0}]", dirDetails.Dir.Name);
+                    expired.Add(dirDetails);
+                }
+                Log.Trace("Found {0} expired dirs.", expired.Count);
+                if (expired.Count > 0)
+                    _expiredDirs.Add(userdir.Name, expired);
+            }
+            _lastUpdateGraceLocation = DateTime.Now;
+
+            if (_expiredDirs.Count > 0) {
+                Log.Debug("Updated storage status: {0} expired directories in grace location.",
+                    _expiredDirs.Count);
+            }
+        }
+
+        /// <summary>
+        /// Update the current storage status in case the last update is already older than the
+        /// configured threshold <see cref="_lastUpdateFreeSpace"/>.
+        /// </summary>
+        /// <param name="force">Update, independently of the last update timestamp.</param>
+        public void Update(bool force = false) {
+            try {
+                UpdateFreeSpace(force);
+                UpdateGraceLocation(force);
+            }
+            catch (Exception ex) {
+                Log.Error("Updating storage status failed: {0}", ex.Message);
+                throw;
+            }
+        }
+
+        /// <summary>
+        /// Create an overall storage summary (free space and grace location).
+        /// </summary>
+        /// <returns>Human-readable string with details on free space + grace location.</returns>
+        public string Summary() {
+            return $"{SpaceSummary()}\n{GraceLocationSummary()}";
+        }
+    }
+}

ATxCommon/SystemChecks.cs

@@ -4,6 +4,7 @@ using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Management;
+using ATxCommon.Serializables;
 using NLog;
 
 namespace ATxCommon
@@ -55,7 +56,7 @@ namespace ATxCommon
         /// Check all configured disks for their free space and generate a
         /// summary with details to be used in a notification message.
         /// </summary>
-        public static string CheckFreeDiskSpace(List<Serializables.DriveToCheck> drives) {
+        public static string CheckFreeDiskSpace(List<DriveToCheck> drives) {
             var msg = "";
             foreach (var driveToCheck in drives) {
                 var freeSpace = GetFreeDriveSpace(driveToCheck.DriveName);
@@ -123,7 +124,7 @@ namespace ATxCommon
         /// log, enclosed by square brackets (e.g. [explorer]). If "longFormat" is set to true,
         /// each process name will be printed on a separate line, followed by the title of the
         /// corresponding main window (if existing).</param>
-        public static void LogRunningProcesses(bool longFormat=false) {
+        public static void LogRunningProcesses(bool longFormat = false) {
             if (!Log.IsDebugEnabled)
                 return;
 
@@ -149,5 +150,33 @@ namespace ATxCommon
                 Log.Debug("Currently running processes: {0}", procs.Substring(2));
             }
         }
+
+        /// <summary>
+        /// Generate an overall system health report with free space, grace location status, etc.
+        /// </summary>
+        /// <param name="storage">StorageStatus object used for space and grace reports.</param>
+        /// <returns>A multi-line string containing the details assembled in the report. These
+        /// comprise system uptime, free RAM, free storage space and current grace location status.
+        /// </returns>
+        public static string HealthReport(StorageStatus storage) {
+            var report = "------ System health report ------\n\n" +
+                         $" - hostname: {Environment.MachineName}\n" +
+                         $" - uptime: {TimeUtils.SecondsToHuman(Uptime(), false)}\n" +
+                         $" - free system memory: {GetFreeMemory()} MB" + "\n\n";
+
+            report += storage.Summary();
+
+            return report;
+        }
+
+        /// <summary>
+        /// Get the current system uptime in seconds. Note that this will miss all times where the
+        /// system had been suspended / hibernated, as it is based on the OS's ticks counter.
+        /// </summary>
+        /// <returns>The time since the last system boot in seconds.</returns>
+        public static long Uptime() {
+            var ticks = Stopwatch.GetTimestamp();
+            return ticks / Stopwatch.Frequency;
+        }
     }
 }

ATxService/AutoTx.cs

@@ -69,6 +69,7 @@ namespace ATxService
         /// <summary>
         /// Counter on how many load monitoring properties are currently exceeding their limit(s).
         /// </summary>
+        // ReSharper disable once RedundantDefaultMemberInitializer
         private int _exceedingLoadLimit = 0;
 
         private DateTime _lastUserDirCheck = DateTime.MinValue;
@@ -100,6 +101,7 @@ namespace ATxService
 
         private ServiceConfig _config;
         private ServiceStatus _status;
+        private StorageStatus _storage;
 
         private static Timer _mainTimer;
 
@@ -124,6 +126,7 @@ namespace ATxService
 
             InitializePerformanceMonitors();
             InitializeDirectories();
+            SetupStorageStatus();
             StartupSummary();
 
             if (_config.DebugRoboSharp) {
@@ -405,28 +408,9 @@ namespace ATxService
                    "\n------ Loaded configuration settings ------\n" + _config.Summary();
 
 
-            msg += "\n------ Current system parameters ------\n" +
-                   "Hostname: " + Environment.MachineName + "\n" +
-                   "Free system memory: " + SystemChecks.GetFreeMemory() + " MB" + "\n";
-            foreach (var driveToCheck in _config.SpaceMonitoring) {
-                msg += "Free space on drive '" + driveToCheck.DriveName + "': " +
-                       Conv.BytesToString(SystemChecks.GetFreeDriveSpace(driveToCheck.DriveName)) + "\n";
-            }
-
-
-            msg += "\n------ Grace location status, threshold: " + _config.GracePeriod + " days " +
-                "(" + TimeUtils.DaysToHuman(_config.GracePeriod) + ") ------\n";
-            try {
-                var tmp = SendGraceLocationSummary(_config.GracePeriod);
-                if (string.IsNullOrEmpty(tmp)) {
-                    msg += " -- NO EXPIRED folders in grace location! --\n";
-                } else {
-                    msg += tmp;
-                }
-            }
-            catch (Exception ex) {
-                Log.Error("GraceLocationSummary() failed: {0}", ex.Message);
-            }
+            var health = SystemChecks.HealthReport(_storage);
+            SendHealthReport(health);
+            msg += "\n" + health;
 
             Log.Debug(msg);
             
@@ -643,7 +627,7 @@ namespace ATxService
             // throw new Exception("just a test exception from RunMainTasks");
 
             // mandatory tasks, run on every call:
-            SendLowSpaceMail(SystemChecks.CheckFreeDiskSpace(_config.SpaceMonitoring));
+            SendLowSpaceMail();
             UpdateServiceState();
             _status.SerializeHeartbeat();
 
@@ -944,9 +928,8 @@ namespace ATxService
                     sourceDirectory.Delete();
                     if (sourceDirectory.Parent != null)
                         sourceDirectory.Parent.Delete();
-                    // check age and size of existing folders in the grace location after
-                    // a transfer has completed, trigger a notification if necessary:
-                    Log.Debug(SendGraceLocationSummary(_config.GracePeriod));
+                    // check grace location and trigger a notification if necessary:
+                    SendGraceLocationSummary();
                     return;
                 }
                 errMsg = "unable to move " + sourceDirectory.FullName;
@@ -970,6 +953,13 @@ namespace ATxService
             _lastUserDirCheck = FsUtils.CreateIncomingDirectories(
                 _config.DestinationDirectory, _config.TmpTransferDir, _config.IncomingPath);
         }
+
+        /// <summary>
+        /// Set up the StorageStatus object using the current configuration.
+        /// </summary>
+        private void SetupStorageStatus() {
+            _storage = new StorageStatus(_config);
+        }
         
         #endregion
 

ATxService/Email.cs

@@ -20,7 +20,8 @@ namespace ATxService
             subject = $"{_config.EmailPrefix}{ServiceName} - {subject} - {_config.HostAlias}";
             body += $"\n\n--\n[{_versionSummary}]\n";
             if (string.IsNullOrEmpty(_config.SmtpHost)) {
-                Log.Debug("SendEmail: {0}\n{1}", subject, body);
+                Log.Debug("SendEmail: config option <SmtpHost> is unset, not sending mail - " +
+                          "content shown below.\n[Subject] {0}\n[Body] {1}", subject, body);
                 return;
             }
             if (!recipient.Contains(@"@")) {
@@ -77,16 +78,17 @@ namespace ATxService
         /// </summary>
         /// <param name="body">The email text.</param>
         /// <param name="subject">Optional subject for the email.</param>
-        private void SendAdminEmail(string body, string subject = "") {
+        /// <returns>True in case an email was sent, false otherwise.</returns>
+        private bool SendAdminEmail(string body, string subject = "") {
             if (_config.SendAdminNotification == false)
-                return;
+                return false;
 
             var delta = TimeUtils.MinutesSince(_status.LastAdminNotification);
             if (delta < _config.AdminNotificationDelta) {
                 Log.Warn("Suppressed admin email, interval too short ({0} vs. {1}):\n\n{2}\n{3}",
                     TimeUtils.MinutesToHuman(delta),
                     TimeUtils.MinutesToHuman(_config.AdminNotificationDelta), subject, body);
-                return;
+                return false;
             }
 
             if (string.IsNullOrWhiteSpace(subject))
@@ -96,16 +98,19 @@ namespace ATxService
             Log.Debug("Sending an admin notification email.");
             SendEmail(_config.AdminEmailAdress, subject, body);
             _status.LastAdminNotification = DateTime.Now;
-
+            Log.Debug("{0} sent to AdminEmailAdress.", subject);
+            return true;
         }
 
         /// <summary>
-        /// Send a notification about low drive space to the admin.
+        /// Send a notification about low drive space to the admin if the time since the last
+        /// notification has elapsed the configured delta. The report will also contain a summary
+        /// of the grace location status. If none of the drives are low on space nothing will be
+        /// done (i.e. only a generic trace-level message will be logged).
         /// </summary>
-        /// <param name="spaceDetails">String describing the drives being low on space.</param>
-        private void SendLowSpaceMail(string spaceDetails) {
-            if (string.IsNullOrWhiteSpace(spaceDetails)) {
-                Log.Trace("SendLowSpaceMail(): spaceDetails emtpy!");
+        private void SendLowSpaceMail() {
+            if (_storage.AllDrivesAboveThreshold()) {
+                Log.Trace("Free space on all drives above threshold.");
                 return;
             }
 
@@ -116,25 +121,24 @@ namespace ATxService
                 return;
             }
 
-            // reaching this point means a notification will be sent to the admin, and in that
-            // case it makes sense to also include details about the grace location:
-            var graceReport = FsUtils.GraceLocationSummary(
-                new DirectoryInfo(_config.DonePath), _config.GracePeriod);
+            // reaching this point means a notification will be sent, so now we can ask for the
+            // full storage status report:
+            var report = _storage.Summary();
 
-
-            Log.Warn("WARNING: {0}", spaceDetails);
+            Log.Warn("WARNING: {0}", report);
             _status.LastStorageNotification = DateTime.Now;
 
             var substitutions = new List<Tuple<string, string>> {
                 Tuple.Create("SERVICE_NAME", ServiceName),
                 Tuple.Create("HOST_ALIAS", _config.HostAlias),
                 Tuple.Create("HOST_NAME", Environment.MachineName),
-                Tuple.Create("LOW_SPACE_DRIVES", spaceDetails)
+                Tuple.Create("LOW_SPACE_DRIVES", report)
             };
             try {
                 var body = LoadMailTemplate("DiskSpace-Low.txt", substitutions);
-                if (graceReport.Length > 0)
-                    body += $"\n\n--\n{graceReport}";
+                // explicitly use SendEmail() instead of SendAdminEmail() here to circumvent the
+                // additional checks done in the latter one and make sure the low space email is
+                // sent out independently of that:
                 SendEmail(_config.AdminEmailAdress, "low disk space", body);
             }
             catch (Exception ex) {
@@ -205,27 +209,48 @@ namespace ATxService
 
         /// <summary>
         /// Send a report on expired folders in the grace location if applicable.
-        /// 
-        /// Create a summary of expired folders and send it to the admin address
-        /// if the configured GraceNotificationDelta has passed since the last email.
+        ///
+        /// A summary of expired folders is created and sent to the admin address if the configured
+        /// GraceNotificationDelta has passed since the last email. The report will also contain a
+        /// summary of free disk space for all configured drives.
         /// </summary>
-        /// <param name="threshold">The number of days used as expiration threshold.</param>
-        /// <returns>The summary report, empty if no expired folders exist.</returns>
-        private string SendGraceLocationSummary(int threshold) {
-            var report = FsUtils.GraceLocationSummary(
-                new DirectoryInfo(_config.DonePath), threshold);
-            if (string.IsNullOrEmpty(report))
-                return "";
-
-            report += $"\n{SystemChecks.CheckFreeDiskSpace(_config.SpaceMonitoring)}" +
-                      "\nTime since last grace notification: " +
-                      $"{TimeUtils.HumanSince(_status.LastGraceNotification)}\n";
-            if (TimeUtils.MinutesSince(_status.LastGraceNotification) < _config.GraceNotificationDelta)
-                return report;
+        /// <returns>True if a report was sent, false otherwise (includes situations where there
+        /// are expired directories but the report has not been sent via email as the grace
+        /// notification delta hasn't expired yet, the report will still be logged then).</returns>
+        private bool SendGraceLocationSummary() {
+            if (_storage.ExpiredDirsCount == 0)
+                return false;
+
+            var report = _storage.Summary() +
+                         "\nTime since last grace notification: " +
+                         $"{TimeUtils.HumanSince(_status.LastGraceNotification)}\n";
+
+            if (TimeUtils.MinutesSince(_status.LastGraceNotification) < _config.GraceNotificationDelta) {
+                Log.Debug(report);
+                return false;
+            }
 
             _status.LastGraceNotification = DateTime.Now;
-            SendAdminEmail(report, "grace location summary");
-            return report + "\nNotification sent to AdminEmailAdress.\n";
+            return SendAdminEmail(report, "grace location summary");
+        }
+
+        /// <summary>
+        /// Send a system health report if enough time has elapsed since the previous one.
+        /// </summary>
+        /// <param name="report">The health report.</param>
+        /// <returns>True in case the report was sent, false otherwise.</returns>
+        private bool SendHealthReport(string report) {
+            var elapsedHuman = TimeUtils.HumanSince(_status.LastStartupNotification);
+
+            if (TimeUtils.MinutesSince(_status.LastStartupNotification) < _config.StartupNotificationDelta) {
+                Log.Trace("Not sending system health report now, last one has been sent {0}",
+                    elapsedHuman);
+                return false;
+            }
+
+            report += $"\nPrevious system health report notification was sent {elapsedHuman}.\n";
+            _status.LastStartupNotification = DateTime.Now;
+            return SendAdminEmail(report, "system health report");
         }
     }
 }
\ No newline at end of file

Resources/conf/config.common.xml

@@ -118,6 +118,11 @@
          in case one of the drives is below the threshold (in minutes) -->
     <StorageNotificationDelta>720</StorageNotificationDelta>
 
+    <!-- StartupNotificationDelta: minimum time (in minutes) between two service
+         startup system health notification emails (default: 2880 (2d). Set to 0 
+         to disable startup health reports. -->
+    <StartupNotificationDelta>2880</StartupNotificationDelta>
+
     <!--  OPTIONAL NOTIFICATION / EMAIL SETTINGS  -->
 
 </ServiceConfig>