diff --git a/ATxCommon/ATxCommon.csproj b/ATxCommon/ATxCommon.csproj
index 7e3dffdf3aaf249295a18f42c9e956c71bd303ea..40c19c2f0fd96c3c5e4f53e0b9865daa7618a329 100644
--- a/ATxCommon/ATxCommon.csproj
+++ b/ATxCommon/ATxCommon.csproj
@@ -46,6 +46,7 @@
<Compile Include="ActiveDirectory.cs" />
<Compile Include="BuildDetails.cs" />
<Compile Include="Conv.cs" />
+ <Compile Include="DirectoryDetails.cs" />
<Compile Include="FsUtils.cs" />
<Compile Include="Monitoring\Cpu.cs" />
<Compile Include="Monitoring\MonitorBase.cs" />
@@ -55,6 +56,7 @@
<Compile Include="Serializables\ServiceConfig.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Serializables\ServiceStatus.cs" />
+ <Compile Include="StorageStatus.cs" />
<Compile Include="SystemChecks.cs" />
<Compile Include="TimeUtils.cs" />
</ItemGroup>
diff --git a/ATxCommon/DirectoryDetails.cs b/ATxCommon/DirectoryDetails.cs
new file mode 100644
index 0000000000000000000000000000000000000000..df0f58223342056f5e9c40cee9070989815afc96
--- /dev/null
+++ b/ATxCommon/DirectoryDetails.cs
@@ -0,0 +1,69 @@
+using System;
+using System.IO;
+using NLog;
+
+namespace ATxCommon
+{
+ public class DirectoryDetails
+ {
+ private static readonly Logger Log = LogManager.GetCurrentClassLogger();
+
+ private long _size = -1;
+ private int _age = -1;
+
+ /// <summary>
+ /// Initialize the DirectoryDetails object from the given DirectoryInfo.
+ /// </summary>
+ /// <param name="dirInfo">The DirectoryInfo object of the directory to investigate.</param>
+ public DirectoryDetails(DirectoryInfo dirInfo) {
+ Dir = dirInfo;
+ }
+
+ /// <summary>
+ /// The underlying DirectoryInfo object.
+ /// </summary>
+ public DirectoryInfo Dir { get; }
+
+ /// <summary>
+ /// The age defined by the directory's NAME in days.
+ /// </summary>
+ public int AgeFromName {
+ get {
+ if (_age < 0)
+ _age = FsUtils.DirNameToAge(Dir, DateTime.Now);
+
+ return _age;
+ }
+ }
+
+ /// <summary>
+ /// The full size of the directory tree in bytes.
+ /// </summary>
+ public long Size {
+ get {
+ if (_size >= 0)
+ return _size;
+
+ try {
+ _size = FsUtils.GetDirectorySize(Dir.FullName);
+ }
+ catch (Exception ex) {
+ Log.Error("ERROR getting directory size of [{0}]: {1}",
+ Dir.FullName, ex.Message);
+ }
+
+ return _size;
+ }
+ }
+
+ /// <summary>
+ /// Human friendly description of the directory's age, derived from its NAME.
+ /// </summary>
+ public string HumanAgeFromName => TimeUtils.DaysToHuman(AgeFromName, false);
+
+ /// <summary>
+ /// Human friendly description of the directory tree size (recursively).
+ /// </summary>
+ public string HumanSize => Conv.BytesToString(Size);
+ }
+}
diff --git a/ATxCommon/FsUtils.cs b/ATxCommon/FsUtils.cs
index 1c530d88bcbc1fa55624dd97f0623b41cd932dff..fd1c10165415ba64f811f891ba2535036f3c9d3b 100644
--- a/ATxCommon/FsUtils.cs
+++ b/ATxCommon/FsUtils.cs
@@ -1,5 +1,4 @@
using System;
-using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
@@ -52,11 +51,11 @@ namespace ATxCommon
Log.Error("ERROR: CheckForDirectory() parameter must not be empty!");
return false;
}
- return FsUtils.CreateNewDirectory(path, false) == path;
+ return CreateNewDirectory(path, false) == path;
}
/// <summary>
- /// Recursively sum up size of all files under a given path.
+ /// Recursively sum up size (in bytes) of all files under a given path.
/// </summary>
/// <param name="path">Full path of the directory.</param>
/// <returns>The total size in bytes.</returns>
@@ -88,67 +87,6 @@ namespace ATxCommon
return (baseTime - dirTimestamp).Days;
}
- /// <summary>
- /// Assemble a dictionary with information about expired directories.
- /// </summary>
- /// <param name="baseDir">The base directory to scan for subdirectories.</param>
- /// <param name="thresh">The number of days used as expiration threshold.</param>
- /// <returns>A dictionary having usernames as keys (of those users that actually do have
- /// expired directories), where the values are lists of tuples with the DirInfo objects,
- /// size (in bytes) and age (in days) of the expired directories.</returns>
- public static Dictionary<string, List<Tuple<DirectoryInfo, long, int>>>
- ExpiredDirs(DirectoryInfo baseDir,int thresh) {
-
- var collection = new Dictionary<string, List<Tuple<DirectoryInfo, long, int>>>();
- var now = DateTime.Now;
- foreach (var userdir in baseDir.GetDirectories()) {
- var expired = new List<Tuple<DirectoryInfo, long, int>>();
- foreach (var subdir in userdir.GetDirectories()) {
- var age = DirNameToAge(subdir, now);
- if (age < thresh)
- continue;
- long size = -1;
- try {
- size = GetDirectorySize(subdir.FullName);
- }
- catch (Exception ex) {
- Log.Error("ERROR getting directory size of [{0}]: {1}",
- subdir.FullName, ex.Message);
- }
- expired.Add(new Tuple<DirectoryInfo, long, int>(subdir, size, age));
- }
- if (expired.Count > 0)
- collection.Add(userdir.Name, expired);
- }
- return collection;
- }
-
- /// <summary>
- /// Generate a report on expired folders in the grace location.
- ///
- /// Check all user-directories in the grace location for subdirectories whose timestamp
- /// (the directory name) exceeds the configured grace period and generate a summary
- /// containing the age and size of those directories.
- /// </summary>
- /// <param name="graceLocation">The location to scan for expired folders.</param>
- /// <param name="threshold">The number of days used as expiration threshold.</param>
- public static string GraceLocationSummary(DirectoryInfo graceLocation, int threshold) {
- var expired = ExpiredDirs(graceLocation, threshold);
- var report = "";
- foreach (var userdir in expired.Keys) {
- report += "\n - user '" + userdir + "'\n";
- foreach (var subdir in expired[userdir]) {
- report += string.Format(" - {0} [age: {2}, size: {1}]\n",
- subdir.Item1, Conv.BytesToString(subdir.Item2),
- TimeUtils.DaysToHuman(subdir.Item3));
- }
- }
- if (string.IsNullOrEmpty(report))
- return "";
-
- return "Expired folders in grace location:\n" + report;
- }
-
/// <summary>
/// Check if a given directory is empty. If a marker file is set in the config a
/// file with this name will be created inside the given directory and will be
diff --git a/ATxCommon/Serializables/DriveToCheck.cs b/ATxCommon/Serializables/DriveToCheck.cs
index 6411bdbc36d8354cdcca054cba9a69872ccc6ef3..632f9948fd069950a4f0ec88abfed114d54fad4a 100644
--- a/ATxCommon/Serializables/DriveToCheck.cs
+++ b/ATxCommon/Serializables/DriveToCheck.cs
@@ -1,4 +1,6 @@
-namespace ATxCommon.Serializables
+using System.Xml.Serialization;
+
+namespace ATxCommon.Serializables
{
/// <summary>
/// Helper class for the nested SpaceMonitoring sections.
@@ -14,5 +16,19 @@
/// Limit (in GB) of free space, lower values will trigger a notification.
/// </summary>
public long SpaceThreshold { get; set; }
+
+ /// <summary>
+ /// Free space of a drive in bytes, set to -1 if unknown or check resulted in an error.
+ /// </summary>
+ [XmlIgnore]
+ public long FreeSpace { get; set; } = -1;
+
+ /// <summary>
+ /// Check if this drive's free space is below its threshold.
+ /// </summary>
+ /// <returns>True if free space is below the threshold, false otherwise.</returns>
+ public bool DiskSpaceLow() {
+ return FreeSpace < SpaceThreshold * Conv.GigaBytes;
+ }
}
}
\ No newline at end of file
diff --git a/ATxCommon/Serializables/ServiceConfig.cs b/ATxCommon/Serializables/ServiceConfig.cs
index b6f1ead758e8fb322a2678cb0801e5d3d5ce71a4..82aace483a7282c77ba5fb7f5948064974d36adc 100644
--- a/ATxCommon/Serializables/ServiceConfig.cs
+++ b/ATxCommon/Serializables/ServiceConfig.cs
@@ -224,6 +224,12 @@ namespace ATxCommon.Serializables
/// Minimum time in minutes between two low-space notifications. Default: 720 (12h).
/// </summary>
public int StorageNotificationDelta { get; set; } = 720;
+
+ /// <summary>
+ /// Minimum time in minutes between two startup system health notifications.
+ /// Default: 2880 (2d).
+ /// </summary>
+ public int StartupNotificationDelta { get; set; } = 2880;
#endregion
@@ -291,6 +297,11 @@ namespace ATxCommon.Serializables
[XmlIgnore]
public static string ValidatorWarnings { get; set; }
+ /// <summary>
+ /// Convenience property converting the grace period to a human-friendly format.
+ /// </summary>
+ [XmlIgnore]
+ public string HumanGracePeriod => TimeUtils.DaysToHuman(GracePeriod, false);
#endregion
@@ -472,6 +483,7 @@ namespace ATxCommon.Serializables
WarnOnHighValue(c.AdminNotificationDelta, nameof(c.AdminNotificationDelta), 1440);
WarnOnHighValue(c.GraceNotificationDelta, nameof(c.GraceNotificationDelta), 10080);
WarnOnHighValue(c.StorageNotificationDelta, nameof(c.StorageNotificationDelta), 10080);
+ WarnOnHighValue(c.StartupNotificationDelta, nameof(c.StartupNotificationDelta), 40320);
WarnOnHighValue(c.GracePeriod, nameof(c.GracePeriod), 100);
if (!c.DestinationDirectory.StartsWith(@"\\"))
@@ -557,6 +569,8 @@ namespace ATxCommon.Serializables
TimeUtils.MinutesToHuman(GraceNotificationDelta, false) + ")\n" +
$"StorageNotificationDelta: {StorageNotificationDelta} min (" +
TimeUtils.MinutesToHuman(StorageNotificationDelta, false) + ")\n" +
+ $"StartupNotificationDelta: {StartupNotificationDelta} min (" +
+ TimeUtils.MinutesToHuman(StartupNotificationDelta, false) + ")\n" +
"";
}
return msg;
diff --git a/ATxCommon/Serializables/ServiceStatus.cs b/ATxCommon/Serializables/ServiceStatus.cs
index 0d43860be4eeb14d8883fa0a6784fa471a43d64e..2c349162ce8c1b80e444400385cea613c4ca728a 100644
--- a/ATxCommon/Serializables/ServiceStatus.cs
+++ b/ATxCommon/Serializables/ServiceStatus.cs
@@ -19,6 +19,7 @@ namespace ATxCommon.Serializables
private DateTime _lastStorageNotification;
private DateTime _lastAdminNotification;
private DateTime _lastGraceNotification;
+ private DateTime _lastStartupNotification;
private string _statusDescription;
private string _currentTransferSrc;
@@ -159,6 +160,18 @@ namespace ATxCommon.Serializables
}
}
+ /// <summary>
+ /// Timestamp indicating when the last startup system health notification has been sent.
+ /// </summary>
+ [XmlElement("LastStartupNotification", DataType = "dateTime")]
+ public DateTime LastStartupNotification {
+ get => _lastStartupNotification;
+ set {
+ _lastStartupNotification = value;
+ Serialize();
+ }
+ }
+
/// <summary>
/// String indicating why the service is currently suspended (empty if not suspended).
/// </summary>
@@ -374,7 +387,10 @@ namespace ATxCommon.Serializables
$"LastAdminNotification: {LastAdminNotification:yyyy-MM-dd HH:mm:ss}" +
$" ({TimeUtils.HumanSince(LastAdminNotification)})\n" +
$"LastGraceNotification: {LastGraceNotification:yyyy-MM-dd HH:mm:ss}" +
- $" ({TimeUtils.HumanSince(LastGraceNotification)})\n";
+ $" ({TimeUtils.HumanSince(LastGraceNotification)})\n" +
+ $"LastStartupNotification: {LastStartupNotification:yyyy-MM-dd HH:mm:ss}" +
+ $" ({TimeUtils.HumanSince(LastStartupNotification)})\n" +
+ "";
}
#endregion validate and report
diff --git a/ATxCommon/StorageStatus.cs b/ATxCommon/StorageStatus.cs
new file mode 100644
index 0000000000000000000000000000000000000000..4a3078cdd9e192c8f2e5775434c8544b717d6eb0
--- /dev/null
+++ b/ATxCommon/StorageStatus.cs
@@ -0,0 +1,218 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using ATxCommon.Serializables;
+using NLog;
+
+namespace ATxCommon
+{
+ public class StorageStatus
+ {
+ /// <summary>
+ /// By default the statuses will only be updated if more than UpdateDelta seconds have
+ /// elapsed since the last update to prevent too many updates causing high system load.
+ /// </summary>
+ public int UpdateDelta = 20;
+
+ private DateTime _lastUpdateFreeSpace = DateTime.MinValue;
+ private DateTime _lastUpdateGraceLocation = DateTime.MinValue;
+
+ private static readonly Logger Log = LogManager.GetCurrentClassLogger();
+
+ private readonly Dictionary<string, List<DirectoryDetails>> _expiredDirs;
+
+ private readonly List<DriveToCheck> _drives;
+ private readonly int _gracePeriod;
+ private readonly string _gracePeriodHuman;
+ private readonly DirectoryInfo _graceLocation;
+
+ /// <summary>
+ /// Initialize the StorageStatus object from the given ServiceConfig.
+ /// </summary>
+ /// <param name="config">The service configuration object.</param>
+ public StorageStatus(ServiceConfig config) {
+ _drives = config.SpaceMonitoring;
+ _gracePeriod = config.GracePeriod;
+ _gracePeriodHuman = config.HumanGracePeriod;
+ _graceLocation = new DirectoryInfo(config.DonePath);
+ _expiredDirs = new Dictionary<string, List<DirectoryDetails>>();
+ Log.Debug("StorageStatus initialization complete, updating status...");
+ Update(true);
+ }
+
+ /// <summary>
+ /// Number of expired directories in the grace location.
+ /// </summary>
+ public int ExpiredDirsCount {
+ get {
+ UpdateGraceLocation();
+ return _expiredDirs.Count;
+ }
+ }
+
+ /// <summary>
+ /// Check if free space on all configured drives is above their threshold.
+ /// </summary>
+ /// <returns>False if any of the drives is below its threshold, true otherwise.</returns>
+ public bool AllDrivesAboveThreshold() {
+ UpdateFreeSpace();
+ foreach (var drive in _drives) {
+ if (drive.DiskSpaceLow()) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /// <summary>
+ /// Get a dictionary of expired directories from the grace location.
+ /// </summary>
+ public Dictionary<string, List<DirectoryDetails>> ExpiredDirs {
+ get {
+ UpdateGraceLocation();
+ return _expiredDirs;
+ }
+ }
+
+ /// <summary>
+ /// Human-friendly summary of expired directories in the grace location.
+ /// </summary>
+ /// <returns>A human-readable (i.e. formatted) string with details on the grace location
+ /// and all expired directories, grouped by the topmost level (i.e. user dirs).</returns>
+ public string GraceLocationSummary() {
+ UpdateGraceLocation();
+ var summary = "------ Grace location status, " +
+ $"threshold: {_gracePeriod} days ({_gracePeriodHuman}) ------\n\n" +
+ $" - location: [{_graceLocation}]\n";
+
+ if (_expiredDirs.Count == 0)
+ return summary + " -- NO EXPIRED folders in grace location! --";
+
+ foreach (var dir in _expiredDirs.Keys) {
+ summary += "\n - directory '" + dir + "'\n";
+ foreach (var subdir in _expiredDirs[dir]) {
+ summary += $" - {subdir.Dir.Name} " +
+ $"[age: {subdir.HumanAgeFromName}, " +
+ $"size: {subdir.HumanSize}]\n";
+ }
+ }
+
+ return summary;
+ }
+
+ /// <summary>
+ /// Human-friendly summary of free disk space on all configured drives.
+ /// </summary>
+ /// <param name="onlyLowSpace"></param>
+ /// <returns>A human-readable (i.e. formatted) string with details on the free space on all
+ /// configured drives. If <paramref name="onlyLowSpace"/> is set to "true", space will only
+ /// be reported for drives that are below their corresponding threshold.</returns>
+ public string SpaceSummary(bool onlyLowSpace = false) {
+ UpdateFreeSpace();
+ var summary = "------ Storage space status ------\n\n";
+ foreach (var drive in _drives) {
+ var msg = $" - drive [{drive.DriveName}] " +
+ $"free space: {Conv.BytesToString(drive.FreeSpace)} " +
+ $"(threshold: {Conv.GigabytesToString(drive.SpaceThreshold)})\n";
+
+ if (onlyLowSpace && !drive.DiskSpaceLow()) {
+ Log.Trace(msg);
+ continue;
+ }
+
+ summary += msg;
+ }
+
+ return summary;
+ }
+
+ /// <summary>
+ /// Update the storage status of free drive space if it's older than its threshold.
+ /// </summary>
+ /// <param name="force">Update, independently of the last update timestamp.</param>
+ public void UpdateFreeSpace(bool force = false) {
+ if (force)
+ _lastUpdateFreeSpace = DateTime.MinValue;
+
+ if (TimeUtils.SecondsSince(_lastUpdateFreeSpace) < UpdateDelta)
+ return;
+
+ Log.Trace("Updating storage status: checking free disk space...");
+ foreach (var drive in _drives) {
+ try {
+ drive.FreeSpace = new DriveInfo(drive.DriveName).TotalFreeSpace;
+ }
+ catch (Exception ex) {
+ // log this as an error which then also gets sent via email (if configured) and
+ // let the rate-limiter take care of not flooding the admin with mails:
+ Log.Error("Error in GetFreeDriveSpace({0}): {1}", drive.DriveName, ex.Message);
+ }
+ }
+
+ _lastUpdateFreeSpace = DateTime.Now;
+ }
+
+ /// <summary>
+ /// Update the storage status of the grace location if it's older than its threshold.
+ /// </summary>
+ /// <param name="force">Update, independently of the last update timestamp.</param>
+ public void UpdateGraceLocation(bool force = false) {
+ if (force)
+ _lastUpdateGraceLocation = DateTime.MinValue;
+
+ if (TimeUtils.SecondsSince(_lastUpdateGraceLocation) < UpdateDelta)
+ return;
+
+ Log.Debug("Updating storage status: checking grace location...");
+ _expiredDirs.Clear();
+ foreach (var userdir in _graceLocation.GetDirectories()) {
+ Log.Trace("Scanning directory [{0}]", userdir.Name);
+ var expired = new List<DirectoryDetails>();
+ foreach (var subdir in userdir.GetDirectories()) {
+ var dirDetails = new DirectoryDetails(subdir);
+ Log.Trace("Checking directory [{0}]: {1}",
+ dirDetails.Dir.Name, dirDetails.HumanAgeFromName);
+ if (dirDetails.AgeFromName < _gracePeriod)
+ continue;
+
+ Log.Trace("Found expired directory [{0}]", dirDetails.Dir.Name);
+ expired.Add(dirDetails);
+ }
+ Log.Trace("Found {0} expired dirs.", expired.Count);
+ if (expired.Count > 0)
+ _expiredDirs.Add(userdir.Name, expired);
+ }
+ _lastUpdateGraceLocation = DateTime.Now;
+
+ if (_expiredDirs.Count > 0) {
+ Log.Debug("Updated storage status: {0} expired directories in grace location.",
+ _expiredDirs.Count);
+ }
+ }
+
+ /// <summary>
+ /// Update the current storage status in case the last update is already older than the
+ /// configured threshold <see cref="_lastUpdateFreeSpace"/>.
+ /// </summary>
+ /// <param name="force">Update, independently of the last update timestamp.</param>
+ public void Update(bool force = false) {
+ try {
+ UpdateFreeSpace(force);
+ UpdateGraceLocation(force);
+ }
+ catch (Exception ex) {
+ Log.Error("Updating storage status failed: {0}", ex.Message);
+ throw;
+ }
+ }
+
+ /// <summary>
+ /// Create an overall storage summary (free space and grace location).
+ /// </summary>
+ /// <returns>Human-readable string with details on free space + grace location.</returns>
+ public string Summary() {
+ return $"{SpaceSummary()}\n{GraceLocationSummary()}";
+ }
+ }
+}
diff --git a/ATxCommon/SystemChecks.cs b/ATxCommon/SystemChecks.cs
index 7f8231ba8a425a51ef2b755b32d48e6abcb401ff..bddd1ae331c73f2f705c86e606bd1c08d7540f3e 100644
--- a/ATxCommon/SystemChecks.cs
+++ b/ATxCommon/SystemChecks.cs
@@ -4,6 +4,7 @@ using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Management;
+using ATxCommon.Serializables;
using NLog;
namespace ATxCommon
@@ -55,7 +56,7 @@ namespace ATxCommon
/// Check all configured disks for their free space and generate a
/// summary with details to be used in a notification message.
/// </summary>
- public static string CheckFreeDiskSpace(List<Serializables.DriveToCheck> drives) {
+ public static string CheckFreeDiskSpace(List<DriveToCheck> drives) {
var msg = "";
foreach (var driveToCheck in drives) {
var freeSpace = GetFreeDriveSpace(driveToCheck.DriveName);
@@ -123,7 +124,7 @@ namespace ATxCommon
/// log, enclosed by square brackets (e.g. [explorer]). If "longFormat" is set to true,
/// each process name will be printed on a separate line, followed by the title of the
/// corresponding main window (if existing).</param>
- public static void LogRunningProcesses(bool longFormat=false) {
+ public static void LogRunningProcesses(bool longFormat = false) {
if (!Log.IsDebugEnabled)
return;
@@ -149,5 +150,33 @@ namespace ATxCommon
Log.Debug("Currently running processes: {0}", procs.Substring(2));
}
}
+
+ /// <summary>
+ /// Generate an overall system health report with free space, grace location status, etc.
+ /// </summary>
+ /// <param name="storage">StorageStatus object used for space and grace reports.</param>
+ /// <returns>A multi-line string containing the details assembled in the report. These
+ /// comprise system uptime, free RAM, free storage space and current grace location status.
+ /// </returns>
+ public static string HealthReport(StorageStatus storage) {
+ var report = "------ System health report ------\n\n" +
+ $" - hostname: {Environment.MachineName}\n" +
+ $" - uptime: {TimeUtils.SecondsToHuman(Uptime(), false)}\n" +
+ $" - free system memory: {GetFreeMemory()} MB" + "\n\n";
+
+ report += storage.Summary();
+
+ return report;
+ }
+
+ /// <summary>
+ /// Get the current system uptime in seconds. Note that this will miss all times where the
+ /// system had been suspended / hibernated, as it is based on the OS's ticks counter.
+ /// </summary>
+ /// <returns>The time since the last system boot in seconds.</returns>
+ public static long Uptime() {
+ var ticks = Stopwatch.GetTimestamp();
+ return ticks / Stopwatch.Frequency;
+ }
}
}
diff --git a/ATxService/AutoTx.cs b/ATxService/AutoTx.cs
index 172ac3f112695c3f21f12b6a727cb7b76b67bc2c..396854067f76244360a1368a388776bdf1b2851f 100644
--- a/ATxService/AutoTx.cs
+++ b/ATxService/AutoTx.cs
@@ -69,6 +69,7 @@ namespace ATxService
/// <summary>
/// Counter on how many load monitoring properties are currently exceeding their limit(s).
/// </summary>
+ // ReSharper disable once RedundantDefaultMemberInitializer
private int _exceedingLoadLimit = 0;
private DateTime _lastUserDirCheck = DateTime.MinValue;
@@ -100,6 +101,7 @@ namespace ATxService
private ServiceConfig _config;
private ServiceStatus _status;
+ private StorageStatus _storage;
private static Timer _mainTimer;
@@ -124,6 +126,7 @@ namespace ATxService
InitializePerformanceMonitors();
InitializeDirectories();
+ SetupStorageStatus();
StartupSummary();
if (_config.DebugRoboSharp) {
@@ -405,28 +408,9 @@ namespace ATxService
"\n------ Loaded configuration settings ------\n" + _config.Summary();
- msg += "\n------ Current system parameters ------\n" +
- "Hostname: " + Environment.MachineName + "\n" +
- "Free system memory: " + SystemChecks.GetFreeMemory() + " MB" + "\n";
- foreach (var driveToCheck in _config.SpaceMonitoring) {
- msg += "Free space on drive '" + driveToCheck.DriveName + "': " +
- Conv.BytesToString(SystemChecks.GetFreeDriveSpace(driveToCheck.DriveName)) + "\n";
- }
-
-
- msg += "\n------ Grace location status, threshold: " + _config.GracePeriod + " days " +
- "(" + TimeUtils.DaysToHuman(_config.GracePeriod) + ") ------\n";
- try {
- var tmp = SendGraceLocationSummary(_config.GracePeriod);
- if (string.IsNullOrEmpty(tmp)) {
- msg += " -- NO EXPIRED folders in grace location! --\n";
- } else {
- msg += tmp;
- }
- }
- catch (Exception ex) {
- Log.Error("GraceLocationSummary() failed: {0}", ex.Message);
- }
+ var health = SystemChecks.HealthReport(_storage);
+ SendHealthReport(health);
+ msg += "\n" + health;
Log.Debug(msg);
@@ -643,7 +627,7 @@ namespace ATxService
// throw new Exception("just a test exception from RunMainTasks");
// mandatory tasks, run on every call:
- SendLowSpaceMail(SystemChecks.CheckFreeDiskSpace(_config.SpaceMonitoring));
+ SendLowSpaceMail();
UpdateServiceState();
_status.SerializeHeartbeat();
@@ -944,9 +928,8 @@ namespace ATxService
sourceDirectory.Delete();
if (sourceDirectory.Parent != null)
sourceDirectory.Parent.Delete();
- // check age and size of existing folders in the grace location after
- // a transfer has completed, trigger a notification if necessary:
- Log.Debug(SendGraceLocationSummary(_config.GracePeriod));
+ // check grace location and trigger a notification if necessary:
+ SendGraceLocationSummary();
return;
}
errMsg = "unable to move " + sourceDirectory.FullName;
@@ -970,6 +953,13 @@ namespace ATxService
_lastUserDirCheck = FsUtils.CreateIncomingDirectories(
_config.DestinationDirectory, _config.TmpTransferDir, _config.IncomingPath);
}
+
+ /// <summary>
+ /// Set up the StorageStatus object using the current configuration.
+ /// </summary>
+ private void SetupStorageStatus() {
+ _storage = new StorageStatus(_config);
+ }
#endregion
diff --git a/ATxService/Email.cs b/ATxService/Email.cs
index dca17e4730f1e27a7271c5602574d60a40e28658..df59f298ecce6c1b020d5a27c7fc2fe49cf5f22d 100644
--- a/ATxService/Email.cs
+++ b/ATxService/Email.cs
@@ -20,7 +20,8 @@ namespace ATxService
subject = $"{_config.EmailPrefix}{ServiceName} - {subject} - {_config.HostAlias}";
body += $"\n\n--\n[{_versionSummary}]\n";
if (string.IsNullOrEmpty(_config.SmtpHost)) {
- Log.Debug("SendEmail: {0}\n{1}", subject, body);
+ Log.Debug("SendEmail: config option <SmtpHost> is unset, not sending mail - " +
+ "content shown below.\n[Subject] {0}\n[Body] {1}", subject, body);
return;
}
if (!recipient.Contains(@"@")) {
@@ -77,16 +78,17 @@ namespace ATxService
/// </summary>
/// <param name="body">The email text.</param>
/// <param name="subject">Optional subject for the email.</param>
- private void SendAdminEmail(string body, string subject = "") {
+ /// <returns>True in case an email was sent, false otherwise.</returns>
+ private bool SendAdminEmail(string body, string subject = "") {
if (_config.SendAdminNotification == false)
- return;
+ return false;
var delta = TimeUtils.MinutesSince(_status.LastAdminNotification);
if (delta < _config.AdminNotificationDelta) {
Log.Warn("Suppressed admin email, interval too short ({0} vs. {1}):\n\n{2}\n{3}",
TimeUtils.MinutesToHuman(delta),
TimeUtils.MinutesToHuman(_config.AdminNotificationDelta), subject, body);
- return;
+ return false;
}
if (string.IsNullOrWhiteSpace(subject))
@@ -96,16 +98,19 @@ namespace ATxService
Log.Debug("Sending an admin notification email.");
SendEmail(_config.AdminEmailAdress, subject, body);
_status.LastAdminNotification = DateTime.Now;
-
+ Log.Debug("{0} sent to AdminEmailAdress.", subject);
+ return true;
}
/// <summary>
- /// Send a notification about low drive space to the admin.
+ /// Send a notification about low drive space to the admin if the time since the last
+ /// notification has elapsed the configured delta. The report will also contain a summary
+ /// of the grace location status. If none of the drives are low on space nothing will be
+ /// done (i.e. only a generic trace-level message will be logged).
/// </summary>
- /// <param name="spaceDetails">String describing the drives being low on space.</param>
- private void SendLowSpaceMail(string spaceDetails) {
- if (string.IsNullOrWhiteSpace(spaceDetails)) {
- Log.Trace("SendLowSpaceMail(): spaceDetails emtpy!");
+ private void SendLowSpaceMail() {
+ if (_storage.AllDrivesAboveThreshold()) {
+ Log.Trace("Free space on all drives above threshold.");
return;
}
@@ -116,25 +121,24 @@ namespace ATxService
return;
}
- // reaching this point means a notification will be sent to the admin, and in that
- // case it makes sense to also include details about the grace location:
- var graceReport = FsUtils.GraceLocationSummary(
- new DirectoryInfo(_config.DonePath), _config.GracePeriod);
+ // reaching this point means a notification will be sent, so now we can ask for the
+ // full storage status report:
+ var report = _storage.Summary();
-
- Log.Warn("WARNING: {0}", spaceDetails);
+ Log.Warn("WARNING: {0}", report);
_status.LastStorageNotification = DateTime.Now;
var substitutions = new List<Tuple<string, string>> {
Tuple.Create("SERVICE_NAME", ServiceName),
Tuple.Create("HOST_ALIAS", _config.HostAlias),
Tuple.Create("HOST_NAME", Environment.MachineName),
- Tuple.Create("LOW_SPACE_DRIVES", spaceDetails)
+ Tuple.Create("LOW_SPACE_DRIVES", report)
};
try {
var body = LoadMailTemplate("DiskSpace-Low.txt", substitutions);
- if (graceReport.Length > 0)
- body += $"\n\n--\n{graceReport}";
+ // explicitly use SendEmail() instead of SendAdminEmail() here to circumvent the
+ // additional checks done in the latter one and make sure the low space email is
+ // sent out independently of that:
SendEmail(_config.AdminEmailAdress, "low disk space", body);
}
catch (Exception ex) {
@@ -205,27 +209,48 @@ namespace ATxService
/// <summary>
/// Send a report on expired folders in the grace location if applicable.
- ///
- /// Create a summary of expired folders and send it to the admin address
- /// if the configured GraceNotificationDelta has passed since the last email.
+ ///
+ /// A summary of expired folders is created and sent to the admin address if the configured
+ /// GraceNotificationDelta has passed since the last email. The report will also contain a
+ /// summary of free disk space for all configured drives.
/// </summary>
- /// <param name="threshold">The number of days used as expiration threshold.</param>
- /// <returns>The summary report, empty if no expired folders exist.</returns>
- private string SendGraceLocationSummary(int threshold) {
- var report = FsUtils.GraceLocationSummary(
- new DirectoryInfo(_config.DonePath), threshold);
- if (string.IsNullOrEmpty(report))
- return "";
-
- report += $"\n{SystemChecks.CheckFreeDiskSpace(_config.SpaceMonitoring)}" +
- "\nTime since last grace notification: " +
- $"{TimeUtils.HumanSince(_status.LastGraceNotification)}\n";
- if (TimeUtils.MinutesSince(_status.LastGraceNotification) < _config.GraceNotificationDelta)
- return report;
+ /// <returns>True if a report was sent, false otherwise (includes situations where there
+ /// are expired directories but the report has not been sent via email as the grace
+ /// notification delta hasn't expired yet, the report will still be logged then).</returns>
+ private bool SendGraceLocationSummary() {
+ if (_storage.ExpiredDirsCount == 0)
+ return false;
+
+ var report = _storage.Summary() +
+ "\nTime since last grace notification: " +
+ $"{TimeUtils.HumanSince(_status.LastGraceNotification)}\n";
+
+ if (TimeUtils.MinutesSince(_status.LastGraceNotification) < _config.GraceNotificationDelta) {
+ Log.Debug(report);
+ return false;
+ }
_status.LastGraceNotification = DateTime.Now;
- SendAdminEmail(report, "grace location summary");
- return report + "\nNotification sent to AdminEmailAdress.\n";
+ return SendAdminEmail(report, "grace location summary");
+ }
+
+ /// <summary>
+ /// Send a system health report if enough time has elapsed since the previous one.
+ /// </summary>
+ /// <param name="report">The health report.</param>
+ /// <returns>True in case the report was sent, false otherwise.</returns>
+ private bool SendHealthReport(string report) {
+ var elapsedHuman = TimeUtils.HumanSince(_status.LastStartupNotification);
+
+ if (TimeUtils.MinutesSince(_status.LastStartupNotification) < _config.StartupNotificationDelta) {
+ Log.Trace("Not sending system health report now, last one has been sent {0}",
+ elapsedHuman);
+ return false;
+ }
+
+ report += $"\nPrevious system health report notification was sent {elapsedHuman}.\n";
+ _status.LastStartupNotification = DateTime.Now;
+ return SendAdminEmail(report, "system health report");
}
}
}
\ No newline at end of file
diff --git a/Resources/conf/config.common.xml b/Resources/conf/config.common.xml
index 43845c1d0abd6dc340ec4b54b1fc464d38b4e988..56655583db873d5d337640aa4b2904932d82182d 100644
--- a/Resources/conf/config.common.xml
+++ b/Resources/conf/config.common.xml
@@ -118,6 +118,11 @@
in case one of the drives is below the threshold (in minutes) -->
<StorageNotificationDelta>720</StorageNotificationDelta>
+ <!-- StartupNotificationDelta: minimum time (in minutes) between two service
+ startup system health notification emails (default: 2880 (2d). Set to 0
+ to disable startup health reports. -->
+ <StartupNotificationDelta>2880</StartupNotificationDelta>
+
<!-- OPTIONAL NOTIFICATION / EMAIL SETTINGS -->
</ServiceConfig>