using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using NLog; namespace ATxCommon { public static class FsUtils { private static readonly Logger Log = LogManager.GetCurrentClassLogger(); /// <summary> /// Create a directory with the given name if it doesn't exist yet, otherwise /// (optionally) create a new one using a date suffix to distinguish it from /// the existing one. /// </summary> /// <param name="dirPath">The full path of the directory to be created.</param> /// <param name="unique">Add a time-suffix to the name if the directory exists.</param> /// <returns>The name of the (created or pre-existing) directory. This will only /// differ from the input parameter "dirPath" if the "unique" parameter is set /// to true (then it will give the newly generated name) or if an error occured /// (in which case it will return an empty string).</returns> public static string CreateNewDirectory(string dirPath, bool unique) { try { if (Directory.Exists(dirPath)) { // if unique was not requested, return the name of the existing dir: if (unique == false) return dirPath; dirPath = dirPath + "_" + TimeUtils.Timestamp(); } Directory.CreateDirectory(dirPath); Log.Debug("Created directory: [{0}]", dirPath); return dirPath; } catch (Exception ex) { Log.Error("Error in CreateNewDirectory({0}): {1}", dirPath, ex.Message); } return ""; } /// <summary> /// Helper method to check if a directory exists, trying to create it if not. /// </summary> /// <param name="path">The full path of the directory to check / create.</param> /// <returns>True if existing or creation was successful, false otherwise.</returns> public static bool CheckForDirectory(string path) { if (string.IsNullOrWhiteSpace(path)) { Log.Error("ERROR: CheckForDirectory() parameter must not be empty!"); return false; } return FsUtils.CreateNewDirectory(path, false) == path; } /// <summary> /// Recursively sum up size of all files under a given path. /// </summary> /// <param name="path">Full path of the directory.</param> /// <returns>The total size in bytes.</returns> public static long GetDirectorySize(string path) { return new DirectoryInfo(path) .GetFiles("*", SearchOption.AllDirectories) .Sum(file => file.Length); } /// <summary> /// Convert the timestamp given by the NAME of a directory into the age in days. /// </summary> /// <param name="dir">The DirectoryInfo object to check for its name-age.</param> /// <param name="baseTime">The DateTime object to compare to.</param> /// <returns>The age in days, or -1 in case of an error.</returns> public static int DirNameToAge(DirectoryInfo dir, DateTime baseTime) { DateTime dirTimestamp; try { dirTimestamp = DateTime.ParseExact(dir.Name, "yyyy-MM-dd__HH-mm-ss", CultureInfo.InvariantCulture); } catch (Exception ex) { // TODO: discuss if this should be an "Error" message to trigger a mail // notification to the AdminDebug address: Log.Warn("Unable to parse time from name [{0}], skipping: {1}", dir.Name, ex.Message); return -1; } return (baseTime - dirTimestamp).Days; } /// <summary> /// Assemble a dictionary with information about expired directories. /// </summary> /// <param name="baseDir">The base directory to scan for subdirectories.</param> /// <param name="thresh">The number of days used as expiration threshold.</param> /// <returns>A dictionary having usernames as keys (of those users that actually do have /// expired directories), where the values are lists of tuples with the DirInfo objects, /// size (in bytes) and age (in days) of the expired directories.</returns> public static Dictionary<string, List<Tuple<DirectoryInfo, long, int>>> ExpiredDirs(DirectoryInfo baseDir,int thresh) { var collection = new Dictionary<string, List<Tuple<DirectoryInfo, long, int>>>(); var now = DateTime.Now; foreach (var userdir in baseDir.GetDirectories()) { var expired = new List<Tuple<DirectoryInfo, long, int>>(); foreach (var subdir in userdir.GetDirectories()) { var age = DirNameToAge(subdir, now); if (age < thresh) continue; long size = -1; try { size = GetDirectorySize(subdir.FullName); } catch (Exception ex) { Log.Error("ERROR getting directory size of [{0}]: {1}", subdir.FullName, ex.Message); } expired.Add(new Tuple<DirectoryInfo, long, int>(subdir, size, age)); } if (expired.Count > 0) collection.Add(userdir.Name, expired); } return collection; } /// <summary> /// Generate a report on expired folders in the grace location. /// /// Check all user-directories in the grace location for subdirectories whose timestamp /// (the directory name) exceeds the configured grace period and generate a summary /// containing the age and size of those directories. /// </summary> /// <param name="graceLocation">The location to scan for expired folders.</param> /// <param name="threshold">The number of days used as expiration threshold.</param> public static string GraceLocationSummary(DirectoryInfo graceLocation, int threshold) { var expired = ExpiredDirs(graceLocation, threshold); var report = ""; foreach (var userdir in expired.Keys) { report += "\n - user '" + userdir + "'\n"; foreach (var subdir in expired[userdir]) { report += string.Format(" - {0} [age: {2}, size: {1}]\n", subdir.Item1, Conv.BytesToString(subdir.Item2), TimeUtils.DaysToHuman(subdir.Item3)); } } if (string.IsNullOrEmpty(report)) return ""; return "Expired folders in grace location:\n" + report; } /// <summary> /// Check if a given directory is empty. If a marker file is set in the config a /// file with this name will be created inside the given directory and will be /// skipped itself when checking for files and directories. /// </summary> /// <param name="dirInfo">The directory to check.</param> /// <param name="ignoredName">A filename that will be ignored.</param> /// <returns>True if access is denied or the dir is empty, false otherwise.</returns> public static bool DirEmptyExcept(DirectoryInfo dirInfo, string ignoredName) { try { var filesInTree = dirInfo.GetFiles("*", SearchOption.AllDirectories); if (string.IsNullOrEmpty(ignoredName)) return filesInTree.Length == 0; // check if there is ONLY the marker file: if (filesInTree.Length == 1 && filesInTree[0].Name.Equals(ignoredName)) return true; // make sure the marker file is there: var markerFilePath = Path.Combine(dirInfo.FullName, ignoredName); if (!File.Exists(markerFilePath)) File.Create(markerFilePath); return filesInTree.Length == 0; } catch (Exception e) { Log.Error("Error accessing directories: {0}", e.Message); } // if nothing triggered before, we pretend the dir is empty: return true; } /// <summary> /// Collect individual files in the root of a given directory tree in a specific /// sub-directory. A file name given as "ignoredName" will be skipped in the checks. /// </summary> /// <param name="userDir">The user directory to check for individual files.</param> /// <param name="ignoredName">A filename that will be ignored.</param> public static void CollectOrphanedFiles(DirectoryInfo userDir, string ignoredName) { var fileList = userDir.GetFiles(); var orphanedDir = Path.Combine(userDir.FullName, "orphaned"); try { if (fileList.Length > 1 || (string.IsNullOrEmpty(ignoredName) && fileList.Length > 0)) { if (Directory.Exists(orphanedDir)) { Log.Info("Orphaned directory already exists, skipping individual files."); return; } Log.Debug("Found individual files, collecting them in 'orphaned' folder."); CreateNewDirectory(orphanedDir, false); } foreach (var file in fileList) { if (file.Name.Equals(ignoredName)) continue; Log.Debug("Collecting orphan: [{0}]", file.Name); file.MoveTo(Path.Combine(orphanedDir, file.Name)); } } catch (Exception ex) { Log.Error("Error collecting orphaned files: {0}\n{1}", ex.Message, ex.StackTrace); } } /// <summary> /// Ensure the required spooling directories (managed/incoming) exist. /// </summary> /// <param name="incoming">The path to the incoming location.</param> /// <param name="managed">The path to the managed location.</param> /// <returns>True if all dirs exist or were created successfully.</returns> public static bool CheckSpoolingDirectories(string incoming, string managed) { var retval = CheckForDirectory(incoming); retval &= CheckForDirectory(managed); retval &= CheckForDirectory(Path.Combine(managed, "PROCESSING")); retval &= CheckForDirectory(Path.Combine(managed, "DONE")); retval &= CheckForDirectory(Path.Combine(managed, "UNMATCHED")); return retval; } /// <summary> /// Move all subdirectories of a given path into a destination directory. The destination /// will be created if it doesn't exist yet. If a subdirectory of the same name already /// exists in the destination, a timestamp-suffix is added to the new one. /// </summary> /// <param name="sourceDir">The source path as DirectoryInfo object.</param> /// <param name="destPath">The destination path as a string.</param> /// <param name="resetAcls">Whether to reset the ACLs on the moved subdirectories.</param> /// <returns>True on success, false otherwise.</returns> public static bool MoveAllSubDirs(DirectoryInfo sourceDir, string destPath, bool resetAcls = false) { // TODO: check whether _transferState should be adjusted while moving dirs! Log.Debug("MoveAllSubDirs: [{0}] to [{1}]", sourceDir.FullName, destPath); try { // make sure the target directory that should hold all subdirectories to // be moved is existing: if (string.IsNullOrEmpty(FsUtils.CreateNewDirectory(destPath, false))) { Log.Warn("WARNING: destination path doesn't exist: {0}", destPath); return false; } foreach (var subDir in sourceDir.GetDirectories()) { var target = Path.Combine(destPath, subDir.Name); // make sure NOT to overwrite the subdirectories: if (Directory.Exists(target)) target += "_" + TimeUtils.Timestamp(); Log.Debug(" - [{0}] > [{1}]", subDir.Name, target); subDir.MoveTo(target); if (!resetAcls) continue; try { var acl = Directory.GetAccessControl(target); acl.SetAccessRuleProtection(false, false); Directory.SetAccessControl(target, acl); Log.Debug("Successfully reset inherited ACLs on [{0}]", target); } catch (Exception ex) { Log.Error("Error resetting inherited ACLs on [{0}]:\n{1}", target, ex.Message); } } } catch (Exception ex) { Log.Error("Error moving directories: [{0}] > [{1}]\n{2}", sourceDir.FullName, destPath, ex.Message); return false; } return true; } } }