Store expiration in file creation time. Cyclic cleaning.

This commit is contained in:
ClemensFischer 2025-02-19 19:48:38 +01:00
parent d31f522c52
commit 11ed14c823
2 changed files with 172 additions and 336 deletions

View file

@ -9,7 +9,6 @@ using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@ -18,25 +17,38 @@ namespace MapControl.Caching
/// <summary>
/// IDistributedCache implementation based on local image files.
/// </summary>
public class ImageFileCache : IDistributedCache
public sealed class ImageFileCache : IDistributedCache, IDisposable
{
private static readonly byte[] expirationTag = Encoding.ASCII.GetBytes("EXPIRES:");
private readonly MemoryDistributedCache memoryCache = new MemoryDistributedCache(Options.Create(new MemoryDistributedCacheOptions()));
private readonly string rootPath;
private readonly DirectoryInfo rootDirectory;
private readonly Timer cleanTimer;
private bool cleaning;
public ImageFileCache(string path)
: this(path, TimeSpan.FromHours(1))
{
}
public ImageFileCache(string path, TimeSpan autoCleanInterval)
{
if (string.IsNullOrEmpty(path))
{
throw new ArgumentException($"The {nameof(path)} argument must not be null or empty.", nameof(path));
}
rootPath = path;
rootDirectory = new DirectoryInfo(path);
Debug.WriteLine($"{nameof(ImageFileCache)}: {rootPath}");
Debug.WriteLine($"{nameof(ImageFileCache)}: {rootDirectory.FullName}");
ThreadPool.QueueUserWorkItem(o => Clean());
if (autoCleanInterval > TimeSpan.Zero)
{
cleanTimer = new Timer(_ => Clean(), null, TimeSpan.Zero, autoCleanInterval);
}
}
public void Dispose()
{
cleanTimer?.Dispose();
}
public byte[] Get(string key)
@ -45,63 +57,66 @@ namespace MapControl.Caching
if (buffer == null)
{
var path = GetPath(key);
var file = GetFile(key);
try
{
if (path != null && File.Exists(path))
if (file != null && file.Exists && file.CreationTime > DateTime.Now)
{
buffer = File.ReadAllBytes(path);
if (CheckExpiration(ref buffer, out DistributedCacheEntryOptions options))
using (var stream = file.OpenRead())
{
memoryCache.Set(key, buffer, options);
buffer = new byte[stream.Length];
var offset = 0;
while (offset < buffer.Length)
{
offset += stream.Read(buffer, offset, buffer.Length - offset);
}
}
var options = new DistributedCacheEntryOptions { AbsoluteExpiration = file.CreationTime };
memoryCache.Set(key, buffer, options);
}
}
catch (Exception ex)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed reading {path}: {ex.Message}");
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed reading {file.FullName}: {ex.Message}");
}
}
return buffer;
}
public async Task<byte[]> GetAsync(string key, CancellationToken cancellationToken = default)
public async Task<byte[]> GetAsync(string key, CancellationToken token = default)
{
var buffer = await memoryCache.GetAsync(key, cancellationToken).ConfigureAwait(false);
var buffer = await memoryCache.GetAsync(key, token).ConfigureAwait(false);
if (buffer == null)
{
var path = GetPath(key);
var file = GetFile(key);
try
{
if (path != null && File.Exists(path) && !cancellationToken.IsCancellationRequested)
if (file != null && file.Exists && file.CreationTime > DateTime.Now)
{
#if NETFRAMEWORK
using (var stream = File.OpenRead(path))
using (var stream = file.OpenRead())
{
buffer = new byte[stream.Length];
var offset = 0;
while (offset < buffer.Length)
{
offset += await stream.ReadAsync(buffer, offset, buffer.Length - offset, cancellationToken).ConfigureAwait(false);
offset += await stream.ReadAsync(buffer, offset, buffer.Length - offset, token).ConfigureAwait(false);
}
}
#else
buffer = await File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false);
#endif
if (CheckExpiration(ref buffer, out DistributedCacheEntryOptions options))
{
await memoryCache.SetAsync(key, buffer, options, cancellationToken).ConfigureAwait(false);
}
var options = new DistributedCacheEntryOptions { AbsoluteExpiration = file.CreationTime };
await memoryCache.SetAsync(key, buffer, options, token).ConfigureAwait(false);
}
}
catch (Exception ex)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed reading {path}: {ex.Message}");
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed reading {file.FullName}: {ex.Message}");
}
}
@ -112,61 +127,43 @@ namespace MapControl.Caching
{
memoryCache.Set(key, buffer, options);
var path = GetPath(key);
if (path != null && buffer?.Length > 0)
{
try
{
Directory.CreateDirectory(Path.GetDirectoryName(path));
using (var stream = File.Create(path))
{
stream.Write(buffer, 0, buffer.Length);
if (GetExpirationBytes(options, out byte[] expiration))
{
stream.Write(expiration, 0, expiration.Length);
}
}
SetAccessControl(path);
}
catch (Exception ex)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed writing {path}: {ex.Message}");
}
}
}
public async Task SetAsync(string key, byte[] buffer, DistributedCacheEntryOptions options, CancellationToken cancellationToken = default)
{
await memoryCache.SetAsync(key, buffer, options, cancellationToken).ConfigureAwait(false);
var path = GetPath(key);
var file = GetFile(key);
try
{
if (path != null && buffer?.Length > 0 && !cancellationToken.IsCancellationRequested)
if (file != null && buffer?.Length > 0)
{
Directory.CreateDirectory(Path.GetDirectoryName(path));
using (var stream = File.Create(path))
using (var stream = CreateFile(file, options))
{
await stream.WriteAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false);
if (GetExpirationBytes(options, out byte[] expiration))
{
await stream.WriteAsync(expiration, 0, expiration.Length, cancellationToken).ConfigureAwait(false);
}
stream.Write(buffer, 0, buffer.Length);
}
SetAccessControl(path);
}
}
catch (Exception ex)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed writing {path}: {ex.Message}");
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed writing {file.FullName}: {ex.Message}");
}
}
public async Task SetAsync(string key, byte[] buffer, DistributedCacheEntryOptions options, CancellationToken token = default)
{
await memoryCache.SetAsync(key, buffer, options, token).ConfigureAwait(false);
var file = GetFile(key);
try
{
if (file != null && buffer?.Length > 0 && !token.IsCancellationRequested)
{
using (var stream = CreateFile(file, options))
{
await stream.WriteAsync(buffer, 0, buffer.Length, token).ConfigureAwait(false);
}
}
}
catch (Exception ex)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed writing {file.FullName}: {ex.Message}");
}
}
@ -184,47 +181,57 @@ namespace MapControl.Caching
{
memoryCache.Remove(key);
var path = GetPath(key);
var file = GetFile(key);
try
{
if (path != null && File.Exists(path))
if (file != null && file.Exists)
{
File.Delete(path);
file.Delete();
}
}
catch (Exception ex)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed deleting {path}: {ex.Message}");
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed deleting {file.FullName}: {ex.Message}");
}
}
public async Task RemoveAsync(string key, CancellationToken cancellationToken = default)
public async Task RemoveAsync(string key, CancellationToken token = default)
{
await memoryCache.RemoveAsync(key, cancellationToken);
await memoryCache.RemoveAsync(key, token);
var path = GetPath(key);
var file = GetFile(key);
try
{
if (path != null && File.Exists(path) && !cancellationToken.IsCancellationRequested)
if (file != null && file.Exists && !token.IsCancellationRequested)
{
File.Delete(path);
file.Delete();
}
}
catch (Exception ex)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed deleting {path}: {ex.Message}");
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed deleting {file.FullName}: {ex.Message}");
}
}
public void Clean()
{
var deletedFileCount = CleanDirectory(new DirectoryInfo(rootPath));
if (deletedFileCount > 0)
if (!cleaning)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Deleted {deletedFileCount} expired files.");
cleaning = true;
foreach (var directory in rootDirectory.EnumerateDirectories())
{
var deletedFileCount = CleanDirectory(directory);
if (deletedFileCount > 0)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Deleted {deletedFileCount} expired files in {directory.Name}.");
}
}
cleaning = false;
}
}
@ -233,11 +240,11 @@ namespace MapControl.Caching
return Task.Run(Clean);
}
private string GetPath(string key)
private FileInfo GetFile(string key)
{
try
{
return Path.Combine(rootPath, Path.Combine(key.Split('/')));
return new FileInfo(Path.Combine(rootDirectory.FullName, Path.Combine(key.Split('/'))));
}
catch (Exception ex)
{
@ -247,15 +254,39 @@ namespace MapControl.Caching
return null;
}
private static FileStream CreateFile(FileInfo file, DistributedCacheEntryOptions options)
{
file.Directory.Create();
var stream = file.Create();
try
{
file.CreationTime = options.AbsoluteExpiration.HasValue
? options.AbsoluteExpiration.Value.LocalDateTime
: DateTime.Now.Add(options.AbsoluteExpirationRelativeToNow ?? (options.SlidingExpiration ?? TimeSpan.FromDays(1)));
}
catch (Exception ex)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed setting creation time of {file.FullName}: {ex.Message}");
}
return stream;
}
private static int CleanDirectory(DirectoryInfo directory)
{
var deletedFileCount = 0;
try
{
deletedFileCount += directory.EnumerateDirectories().Sum(dir => CleanDirectory(dir));
deletedFileCount = directory.EnumerateDirectories().Sum(CleanDirectory);
deletedFileCount += directory.EnumerateFiles().Sum(file => CleanFile(file));
foreach (var file in directory.EnumerateFiles().Where(f => f.CreationTime <= DateTime.Now))
{
file.Delete();
deletedFileCount++;
}
if (!directory.EnumerateFileSystemInfos().Any())
{
@ -269,120 +300,5 @@ namespace MapControl.Caching
return deletedFileCount;
}
private static int CleanFile(FileInfo file)
{
var deletedFileCount = 0;
if (file.Length > 16)
{
try
{
var hasExpired = false;
using (var stream = file.OpenRead())
{
stream.Seek(-16, SeekOrigin.End);
var buffer = new byte[16];
hasExpired = stream.Read(buffer, 0, 16) == 16
&& GetExpirationTicks(buffer, out long expiration)
&& expiration <= DateTimeOffset.UtcNow.Ticks;
}
if (hasExpired)
{
file.Delete();
deletedFileCount = 1;
}
}
catch (Exception ex)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed cleaning {file.FullName}: {ex.Message}");
}
}
return deletedFileCount;
}
private static bool CheckExpiration(ref byte[] buffer, out DistributedCacheEntryOptions options)
{
if (GetExpirationTicks(buffer, out long expiration))
{
if (expiration > DateTimeOffset.UtcNow.Ticks)
{
Array.Resize(ref buffer, buffer.Length - 16);
options = new DistributedCacheEntryOptions
{
AbsoluteExpiration = new DateTimeOffset(expiration, TimeSpan.Zero)
};
return true;
}
buffer = null; // buffer has expired
}
options = null;
return false;
}
private static bool GetExpirationTicks(byte[] buffer, out long expirationTicks)
{
if (buffer.Length >= 16 &&
expirationTag.SequenceEqual(buffer.Skip(buffer.Length - 16).Take(8)))
{
expirationTicks = BitConverter.ToInt64(buffer, buffer.Length - 8);
return true;
}
expirationTicks = 0;
return false;
}
private static bool GetExpirationBytes(DistributedCacheEntryOptions options, out byte[] expirationBytes)
{
long expirationTicks;
if (options.AbsoluteExpiration.HasValue)
{
expirationTicks = options.AbsoluteExpiration.Value.Ticks;
}
else if (options.AbsoluteExpirationRelativeToNow.HasValue)
{
expirationTicks = DateTimeOffset.UtcNow.Add(options.AbsoluteExpirationRelativeToNow.Value).Ticks;
}
else if (options.SlidingExpiration.HasValue)
{
expirationTicks = DateTimeOffset.UtcNow.Add(options.SlidingExpiration.Value).Ticks;
}
else
{
expirationBytes = null;
return false;
}
expirationBytes = expirationTag.Concat(BitConverter.GetBytes(expirationTicks)).ToArray();
return true;
}
private static void SetAccessControl(string path)
{
#if AVALONIA
if (!OperatingSystem.IsWindows()) return;
#endif
var fileInfo = new FileInfo(path);
var fileSecurity = fileInfo.GetAccessControl();
var fullControlRule = new System.Security.AccessControl.FileSystemAccessRule(
new System.Security.Principal.SecurityIdentifier(
System.Security.Principal.WellKnownSidType.BuiltinUsersSid, null),
System.Security.AccessControl.FileSystemRights.FullControl,
System.Security.AccessControl.AccessControlType.Allow);
fileSecurity.AddAccessRule(fullControlRule);
fileInfo.SetAccessControl(fileSecurity);
}
}
}

View file

@ -7,34 +7,45 @@ using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Options;
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Windows.Storage;
using Windows.Storage.Streams;
using Buffer = Windows.Storage.Streams.Buffer;
namespace MapControl.Caching
{
/// <summary>
/// IDistributedCache implementation based on local image files.
/// </summary>
public partial class ImageFileCache : IDistributedCache
public sealed class ImageFileCache : IDistributedCache, IDisposable
{
private static readonly byte[] expirationTag = Encoding.ASCII.GetBytes("EXPIRES:");
private readonly MemoryDistributedCache memoryCache = new MemoryDistributedCache(Options.Create(new MemoryDistributedCacheOptions()));
private readonly StorageFolder rootFolder;
private readonly Timer cleanTimer;
private bool cleaning;
public ImageFileCache(StorageFolder folder)
: this(folder, TimeSpan.FromHours(1))
{
}
public ImageFileCache(StorageFolder folder, TimeSpan autoCleanInterval)
{
rootFolder = folder ?? throw new ArgumentException($"The {nameof(folder)} argument must not be null or empty.", nameof(folder));
Debug.WriteLine($"{nameof(ImageFileCache)}: {rootFolder.Path}");
_ = Task.Factory.StartNew(CleanAsync, TaskCreationOptions.LongRunning);
if (autoCleanInterval > TimeSpan.Zero)
{
cleanTimer = new Timer(_ => CleanAsync().Wait(), null, TimeSpan.Zero, autoCleanInterval);
}
}
public void Dispose()
{
cleanTimer?.Dispose();
}
public byte[] Get(string key)
@ -46,7 +57,7 @@ namespace MapControl.Caching
{
throw new NotSupportedException();
}
public void Remove(string key)
{
throw new NotSupportedException();
@ -77,14 +88,13 @@ namespace MapControl.Caching
{
var item = await rootFolder.TryGetItemAsync(key.Replace('/', '\\'));
if (item is StorageFile file)
if (item is StorageFile file && file.DateCreated > DateTimeOffset.Now)
{
buffer = (await FileIO.ReadBufferAsync(file)).ToArray();
if (CheckExpiration(ref buffer, out DistributedCacheEntryOptions options))
{
await memoryCache.SetAsync(key, buffer, options, token).ConfigureAwait(false);
}
var options = new DistributedCacheEntryOptions { AbsoluteExpiration = file.DateCreated };
await memoryCache.SetAsync(key, buffer, options, token).ConfigureAwait(false);
}
}
catch (Exception ex)
@ -114,15 +124,13 @@ namespace MapControl.Caching
var file = await folder.CreateFileAsync(keyComponents[keyComponents.Length - 1], CreationCollisionOption.OpenIfExists);
using (var stream = await file.OpenAsync(FileAccessMode.ReadWrite))
{
await stream.WriteAsync(buffer.AsBuffer());
await FileIO.WriteBytesAsync(file, buffer);
if (GetExpirationBytes(options, out byte[] expiration))
{
await stream.WriteAsync(expiration.AsBuffer());
}
}
var expiration = options.AbsoluteExpiration.HasValue
? options.AbsoluteExpiration.Value.LocalDateTime
: DateTime.Now.Add(options.AbsoluteExpirationRelativeToNow ?? (options.SlidingExpiration ?? TimeSpan.FromDays(1)));
File.SetCreationTime(file.Path, expiration);
}
catch (Exception ex)
{
@ -133,11 +141,21 @@ namespace MapControl.Caching
public async Task CleanAsync()
{
var deletedFileCount = await CleanFolder(rootFolder);
if (deletedFileCount > 0)
if (!cleaning)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Deleted {deletedFileCount} expired files.");
cleaning = true;
foreach (var folder in await rootFolder.GetFoldersAsync())
{
var deletedFileCount = await CleanFolder(folder);
if (deletedFileCount > 0)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Deleted {deletedFileCount} expired files in {folder.Name}.");
}
}
cleaning = false;
}
}
@ -147,14 +165,15 @@ namespace MapControl.Caching
try
{
foreach (var f in await folder.GetFoldersAsync())
foreach (var subFolder in await folder.GetFoldersAsync())
{
deletedFileCount += await CleanFolder(f);
deletedFileCount += await CleanFolder(subFolder);
}
foreach (var f in await folder.GetFilesAsync())
foreach (var file in (await folder.GetFilesAsync()).Where(f => f.DateCreated <= DateTime.Now))
{
deletedFileCount += await CleanFile(f);
await file.DeleteAsync();
deletedFileCount++;
}
if ((await folder.GetItemsAsync()).Count == 0)
@ -169,104 +188,5 @@ namespace MapControl.Caching
return deletedFileCount;
}
private static async Task<int> CleanFile(StorageFile file)
{
var deletedFileCount = 0;
var size = (await file.GetBasicPropertiesAsync()).Size;
if (size > 16)
{
try
{
var hasExpired = false;
using (var stream = await file.OpenReadAsync())
{
stream.Seek(size - 16);
var buffer = await stream.ReadAsync(new Buffer(16), 16, InputStreamOptions.None);
hasExpired = buffer.Length == 16
&& GetExpirationTicks(buffer.ToArray(), out long expiration)
&& expiration <= DateTimeOffset.UtcNow.Ticks;
}
if (hasExpired)
{
await file.DeleteAsync();
deletedFileCount = 1;
}
}
catch (Exception ex)
{
Debug.WriteLine($"{nameof(ImageFileCache)}: Failed cleaning {file.Path}: {ex.Message}");
}
}
return deletedFileCount;
}
private static bool CheckExpiration(ref byte[] buffer, out DistributedCacheEntryOptions options)
{
if (GetExpirationTicks(buffer, out long expiration))
{
if (expiration > DateTimeOffset.UtcNow.Ticks)
{
Array.Resize(ref buffer, buffer.Length - 16);
options = new DistributedCacheEntryOptions
{
AbsoluteExpiration = new DateTimeOffset(expiration, TimeSpan.Zero)
};
return true;
}
buffer = null; // buffer has expired
}
options = null;
return false;
}
private static bool GetExpirationTicks(byte[] buffer, out long expirationTicks)
{
if (buffer.Length >= 16 &&
expirationTag.SequenceEqual(buffer.Skip(buffer.Length - 16).Take(8)))
{
expirationTicks = BitConverter.ToInt64(buffer, buffer.Length - 8);
return true;
}
expirationTicks = 0;
return false;
}
private static bool GetExpirationBytes(DistributedCacheEntryOptions options, out byte[] expirationBytes)
{
long expirationTicks;
if (options.AbsoluteExpiration.HasValue)
{
expirationTicks = options.AbsoluteExpiration.Value.Ticks;
}
else if (options.AbsoluteExpirationRelativeToNow.HasValue)
{
expirationTicks = DateTimeOffset.UtcNow.Add(options.AbsoluteExpirationRelativeToNow.Value).Ticks;
}
else if (options.SlidingExpiration.HasValue)
{
expirationTicks = DateTimeOffset.UtcNow.Add(options.SlidingExpiration.Value).Ticks;
}
else
{
expirationBytes = null;
return false;
}
expirationBytes = expirationTag.Concat(BitConverter.GetBytes(expirationTicks)).ToArray();
return true;
}
}
}