highperfocused/void.cat
1
0
Fork 0
mirror of https://git.v0l.io/Kieran/void.cat.git synced 2025-11-15 06:36:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

Delete expired files

Browse Source
This commit is contained in:
Kieran
2022-09-06 22:32:22 +01:00
parent ebe5a0e106
commit 1d451aac82
18 changed files with 128 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
VoidCat/Services/Abstractions/IFileInfoManager.cs
View File

@@ -1,38 +0,0 @@
using VoidCat.Model;
namespace VoidCat.Services.Abstractions;
/// <summary>
/// Main interface for getting file info to serve to clients.
/// This interface should wrap all stores and return the combined result
/// </summary>
public interface IFileInfoManager
{
/// <summary>
/// Get all metadata for a single file
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask<PublicVoidFile?> Get(Guid id);
/// <summary>
/// Get all private metadata for a single file
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask<PrivateVoidFile?> GetPrivate(Guid id);
/// <summary>
/// Get all metadata for multiple files
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids);
/// <summary>
/// Deletes all file metadata
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask Delete(Guid id);
}

52
VoidCat/Services/Background/DeleteExpiredFiles.cs Normal file
View File

@@ -0,0 +1,52 @@
using VoidCat.Model;
using VoidCat.Services.Abstractions;
using VoidCat.Services.Files;
namespace VoidCat.Services.Background;
/// <summary>
/// Delete expired files
/// </summary>
public sealed class DeleteExpiredFiles : BackgroundService
{
private readonly ILogger<DeleteExpiredFiles> _logger;
private readonly IServiceScopeFactory _scopeFactory;
public DeleteExpiredFiles(ILogger<DeleteExpiredFiles> logger, IServiceScopeFactory scopeFactory)
{
_logger = logger;
_scopeFactory = scopeFactory;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
using var scope = _scopeFactory.CreateScope();
var metadata = scope.ServiceProvider.GetRequiredService<IFileMetadataStore>();
var fileInfoManager = scope.ServiceProvider.GetRequiredService<FileInfoManager>();
var fileStoreFactory = scope.ServiceProvider.GetRequiredService<FileStoreFactory>();
var files = await metadata.ListFiles<SecretVoidFileMeta>(new(0, int.MaxValue));
await foreach (var f in files.Results.WithCancellation(stoppingToken))
{
try
{
if (f.Expires < DateTime.Now)
{
await fileStoreFactory.DeleteFile(f.Id);
await fileInfoManager.Delete(f.Id);
_logger.LogInformation("Deleted file: {Id}", f.Id);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to delete file: {Id}", f.Id);
}
}
await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken);
}
}
}

2
VoidCat/Services/Background/DeleteUnverifiedAccounts.cs
View File

@@ -25,7 +25,7 @@ public class DeleteUnverifiedAccounts : BackgroundService
var userStore = scope.ServiceProvider.GetRequiredService<IUserStore>();
var userUploads = scope.ServiceProvider.GetRequiredService<IUserUploadsStore>();
var fileStore = scope.ServiceProvider.GetRequiredService<FileStoreFactory>();
var fileInfoManager = scope.ServiceProvider.GetRequiredService<IFileInfoManager>();
var fileInfoManager = scope.ServiceProvider.GetRequiredService<FileInfoManager>();
var accounts = await userStore.ListUsers(new(0, Int32.MaxValue));

31
VoidCat/Services/Files/FileInfoManager.cs
View File

@@ -3,8 +3,11 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files;
/// <inheritdoc />
public class FileInfoManager : IFileInfoManager
/// <summary>
/// Main interface for getting file info to serve to clients.
/// This interface should wrap all stores and return the combined result
/// </summary>
public sealed class FileInfoManager
{
private readonly IFileMetadataStore _metadataStore;
private readonly IPaywallStore _paywallStore;
@@ -24,19 +27,31 @@ public class FileInfoManager : IFileInfoManager
_userUploadsStore = userUploadsStore;
}
/// <inheritdoc />
/// <summary>
/// Get all metadata for a single file
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public ValueTask<PublicVoidFile?> Get(Guid id)
{
return Get<PublicVoidFile, VoidFileMeta>(id);
}
/// <inheritdoc />
/// <summary>
/// Get all private metadata for a single file
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public ValueTask<PrivateVoidFile?> GetPrivate(Guid id)
{
return Get<PrivateVoidFile, SecretVoidFileMeta>(id);
}
/// <inheritdoc />
/// <summary>
/// Get all metadata for multiple files
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
public async ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids)
{
var ret = new List<PublicVoidFile>();
@@ -52,7 +67,11 @@ public class FileInfoManager : IFileInfoManager
return ret;
}
/// <inheritdoc />
/// <summary>
/// Deletes all file metadata
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public async ValueTask Delete(Guid id)
{
await _metadataStore.Delete(id);

4
VoidCat/Services/Files/FileStorageStartup.cs
View File

@@ -8,7 +8,7 @@ public static class FileStorageStartup
{
public static void AddStorage(this IServiceCollection services, VoidSettings settings)
{
services.AddTransient<IFileInfoManager, FileInfoManager>();
services.AddTransient<FileInfoManager>();
services.AddTransient<FileStoreFactory>();
if (settings.CloudStorage != default)
@@ -19,7 +19,7 @@ public static class FileStorageStartup
services.AddTransient<IFileStore>((svc) =>
new S3FileStore(s3,
svc.GetRequiredService<IAggregateStatsCollector>(),
svc.GetRequiredService<IFileInfoManager>(),
svc.GetRequiredService<FileInfoManager>(),
svc.GetRequiredService<ICache>()));
if (settings.MetadataStore == s3.Name)

2
VoidCat/Services/Files/LocalDiskFileMetadataStore.cs
View File

@@ -52,8 +52,8 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
await Set(id, oldMeta);
}

5
VoidCat/Services/Files/LocalDiskFileStorage.cs
View File

@@ -8,14 +8,12 @@ namespace VoidCat.Services.Files;
public class LocalDiskFileStore : StreamFileStore, IFileStore
{
private const string FilesDir = "files-v1";
private readonly ILogger<LocalDiskFileStore> _logger;
private readonly VoidSettings _settings;
public LocalDiskFileStore(ILogger<LocalDiskFileStore> logger, VoidSettings settings, IAggregateStatsCollector stats)
public LocalDiskFileStore(VoidSettings settings, IAggregateStatsCollector stats)
: base(stats)
{
_settings = settings;
_logger = logger;
var dir = Path.Combine(_settings.DataDirectory, FilesDir);
if (!Directory.Exists(dir))
@@ -55,7 +53,6 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
var fp = MapPath(id);
if (File.Exists(fp))
{
_logger.LogInformation("Deleting file: {Path}", fp);
File.Delete(fp);
}

4
VoidCat/Services/Files/PostgresFileMetadataStore.cs
View File

@@ -54,7 +54,7 @@ on conflict (""Id"") do update set
mimeType = obj.MimeType,
digest = obj.Digest,
editSecret = obj.EditSecret,
expires = obj.Expires,
expires = obj.Expires?.ToUniversalTime(),
store = obj.Storage
});
}
@@ -91,8 +91,8 @@ on conflict (""Id"") do update set
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
await Set(id, oldMeta);
}

2
VoidCat/Services/Files/S3FileMetadataStore.cs
View File

@@ -53,8 +53,8 @@ public class S3FileMetadataStore : IFileMetadataStore
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
await Set(id, oldMeta);
}

4
VoidCat/Services/Files/S3FileStore.cs
View File

@@ -9,13 +9,13 @@ namespace VoidCat.Services.Files;
/// <inheritdoc cref="VoidCat.Services.Abstractions.IFileStore" />
public class S3FileStore : StreamFileStore, IFileStore
{
private readonly IFileInfoManager _fileInfo;
private readonly FileInfoManager _fileInfo;
private readonly AmazonS3Client _client;
private readonly S3BlobConfig _config;
private readonly IAggregateStatsCollector _statsCollector;
private readonly ICache _cache;
public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, IFileInfoManager fileInfo, ICache cache) : base(stats)
public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, FileInfoManager fileInfo, ICache cache) : base(stats)
{
_fileInfo = fileInfo;
_cache = cache;