highperfocused/void.cat
1
0
Fork 0
mirror of https://git.v0l.io/Kieran/void.cat.git synced 2025-11-15 08:17:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Multi-Storage backend support

Browse Source 
Api Keys support
This commit is contained in:
Kieran
2022-07-25 18:59:32 +01:00
parent 677c3593f1
commit 99907fce8b
43 changed files with 753 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
VoidCat/Services/Files/FileStorageStartup.cs
View File

@@ -9,29 +9,42 @@ public static class FileStorageStartup
public static void AddStorage(this IServiceCollection services, VoidSettings settings)
{
services.AddTransient<IFileInfoManager, FileInfoManager>();
services.AddTransient<FileStoreFactory>();
if (settings.CloudStorage != default)
{
services.AddTransient<IUserUploadsStore, CacheUserUploadStore>();
// cloud storage
if (settings.CloudStorage.S3 != default)
// S3 storage
foreach (var s3 in settings.CloudStorage.S3 ?? Array.Empty<S3BlobConfig>())
{
services.AddSingleton<IFileStore, S3FileStore>();
services.AddSingleton<IFileMetadataStore, S3FileMetadataStore>();
services.AddTransient<IFileStore>((svc) =>
new S3FileStore(s3, svc.GetRequiredService<IAggregateStatsCollector>(),
svc.GetRequiredService<IFileInfoManager>()));
if (settings.MetadataStore == s3.Name)
{
services.AddSingleton<IFileMetadataStore>((svc) =>
new S3FileMetadataStore(s3, svc.GetRequiredService<ILogger<S3FileMetadataStore>>()));
}
}
}
else if (!string.IsNullOrEmpty(settings.Postgres))
if (!string.IsNullOrEmpty(settings.Postgres))
{
services.AddTransient<IUserUploadsStore, PostgresUserUploadStore>();
services.AddTransient<IFileStore, LocalDiskFileStore>();
services.AddTransient<IFileMetadataStore, PostgresFileMetadataStore>();
if (settings.MetadataStore == "postgres")
{
services.AddSingleton<IFileMetadataStore, PostgresFileMetadataStore>();
}
}
else
{
services.AddTransient<IUserUploadsStore, CacheUserUploadStore>();
services.AddTransient<IFileStore, LocalDiskFileStore>();
services.AddTransient<IFileMetadataStore, LocalDiskFileMetadataStore>();
if (settings.MetadataStore == "local-disk")
{
services.AddSingleton<IFileMetadataStore, LocalDiskFileMetadataStore>();
}
}
}
}
}

89
VoidCat/Services/Files/FileSystemFactory.cs Normal file
View File

@@ -0,0 +1,89 @@
using VoidCat.Model;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files;
/// <summary>
/// Primary class for accessing <see cref="IFileStore"/> implementations
/// </summary>
public class FileStoreFactory : IFileStore
{
private readonly IFileMetadataStore _metadataStore;
private readonly IEnumerable<IFileStore> _fileStores;
public FileStoreFactory(IEnumerable<IFileStore> fileStores, IFileMetadataStore metadataStore)
{
_fileStores = fileStores;
_metadataStore = metadataStore;
}
/// <summary>
/// Get files store interface by key
/// </summary>
/// <param name="key"></param>
/// <returns></returns>
public IFileStore? GetFileStore(string? key)
{
if (key == default && _fileStores.Count() == 1)
{
return _fileStores.First();
}
return _fileStores.FirstOrDefault(a => a.Key == key);
}
/// <inheritdoc />
public string? Key => null;
/// <inheritdoc />
public ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
{
var store = GetFileStore(payload.Meta.Storage!);
if (store == default)
{
throw new InvalidOperationException($"Cannot find store '{payload.Meta.Storage}'");
}
return store.Ingress(payload, cts);
}
/// <inheritdoc />
public async ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts)
{
var store = await GetStore(request.Id);
await store.Egress(request, outStream, cts);
}
/// <inheritdoc />
public async ValueTask DeleteFile(Guid id)
{
var store = await GetStore(id);
await store.DeleteFile(id);
}
/// <inheritdoc />
public async ValueTask<Stream> Open(EgressRequest request, CancellationToken cts)
{
var store = await GetStore(request.Id);
return await store.Open(request, cts);
}
/// <summary>
/// Get file store for a file by id
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
/// <exception cref="InvalidOperationException"></exception>
private async Task<IFileStore> GetStore(Guid id)
{
var meta = await _metadataStore.Get(id);
var store = GetFileStore(meta?.Storage);
if (store == default)
{
throw new InvalidOperationException($"Cannot find store '{meta?.Storage}'");
}
return store;
}
}

1
VoidCat/Services/Files/LocalDiskFileMetadataStore.cs
View File

@@ -53,6 +53,7 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
await Set(id, oldMeta);
}

3
VoidCat/Services/Files/LocalDiskFileStorage.cs
View File

@@ -31,6 +31,9 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
await EgressFromStream(fs, request, outStream, cts);
}
/// <inheritdoc />
public string Key => "local-disk";
/// <inheritdoc />
public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
{

21
VoidCat/Services/Files/PostgresFileMetadataStore.cs
View File

@@ -13,7 +13,10 @@ public class PostgresFileMetadataStore : IFileMetadataStore
{
_connection = connection;
}
/// <inheritdoc />
public string? Key => "postgres";
/// <inheritdoc />
public ValueTask<VoidFileMeta?> Get(Guid id)
{
@@ -32,9 +35,15 @@ public class PostgresFileMetadataStore : IFileMetadataStore
await using var conn = await _connection.Get();
await conn.ExecuteAsync(
@"insert into
""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"", ""Expires"")
values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret, :expires)
on conflict (""Id"") do update set ""Name"" = :name, ""Size"" = :size, ""Description"" = :description, ""MimeType"" = :mimeType, ""Expires"" = :expires",
""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"", ""Expires"", ""Storage"")
values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret, :expires, :store)
on conflict (""Id"") do update set
""Name"" = :name,
""Size"" = :size,
""Description"" = :description,
""MimeType"" = :mimeType,
""Expires"" = :expires,
""Storage"" = :store",
new
{
id,
@@ -45,7 +54,8 @@ on conflict (""Id"") do update set ""Name"" = :name, ""Size"" = :size, ""Descrip
mimeType = obj.MimeType,
digest = obj.Digest,
editSecret = obj.EditSecret,
expires = obj.Expires
expires = obj.Expires,
store = obj.Storage
});
}
@@ -82,6 +92,7 @@ on conflict (""Id"") do update set ""Name"" = :name, ""Size"" = :size, ""Descrip
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
await Set(id, oldMeta);
}

12
VoidCat/Services/Files/S3FileMetadataStore.cs
View File

@@ -11,16 +11,17 @@ public class S3FileMetadataStore : IFileMetadataStore
private readonly ILogger<S3FileMetadataStore> _logger;
private readonly AmazonS3Client _client;
private readonly S3BlobConfig _config;
private readonly bool _includeUrl;
public S3FileMetadataStore(VoidSettings settings, ILogger<S3FileMetadataStore> logger)
public S3FileMetadataStore(S3BlobConfig settings, ILogger<S3FileMetadataStore> logger)
{
_logger = logger;
_includeUrl = settings.CloudStorage?.ServeFromCloud ?? false;
_config = settings.CloudStorage!.S3!;
_config = settings;
_client = _config.CreateClient();
}
/// <inheritdoc />
public string? Key => _config.Name;
/// <inheritdoc />
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
{
@@ -53,6 +54,7 @@ public class S3FileMetadataStore : IFileMetadataStore
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Expires = meta.Expires ?? oldMeta.Expires;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
await Set(id, oldMeta);
}
@@ -141,7 +143,7 @@ public class S3FileMetadataStore : IFileMetadataStore
if (ret != default)
{
ret.Id = id;
if (_includeUrl)
if (_config.Direct)
{
var ub = new UriBuilder(_config.ServiceUrl!)
{

16
VoidCat/Services/Files/S3FileStore.cs
View File

@@ -5,6 +5,7 @@ using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files;
/// <inheritdoc cref="VoidCat.Services.Abstractions.IFileStore" />
public class S3FileStore : StreamFileStore, IFileStore
{
private readonly IFileInfoManager _fileInfo;
@@ -12,22 +13,28 @@ public class S3FileStore : StreamFileStore, IFileStore
private readonly S3BlobConfig _config;
private readonly IAggregateStatsCollector _statsCollector;
public S3FileStore(VoidSettings settings, IAggregateStatsCollector stats, IFileInfoManager fileInfo) : base(stats)
public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, IFileInfoManager fileInfo) : base(stats)
{
_fileInfo = fileInfo;
_statsCollector = stats;
_config = settings.CloudStorage!.S3!;
_config = settings;
_client = _config.CreateClient();
}
/// <inheritdoc />
public string Key => _config.Name;
/// <inheritdoc />
public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
{
if (payload.IsAppend) throw new InvalidOperationException("Cannot append to S3 store");
var req = new PutObjectRequest
{
BucketName = _config.BucketName,
Key = payload.Id.ToString(),
InputStream = payload.InStream,
ContentType = "application/octet-stream",
ContentType = payload.Meta.MimeType ?? "application/octet-stream",
AutoResetStreamPosition = false,
AutoCloseStream = false,
ChecksumAlgorithm = ChecksumAlgorithm.SHA256,
@@ -47,6 +54,7 @@ public class S3FileStore : StreamFileStore, IFileStore
return HandleCompletedUpload(payload, payload.Meta.Size);
}
/// <inheritdoc />
public async ValueTask Egress(EgressRequest request, Stream outStream, CancellationToken cts)
{
await using var stream = await Open(request, cts);
@@ -108,11 +116,13 @@ public class S3FileStore : StreamFileStore, IFileStore
}
}
/// <inheritdoc />
public async ValueTask DeleteFile(Guid id)
{
await _client.DeleteObjectAsync(_config.BucketName, id.ToString());
}
/// <inheritdoc />
public async ValueTask<Stream> Open(EgressRequest request, CancellationToken cts)
{
var req = new GetObjectRequest()