highperfocused/void.cat
1
0
Fork 0
mirror of https://git.v0l.io/Kieran/void.cat.git synced 2025-11-15 06:36:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

Client side encryption completed

Browse Source
This commit is contained in:
Kieran
2022-09-11 20:07:38 +01:00
parent d0a92fa115
commit 78ced7f4f3
26 changed files with 425 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
VoidCat/Services/Abstractions/IFileMetadataStore.cs
View File

@@ -5,7 +5,7 @@ namespace VoidCat.Services.Abstractions;
/// <summary>
/// File metadata contains all data about a file except for the file data itself
/// </summary>
public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVoidFileMeta>
public interface IFileMetadataStore : IPublicPrivateStore<FileMeta, SecretFileMeta>
{
/// <summary>
/// Get metadata for a single file
@@ -13,7 +13,7 @@ public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVo
/// <param name="id"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta;
ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : FileMeta;
/// <summary>
/// Get metadata for multiple files
@@ -21,7 +21,7 @@ public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVo
/// <param name="ids"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta;
ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : FileMeta;
/// <summary>
/// Update file metadata
@@ -30,7 +30,7 @@ public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVo
/// <param name="meta"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta;
ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : FileMeta;
/// <summary>
/// List all files in the store
@@ -38,7 +38,7 @@ public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVo
/// <param name="request"></param>
/// <typeparam name="TMeta"></typeparam>
/// <returns></returns>
ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta;
ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : FileMeta;
/// <summary>
/// Returns basic stats about the file store

2
VoidCat/Services/Background/DeleteExpiredFiles.cs
View File

@@ -27,7 +27,7 @@ public sealed class DeleteExpiredFiles : BackgroundService
var fileInfoManager = scope.ServiceProvider.GetRequiredService<FileInfoManager>();
var fileStoreFactory = scope.ServiceProvider.GetRequiredService<FileStoreFactory>();
var files = await metadata.ListFiles<SecretVoidFileMeta>(new(0, int.MaxValue));
var files = await metadata.ListFiles<SecretFileMeta>(new(0, int.MaxValue));
await foreach (var f in files.Results.WithCancellation(stoppingToken))
{
try

2
VoidCat/Services/Background/VirusScannerService.cs
View File

@@ -29,7 +29,7 @@ public class VirusScannerService : BackgroundService
var page = 0;
while (true)
{
var files = await _fileStore.ListFiles<VoidFileMeta>(new(page, 1_000));
var files = await _fileStore.ListFiles<FileMeta>(new(page, 1_000));
if (files.Pages < page) break;
page++;

6
VoidCat/Services/Files/FileInfoManager.cs
View File

@@ -35,7 +35,7 @@ public sealed class FileInfoManager
/// <returns></returns>
public ValueTask<PublicVoidFile?> Get(Guid id)
{
return Get<PublicVoidFile, VoidFileMeta>(id);
return Get<PublicVoidFile, FileMeta>(id);
}
/// <summary>
@@ -45,7 +45,7 @@ public sealed class FileInfoManager
/// <returns></returns>
public ValueTask<PrivateVoidFile?> GetPrivate(Guid id)
{
return Get<PrivateVoidFile, SecretVoidFileMeta>(id);
return Get<PrivateVoidFile, SecretFileMeta>(id);
}
/// <summary>
@@ -82,7 +82,7 @@ public sealed class FileInfoManager
}
private async ValueTask<TFile?> Get<TFile, TMeta>(Guid id)
where TMeta : VoidFileMeta where TFile : VoidFile<TMeta>, new()
where TMeta : FileMeta where TFile : VoidFile<TMeta>, new()
{
var meta = _metadataStore.Get<TMeta>(id);
var payment = _paymentStore.Get(id);

29
VoidCat/Services/Files/LocalDiskFileMetadataStore.cs
View File

@@ -22,13 +22,13 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
}
/// <inheritdoc />
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : FileMeta
{
return GetMeta<TMeta>(id);
}
/// <inheritdoc />
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : FileMeta
{
var ret = new List<TMeta>();
foreach (var id in ids)
@@ -44,22 +44,17 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
}
/// <inheritdoc />
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : FileMeta
{
var oldMeta = await Get<SecretVoidFileMeta>(id);
var oldMeta = await Get<SecretFileMeta>(id);
if (oldMeta == default) return;
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
oldMeta.Patch(meta);
await Set(id, oldMeta);
}
/// <inheritdoc />
public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta
public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : FileMeta
{
async IAsyncEnumerable<TMeta> EnumerateFiles()
{
@@ -102,26 +97,26 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
/// <inheritdoc />
public async ValueTask<IFileMetadataStore.StoreStats> Stats()
{
var files = await ListFiles<VoidFileMeta>(new(0, Int32.MaxValue));
var files = await ListFiles<FileMeta>(new(0, Int32.MaxValue));
var count = await files.Results.CountAsync();
var size = await files.Results.SumAsync(a => (long) a.Size);
return new(count, (ulong) size);
}
/// <inheritdoc />
public ValueTask<VoidFileMeta?> Get(Guid id)
public ValueTask<FileMeta?> Get(Guid id)
{
return GetMeta<VoidFileMeta>(id);
return GetMeta<FileMeta>(id);
}
/// <inheritdoc />
public ValueTask<SecretVoidFileMeta?> GetPrivate(Guid id)
public ValueTask<SecretFileMeta?> GetPrivate(Guid id)
{
return GetMeta<SecretVoidFileMeta>(id);
return GetMeta<SecretFileMeta>(id);
}
/// <inheritdoc />
public async ValueTask Set(Guid id, SecretVoidFileMeta meta)
public async ValueTask Set(Guid id, SecretFileMeta meta)
{
var path = MapMeta(id);
var json = JsonConvert.SerializeObject(meta);

37
VoidCat/Services/Files/PostgresFileMetadataStore.cs
View File

@@ -18,32 +18,33 @@ public class PostgresFileMetadataStore : IFileMetadataStore
public string? Key => "postgres";
/// <inheritdoc />
public ValueTask<VoidFileMeta?> Get(Guid id)
public ValueTask<FileMeta?> Get(Guid id)
{
return Get<VoidFileMeta>(id);
return Get<FileMeta>(id);
}
/// <inheritdoc />
public ValueTask<SecretVoidFileMeta?> GetPrivate(Guid id)
public ValueTask<SecretFileMeta?> GetPrivate(Guid id)
{
return Get<SecretVoidFileMeta>(id);
return Get<SecretFileMeta>(id);
}
/// <inheritdoc />
public async ValueTask Set(Guid id, SecretVoidFileMeta obj)
public async ValueTask Set(Guid id, SecretFileMeta obj)
{
await using var conn = await _connection.Get();
await conn.ExecuteAsync(
@"insert into
""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"", ""Expires"", ""Storage"")
values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret, :expires, :store)
""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"", ""Expires"", ""Storage"", ""EncryptionParams"")
values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret, :expires, :store, :encryptionParams)
on conflict (""Id"") do update set
""Name"" = :name,
""Size"" = :size,
""Description"" = :description,
""MimeType"" = :mimeType,
""Expires"" = :expires,
""Storage"" = :store",
""Storage"" = :store,
""EncryptionParams"" = :encryptionParams",
new
{
id,
@@ -55,7 +56,8 @@ on conflict (""Id"") do update set
digest = obj.Digest,
editSecret = obj.EditSecret,
expires = obj.Expires?.ToUniversalTime(),
store = obj.Storage
store = obj.Storage,
encryptionParams = obj.EncryptionParams
});
}
@@ -67,7 +69,7 @@ on conflict (""Id"") do update set
}
/// <inheritdoc />
public async ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
public async ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : FileMeta
{
await using var conn = await _connection.Get();
return await conn.QuerySingleOrDefaultAsync<TMeta?>(@"select * from ""Files"" where ""Id"" = :id",
@@ -75,7 +77,7 @@ on conflict (""Id"") do update set
}
/// <inheritdoc />
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : FileMeta
{
await using var conn = await _connection.Get();
var ret = await conn.QueryAsync<TMeta>("select * from \"Files\" where \"Id\" in :ids", new {ids});
@@ -83,22 +85,17 @@ on conflict (""Id"") do update set
}
/// <inheritdoc />
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : FileMeta
{
var oldMeta = await Get<SecretVoidFileMeta>(id);
var oldMeta = await Get<SecretFileMeta>(id);
if (oldMeta == default) return;
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
oldMeta.Patch(meta);
await Set(id, oldMeta);
}
/// <inheritdoc />
public async ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta
public async ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : FileMeta
{
await using var conn = await _connection.Get();
var count = await conn.ExecuteScalarAsync<int>(@"select count(*) from ""Files""");

31
VoidCat/Services/Files/S3FileMetadataStore.cs
View File

@@ -23,13 +23,13 @@ public class S3FileMetadataStore : IFileMetadataStore
public string? Key => _config.Name;
/// <inheritdoc />
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : FileMeta
{
return GetMeta<TMeta>(id);
}
/// <inheritdoc />
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : FileMeta
{
var ret = new List<TMeta>();
foreach (var id in ids)
@@ -45,22 +45,17 @@ public class S3FileMetadataStore : IFileMetadataStore
}
/// <inheritdoc />
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : FileMeta
{
var oldMeta = await GetMeta<SecretVoidFileMeta>(id);
var oldMeta = await Get<SecretFileMeta>(id);
if (oldMeta == default) return;
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
oldMeta.Storage = meta.Storage ?? oldMeta.Storage;
oldMeta.Expires = meta.Expires;
oldMeta.Patch(meta);
await Set(id, oldMeta);
}
/// <inheritdoc />
public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : VoidFileMeta
public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : FileMeta
{
async IAsyncEnumerable<TMeta> Enumerate()
{
@@ -95,26 +90,26 @@ public class S3FileMetadataStore : IFileMetadataStore
/// <inheritdoc />
public async ValueTask<IFileMetadataStore.StoreStats> Stats()
{
var files = await ListFiles<VoidFileMeta>(new(0, Int32.MaxValue));
var files = await ListFiles<FileMeta>(new(0, Int32.MaxValue));
var count = await files.Results.CountAsync();
var size = await files.Results.SumAsync(a => (long) a.Size);
return new(count, (ulong) size);
}
/// <inheritdoc />
public ValueTask<VoidFileMeta?> Get(Guid id)
public ValueTask<FileMeta?> Get(Guid id)
{
return GetMeta<VoidFileMeta>(id);
return GetMeta<FileMeta>(id);
}
/// <inheritdoc />
public ValueTask<SecretVoidFileMeta?> GetPrivate(Guid id)
public ValueTask<SecretFileMeta?> GetPrivate(Guid id)
{
return GetMeta<SecretVoidFileMeta>(id);
return GetMeta<SecretFileMeta>(id);
}
/// <inheritdoc />
public async ValueTask Set(Guid id, SecretVoidFileMeta meta)
public async ValueTask Set(Guid id, SecretFileMeta meta)
{
await _client.PutObjectAsync(new()
{
@@ -131,7 +126,7 @@ public class S3FileMetadataStore : IFileMetadataStore
await _client.DeleteObjectAsync(_config.BucketName, ToKey(id));
}
private async ValueTask<TMeta?> GetMeta<TMeta>(Guid id) where TMeta : VoidFileMeta
private async ValueTask<TMeta?> GetMeta<TMeta>(Guid id) where TMeta : FileMeta
{
try
{

20
VoidCat/Services/Migrations/Database/06-EncryptionParams.cs Normal file
View File

@@ -0,0 +1,20 @@
using FluentMigrator;
namespace VoidCat.Services.Migrations.Database;
[Migration(20220911_1635)]
public class EncryptionParams : Migration{
public override void Up()
{
Create.Column("EncryptionParams")
.OnTable("Files")
.AsString()
.Nullable();
}
public override void Down()
{
Delete.Column("EncryptionParams")
.FromTable("Files");
}
}

2
VoidCat/Services/Migrations/FixSize.cs
View File

@@ -24,7 +24,7 @@ public class FixSize : IMigration
/// <inheritdoc />
public async ValueTask<IMigration.MigrationResult> Migrate(string[] args)
{
var files = await _fileMetadata.ListFiles<SecretVoidFileMeta>(new(0, int.MaxValue));
var files = await _fileMetadata.ListFiles<SecretFileMeta>(new(0, int.MaxValue));
await foreach (var file in files.Results)
{
try

4
VoidCat/Services/Migrations/MigrateToPostgres.cs
View File

@@ -102,7 +102,7 @@ public class MigrateToPostgres : IMigration
{
var cachePaywallStore = new CachePaymentStore(_cache);
var files = await _fileMetadata.ListFiles<VoidFileMeta>(new(0, int.MaxValue));
var files = await _fileMetadata.ListFiles<FileMeta>(new(0, int.MaxValue));
await foreach (var file in files.Results)
{
try
@@ -161,7 +161,7 @@ public class MigrateToPostgres : IMigration
public string? Password { get; set; }
}
private record UploaderSecretVoidFileMeta : SecretVoidFileMeta
private record UploaderSecretVoidFileMeta : SecretFileMeta
{
[JsonConverter(typeof(Base58GuidConverter))]
public Guid? Uploader { get; set; }

2
VoidCat/Services/Migrations/PopulateMetadataId.cs
View File

@@ -24,7 +24,7 @@ public class PopulateMetadataId : IMigration
return IMigration.MigrationResult.Skipped;
}
var files = await _metadataStore.ListFiles<SecretVoidFileMeta>(new(0, Int32.MaxValue));
var files = await _metadataStore.ListFiles<SecretFileMeta>(new(0, Int32.MaxValue));
await foreach (var file in files.Results)
{
// read-write file metadata

2
VoidCat/Services/VirusScanner/CacheVirusScanStore.cs
View File

@@ -23,7 +23,7 @@ public class CacheVirusScanStore : BasicCacheStore<VirusScanResult>, IVirusScanS
var scans = await _cache.GetList(MapFilesKey(id));
if (scans.Length > 0)
{
return await Get(Guid.Parse(scans.First()));
return await Get(Guid.Parse(scans.Last()));
}
return default;