v5 (#65)

Co-authored-by: Kieran <kieran@harkin.me>
Reviewed-on: https://git.v0l.io/Kieran/void.cat/pulls/65

VoidCat/Services/Files/FileInfoManager.cs

@@ -1,5 +1,6 @@
+using System.Collections.Immutable;
+using VoidCat.Database;
 using VoidCat.Model;
-using VoidCat.Model.User;
 using VoidCat.Services.Abstractions;
 
 namespace VoidCat.Services.Files;
@@ -32,36 +33,47 @@ public sealed class FileInfoManager
     /// Get all metadata for a single file
     /// </summary>
     /// <param name="id"></param>
+    /// <param name="withEditSecret"></param>
     /// <returns></returns>
-    public ValueTask<PublicVoidFile?> Get(Guid id)
+    public async ValueTask<VoidFileResponse?> Get(Guid id, bool withEditSecret)
     {
-        return Get<PublicVoidFile, FileMeta>(id);
-    }
+        var meta = await _metadataStore.Get(id);
+        if (meta == default) return default;
 
-    /// <summary>
-    /// Get all private metadata for a single file
-    /// </summary>
-    /// <param name="id"></param>
-    /// <returns></returns>
-    public ValueTask<PrivateVoidFile?> GetPrivate(Guid id)
-    {
-        return Get<PrivateVoidFile, SecretFileMeta>(id);
+        var payment = await _paymentStore.Get(id);
+        var bandwidth = await _statsReporter.GetBandwidth(id);
+        var virusScan = await _virusScanStore.GetByFile(id);
+        var uploader = await _userUploadsStore.Uploader(id);
+
+        var user = uploader.HasValue ? await _userStore.Get(uploader.Value) : null;
+
+        return new VoidFileResponse
+        {
+            Id = id,
+            Metadata = meta.ToMeta(withEditSecret),
+            Payment = payment,
+            Bandwidth = bandwidth,
+            Uploader = user?.Flags.HasFlag(UserFlags.PublicProfile) == true || withEditSecret ? user?.ToApiUser(false) : null,
+            VirusScan = virusScan?.ToVirusStatus()
+        };
     }
 
     /// <summary>
     /// Get all metadata for multiple files
     /// </summary>
     /// <param name="ids"></param>
+    /// <param name="withEditSecret"></param>
     /// <returns></returns>
-    public async ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids)
+    public async ValueTask<IReadOnlyList<VoidFileResponse>> Get(Guid[] ids, bool withEditSecret)
     {
-        var ret = new List<PublicVoidFile>();
-        foreach (var id in ids)
+        //todo: improve this
+        var ret = new List<VoidFileResponse>();
+        foreach (var i in ids)
         {
-            var v = await Get(id);
-            if (v != default)
+            var x = await Get(i, withEditSecret);
+            if (x != default)
             {
-                ret.Add(v);
+                ret.Add(x);
             }
         }
 
@@ -80,28 +92,4 @@ public sealed class FileInfoManager
         await _statsReporter.Delete(id);
         await _virusScanStore.Delete(id);
     }
-
-    private async ValueTask<TFile?> Get<TFile, TMeta>(Guid id)
-        where TMeta : FileMeta where TFile : VoidFile<TMeta>, new()
-    {
-        var meta = _metadataStore.Get<TMeta>(id);
-        var payment = _paymentStore.Get(id);
-        var bandwidth = _statsReporter.GetBandwidth(id);
-        var virusScan = _virusScanStore.GetByFile(id);
-        var uploader = _userUploadsStore.Uploader(id);
-        await Task.WhenAll(meta.AsTask(), payment.AsTask(), bandwidth.AsTask(), virusScan.AsTask(), uploader.AsTask());
-
-        if (meta.Result == default) return default;
-        var user = uploader.Result.HasValue ? await _userStore.Get<PublicUser>(uploader.Result.Value) : null;
-
-        return new TFile()
-        {
-            Id = id,
-            Metadata = meta.Result,
-            Payment = payment.Result,
-            Bandwidth = bandwidth.Result,
-            Uploader = user?.Flags.HasFlag(UserFlags.PublicProfile) == true ? user : null,
-            VirusScan = virusScan.Result
-        };
-    }
-}
+}

VoidCat/Services/Files/FileStorageStartup.cs

@@ -11,23 +11,23 @@ public static class FileStorageStartup
         services.AddTransient<FileInfoManager>();
         services.AddTransient<FileStoreFactory>();
         services.AddTransient<CompressContent>();
-        
+
         if (settings.CloudStorage != default)
         {
             // S3 storage
             foreach (var s3 in settings.CloudStorage.S3 ?? Array.Empty<S3BlobConfig>())
             {
-                services.AddTransient<IFileStore>((svc) =>
-                    new S3FileStore(s3, 
-                        svc.GetRequiredService<IAggregateStatsCollector>(),
-                        svc.GetRequiredService<FileInfoManager>(),
-                        svc.GetRequiredService<ICache>()));
-
                 if (settings.MetadataStore == s3.Name)
                 {
                     services.AddSingleton<IFileMetadataStore>((svc) =>
                         new S3FileMetadataStore(s3, svc.GetRequiredService<ILogger<S3FileMetadataStore>>()));
                 }
+
+                services.AddTransient<IFileStore>((svc) =>
+                    new S3FileStore(s3,
+                        svc.GetRequiredService<IAggregateStatsCollector>(),
+                        svc.GetRequiredService<IFileMetadataStore>(),
+                        svc.GetRequiredService<ICache>()));
             }
         }
 
@@ -37,7 +37,7 @@ public static class FileStorageStartup
             services.AddTransient<IFileStore, LocalDiskFileStore>();
             if (settings.MetadataStore is "postgres" or "local-disk")
             {
-                services.AddSingleton<IFileMetadataStore, PostgresFileMetadataStore>();
+                services.AddTransient<IFileMetadataStore, PostgresFileMetadataStore>();
             }
         }
         else

VoidCat/Services/Files/FileSystemFactory.cs

@@ -36,7 +36,7 @@ public class FileStoreFactory : IFileStore
     public string? Key => null;
 
     /// <inheritdoc />
-    public ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
+    public ValueTask<Database.File> Ingress(IngressPayload payload, CancellationToken cts)
     {
         var store = GetFileStore(payload.Meta.Storage!);
         if (store == default)

VoidCat/Services/Files/LocalDiskFileMetadataStore.cs

@@ -22,18 +22,18 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
     }
 
     /// <inheritdoc />
-    public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : FileMeta
+    public ValueTask<Database.File?> Get(Guid id)
     {
-        return GetMeta<TMeta>(id);
+        return GetMeta<Database.File>(id);
     }
 
     /// <inheritdoc />
-    public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : FileMeta
+    public async ValueTask<IReadOnlyList<Database.File>> Get(Guid[] ids)
     {
-        var ret = new List<TMeta>();
+        var ret = new List<Database.File>();
         foreach (var id in ids)
         {
-            var r = await GetMeta<TMeta>(id);
+            var r = await GetMeta<Database.File>(id);
             if (r != null)
             {
                 ret.Add(r);
@@ -42,11 +42,16 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
 
         return ret;
     }
+    
+    public ValueTask Add(Database.File f)
+    {
+        return Set(f.Id, f);
+    }
 
     /// <inheritdoc />
-    public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : FileMeta
+    public async ValueTask Update(Guid id, Database.File meta)
     {
-        var oldMeta = await Get<SecretFileMeta>(id);
+        var oldMeta = await Get(id);
         if (oldMeta == default) return;
 
         oldMeta.Patch(meta);
@@ -54,22 +59,18 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
     }
 
     /// <inheritdoc />
-    public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : FileMeta
+    public ValueTask<PagedResult<Database.File>> ListFiles(PagedRequest request)
     {
-        async IAsyncEnumerable<TMeta> EnumerateFiles()
+        async IAsyncEnumerable<Database.File> EnumerateFiles()
         {
             foreach (var metaFile in
                      Directory.EnumerateFiles(Path.Join(_settings.DataDirectory, MetadataDir), "*.json"))
             {
                 var json = await File.ReadAllTextAsync(metaFile);
-                var meta = JsonConvert.DeserializeObject<TMeta>(json);
+                var meta = JsonConvert.DeserializeObject<Database.File>(json);
                 if (meta != null)
                 {
-                    yield return meta with
-                    {
-                        // TODO: remove after migration decay
-                        Id = Guid.Parse(Path.GetFileNameWithoutExtension(metaFile))
-                    };
+                    yield return meta;
                 }
             }
         }
@@ -86,7 +87,7 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
             _ => results
         };
 
-        return ValueTask.FromResult(new PagedResult<TMeta>
+        return ValueTask.FromResult(new PagedResult<Database.File>
         {
             Page = request.Page,
             PageSize = request.PageSize,
@@ -97,26 +98,14 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
     /// <inheritdoc />
     public async ValueTask<IFileMetadataStore.StoreStats> Stats()
     {
-        var files = await ListFiles<FileMeta>(new(0, Int32.MaxValue));
+        var files = await ListFiles(new(0, Int32.MaxValue));
         var count = await files.Results.CountAsync();
         var size = await files.Results.SumAsync(a => (long) a.Size);
         return new(count, (ulong) size);
     }
 
     /// <inheritdoc />
-    public ValueTask<FileMeta?> Get(Guid id)
-    {
-        return GetMeta<FileMeta>(id);
-    }
-
-    /// <inheritdoc />
-    public ValueTask<SecretFileMeta?> GetPrivate(Guid id)
-    {
-        return GetMeta<SecretFileMeta>(id);
-    }
-
-    /// <inheritdoc />
-    public async ValueTask Set(Guid id, SecretFileMeta meta)
+    public async ValueTask Set(Guid id, Database.File meta)
     {
         var path = MapMeta(id);
         var json = JsonConvert.SerializeObject(meta);

VoidCat/Services/Files/LocalDiskFileStorage.cs

@@ -42,7 +42,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
     public string Key => "local-disk";
 
     /// <inheritdoc />
-    public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
+    public async ValueTask<Database.File> Ingress(IngressPayload payload, CancellationToken cts)
     {
         var finalPath = MapPath(payload.Id);
         await using var fsTemp = new FileStream(finalPath,
@@ -53,7 +53,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
         if (payload.ShouldStripMetadata && payload.Segment == payload.TotalSegments)
         {
             fsTemp.Close();
-            var ext = Path.GetExtension(vf.Metadata!.Name);
+            var ext = Path.GetExtension(vf.Name);
             var srcPath = $"{finalPath}_orig{ext}";
             File.Move(finalPath, srcPath);
 
@@ -69,12 +69,9 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
                 var hash = await SHA256.Create().ComputeHashAsync(fInfo.OpenRead(), cts);
                 vf = vf with
                 {
-                    Metadata = vf.Metadata! with
-                    {
-                        Size = (ulong)fInfo.Length,
-                        Digest = hash.ToHex(),
-                        MimeType = res.MimeType ?? vf.Metadata.MimeType
-                    }
+                    Size = (ulong)fInfo.Length,
+                    Digest = hash.ToHex(),
+                    MimeType = res.MimeType ?? vf.MimeType
                 };
             }
             else
@@ -86,7 +83,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
 
         if (payload.Segment == payload.TotalSegments)
         {
-            var t = await vf.Metadata!.MakeTorrent(
+            var t = await vf.ToMeta(false).MakeTorrent(vf.Id,
                 new FileStream(finalPath, FileMode.Open),
                 _settings.SiteUrl,
                 _settings.TorrentTrackers);
@@ -94,7 +91,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
             var ub = new UriBuilder(_settings.SiteUrl);
             ub.Path = $"/d/{vf.Id.ToBase58()}.torrent";
 
-            vf.Metadata!.MagnetLink = $"{t.GetMagnetLink()}&xs={Uri.EscapeDataString(ub.ToString())}";
+            vf.MagnetLink = $"{t.GetMagnetLink()}&xs={Uri.EscapeDataString(ub.ToString())}";
         }
 
         return vf;

VoidCat/Services/Files/PostgresFileMetadataStore.cs

@@ -1,4 +1,4 @@
-using Dapper;
+using Microsoft.EntityFrameworkCore;
 using VoidCat.Model;
 using VoidCat.Services.Abstractions;
 
@@ -7,119 +7,103 @@ namespace VoidCat.Services.Files;
 /// <inheritdoc />
 public class PostgresFileMetadataStore : IFileMetadataStore
 {
-    private readonly PostgresConnectionFactory _connection;
+    private readonly VoidContext _db;
+    private readonly IServiceScopeFactory _scopeFactory;
 
-    public PostgresFileMetadataStore(PostgresConnectionFactory connection)
+    public PostgresFileMetadataStore(VoidContext db, IServiceScopeFactory scopeFactory)
     {
-        _connection = connection;
+        _db = db;
+        _scopeFactory = scopeFactory;
     }
-    
-    /// <inheritdoc />
+
     public string? Key => "postgres";
-    
-    /// <inheritdoc />
-    public ValueTask<FileMeta?> Get(Guid id)
-    {
-        return Get<FileMeta>(id);
-    }
 
     /// <inheritdoc />
-    public ValueTask<SecretFileMeta?> GetPrivate(Guid id)
+    public async ValueTask<Database.File?> Get(Guid id)
     {
-        return Get<SecretFileMeta>(id);
+        return await _db.Files
+            .AsNoTracking()
+            .Include(a => a.Paywall)
+            .SingleOrDefaultAsync(a => a.Id == id);
     }
 
-    /// <inheritdoc />
-    public async ValueTask Set(Guid id, SecretFileMeta obj)
+    public async ValueTask Add(Database.File f)
     {
-        await using var conn = await _connection.Get();
-        await conn.ExecuteAsync(
-            @"insert into 
-""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"", ""Expires"", ""Storage"", ""EncryptionParams"", ""MagnetLink"")
-values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret, :expires, :store, :encryptionParams, :magnetLink)
-on conflict (""Id"") do update set 
-""Name"" = :name, 
-""Size"" = :size, 
-""Description"" = :description, 
-""MimeType"" = :mimeType, 
-""Expires"" = :expires,
-""Storage"" = :store,
-""EncryptionParams"" = :encryptionParams,
-""MagnetLink"" = :magnetLink",
-            new
-            {
-                id,
-                name = obj.Name,
-                size = (long) obj.Size,
-                uploaded = obj.Uploaded.ToUniversalTime(),
-                description = obj.Description,
-                mimeType = obj.MimeType,
-                digest = obj.Digest,
-                editSecret = obj.EditSecret,
-                expires = obj.Expires?.ToUniversalTime(),
-                store = obj.Storage,
-                encryptionParams = obj.EncryptionParams,
-                magnetLink = obj.MagnetLink,
-            });
+        _db.Files.Add(f);
+        await _db.SaveChangesAsync();
     }
 
     /// <inheritdoc />
     public async ValueTask Delete(Guid id)
     {
-        await using var conn = await _connection.Get();
-        await conn.ExecuteAsync("delete from \"Files\" where \"Id\" = :id", new {id});
+        await _db.Files
+            .Where(a => a.Id == id)
+            .ExecuteDeleteAsync();
     }
 
     /// <inheritdoc />
-    public async ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : FileMeta
+    public async ValueTask<IReadOnlyList<Database.File>> Get(Guid[] ids)
     {
-        await using var conn = await _connection.Get();
-        return await conn.QuerySingleOrDefaultAsync<TMeta?>(@"select * from ""Files"" where ""Id"" = :id",
-            new {id});
+        return await _db.Files
+            .Include(a => a.Paywall)
+            .Where(a => ids.Contains(a.Id))
+            .ToArrayAsync();
     }
 
     /// <inheritdoc />
-    public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : FileMeta
+    public async ValueTask Update(Guid id, Database.File obj)
     {
-        await using var conn = await _connection.Get();
-        var ret = await conn.QueryAsync<TMeta>("select * from \"Files\" where \"Id\" in :ids", new {ids});
-        return ret.ToList();
-    }
-
-    /// <inheritdoc />
-    public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : FileMeta
-    {
-        var oldMeta = await Get<SecretFileMeta>(id);
-        if (oldMeta == default) return;
-
-        oldMeta.Patch(meta);
-        await Set(id, oldMeta);
-    }
-
-    /// <inheritdoc />
-    public async ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : FileMeta
-    {
-        await using var conn = await _connection.Get();
-        var count = await conn.ExecuteScalarAsync<int>(@"select count(*) from ""Files""");
-
-        async IAsyncEnumerable<TMeta> Enumerate()
+        var existing = await _db.Files.FindAsync(id);
+        if (existing == default)
         {
-            var orderBy = request.SortBy switch
-            {
-                PagedSortBy.Date => "Uploaded",
-                PagedSortBy.Name => "Name",
-                PagedSortBy.Size => "Size",
-                _ => "Id"
-            };
-            await using var iconn = await _connection.Get();
-            var orderDirection = request.SortOrder == PageSortOrder.Asc ? "asc" : "desc";
-            var results = await iconn.QueryAsync<TMeta>(
-                $"select * from \"Files\" order by \"{orderBy}\" {orderDirection} offset @offset limit @limit",
-                new {offset = request.PageSize * request.Page, limit = request.PageSize});
+            return;
+        }
 
-            foreach (var meta in results)
+        existing.Patch(obj);
+        await _db.SaveChangesAsync();
+    }
+
+    /// <inheritdoc />
+    public async ValueTask<PagedResult<Database.File>> ListFiles(PagedRequest request)
+    {
+        var count = await _db.Files.CountAsync();
+
+        async IAsyncEnumerable<Database.File> Enumerate()
+        {
+            using var scope = _scopeFactory.CreateScope();
+            var db = scope.ServiceProvider.GetRequiredService<VoidContext>();
+            var q = db.Files.AsNoTracking().AsQueryable();
+            switch (request.SortBy, request.SortOrder)
             {
-                yield return meta;
+                case (PagedSortBy.Id, PageSortOrder.Asc):
+                    q = q.OrderBy(a => a.Id);
+                    break;
+                case (PagedSortBy.Id, PageSortOrder.Dsc):
+                    q = q.OrderByDescending(a => a.Id);
+                    break;
+                case (PagedSortBy.Name, PageSortOrder.Asc):
+                    q = q.OrderBy(a => a.Name);
+                    break;
+                case (PagedSortBy.Name, PageSortOrder.Dsc):
+                    q = q.OrderByDescending(a => a.Name);
+                    break;
+                case (PagedSortBy.Date, PageSortOrder.Asc):
+                    q = q.OrderBy(a => a.Uploaded);
+                    break;
+                case (PagedSortBy.Date, PageSortOrder.Dsc):
+                    q = q.OrderByDescending(a => a.Uploaded);
+                    break;
+                case (PagedSortBy.Size, PageSortOrder.Asc):
+                    q = q.OrderBy(a => a.Size);
+                    break;
+                case (PagedSortBy.Size, PageSortOrder.Dsc):
+                    q = q.OrderByDescending(a => a.Size);
+                    break;
+            }
+
+            await foreach (var r in q.Skip(request.Page * request.PageSize).Take(request.PageSize).AsAsyncEnumerable())
+            {
+                yield return r;
             }
         }
 
@@ -135,9 +119,11 @@ on conflict (""Id"") do update set
     /// <inheritdoc />
     public async ValueTask<IFileMetadataStore.StoreStats> Stats()
     {
-        await using var conn = await _connection.Get();
-        var v = await conn.QuerySingleAsync<(long Files, long Size)>(
-            @"select count(1) ""Files"", cast(sum(""Size"") as bigint) ""Size"" from ""Files""");
-        return new(v.Files, (ulong) v.Size);
+        var size = await _db.Files
+            .AsNoTracking()
+            .SumAsync(a => (long)a.Size);
+
+        var count = await _db.Files.CountAsync();
+        return new(count, (ulong)size);
     }
-}
+}

VoidCat/Services/Files/S3FileMetadataStore.cs

@@ -19,22 +19,35 @@ public class S3FileMetadataStore : IFileMetadataStore
         _client = _config.CreateClient();
     }
 
-    /// <inheritdoc />
     public string? Key => _config.Name;
     
     /// <inheritdoc />
-    public ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : FileMeta
+    public async ValueTask<Database.File?> Get(Guid id)
     {
-        return GetMeta<TMeta>(id);
+        try
+        {
+            var obj = await _client.GetObjectAsync(_config.BucketName, ToKey(id));
+
+            using var sr = new StreamReader(obj.ResponseStream);
+            var json = await sr.ReadToEndAsync();
+            var ret = JsonConvert.DeserializeObject<Database.File>(json);
+            return ret;
+        }
+        catch (AmazonS3Exception aex)
+        {
+            _logger.LogError(aex, "Failed to get metadata for {Id}, {Error}", id, aex.Message);
+        }
+
+        return default;
     }
 
     /// <inheritdoc />
-    public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : FileMeta
+    public async ValueTask<IReadOnlyList<Database.File>> Get(Guid[] ids)
     {
-        var ret = new List<TMeta>();
+        var ret = new List<Database.File>();
         foreach (var id in ids)
         {
-            var r = await GetMeta<TMeta>(id);
+            var r = await Get(id);
             if (r != null)
             {
                 ret.Add(r);
@@ -43,11 +56,15 @@ public class S3FileMetadataStore : IFileMetadataStore
 
         return ret;
     }
+    public ValueTask Add(Database.File f)
+    {
+        return Set(f.Id, f);
+    }
 
     /// <inheritdoc />
-    public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : FileMeta
+    public async ValueTask Update(Guid id, Database.File meta)
     {
-        var oldMeta = await Get<SecretFileMeta>(id);
+        var oldMeta = await Get(id);
         if (oldMeta == default) return;
 
         oldMeta.Patch(meta);
@@ -55,9 +72,9 @@ public class S3FileMetadataStore : IFileMetadataStore
     }
 
     /// <inheritdoc />
-    public ValueTask<PagedResult<TMeta>> ListFiles<TMeta>(PagedRequest request) where TMeta : FileMeta
+    public ValueTask<PagedResult<Database.File>> ListFiles(PagedRequest request)
     {
-        async IAsyncEnumerable<TMeta> Enumerate()
+        async IAsyncEnumerable<Database.File> Enumerate()
         {
             var obj = await _client.ListObjectsV2Async(new()
             {
@@ -70,7 +87,7 @@ public class S3FileMetadataStore : IFileMetadataStore
             {
                 if (Guid.TryParse(file.Key.Split("metadata_")[1], out var id))
                 {
-                    var meta = await GetMeta<TMeta>(id);
+                    var meta = await Get(id);
                     if (meta != default)
                     {
                         yield return meta;
@@ -79,7 +96,7 @@ public class S3FileMetadataStore : IFileMetadataStore
             }
         }
 
-        return ValueTask.FromResult(new PagedResult<TMeta>
+        return ValueTask.FromResult(new PagedResult<Database.File>
         {
             Page = request.Page,
             PageSize = request.PageSize,
@@ -90,26 +107,19 @@ public class S3FileMetadataStore : IFileMetadataStore
     /// <inheritdoc />
     public async ValueTask<IFileMetadataStore.StoreStats> Stats()
     {
-        var files = await ListFiles<FileMeta>(new(0, Int32.MaxValue));
+        var files = await ListFiles(new(0, Int32.MaxValue));
         var count = await files.Results.CountAsync();
         var size = await files.Results.SumAsync(a => (long) a.Size);
         return new(count, (ulong) size);
     }
 
     /// <inheritdoc />
-    public ValueTask<FileMeta?> Get(Guid id)
+    public async ValueTask Delete(Guid id)
     {
-        return GetMeta<FileMeta>(id);
+        await _client.DeleteObjectAsync(_config.BucketName, ToKey(id));
     }
 
-    /// <inheritdoc />
-    public ValueTask<SecretFileMeta?> GetPrivate(Guid id)
-    {
-        return GetMeta<SecretFileMeta>(id);
-    }
-
-    /// <inheritdoc />
-    public async ValueTask Set(Guid id, SecretFileMeta meta)
+    private async ValueTask Set(Guid id, Database.File meta)
     {
         await _client.PutObjectAsync(new()
         {
@@ -120,35 +130,5 @@ public class S3FileMetadataStore : IFileMetadataStore
         });
     }
 
-    /// <inheritdoc />
-    public async ValueTask Delete(Guid id)
-    {
-        await _client.DeleteObjectAsync(_config.BucketName, ToKey(id));
-    }
-
-    private async ValueTask<TMeta?> GetMeta<TMeta>(Guid id) where TMeta : FileMeta
-    {
-        try
-        {
-            var obj = await _client.GetObjectAsync(_config.BucketName, ToKey(id));
-
-            using var sr = new StreamReader(obj.ResponseStream);
-            var json = await sr.ReadToEndAsync();
-            var ret = JsonConvert.DeserializeObject<TMeta>(json);
-            if (ret != default)
-            {
-                ret.Id = id;
-            }
-
-            return ret;
-        }
-        catch (AmazonS3Exception aex)
-        {
-            _logger.LogError(aex, "Failed to get metadata for {Id}, {Error}", id, aex.Message);
-        }
-
-        return default;
-    }
-
     private static string ToKey(Guid id) => $"metadata_{id}";
 }

VoidCat/Services/Files/S3FileStore.cs

@@ -9,13 +9,13 @@ namespace VoidCat.Services.Files;
 /// <inheritdoc cref="VoidCat.Services.Abstractions.IFileStore" />
 public class S3FileStore : StreamFileStore, IFileStore
 {
-    private readonly FileInfoManager _fileInfo;
+    private readonly IFileMetadataStore _fileInfo;
     private readonly AmazonS3Client _client;
     private readonly S3BlobConfig _config;
     private readonly IAggregateStatsCollector _statsCollector;
     private readonly ICache _cache;
 
-    public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, FileInfoManager fileInfo, ICache cache) : base(stats)
+    public S3FileStore(S3BlobConfig settings, IAggregateStatsCollector stats, IFileMetadataStore fileInfo, ICache cache) : base(stats)
     {
         _fileInfo = fileInfo;
         _cache = cache;
@@ -28,7 +28,7 @@ public class S3FileStore : StreamFileStore, IFileStore
     public string Key => _config.Name;
 
     /// <inheritdoc />
-    public async ValueTask<PrivateVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
+    public async ValueTask<Database.File> Ingress(IngressPayload payload, CancellationToken cts)
     {
         if (payload.IsMultipart) return await IngressMultipart(payload, cts);
 
@@ -75,15 +75,15 @@ public class S3FileStore : StreamFileStore, IFileStore
             Key = request.Id.ToString(),
             ResponseHeaderOverrides = new()
             {
-                ContentDisposition = $"inline; filename=\"{meta?.Metadata?.Name}\"",
-                ContentType = meta?.Metadata?.MimeType
+                ContentDisposition = $"inline; filename=\"{meta?.Name}\"",
+                ContentType = meta?.MimeType
             }
         });
 
         return new(new Uri(url));
     }
 
-    public async ValueTask<PagedResult<PublicVoidFile>> ListFiles(PagedRequest request)
+    public async ValueTask<PagedResult<Database.File>> ListFiles(PagedRequest request)
     {
         try
         {
@@ -103,7 +103,7 @@ public class S3FileStore : StreamFileStore, IFileStore
                 _ => objs.S3Objects.AsEnumerable()
             };
 
-            async IAsyncEnumerable<PublicVoidFile> EnumerateFiles(IEnumerable<S3Object> page)
+            async IAsyncEnumerable<Database.File> EnumerateFiles(IEnumerable<S3Object> page)
             {
                 foreach (var item in page)
                 {
@@ -133,7 +133,7 @@ public class S3FileStore : StreamFileStore, IFileStore
                 Page = request.Page,
                 PageSize = request.PageSize,
                 TotalResults = 0,
-                Results = AsyncEnumerable.Empty<PublicVoidFile>()
+                Results = AsyncEnumerable.Empty<Database.File>()
             };
         }
     }
@@ -163,7 +163,7 @@ public class S3FileStore : StreamFileStore, IFileStore
         return obj.ResponseStream;
     }
 
-    private async Task<PrivateVoidFile> IngressMultipart(IngressPayload payload, CancellationToken cts)
+    private async Task<Database.File> IngressMultipart(IngressPayload payload, CancellationToken cts)
     {
         string? uploadId = null;
         var cacheKey = $"s3:{_config.Name}:multipart-upload-id:{payload.Id}";

VoidCat/Services/Files/StreamFileStore.cs

@@ -31,7 +31,7 @@ public abstract class StreamFileStore
         }
     }
 
-    protected async ValueTask<PrivateVoidFile> IngressToStream(Stream outStream, IngressPayload payload,
+    protected async ValueTask<Database.File> IngressToStream(Stream outStream, IngressPayload payload,
         CancellationToken cts)
     {
         var id = payload.Id;
@@ -48,13 +48,14 @@ public abstract class StreamFileStore
         return HandleCompletedUpload(payload, total);
     }
 
-    protected PrivateVoidFile HandleCompletedUpload(IngressPayload payload, ulong totalSize)
+    protected Database.File HandleCompletedUpload(IngressPayload payload, ulong totalSize)
     {
         var meta = payload.Meta;
         if (payload.IsAppend)
         {
             meta = meta with
             {
+                Id = payload.Id,
                 Size = meta.Size + totalSize
             };
         }
@@ -62,19 +63,14 @@ public abstract class StreamFileStore
         {
             meta = meta with
             {
-                Uploaded = DateTimeOffset.UtcNow,
+                Id = payload.Id,
+                Uploaded = DateTime.UtcNow,
                 EditSecret = Guid.NewGuid(),
                 Size = totalSize
             };
         }
 
-        var vf = new PrivateVoidFile()
-        {
-            Id = payload.Id,
-            Metadata = meta
-        };
-
-        return vf;
+        return meta;
     }
 
     private async Task<ulong> IngressInternal(Guid id, Stream ingress, Stream outStream,