Add postgres support for users/files

README.md

@@ -31,4 +31,15 @@ curl -X POST \
 ```
 
 This command will return the direct download URL only. 
-To get the json output simply remove the `?cli=true` from the url.
+To get the json output simply remove the `?cli=true` from the url.
+
+### Development
+To run postgres in local use:
+```
+docker run --rm -it -p 5432:5432 -e POSTGRES_DB=void -e POSTGRES_PASSWORD=postgres postgres -d postgres
+```
+
+To run MinIO in local use:
+```
+docker run --rm -it -p 9000:9000 -p 9001:9001 minio/minio -- server /data --console-address ":9001"
+```

VoidCat/Controllers/InfoController.cs

@@ -8,13 +8,13 @@ namespace VoidCat.Controllers;
 public class InfoController : Controller
 {
     private readonly IStatsReporter _statsReporter;
-    private readonly IFileStore _fileStore;
+    private readonly IFileMetadataStore _fileMetadata;
     private readonly VoidSettings _settings;
 
-    public InfoController(IStatsReporter statsReporter, IFileStore fileStore, VoidSettings settings)
+    public InfoController(IStatsReporter statsReporter, IFileMetadataStore fileMetadata, VoidSettings settings)
     {
         _statsReporter = statsReporter;
-        _fileStore = fileStore;
+        _fileMetadata = fileMetadata;
         _settings = settings;
     }
 
@@ -27,18 +27,11 @@ public class InfoController : Controller
     public async Task<GlobalInfo> GetGlobalStats()
     {
         var bw = await _statsReporter.GetBandwidth();
-        var bytes = 0UL;
+        var storeStats = await _fileMetadata.Stats();
-        var count = 0;
+        
-        var files = await _fileStore.ListFiles(new(0, Int32.MaxValue));
+        return new(bw, (ulong)storeStats.Size, storeStats.Files, BuildInfo.GetBuildInfo(), _settings.CaptchaSettings?.SiteKey);
-        await foreach (var vf in files.Results)
-        {
-            bytes += vf.Metadata?.Size ?? 0;
-            count++;
-        }
-
-        return new(bw, bytes, count, BuildInfo.GetBuildInfo(), _settings.CaptchaSettings?.SiteKey);
     }
 
-    public sealed record GlobalInfo(Bandwidth Bandwidth, ulong TotalBytes, int Count, BuildInfo BuildInfo,
+    public sealed record GlobalInfo(Bandwidth Bandwidth, ulong TotalBytes, long Count, BuildInfo BuildInfo,
         string? CaptchaSiteKey);
 }

VoidCat/Controllers/UploadController.cs

@@ -60,7 +60,7 @@ namespace VoidCat.Controllers
                     }
                 }
 
-                var meta = new SecretVoidFileMeta()
+                var meta = new SecretVoidFileMeta
                 {
                     MimeType = mime,
                     Name = filename,

VoidCat/Model/VoidFileMeta.cs

@@ -1,6 +1,8 @@
 using Newtonsoft.Json;
 using VoidCat.Services.Abstractions;
 
+// ReSharper disable InconsistentNaming
+
 namespace VoidCat.Model;
 
 /// <summary>
@@ -9,7 +11,7 @@ namespace VoidCat.Model;
 public interface IVoidFileMeta
 {
     const int CurrentVersion = 3;
-    
+
     int Version { get; init; }
 }
 
@@ -22,7 +24,7 @@ public record VoidFileMeta : IVoidFileMeta
     /// Metadata version
     /// </summary>
     public int Version { get; init; } = IVoidFileMeta.CurrentVersion;
-    
+
     /// <summary>
     /// Filename
     /// </summary>
@@ -52,12 +54,12 @@ public record VoidFileMeta : IVoidFileMeta
     /// SHA-256 hash of the file
     /// </summary>
     public string? Digest { get; init; }
-    
+
     /// <summary>
     /// Url to download the file
     /// </summary>
     public Uri? Url { get; set; }
-    
+
     /// <summary>
     /// User who uploaded the file
     /// </summary>

VoidCat/Model/VoidSettings.cs

@@ -29,6 +29,8 @@ namespace VoidCat.Model
         public IEnumerable<string>? RequestHeadersLog { get; init; }
         
         public CaptchaSettings? CaptchaSettings { get; init; }
+        
+        public string? Postgres { get; init; }
     }
 
     public sealed class TorSettings

VoidCat/Model/VoidUser.cs

@@ -1,7 +1,12 @@
 using Newtonsoft.Json;
 
+// ReSharper disable InconsistentNaming
+
 namespace VoidCat.Model;
 
+/// <summary>
+/// The base user object for the system
+/// </summary>
 public abstract class VoidUser
 {
     protected VoidUser(Guid id)
@@ -9,20 +14,32 @@ public abstract class VoidUser
         Id = id;
     }
 
+    /// <summary>
+    /// Unique Id of the user
+    /// </summary>
     [JsonConverter(typeof(Base58GuidConverter))]
     public Guid Id { get; }
 
+    /// <summary>
+    /// Roles assigned to this user which grant them extra permissions
+    /// </summary>
     public HashSet<string> Roles { get; init; } = new() {Model.Roles.User};
 
+    /// <summary>
+    /// When the user account was created
+    /// </summary>
     public DateTimeOffset Created { get; init; }
 
+    /// <summary>
+    /// The last time the user logged in
+    /// </summary>
     public DateTimeOffset LastLogin { get; set; }
 
     /// <summary>
     /// Display avatar for user profile
     /// </summary>
     public string? Avatar { get; set; }
-    
+
     /// <summary>
     /// Display name for user profile
     /// </summary>
@@ -32,7 +49,11 @@ public abstract class VoidUser
     /// Profile flags
     /// </summary>
     public VoidUserFlags Flags { get; set; } = VoidUserFlags.PublicProfile;
-    
+
+    /// <summary>
+    /// Returns the Public object for this user
+    /// </summary>
+    /// <returns></returns>
     public PublicVoidUser ToPublic()
     {
         return new(Id)
@@ -45,28 +66,71 @@ public abstract class VoidUser
     }
 }
 
+/// <summary>
+/// Internal user object used by the system
+/// </summary>
 public sealed class InternalVoidUser : PrivateVoidUser
 {
+    /// <inheritdoc />
     public InternalVoidUser(Guid id, string email, string passwordHash) : base(id, email)
     {
         PasswordHash = passwordHash;
     }
 
+    /// <inheritdoc />
+    public InternalVoidUser(Guid Id, string Email, string Password, DateTime Created, DateTime LastLogin,
+        string Avatar,
+        string DisplayName, int Flags) : base(Id, Email)
+    {
+        PasswordHash = Password;
+    }
+
+    /// <summary>
+    /// A password hash for the user in the format <see cref="Extensions.HashPassword"/>
+    /// </summary>
     public string PasswordHash { get; }
 }
 
+/// <summary>
+/// A user object which includes the Email
+/// </summary>
 public class PrivateVoidUser : VoidUser
 {
+    /// <inheritdoc />
     public PrivateVoidUser(Guid id, string email) : base(id)
     {
         Email = email;
     }
 
+    /// <summary>
+    /// Full constructor for Dapper
+    /// </summary>
+    /// <param name="id"></param>
+    /// <param name="email"></param>
+    /// <param name="password"></param>
+    /// <param name="created"></param>
+    /// <param name="last_login"></param>
+    /// <param name="avatar"></param>
+    /// <param name="display_name"></param>
+    /// <param name="flags"></param>
+    public PrivateVoidUser(Guid Id, String Email, string Password, DateTime Created, DateTime LastLogin, string Avatar,
+        string DisplayName, int Flags) : base(Id)
+    {
+        this.Email = Email;
+        this.Created = Created;
+        this.LastLogin = LastLogin;
+        this.Avatar = Avatar;
+        this.DisplayName = DisplayName;
+        this.Flags = (VoidUserFlags) Flags;
+    }
+
     public string Email { get; }
 }
 
+/// <inheritdoc />
 public sealed class PublicVoidUser : VoidUser
 {
+    /// <inheritdoc />
     public PublicVoidUser(Guid id) : base(id)
     {
     }
@@ -78,4 +142,4 @@ public enum VoidUserFlags
     PublicProfile = 1,
     PublicUploads = 2,
     EmailVerified = 4
-}
+}

VoidCat/Program.cs

@@ -1,10 +1,13 @@
+using System.Data;
 using System.Reflection;
 using System.Text;
+using FluentMigrator.Runner;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.AspNetCore.HttpLogging;
 using Microsoft.IdentityModel.Tokens;
 using Microsoft.OpenApi.Models;
 using Newtonsoft.Json;
+using Npgsql;
 using Prometheus;
 using StackExchange.Redis;
 using VoidCat.Model;
@@ -56,10 +59,10 @@ services.AddHttpLogging((o) =>
     o.LoggingFields = HttpLoggingFields.RequestPropertiesAndHeaders | HttpLoggingFields.ResponsePropertiesAndHeaders;
     o.RequestBodyLogLimit = 4096;
     o.ResponseBodyLogLimit = 4096;
-    
+
     o.MediaTypeOptions.Clear();
     o.MediaTypeOptions.AddText("application/json");
-    
+
     foreach (var h in voidSettings.RequestHeadersLog ?? Enumerable.Empty<string>())
     {
         o.RequestHeaders.Add(h);
@@ -140,9 +143,7 @@ services.AddTransient<IStatsCollector, PrometheusStatsCollector>();
 services.AddVoidPaywall();
 
 // users
-services.AddTransient<IUserStore, UserStore>();
+services.AddUserServices(voidSettings);
-services.AddTransient<IUserManager, UserManager>();
-services.AddTransient<IEmailVerification, EmailVerification>();
 
 // background services
 services.AddHostedService<DeleteUnverifiedAccounts>();
@@ -153,13 +154,30 @@ services.AddVirusScanner(voidSettings);
 // captcha
 services.AddCaptcha(voidSettings);
 
+// postgres
+if (!string.IsNullOrEmpty(voidSettings.Postgres))
+{
+    services.AddScoped<OpenDatabase>();
+    services.AddScoped((_) => new NpgsqlConnection(voidSettings.Postgres));
+    services.AddScoped<IDbConnection>((svc) => svc.GetRequiredService<NpgsqlConnection>());
+
+    // fluent migrations
+    services.AddTransient<IMigration, FluentMigrationRunner>();
+    services.AddFluentMigratorCore()
+        .ConfigureRunner(r =>
+            r.AddPostgres11_0()
+                .WithGlobalConnectionString(voidSettings.Postgres)
+                .ScanIn(typeof(Program).Assembly).For.Migrations())
+        .AddLogging(l => l.AddFluentMigratorConsole());
+}
+
 if (useRedis)
 {
     services.AddTransient<ICache, RedisCache>();
     services.AddTransient<RedisStatsController>();
     services.AddTransient<IStatsCollector>(svc => svc.GetRequiredService<RedisStatsController>());
     services.AddTransient<IStatsReporter>(svc => svc.GetRequiredService<RedisStatsController>());
-    
+
     // redis specific migrations
     services.AddTransient<IMigration, UserLookupKeyHashMigration>();
 }
@@ -175,10 +193,17 @@ else
 var app = builder.Build();
 
 // run migrations
-var migrations = app.Services.GetServices<IMigration>();
+using (var migrationScope = app.Services.CreateScope())
-foreach (var migration in migrations)
 {
-    await migration.Migrate();
+    var migrations = migrationScope.ServiceProvider.GetServices<IMigration>();
+    foreach (var migration in migrations)
+    {
+        await migration.Migrate(args);
+        if (migration.ExitOnComplete)
+        {
+            return;
+        }
+    }
 }
 
 #if HostSPA
@@ -193,6 +218,11 @@ app.UseSwaggerUI();
 app.UseAuthentication();
 app.UseAuthorization();
 
+if (!string.IsNullOrEmpty(voidSettings.Postgres))
+{
+    app.UseMiddleware<OpenDatabase>();
+}
+
 app.UseEndpoints(ep =>
 {
     ep.MapControllers();

VoidCat/Services/Abstractions/IBasicStore.cs

@@ -4,6 +4,8 @@ public interface IBasicStore<T>
 {
     ValueTask<T?> Get(Guid id);
     
+    ValueTask<IReadOnlyList<T>> Get(Guid[] ids);
+    
     ValueTask Set(Guid id, T obj);
 
     ValueTask Delete(Guid id);

VoidCat/Services/Abstractions/IFileInfoManager.cs

@@ -9,5 +9,12 @@ namespace VoidCat.Services.Abstractions;
 public interface IFileInfoManager
 {
     ValueTask<PublicVoidFile?> Get(Guid id);
+    ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids);
+    
+    /// <summary>
+    /// Deletes all file metadata
+    /// </summary>
+    /// <param name="id"></param>
+    /// <returns></returns>
     ValueTask Delete(Guid id);
 }

VoidCat/Services/Abstractions/IFileMetadataStore.cs

@@ -5,5 +5,14 @@ namespace VoidCat.Services.Abstractions;
 public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVoidFileMeta>
 {
     ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta;
+    ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta;
     ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta;
-}
+
+    /// <summary>
+    /// Returns basic stats about the file store
+    /// </summary>
+    /// <returns></returns>
+    ValueTask<StoreStats> Stats();
+
+    public sealed record StoreStats(long Files, ulong Size);
+}

VoidCat/Services/Abstractions/IFileStore.cs

@@ -10,6 +10,11 @@ public interface IFileStore
 
     ValueTask<PagedResult<PublicVoidFile>> ListFiles(PagedRequest request);
 
+    /// <summary>
+    /// Deletes file data only, metadata must be deleted with <see cref="IFileInfoManager.Delete"/>
+    /// </summary>
+    /// <param name="id"></param>
+    /// <returns></returns>
     ValueTask DeleteFile(Guid id);
 
     ValueTask<Stream> Open(EgressRequest request, CancellationToken cts);

VoidCat/Services/Abstractions/IUserStore.cs

@@ -5,7 +5,6 @@ namespace VoidCat.Services.Abstractions;
 public interface IUserStore : IPublicPrivateStore<VoidUser, InternalVoidUser>
 {
     ValueTask<T?> Get<T>(Guid id) where T : VoidUser;
-    ValueTask Delete(PrivateVoidUser user);
     
     ValueTask<Guid?> LookupUser(string email);
     ValueTask<PagedResult<PrivateVoidUser>> ListUsers(PagedRequest request);

VoidCat/Services/Background/DeleteUnverifiedAccounts.cs

@@ -6,43 +6,54 @@ namespace VoidCat.Services.Background;
 public class DeleteUnverifiedAccounts : BackgroundService
 {
     private readonly ILogger<DeleteUnverifiedAccounts> _logger;
-    private readonly IUserStore _userStore;
+    private readonly IServiceScopeFactory _scopeFactory;
-    private readonly IUserUploadsStore _userUploads;
-    private readonly IFileInfoManager _fileInfo;
-    private readonly IFileStore _fileStore;
 
-    public DeleteUnverifiedAccounts(ILogger<DeleteUnverifiedAccounts> logger, IUserStore userStore,
+    public DeleteUnverifiedAccounts(ILogger<DeleteUnverifiedAccounts> logger, IServiceScopeFactory scopeFactory)
-        IUserUploadsStore uploadsStore, IFileInfoManager fileInfo, IFileStore fileStore)
     {
-        _userStore = userStore;
         _logger = logger;
-        _userUploads = uploadsStore;
+        _scopeFactory = scopeFactory;
-        _fileInfo = fileInfo;
-        _fileStore = fileStore;
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
-        var accounts = await _userStore.ListUsers(new(0, Int32.MaxValue));
+        while (!stoppingToken.IsCancellationRequested)
-
-        await foreach (var account in accounts.Results.WithCancellation(stoppingToken))
         {
-            if (!account.Flags.HasFlag(VoidUserFlags.EmailVerified) &&
+            try
-                account.Created.AddDays(7) < DateTimeOffset.UtcNow)
             {
-                _logger.LogInformation("Deleting un-verified account: {Id}", account.Id.ToBase58());
+                using var scope = _scopeFactory.CreateScope();
-                await _userStore.Delete(account);
+                var userStore = scope.ServiceProvider.GetRequiredService<IUserStore>();
+                var userUploads = scope.ServiceProvider.GetRequiredService<IUserUploadsStore>();
+                var fileStore = scope.ServiceProvider.GetRequiredService<IFileStore>();
+                var fileInfoManager = scope.ServiceProvider.GetRequiredService<IFileInfoManager>();
 
-                var files = await _userUploads.ListFiles(account.Id, new(0, Int32.MinValue));
+                var accounts = await userStore.ListUsers(new(0, Int32.MaxValue));
-                // ReSharper disable once UseCancellationTokenForIAsyncEnumerable
+
-                await foreach (var file in files.Results)
+                await foreach (var account in accounts.Results.WithCancellation(stoppingToken))
                 {
-                    await _fileStore.DeleteFile(file.Id);
+                    if (!account.Flags.HasFlag(VoidUserFlags.EmailVerified) &&
-                    await _fileInfo.Delete(file.Id);
+                        account.Created.AddDays(7) < DateTimeOffset.UtcNow)
+                    {
+                        _logger.LogInformation("Deleting un-verified account: {Id}", account.Id.ToBase58());
+                        await userStore.Delete(account.Id);
+
+                        var files = await userUploads.ListFiles(account.Id, new(0, Int32.MinValue));
+                        // ReSharper disable once UseCancellationTokenForIAsyncEnumerable
+                        await foreach (var file in files.Results)
+                        {
+                            await fileStore.DeleteFile(file.Id);
+                            await fileInfoManager.Delete(file.Id);
+                        }
+                    }
                 }
             }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Failed to delete unverified accounts");
+            }
+            finally
+            {
+                await Task.Delay(TimeSpan.FromHours(1), stoppingToken);
+            }
         }
-
-        await Task.Delay(TimeSpan.FromHours(1), stoppingToken);
     }
 }

VoidCat/Services/BasicCacheStore.cs

@@ -16,6 +16,21 @@ public abstract class BasicCacheStore<TStore> : IBasicStore<TStore>
         return _cache.Get<TStore>(MapKey(id));
     }
 
+    public virtual async ValueTask<IReadOnlyList<TStore>> Get(Guid[] ids)
+    {
+        var ret = new List<TStore>();
+        foreach (var id in ids)
+        {
+            var r = await _cache.Get<TStore>(MapKey(id));
+            if (r != null)
+            {
+                ret.Add(r);
+            }
+        }
+
+        return ret;
+    }
+
     public virtual ValueTask Set(Guid id, TStore obj)
     {
         return _cache.Set(MapKey(id), obj);

VoidCat/Services/Files/FileInfoManager.cs

@@ -30,7 +30,7 @@ public class FileInfoManager : IFileInfoManager
         await Task.WhenAll(meta.AsTask(), paywall.AsTask(), bandwidth.AsTask(), virusScan.AsTask());
 
         if (meta.Result == default) return default;
-        
+
         var uploader = meta.Result?.Uploader;
         var user = uploader.HasValue ? await _userStore.Get<PublicVoidUser>(uploader.Value) : null;
 
@@ -45,6 +45,21 @@ public class FileInfoManager : IFileInfoManager
         };
     }
 
+    public async ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids)
+    {
+        var ret = new List<PublicVoidFile>();
+        foreach (var id in ids)
+        {
+            var v = await Get(id);
+            if (v != default)
+            {
+                ret.Add(v);
+            }
+        }
+
+        return ret;
+    }
+
     public async ValueTask Delete(Guid id)
     {
         await _metadataStore.Delete(id);
@@ -52,4 +67,4 @@ public class FileInfoManager : IFileInfoManager
         await _statsReporter.Delete(id);
         await _virusScanStore.Delete(id);
     }
-}
+}

VoidCat/Services/Files/FileStorageStartup.cs

@@ -9,10 +9,11 @@ public static class FileStorageStartup
     public static void AddStorage(this IServiceCollection services, VoidSettings settings)
     {
         services.AddTransient<IFileInfoManager, FileInfoManager>();
-        services.AddTransient<IUserUploadsStore, UserUploadStore>();
 
         if (settings.CloudStorage != default)
         {
+            services.AddTransient<IUserUploadsStore, UserUploadStore>();
+            
             // cloud storage
             if (settings.CloudStorage.S3 != default)
             {
@@ -20,8 +21,15 @@ public static class FileStorageStartup
                 services.AddSingleton<IFileMetadataStore, S3FileMetadataStore>();
             }
         }
+        else if (!string.IsNullOrEmpty(settings.Postgres))
+        {
+            services.AddTransient<IUserUploadsStore, PostgresUserUploadStore>();
+            services.AddTransient<IFileStore, LocalDiskFileStore>();
+            services.AddTransient<IFileMetadataStore, PostgreFileMetadataStore>();
+        }
         else
         {
+            services.AddTransient<IUserUploadsStore, UserUploadStore>();
             services.AddTransient<IFileStore, LocalDiskFileStore>();
             services.AddTransient<IFileMetadataStore, LocalDiskFileMetadataStore>();
         }

VoidCat/Services/Files/LocalDiskFileMetadataStore.cs

@@ -27,18 +27,59 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
         return GetMeta<TMeta>(id);
     }
 
+    public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
+    {
+        var ret = new List<TMeta>();
+        foreach (var id in ids)
+        {
+            var r = await GetMeta<TMeta>(id);
+            if (r != null)
+            {
+                ret.Add(r);
+            }
+        }
+
+        return ret;
+    }
+
     public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
     {
-        var oldMeta = await GetMeta<SecretVoidFileMeta>(id);
+        var oldMeta = await Get<SecretVoidFileMeta>(id);
         if (oldMeta == default) return;
-        
+
         oldMeta.Description = meta.Description ?? oldMeta.Description;
         oldMeta.Name = meta.Name ?? oldMeta.Name;
         oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
-        
+
         await Set(id, oldMeta);
     }
 
+    public async ValueTask<IFileMetadataStore.StoreStats> Stats()
+    {
+        var count = 0;
+        var size = 0UL;
+        foreach (var metaFile in Directory.EnumerateFiles(Path.Join(_settings.DataDirectory, MetadataDir), "*.json"))
+        {
+            try
+            {
+                var json = await File.ReadAllTextAsync(metaFile);
+                var meta = JsonConvert.DeserializeObject<VoidFileMeta>(json);
+
+                if (meta != null)
+                {
+                    count++;
+                    size += meta.Size;
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Failed to load metadata file: {File}", metaFile);
+            }
+        }
+
+        return new(count, size);
+    }
+
     public ValueTask<VoidFileMeta?> Get(Guid id)
     {
         return GetMeta<VoidFileMeta>(id);
@@ -74,4 +115,4 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
 
     private string MapMeta(Guid id) =>
         Path.ChangeExtension(Path.Join(_settings.DataDirectory, MetadataDir, id.ToString()), ".json");
-}
+}

VoidCat/Services/Files/LocalDiskFileStorage.cs

@@ -85,7 +85,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
         });
     }
 
-    public async ValueTask DeleteFile(Guid id)
+    public ValueTask DeleteFile(Guid id)
     {
         var fp = MapPath(id);
         if (File.Exists(fp))
@@ -93,8 +93,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
             _logger.LogInformation("Deleting file: {Path}", fp);
             File.Delete(fp);
         }
-
+        return ValueTask.CompletedTask;
-        await _metadataStore.Delete(id);
     }
 
     public ValueTask<Stream> Open(EgressRequest request, CancellationToken cts)

VoidCat/Services/Files/PostgreFileMetadataStore.cs

@@ -0,0 +1,76 @@
+using Dapper;
+using Npgsql;
+using VoidCat.Model;
+using VoidCat.Services.Abstractions;
+
+namespace VoidCat.Services.Files;
+
+public class PostgreFileMetadataStore : IFileMetadataStore
+{
+    private readonly NpgsqlConnection _connection;
+
+    public PostgreFileMetadataStore(NpgsqlConnection connection)
+    {
+        _connection = connection;
+    }
+
+    public ValueTask<VoidFileMeta?> Get(Guid id)
+    {
+        return Get<VoidFileMeta>(id);
+    }
+
+    public async ValueTask Set(Guid id, SecretVoidFileMeta obj)
+    {
+        await _connection.ExecuteAsync(
+            @"insert into 
+""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"")
+values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret)
+on conflict (""Id"") do update set ""Name"" = :name, ""Description"" = :description, ""MimeType"" = :mimeType", new
+            {
+                id,
+                name = obj.Name,
+                size = (long) obj.Size,
+                uploaded = obj.Uploaded.ToUniversalTime(),
+                description = obj.Description,
+                mimeType = obj.MimeType,
+                digest = obj.Digest,
+                editSecret = obj.EditSecret
+            });
+    }
+
+    public async ValueTask Delete(Guid id)
+    {
+        await _connection.ExecuteAsync("delete from \"Files\" where \"Id\" = :id", new {id});
+    }
+
+    public async ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
+    {
+        return await _connection.QuerySingleOrDefaultAsync<TMeta?>(@"select * from ""Files"" where ""Id"" = :id",
+            new {id});
+    }
+
+    public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
+    {
+        var ret = await _connection.QueryAsync<TMeta>("select * from \"Files\" where \"Id\" in :ids", new {ids});
+        return ret.ToList();
+    }
+
+    public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
+    {
+        var oldMeta = await Get<SecretVoidFileMeta>(id);
+        if (oldMeta == default) return;
+
+        oldMeta.Description = meta.Description ?? oldMeta.Description;
+        oldMeta.Name = meta.Name ?? oldMeta.Name;
+        oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
+
+        await Set(id, oldMeta);
+    }
+
+    public async ValueTask<IFileMetadataStore.StoreStats> Stats()
+    {
+        var v = await _connection.QuerySingleAsync<(long Files, long Size)>(
+            @"select count(1) ""Files"", cast(sum(""Size"") as bigint) ""Size"" from ""Files""");
+        return new(v.Files, (ulong) v.Size);
+    }
+}

VoidCat/Services/Files/S3FileMetadataStore.cs

@@ -25,18 +25,65 @@ public class S3FileMetadataStore : IFileMetadataStore
         return GetMeta<TMeta>(id);
     }
 
+    public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
+    {
+        var ret = new List<TMeta>();
+        foreach (var id in ids)
+        {
+            var r = await GetMeta<TMeta>(id);
+            if (r != null)
+            {
+                ret.Add(r);
+            }
+        }
+
+        return ret;
+    }
+
     public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
     {
         var oldMeta = await GetMeta<SecretVoidFileMeta>(id);
         if (oldMeta == default) return;
-        
+
         oldMeta.Description = meta.Description ?? oldMeta.Description;
         oldMeta.Name = meta.Name ?? oldMeta.Name;
         oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
-        
+
         await Set(id, oldMeta);
     }
 
+    public async ValueTask<IFileMetadataStore.StoreStats> Stats()
+    {
+        var count = 0;
+        var size = 0UL;
+        try
+        {
+            var obj = await _client.ListObjectsV2Async(new()
+            {
+                BucketName = _config.BucketName,
+            });
+
+            foreach (var file in obj.S3Objects)
+            {
+                if (file.Key.EndsWith("-metadata") && Guid.TryParse(file.Key.Split('-')[0], out var id))
+                {
+                    var meta = await GetMeta<VoidFileMeta>(id);
+                    if (meta != default)
+                    {
+                        count++;
+                        size += meta.Size;
+                    }
+                }
+            }
+        }
+        catch (AmazonS3Exception aex)
+        {
+            _logger.LogError(aex, "Failed to list files: {Error}", aex.Message);
+        }
+
+        return new(count, size);
+    }
+
     public ValueTask<VoidFileMeta?> Get(Guid id)
     {
         return GetMeta<VoidFileMeta>(id);
@@ -86,6 +133,6 @@ public class S3FileMetadataStore : IFileMetadataStore
 
         return default;
     }
-    
+
     private static string ToKey(Guid id) => $"{id}-metadata";
-}
+}

VoidCat/Services/Migrations/Database/00-Init.cs

@@ -0,0 +1,57 @@
+using FluentMigrator;
+using VoidCat.Model;
+
+namespace VoidCat.Services.Migrations.Database;
+
+[Migration(20220604_2232)]
+public class Init : Migration {
+    public override void Up()
+    {
+        Create.Table("Users")
+            .WithColumn("Id").AsGuid().PrimaryKey()
+            .WithColumn("Email").AsString().NotNullable().Indexed()
+            .WithColumn("Password").AsString()
+            .WithColumn("Created").AsDateTime().WithDefault(SystemMethods.CurrentDateTime)
+            .WithColumn("LastLogin").AsDateTime().Nullable()
+            .WithColumn("Avatar").AsString().Nullable()
+            .WithColumn("DisplayName").AsString().WithDefaultValue("void user")
+            .WithColumn("Flags").AsInt32().WithDefaultValue((int)VoidUserFlags.PublicProfile);
+
+        Create.Table("Files")
+            .WithColumn("Id").AsGuid().PrimaryKey()
+            .WithColumn("Name").AsString()
+            .WithColumn("Size").AsInt64()
+            .WithColumn("Uploaded").AsDateTime().Indexed().WithDefault(SystemMethods.CurrentDateTime)
+            .WithColumn("Description").AsString().Nullable()
+            .WithColumn("MimeType").AsString().WithDefaultValue("application/octet-stream")
+            .WithColumn("Digest").AsString()
+            .WithColumn("EditSecret").AsGuid();
+
+        Create.Table("UserFiles")
+            .WithColumn("File").AsGuid().ForeignKey("Files", "Id")
+            .WithColumn("User").AsGuid().ForeignKey("Users", "Id").Indexed();
+
+        Create.UniqueConstraint()
+            .OnTable("UserFiles")
+            .Columns("File", "User");
+
+        Create.Table("Paywall")
+            .WithColumn("File").AsGuid().ForeignKey("Files", "Id").Unique()
+            .WithColumn("Type").AsInt16()
+            .WithColumn("Currency").AsInt16()
+            .WithColumn("Amount").AsDecimal();
+
+        Create.Table("PaywallStrike")
+            .WithColumn("File").AsGuid().ForeignKey("Files", "Id").Unique()
+            .WithColumn("Handle").AsString();
+    }
+
+    public override void Down()
+    {
+        Delete.Table("Users");
+        Delete.Table("Files");
+        Delete.Table("UsersFiles");
+        Delete.Table("Paywall");
+        Delete.Table("PaywallStrike");
+    }
+}

VoidCat/Services/Migrations/FluentMigrationRunner.cs

@@ -0,0 +1,21 @@
+using FluentMigrator.Runner;
+
+namespace VoidCat.Services.Migrations;
+
+public class FluentMigrationRunner : IMigration
+{
+    private readonly IMigrationRunner _runner;
+
+    public FluentMigrationRunner(IMigrationRunner runner)
+    {
+        _runner = runner;
+    }
+
+    public ValueTask Migrate(string[] args)
+    {
+        _runner.MigrateUp();
+        return ValueTask.CompletedTask;
+    }
+
+    public bool ExitOnComplete => false;
+}

VoidCat/Services/Migrations/IMigration.cs

@@ -2,5 +2,6 @@
 
 public interface IMigration
 {
-    ValueTask Migrate();
+    ValueTask Migrate(string[] args);
+    bool ExitOnComplete { get; }
 }

VoidCat/Services/Migrations/MetadataMigrator.cs

@@ -14,7 +14,7 @@ public abstract class MetadataMigrator<TOld, TNew> : IMigration
         _logger = logger;
     }
 
-    public async ValueTask Migrate()
+    public async ValueTask Migrate(string[] args)
     {
         var newMeta = Path.Combine(_settings.DataDirectory, OldPath);
         if (!Directory.Exists(newMeta))
@@ -64,4 +64,6 @@ public abstract class MetadataMigrator<TOld, TNew> : IMigration
 
     private string MapNewMeta(Guid id) =>
         Path.ChangeExtension(Path.Join(_settings.DataDirectory, NewPath, id.ToString()), ".json");
+
+    public bool ExitOnComplete => false;
 }

VoidCat/Services/Migrations/UserLookupKeyHashMigration.cs

@@ -13,7 +13,7 @@ public class UserLookupKeyHashMigration : IMigration
         _database = database;
     }
 
-    public async ValueTask Migrate()
+    public async ValueTask Migrate(string[] args)
     {
         var users = await _database.SetMembersAsync("users");
         foreach (var userId in users)
@@ -41,4 +41,6 @@ public class UserLookupKeyHashMigration : IMigration
 
         public string Email { get; init; }
     }
+
+    public bool ExitOnComplete => false;
 }

VoidCat/Services/OpenDatabase.cs

@@ -0,0 +1,24 @@
+using Npgsql;
+
+public class OpenDatabase : IMiddleware
+{
+    private readonly NpgsqlConnection _connection;
+
+    public OpenDatabase(NpgsqlConnection connection)
+    {
+        _connection = connection;
+    }
+
+    public async Task InvokeAsync(HttpContext context, RequestDelegate next)
+    {
+        await _connection.OpenAsync();
+        try
+        {
+            await next(context);
+        }
+        finally
+        {
+            await _connection.CloseAsync();
+        }
+    }
+}

VoidCat/Services/Paywall/PaywallFactory.cs

@@ -25,16 +25,14 @@ public class PaywallFactory : IPaywallFactory
 
 public static class Paywall
 {
-    public static IServiceCollection AddVoidPaywall(this IServiceCollection services)
+    public static void AddVoidPaywall(this IServiceCollection services)
     {
         services.AddTransient<IPaywallFactory, PaywallFactory>();
         services.AddTransient<IPaywallStore, PaywallStore>();
-        
+
         // strike
         services.AddTransient<StrikeApi>();
         services.AddTransient<StrikePaywallProvider>();
         services.AddTransient<IPaywallProvider>((svc) => svc.GetRequiredService<StrikePaywallProvider>());
-        
-        return services;
     }
 }

VoidCat/Services/Users/EmailVerification.cs

@@ -12,7 +12,8 @@ public class EmailVerification : IEmailVerification
     private readonly ILogger<EmailVerification> _logger;
     private readonly RazorPartialToStringRenderer _renderer;
 
-    public EmailVerification(ICache cache, ILogger<EmailVerification> logger, VoidSettings settings, RazorPartialToStringRenderer renderer)
+    public EmailVerification(ICache cache, ILogger<EmailVerification> logger, VoidSettings settings,
+        RazorPartialToStringRenderer renderer)
     {
         _cache = cache;
         _logger = logger;
@@ -30,7 +31,7 @@ public class EmailVerification : IEmailVerification
         };
         await _cache.Set(MapToken(code.Id), code, TimeSpan.FromHours(codeExpire));
         _logger.LogInformation("Saved email verification token for User={Id} Token={Token}", user.Id, code.Id);
-        
+
         // send email
         try
         {
@@ -40,7 +41,7 @@ public class EmailVerification : IEmailVerification
             sc.Port = conf?.Server?.Port ?? 25;
             sc.EnableSsl = conf?.Server?.Scheme == "tls";
             sc.Credentials = new NetworkCredential(conf?.Username, conf?.Password);
-            
+
             var msgContent = await _renderer.RenderPartialToStringAsync("~/Pages/EmailCode.cshtml", code);
             var msg = new MailMessage();
             msg.From = new MailAddress(conf?.Username ?? "no-reply@void.cat");
@@ -65,7 +66,7 @@ public class EmailVerification : IEmailVerification
     {
         var token = await _cache.Get<EmailVerificationCode>(MapToken(code));
         if (token == default) return false;
-        
+
         var isValid = user.Id == token.UserId && token.Expires > DateTimeOffset.UtcNow;
         if (isValid)
         {

VoidCat/Services/Users/PostgresUserStore.cs

@@ -0,0 +1,103 @@
+using Dapper;
+using Npgsql;
+using VoidCat.Model;
+using VoidCat.Services.Abstractions;
+
+namespace VoidCat.Services.Users;
+
+public class PostgresUserStore : IUserStore
+{
+    private readonly NpgsqlConnection _connection;
+
+    public PostgresUserStore(NpgsqlConnection connection)
+    {
+        _connection = connection;
+    }
+
+    public ValueTask<VoidUser?> Get(Guid id)
+    {
+        return Get<VoidUser>(id);
+    }
+
+    public async ValueTask Set(Guid id, InternalVoidUser obj)
+    {
+        await _connection.ExecuteAsync(
+            @"insert into 
+""Users""(""Id"", ""Email"", ""Password"", ""LastLogin"", ""DisplayName"", ""Avatar"", ""Flags"") 
+values(:id, :email, :password, :lastLogin, :displayName, :avatar, :flags)
+on conflict (""Id"") do update set ""LastLogin"" = :lastLogin, ""DisplayName"" = :displayName, ""Avatar"" = :avatar, ""Flags"" = :flags",
+            new
+            {
+                Id = id,
+                email = obj.Email,
+                password = obj.PasswordHash,
+                displayName = obj.DisplayName,
+                lastLogin = obj.LastLogin,
+                avatar = obj.Avatar,
+                flags = (int) obj.Flags
+            });
+    }
+
+    public async ValueTask Delete(Guid id)
+    {
+        await _connection.ExecuteAsync(@"delete from ""Users"" where ""Id"" = :id", new {id});
+    }
+
+    public async ValueTask<T?> Get<T>(Guid id) where T : VoidUser
+    {
+        return await _connection.QuerySingleOrDefaultAsync<T?>(@"select * from ""Users"" where ""Id"" = :id", new {id});
+    }
+
+    public async ValueTask<Guid?> LookupUser(string email)
+    {
+        return await _connection.QuerySingleOrDefaultAsync<Guid?>(
+            @"select ""Id"" from ""Users"" where ""Email"" = :email",
+            new {email});
+    }
+
+    public async ValueTask<PagedResult<PrivateVoidUser>> ListUsers(PagedRequest request)
+    {
+        var orderBy = request.SortBy switch
+        {
+            PagedSortBy.Date => "Created",
+            PagedSortBy.Name => "DisplayName",
+            _ => "Id"
+        };
+        var sortBy = request.SortOrder switch
+        {
+            PageSortOrder.Dsc => "desc",
+            _ => "asc"
+        };
+        var totalUsers = await _connection.ExecuteScalarAsync<int>(@"select count(*) from ""Users""");
+        var users = await _connection.QueryAsync<PrivateVoidUser>(
+            $@"select * from ""Users"" order by ""{orderBy}"" {sortBy} offset :offset limit :limit",
+            new
+            {
+                offset = request.PageSize * request.Page,
+                limit = request.PageSize
+            });
+
+        async IAsyncEnumerable<PrivateVoidUser> Enumerate()
+        {
+            foreach (var u in users ?? Enumerable.Empty<PrivateVoidUser>())
+            {
+                yield return u;
+            }
+        }
+
+        return new()
+        {
+            Page = request.Page,
+            PageSize = request.PageSize,
+            TotalResults = totalUsers,
+            Results = Enumerate()
+        };
+    }
+
+    public async ValueTask UpdateProfile(PublicVoidUser newUser)
+    {
+        await _connection.ExecuteAsync(
+            @"update ""Users"" set ""DisplayName"" = @displayName, ""Avatar"" = @avatar where ""Id"" = :id",
+            new {id = newUser.Id, displayName = newUser.DisplayName, avatar = newUser.Avatar});
+    }
+}

VoidCat/Services/Users/PostgresUserUploadStore.cs

@@ -0,0 +1,73 @@
+using Dapper;
+using Npgsql;
+using VoidCat.Model;
+using VoidCat.Services.Abstractions;
+
+namespace VoidCat.Services.Users;
+
+public class PostgresUserUploadStore : IUserUploadsStore
+{
+    private readonly NpgsqlConnection _connection;
+    private readonly IFileInfoManager _fileInfoManager;
+
+    public PostgresUserUploadStore(NpgsqlConnection connection, IFileInfoManager fileInfoManager)
+    {
+        _connection = connection;
+        _fileInfoManager = fileInfoManager;
+    }
+
+    public async ValueTask<PagedResult<PublicVoidFile>> ListFiles(Guid user, PagedRequest request)
+    {
+        var query = @"select {0} 
+from ""UserFiles"" uf, ""Files"" f
+where uf.""User"" = :user
+and uf.""File"" = f.""Id""";
+        var queryOrder = @"order by f.""{1}"" {2} limit :limit offset :offset";
+
+        var orderBy = request.SortBy switch
+        {
+            PagedSortBy.Name => "Name",
+            PagedSortBy.Date => "Uploaded",
+            PagedSortBy.Size => "Size",
+            _ => "Id"
+        };
+        var sortOrder = request.SortOrder switch
+        {
+            PageSortOrder.Dsc => "desc",
+            _ => "asc"
+        };
+        var count = await _connection.ExecuteScalarAsync<int>(string.Format(query, "count(*)"), new {user});
+        var files = await _connection.QueryAsync<Guid>(
+            string.Format(query + queryOrder, "uf.\"File\"", orderBy, sortOrder),
+            new {user, offset = request.Page * request.PageSize, limit = request.PageSize});
+
+        async IAsyncEnumerable<PublicVoidFile> EnumerateFiles()
+        {
+            foreach (var file in files ?? Enumerable.Empty<Guid>())
+            {
+                var v = await _fileInfoManager.Get(file);
+                if (v != default)
+                {
+                    yield return v;
+                }
+            }
+        }
+
+        return new()
+        {
+            Page = request.Page,
+            PageSize = request.PageSize,
+            TotalResults = count,
+            Results = EnumerateFiles()
+        };
+    }
+
+    public async ValueTask AddFile(Guid user, PrivateVoidFile voidFile)
+    {
+        await _connection.ExecuteAsync(@"insert into ""UserFiles""(""File"", ""User"") values(:file, :user)", new
+        {
+            file = voidFile.Id,
+            user
+        });
+    }
+}

VoidCat/Services/Users/UserManager.cs

@@ -32,7 +32,7 @@ public class UserManager : IUserManager
     public async ValueTask<InternalVoidUser> Register(string email, string password)
     {
         var existingUser = await _store.LookupUser(email);
-        if (existingUser != Guid.Empty) throw new InvalidOperationException("User already exists");
+        if (existingUser != Guid.Empty && existingUser != null) throw new InvalidOperationException("User already exists");
 
         var newUser = new InternalVoidUser(Guid.NewGuid(), email, password.HashPassword())
         {
@@ -50,9 +50,9 @@ public class UserManager : IUserManager
                 newUser.Roles.Add(Roles.Admin);
             }
         }
-        
+
         await _store.Set(newUser.Id, newUser);
         await _emailVerification.SendNewCode(newUser);
         return newUser;
     }
-}
+}

VoidCat/Services/Users/UserUploadStore.cs

@@ -51,4 +51,4 @@ public class UserUploadStore : IUserUploadsStore
     }
 
     private static string MapKey(Guid id) => $"user:{id}:uploads";
-}
+}

VoidCat/Services/Users/UsersStartup.cs

@@ -0,0 +1,21 @@
+using VoidCat.Model;
+using VoidCat.Services.Abstractions;
+
+namespace VoidCat.Services.Users;
+
+public static class UsersStartup
+{
+    public static void AddUserServices(this IServiceCollection services, VoidSettings settings)
+    {
+        services.AddTransient<IUserManager, UserManager>();
+        services.AddTransient<IEmailVerification, EmailVerification>();
+        if (settings.Postgres != default)
+        {
+            services.AddTransient<IUserStore, PostgresUserStore>();
+        }
+        else
+        {
+            services.AddTransient<IUserStore, UserStore>();
+        }
+    }
+}

VoidCat/VoidCat.csproj

@@ -15,6 +15,10 @@
 
 	<ItemGroup>
 		<PackageReference Include="AWSSDK.S3" Version="3.7.8.2" />
+		<PackageReference Include="Dapper" Version="2.0.123" />
+		<PackageReference Include="FluentMigrator" Version="3.3.2" />
+		<PackageReference Include="FluentMigrator.Runner" Version="3.3.2" />
+		<PackageReference Include="FluentMigrator.Runner.Postgres" Version="3.3.2" />
 		<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.1" />
 		<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.1" />
 		<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.14.0" />
@@ -24,6 +28,7 @@
 		</PackageReference>
 		<PackageReference Include="NBitcoin" Version="6.0.19" />
 		<PackageReference Include="nClam" Version="7.0.0" />
+		<PackageReference Include="Npgsql" Version="6.0.4" />
 		<PackageReference Include="prometheus-net.AspNetCore" Version="5.0.2" />
 		<PackageReference Include="Seq.Extensions.Logging" Version="6.0.0" />
 		<PackageReference Include="StackExchange.Redis" Version="2.5.27-prerelease" />

VoidCat/spa/src/Admin/UserList.js

@@ -18,7 +18,7 @@ export function UserList() {
         let pageReq = {
             page: page,
             pageSize,
-            sortBy: PagedSortBy.Id,
+            sortBy: PagedSortBy.Date,
             sortOrder: PageSortOrder.Asc
         };
         let req = await AdminApi.userList(pageReq);
@@ -33,7 +33,7 @@ export function UserList() {
 
     function renderUser(u) {
         return (
-            <tr>
+            <tr key={u.id}>
                 <td><a href={`/u/${u.id}`}>{u.id.substring(0, 4)}..</a></td>
                 <td>{moment(u.created).fromNow()}</td>
                 <td>{moment(u.lastLogin).fromNow()}</td>