highperfocused/void.cat
1
0
Fork 0
mirror of https://git.v0l.io/Kieran/void.cat.git synced 2025-04-09 18:29:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add postgres support for users/files

Browse Source
This commit is contained in:
Kieran 2022-06-06 22:51:25 +01:00
parent 56a4c52e24
commit c30256bacc
No known key found for this signature in database
GPG Key ID: DE71CEB3925BE941
35 changed files with 733 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@ -31,4 +31,15 @@ curl -X POST \
```
This command will return the direct download URL only.
To get the json output simply remove the `?cli=true` from the url.
To get the json output simply remove the `?cli=true` from the url.
### Development
To run postgres in local use:
```
docker run --rm -it -p 5432:5432 -e POSTGRES_DB=void -e POSTGRES_PASSWORD=postgres postgres -d postgres
```
To run MinIO in local use:
```
docker run --rm -it -p 9000:9000 -p 9001:9001 minio/minio -- server /data --console-address ":9001"
```

21
VoidCat/Controllers/InfoController.cs
View File

@ -8,13 +8,13 @@ namespace VoidCat.Controllers;
public class InfoController : Controller
{
private readonly IStatsReporter _statsReporter;
private readonly IFileStore _fileStore;
private readonly IFileMetadataStore _fileMetadata;
private readonly VoidSettings _settings;
public InfoController(IStatsReporter statsReporter, IFileStore fileStore, VoidSettings settings)
public InfoController(IStatsReporter statsReporter, IFileMetadataStore fileMetadata, VoidSettings settings)
{
_statsReporter = statsReporter;
_fileStore = fileStore;
_fileMetadata = fileMetadata;
_settings = settings;
}
@ -27,18 +27,11 @@ public class InfoController : Controller
public async Task<GlobalInfo> GetGlobalStats()
{
var bw = await _statsReporter.GetBandwidth();
var bytes = 0UL;
var count = 0;
var files = await _fileStore.ListFiles(new(0, Int32.MaxValue));
await foreach (var vf in files.Results)
{
bytes += vf.Metadata?.Size ?? 0;
count++;
}
return new(bw, bytes, count, BuildInfo.GetBuildInfo(), _settings.CaptchaSettings?.SiteKey);
var storeStats = await _fileMetadata.Stats();
return new(bw, (ulong)storeStats.Size, storeStats.Files, BuildInfo.GetBuildInfo(), _settings.CaptchaSettings?.SiteKey);
}
public sealed record GlobalInfo(Bandwidth Bandwidth, ulong TotalBytes, int Count, BuildInfo BuildInfo,
public sealed record GlobalInfo(Bandwidth Bandwidth, ulong TotalBytes, long Count, BuildInfo BuildInfo,
string? CaptchaSiteKey);
}

2
VoidCat/Controllers/UploadController.cs
View File

@ -60,7 +60,7 @@ namespace VoidCat.Controllers
}
}
var meta = new SecretVoidFileMeta()
var meta = new SecretVoidFileMeta
{
MimeType = mime,
Name = filename,

10
VoidCat/Model/VoidFileMeta.cs
View File

@ -1,6 +1,8 @@
using Newtonsoft.Json;
using VoidCat.Services.Abstractions;
// ReSharper disable InconsistentNaming
namespace VoidCat.Model;
/// <summary>
@ -9,7 +11,7 @@ namespace VoidCat.Model;
public interface IVoidFileMeta
{
const int CurrentVersion = 3;
int Version { get; init; }
}
@ -22,7 +24,7 @@ public record VoidFileMeta : IVoidFileMeta
/// Metadata version
/// </summary>
public int Version { get; init; } = IVoidFileMeta.CurrentVersion;
/// <summary>
/// Filename
/// </summary>
@ -52,12 +54,12 @@ public record VoidFileMeta : IVoidFileMeta
/// SHA-256 hash of the file
/// </summary>
public string? Digest { get; init; }
/// <summary>
/// Url to download the file
/// </summary>
public Uri? Url { get; set; }
/// <summary>
/// User who uploaded the file
/// </summary>

2
VoidCat/Model/VoidSettings.cs
View File

@ -29,6 +29,8 @@ namespace VoidCat.Model
public IEnumerable<string>? RequestHeadersLog { get; init; }
public CaptchaSettings? CaptchaSettings { get; init; }
public string? Postgres { get; init; }
}
public sealed class TorSettings

70
VoidCat/Model/VoidUser.cs
View File

@ -1,7 +1,12 @@
using Newtonsoft.Json;
// ReSharper disable InconsistentNaming
namespace VoidCat.Model;
/// <summary>
/// The base user object for the system
/// </summary>
public abstract class VoidUser
{
protected VoidUser(Guid id)
@ -9,20 +14,32 @@ public abstract class VoidUser
Id = id;
}
/// <summary>
/// Unique Id of the user
/// </summary>
[JsonConverter(typeof(Base58GuidConverter))]
public Guid Id { get; }
/// <summary>
/// Roles assigned to this user which grant them extra permissions
/// </summary>
public HashSet<string> Roles { get; init; } = new() {Model.Roles.User};
/// <summary>
/// When the user account was created
/// </summary>
public DateTimeOffset Created { get; init; }
/// <summary>
/// The last time the user logged in
/// </summary>
public DateTimeOffset LastLogin { get; set; }
/// <summary>
/// Display avatar for user profile
/// </summary>
public string? Avatar { get; set; }
/// <summary>
/// Display name for user profile
/// </summary>
@ -32,7 +49,11 @@ public abstract class VoidUser
/// Profile flags
/// </summary>
public VoidUserFlags Flags { get; set; } = VoidUserFlags.PublicProfile;
/// <summary>
/// Returns the Public object for this user
/// </summary>
/// <returns></returns>
public PublicVoidUser ToPublic()
{
return new(Id)
@ -45,28 +66,71 @@ public abstract class VoidUser
}
}
/// <summary>
/// Internal user object used by the system
/// </summary>
public sealed class InternalVoidUser : PrivateVoidUser
{
/// <inheritdoc />
public InternalVoidUser(Guid id, string email, string passwordHash) : base(id, email)
{
PasswordHash = passwordHash;
}
/// <inheritdoc />
public InternalVoidUser(Guid Id, string Email, string Password, DateTime Created, DateTime LastLogin,
string Avatar,
string DisplayName, int Flags) : base(Id, Email)
{
PasswordHash = Password;
}
/// <summary>
/// A password hash for the user in the format <see cref="Extensions.HashPassword"/>
/// </summary>
public string PasswordHash { get; }
}
/// <summary>
/// A user object which includes the Email
/// </summary>
public class PrivateVoidUser : VoidUser
{
/// <inheritdoc />
public PrivateVoidUser(Guid id, string email) : base(id)
{
Email = email;
}
/// <summary>
/// Full constructor for Dapper
/// </summary>
/// <param name="id"></param>
/// <param name="email"></param>
/// <param name="password"></param>
/// <param name="created"></param>
/// <param name="last_login"></param>
/// <param name="avatar"></param>
/// <param name="display_name"></param>
/// <param name="flags"></param>
public PrivateVoidUser(Guid Id, String Email, string Password, DateTime Created, DateTime LastLogin, string Avatar,
string DisplayName, int Flags) : base(Id)
{
this.Email = Email;
this.Created = Created;
this.LastLogin = LastLogin;
this.Avatar = Avatar;
this.DisplayName = DisplayName;
this.Flags = (VoidUserFlags) Flags;
}
public string Email { get; }
}
/// <inheritdoc />
public sealed class PublicVoidUser : VoidUser
{
/// <inheritdoc />
public PublicVoidUser(Guid id) : base(id)
{
}
@ -78,4 +142,4 @@ public enum VoidUserFlags
PublicProfile = 1,
PublicUploads = 2,
EmailVerified = 4
}
}

48
VoidCat/Program.cs
View File

@ -1,10 +1,13 @@
using System.Data;
using System.Reflection;
using System.Text;
using FluentMigrator.Runner;
using Microsoft.AspNetCore.Authentication.JwtBearer;
using Microsoft.AspNetCore.HttpLogging;
using Microsoft.IdentityModel.Tokens;
using Microsoft.OpenApi.Models;
using Newtonsoft.Json;
using Npgsql;
using Prometheus;
using StackExchange.Redis;
using VoidCat.Model;
@ -56,10 +59,10 @@ services.AddHttpLogging((o) =>
o.LoggingFields = HttpLoggingFields.RequestPropertiesAndHeaders | HttpLoggingFields.ResponsePropertiesAndHeaders;
o.RequestBodyLogLimit = 4096;
o.ResponseBodyLogLimit = 4096;
o.MediaTypeOptions.Clear();
o.MediaTypeOptions.AddText("application/json");
foreach (var h in voidSettings.RequestHeadersLog ?? Enumerable.Empty<string>())
{
o.RequestHeaders.Add(h);
@ -140,9 +143,7 @@ services.AddTransient<IStatsCollector, PrometheusStatsCollector>();
services.AddVoidPaywall();
// users
services.AddTransient<IUserStore, UserStore>();
services.AddTransient<IUserManager, UserManager>();
services.AddTransient<IEmailVerification, EmailVerification>();
services.AddUserServices(voidSettings);
// background services
services.AddHostedService<DeleteUnverifiedAccounts>();
@ -153,13 +154,30 @@ services.AddVirusScanner(voidSettings);
// captcha
services.AddCaptcha(voidSettings);
// postgres
if (!string.IsNullOrEmpty(voidSettings.Postgres))
{
services.AddScoped<OpenDatabase>();
services.AddScoped((_) => new NpgsqlConnection(voidSettings.Postgres));
services.AddScoped<IDbConnection>((svc) => svc.GetRequiredService<NpgsqlConnection>());
// fluent migrations
services.AddTransient<IMigration, FluentMigrationRunner>();
services.AddFluentMigratorCore()
.ConfigureRunner(r =>
r.AddPostgres11_0()
.WithGlobalConnectionString(voidSettings.Postgres)
.ScanIn(typeof(Program).Assembly).For.Migrations())
.AddLogging(l => l.AddFluentMigratorConsole());
}
if (useRedis)
{
services.AddTransient<ICache, RedisCache>();
services.AddTransient<RedisStatsController>();
services.AddTransient<IStatsCollector>(svc => svc.GetRequiredService<RedisStatsController>());
services.AddTransient<IStatsReporter>(svc => svc.GetRequiredService<RedisStatsController>());
// redis specific migrations
services.AddTransient<IMigration, UserLookupKeyHashMigration>();
}
@ -175,10 +193,17 @@ else
var app = builder.Build();
// run migrations
var migrations = app.Services.GetServices<IMigration>();
foreach (var migration in migrations)
using (var migrationScope = app.Services.CreateScope())
{
await migration.Migrate();
var migrations = migrationScope.ServiceProvider.GetServices<IMigration>();
foreach (var migration in migrations)
{
await migration.Migrate(args);
if (migration.ExitOnComplete)
{
return;
}
}
}
#if HostSPA
@ -193,6 +218,11 @@ app.UseSwaggerUI();
app.UseAuthentication();
app.UseAuthorization();
if (!string.IsNullOrEmpty(voidSettings.Postgres))
{
app.UseMiddleware<OpenDatabase>();
}
app.UseEndpoints(ep =>
{
ep.MapControllers();

2
VoidCat/Services/Abstractions/IBasicStore.cs
View File

@ -4,6 +4,8 @@ public interface IBasicStore<T>
{
ValueTask<T?> Get(Guid id);
ValueTask<IReadOnlyList<T>> Get(Guid[] ids);
ValueTask Set(Guid id, T obj);
ValueTask Delete(Guid id);

7
VoidCat/Services/Abstractions/IFileInfoManager.cs
View File

@ -9,5 +9,12 @@ namespace VoidCat.Services.Abstractions;
public interface IFileInfoManager
{
ValueTask<PublicVoidFile?> Get(Guid id);
ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids);
/// <summary>
/// Deletes all file metadata
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask Delete(Guid id);
}

11
VoidCat/Services/Abstractions/IFileMetadataStore.cs
View File

@ -5,5 +5,14 @@ namespace VoidCat.Services.Abstractions;
public interface IFileMetadataStore : IPublicPrivateStore<VoidFileMeta, SecretVoidFileMeta>
{
ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta;
ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta;
ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta;
}
/// <summary>
/// Returns basic stats about the file store
/// </summary>
/// <returns></returns>
ValueTask<StoreStats> Stats();
public sealed record StoreStats(long Files, ulong Size);
}

5
VoidCat/Services/Abstractions/IFileStore.cs
View File

@ -10,6 +10,11 @@ public interface IFileStore
ValueTask<PagedResult<PublicVoidFile>> ListFiles(PagedRequest request);
/// <summary>
/// Deletes file data only, metadata must be deleted with <see cref="IFileInfoManager.Delete"/>
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
ValueTask DeleteFile(Guid id);
ValueTask<Stream> Open(EgressRequest request, CancellationToken cts);

1
VoidCat/Services/Abstractions/IUserStore.cs
View File

@ -5,7 +5,6 @@ namespace VoidCat.Services.Abstractions;
public interface IUserStore : IPublicPrivateStore<VoidUser, InternalVoidUser>
{
ValueTask<T?> Get<T>(Guid id) where T : VoidUser;
ValueTask Delete(PrivateVoidUser user);
ValueTask<Guid?> LookupUser(string email);
ValueTask<PagedResult<PrivateVoidUser>> ListUsers(PagedRequest request);

59
VoidCat/Services/Background/DeleteUnverifiedAccounts.cs
View File

@ -6,43 +6,54 @@ namespace VoidCat.Services.Background;
public class DeleteUnverifiedAccounts : BackgroundService
{
private readonly ILogger<DeleteUnverifiedAccounts> _logger;
private readonly IUserStore _userStore;
private readonly IUserUploadsStore _userUploads;
private readonly IFileInfoManager _fileInfo;
private readonly IFileStore _fileStore;
private readonly IServiceScopeFactory _scopeFactory;
public DeleteUnverifiedAccounts(ILogger<DeleteUnverifiedAccounts> logger, IUserStore userStore,
IUserUploadsStore uploadsStore, IFileInfoManager fileInfo, IFileStore fileStore)
public DeleteUnverifiedAccounts(ILogger<DeleteUnverifiedAccounts> logger, IServiceScopeFactory scopeFactory)
{
_userStore = userStore;
_logger = logger;
_userUploads = uploadsStore;
_fileInfo = fileInfo;
_fileStore = fileStore;
_scopeFactory = scopeFactory;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var accounts = await _userStore.ListUsers(new(0, Int32.MaxValue));
await foreach (var account in accounts.Results.WithCancellation(stoppingToken))
while (!stoppingToken.IsCancellationRequested)
{
if (!account.Flags.HasFlag(VoidUserFlags.EmailVerified) &&
account.Created.AddDays(7) < DateTimeOffset.UtcNow)
try
{
_logger.LogInformation("Deleting un-verified account: {Id}", account.Id.ToBase58());
await _userStore.Delete(account);
using var scope = _scopeFactory.CreateScope();
var userStore = scope.ServiceProvider.GetRequiredService<IUserStore>();
var userUploads = scope.ServiceProvider.GetRequiredService<IUserUploadsStore>();
var fileStore = scope.ServiceProvider.GetRequiredService<IFileStore>();
var fileInfoManager = scope.ServiceProvider.GetRequiredService<IFileInfoManager>();
var files = await _userUploads.ListFiles(account.Id, new(0, Int32.MinValue));
// ReSharper disable once UseCancellationTokenForIAsyncEnumerable
await foreach (var file in files.Results)
var accounts = await userStore.ListUsers(new(0, Int32.MaxValue));
await foreach (var account in accounts.Results.WithCancellation(stoppingToken))
{
await _fileStore.DeleteFile(file.Id);
await _fileInfo.Delete(file.Id);
if (!account.Flags.HasFlag(VoidUserFlags.EmailVerified) &&
account.Created.AddDays(7) < DateTimeOffset.UtcNow)
{
_logger.LogInformation("Deleting un-verified account: {Id}", account.Id.ToBase58());
await userStore.Delete(account.Id);
var files = await userUploads.ListFiles(account.Id, new(0, Int32.MinValue));
// ReSharper disable once UseCancellationTokenForIAsyncEnumerable
await foreach (var file in files.Results)
{
await fileStore.DeleteFile(file.Id);
await fileInfoManager.Delete(file.Id);
}
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to delete unverified accounts");
}
finally
{
await Task.Delay(TimeSpan.FromHours(1), stoppingToken);
}
}
await Task.Delay(TimeSpan.FromHours(1), stoppingToken);
}
}

15
VoidCat/Services/BasicCacheStore.cs
View File

@ -16,6 +16,21 @@ public abstract class BasicCacheStore<TStore> : IBasicStore<TStore>
return _cache.Get<TStore>(MapKey(id));
}
public virtual async ValueTask<IReadOnlyList<TStore>> Get(Guid[] ids)
{
var ret = new List<TStore>();
foreach (var id in ids)
{
var r = await _cache.Get<TStore>(MapKey(id));
if (r != null)
{
ret.Add(r);
}
}
return ret;
}
public virtual ValueTask Set(Guid id, TStore obj)
{
return _cache.Set(MapKey(id), obj);

19
VoidCat/Services/Files/FileInfoManager.cs
View File

@ -30,7 +30,7 @@ public class FileInfoManager : IFileInfoManager
await Task.WhenAll(meta.AsTask(), paywall.AsTask(), bandwidth.AsTask(), virusScan.AsTask());
if (meta.Result == default) return default;
var uploader = meta.Result?.Uploader;
var user = uploader.HasValue ? await _userStore.Get<PublicVoidUser>(uploader.Value) : null;
@ -45,6 +45,21 @@ public class FileInfoManager : IFileInfoManager
};
}
public async ValueTask<IReadOnlyList<PublicVoidFile>> Get(Guid[] ids)
{
var ret = new List<PublicVoidFile>();
foreach (var id in ids)
{
var v = await Get(id);
if (v != default)
{
ret.Add(v);
}
}
return ret;
}
public async ValueTask Delete(Guid id)
{
await _metadataStore.Delete(id);
@ -52,4 +67,4 @@ public class FileInfoManager : IFileInfoManager
await _statsReporter.Delete(id);
await _virusScanStore.Delete(id);
}
}
}

10
VoidCat/Services/Files/FileStorageStartup.cs
View File

@ -9,10 +9,11 @@ public static class FileStorageStartup
public static void AddStorage(this IServiceCollection services, VoidSettings settings)
{
services.AddTransient<IFileInfoManager, FileInfoManager>();
services.AddTransient<IUserUploadsStore, UserUploadStore>();
if (settings.CloudStorage != default)
{
services.AddTransient<IUserUploadsStore, UserUploadStore>();
// cloud storage
if (settings.CloudStorage.S3 != default)
{
@ -20,8 +21,15 @@ public static class FileStorageStartup
services.AddSingleton<IFileMetadataStore, S3FileMetadataStore>();
}
}
else if (!string.IsNullOrEmpty(settings.Postgres))
{
services.AddTransient<IUserUploadsStore, PostgresUserUploadStore>();
services.AddTransient<IFileStore, LocalDiskFileStore>();
services.AddTransient<IFileMetadataStore, PostgreFileMetadataStore>();
}
else
{
services.AddTransient<IUserUploadsStore, UserUploadStore>();
services.AddTransient<IFileStore, LocalDiskFileStore>();
services.AddTransient<IFileMetadataStore, LocalDiskFileMetadataStore>();
}

49
VoidCat/Services/Files/LocalDiskFileMetadataStore.cs
View File

@ -27,18 +27,59 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
return GetMeta<TMeta>(id);
}
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
{
var ret = new List<TMeta>();
foreach (var id in ids)
{
var r = await GetMeta<TMeta>(id);
if (r != null)
{
ret.Add(r);
}
}
return ret;
}
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
{
var oldMeta = await GetMeta<SecretVoidFileMeta>(id);
var oldMeta = await Get<SecretVoidFileMeta>(id);
if (oldMeta == default) return;
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
await Set(id, oldMeta);
}
public async ValueTask<IFileMetadataStore.StoreStats> Stats()
{
var count = 0;
var size = 0UL;
foreach (var metaFile in Directory.EnumerateFiles(Path.Join(_settings.DataDirectory, MetadataDir), "*.json"))
{
try
{
var json = await File.ReadAllTextAsync(metaFile);
var meta = JsonConvert.DeserializeObject<VoidFileMeta>(json);
if (meta != null)
{
count++;
size += meta.Size;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to load metadata file: {File}", metaFile);
}
}
return new(count, size);
}
public ValueTask<VoidFileMeta?> Get(Guid id)
{
return GetMeta<VoidFileMeta>(id);
@ -74,4 +115,4 @@ public class LocalDiskFileMetadataStore : IFileMetadataStore
private string MapMeta(Guid id) =>
Path.ChangeExtension(Path.Join(_settings.DataDirectory, MetadataDir, id.ToString()), ".json");
}
}

5
VoidCat/Services/Files/LocalDiskFileStorage.cs
View File

@ -85,7 +85,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
});
}
public async ValueTask DeleteFile(Guid id)
public ValueTask DeleteFile(Guid id)
{
var fp = MapPath(id);
if (File.Exists(fp))
@ -93,8 +93,7 @@ public class LocalDiskFileStore : StreamFileStore, IFileStore
_logger.LogInformation("Deleting file: {Path}", fp);
File.Delete(fp);
}
await _metadataStore.Delete(id);
return ValueTask.CompletedTask;
}
public ValueTask<Stream> Open(EgressRequest request, CancellationToken cts)

76
VoidCat/Services/Files/PostgreFileMetadataStore.cs Normal file
View File

@ -0,0 +1,76 @@
using Dapper;
using Npgsql;
using VoidCat.Model;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Files;
public class PostgreFileMetadataStore : IFileMetadataStore
{
private readonly NpgsqlConnection _connection;
public PostgreFileMetadataStore(NpgsqlConnection connection)
{
_connection = connection;
}
public ValueTask<VoidFileMeta?> Get(Guid id)
{
return Get<VoidFileMeta>(id);
}
public async ValueTask Set(Guid id, SecretVoidFileMeta obj)
{
await _connection.ExecuteAsync(
@"insert into
""Files""(""Id"", ""Name"", ""Size"", ""Uploaded"", ""Description"", ""MimeType"", ""Digest"", ""EditSecret"")
values(:id, :name, :size, :uploaded, :description, :mimeType, :digest, :editSecret)
on conflict (""Id"") do update set ""Name"" = :name, ""Description"" = :description, ""MimeType"" = :mimeType", new
{
id,
name = obj.Name,
size = (long) obj.Size,
uploaded = obj.Uploaded.ToUniversalTime(),
description = obj.Description,
mimeType = obj.MimeType,
digest = obj.Digest,
editSecret = obj.EditSecret
});
}
public async ValueTask Delete(Guid id)
{
await _connection.ExecuteAsync("delete from \"Files\" where \"Id\" = :id", new {id});
}
public async ValueTask<TMeta?> Get<TMeta>(Guid id) where TMeta : VoidFileMeta
{
return await _connection.QuerySingleOrDefaultAsync<TMeta?>(@"select * from ""Files"" where ""Id"" = :id",
new {id});
}
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
{
var ret = await _connection.QueryAsync<TMeta>("select * from \"Files\" where \"Id\" in :ids", new {ids});
return ret.ToList();
}
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
{
var oldMeta = await Get<SecretVoidFileMeta>(id);
if (oldMeta == default) return;
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
await Set(id, oldMeta);
}
public async ValueTask<IFileMetadataStore.StoreStats> Stats()
{
var v = await _connection.QuerySingleAsync<(long Files, long Size)>(
@"select count(1) ""Files"", cast(sum(""Size"") as bigint) ""Size"" from ""Files""");
return new(v.Files, (ulong) v.Size);
}
}

55
VoidCat/Services/Files/S3FileMetadataStore.cs
View File

@ -25,18 +25,65 @@ public class S3FileMetadataStore : IFileMetadataStore
return GetMeta<TMeta>(id);
}
public async ValueTask<IReadOnlyList<TMeta>> Get<TMeta>(Guid[] ids) where TMeta : VoidFileMeta
{
var ret = new List<TMeta>();
foreach (var id in ids)
{
var r = await GetMeta<TMeta>(id);
if (r != null)
{
ret.Add(r);
}
}
return ret;
}
public async ValueTask Update<TMeta>(Guid id, TMeta meta) where TMeta : VoidFileMeta
{
var oldMeta = await GetMeta<SecretVoidFileMeta>(id);
if (oldMeta == default) return;
oldMeta.Description = meta.Description ?? oldMeta.Description;
oldMeta.Name = meta.Name ?? oldMeta.Name;
oldMeta.MimeType = meta.MimeType ?? oldMeta.MimeType;
await Set(id, oldMeta);
}
public async ValueTask<IFileMetadataStore.StoreStats> Stats()
{
var count = 0;
var size = 0UL;
try
{
var obj = await _client.ListObjectsV2Async(new()
{
BucketName = _config.BucketName,
});
foreach (var file in obj.S3Objects)
{
if (file.Key.EndsWith("-metadata") && Guid.TryParse(file.Key.Split('-')[0], out var id))
{
var meta = await GetMeta<VoidFileMeta>(id);
if (meta != default)
{
count++;
size += meta.Size;
}
}
}
}
catch (AmazonS3Exception aex)
{
_logger.LogError(aex, "Failed to list files: {Error}", aex.Message);
}
return new(count, size);
}
public ValueTask<VoidFileMeta?> Get(Guid id)
{
return GetMeta<VoidFileMeta>(id);
@ -86,6 +133,6 @@ public class S3FileMetadataStore : IFileMetadataStore
return default;
}
private static string ToKey(Guid id) => $"{id}-metadata";
}
}

57
VoidCat/Services/Migrations/Database/00-Init.cs Normal file
View File

@ -0,0 +1,57 @@
using FluentMigrator;
using VoidCat.Model;
namespace VoidCat.Services.Migrations.Database;
[Migration(20220604_2232)]
public class Init : Migration {
public override void Up()
{
Create.Table("Users")
.WithColumn("Id").AsGuid().PrimaryKey()
.WithColumn("Email").AsString().NotNullable().Indexed()
.WithColumn("Password").AsString()
.WithColumn("Created").AsDateTime().WithDefault(SystemMethods.CurrentDateTime)
.WithColumn("LastLogin").AsDateTime().Nullable()
.WithColumn("Avatar").AsString().Nullable()
.WithColumn("DisplayName").AsString().WithDefaultValue("void user")
.WithColumn("Flags").AsInt32().WithDefaultValue((int)VoidUserFlags.PublicProfile);
Create.Table("Files")
.WithColumn("Id").AsGuid().PrimaryKey()
.WithColumn("Name").AsString()
.WithColumn("Size").AsInt64()
.WithColumn("Uploaded").AsDateTime().Indexed().WithDefault(SystemMethods.CurrentDateTime)
.WithColumn("Description").AsString().Nullable()
.WithColumn("MimeType").AsString().WithDefaultValue("application/octet-stream")
.WithColumn("Digest").AsString()
.WithColumn("EditSecret").AsGuid();
Create.Table("UserFiles")
.WithColumn("File").AsGuid().ForeignKey("Files", "Id")
.WithColumn("User").AsGuid().ForeignKey("Users", "Id").Indexed();
Create.UniqueConstraint()
.OnTable("UserFiles")
.Columns("File", "User");
Create.Table("Paywall")
.WithColumn("File").AsGuid().ForeignKey("Files", "Id").Unique()
.WithColumn("Type").AsInt16()
.WithColumn("Currency").AsInt16()
.WithColumn("Amount").AsDecimal();
Create.Table("PaywallStrike")
.WithColumn("File").AsGuid().ForeignKey("Files", "Id").Unique()
.WithColumn("Handle").AsString();
}
public override void Down()
{
Delete.Table("Users");
Delete.Table("Files");
Delete.Table("UsersFiles");
Delete.Table("Paywall");
Delete.Table("PaywallStrike");
}
}

21
VoidCat/Services/Migrations/FluentMigrationRunner.cs Normal file
View File

@ -0,0 +1,21 @@
using FluentMigrator.Runner;
namespace VoidCat.Services.Migrations;
public class FluentMigrationRunner : IMigration
{
private readonly IMigrationRunner _runner;
public FluentMigrationRunner(IMigrationRunner runner)
{
_runner = runner;
}
public ValueTask Migrate(string[] args)
{
_runner.MigrateUp();
return ValueTask.CompletedTask;
}
public bool ExitOnComplete => false;
}

3
VoidCat/Services/Migrations/IMigration.cs
View File

@ -2,5 +2,6 @@
public interface IMigration
{
ValueTask Migrate();
ValueTask Migrate(string[] args);
bool ExitOnComplete { get; }
}

4
VoidCat/Services/Migrations/MetadataMigrator.cs
View File

@ -14,7 +14,7 @@ public abstract class MetadataMigrator<TOld, TNew> : IMigration
_logger = logger;
}
public async ValueTask Migrate()
public async ValueTask Migrate(string[] args)
{
var newMeta = Path.Combine(_settings.DataDirectory, OldPath);
if (!Directory.Exists(newMeta))
@ -64,4 +64,6 @@ public abstract class MetadataMigrator<TOld, TNew> : IMigration
private string MapNewMeta(Guid id) =>
Path.ChangeExtension(Path.Join(_settings.DataDirectory, NewPath, id.ToString()), ".json");
public bool ExitOnComplete => false;
}

4
VoidCat/Services/Migrations/UserLookupKeyHashMigration.cs
View File

@ -13,7 +13,7 @@ public class UserLookupKeyHashMigration : IMigration
_database = database;
}
public async ValueTask Migrate()
public async ValueTask Migrate(string[] args)
{
var users = await _database.SetMembersAsync("users");
foreach (var userId in users)
@ -41,4 +41,6 @@ public class UserLookupKeyHashMigration : IMigration
public string Email { get; init; }
}
public bool ExitOnComplete => false;
}

24
VoidCat/Services/OpenDatabase.cs Normal file
View File

@ -0,0 +1,24 @@
using Npgsql;
public class OpenDatabase : IMiddleware
{
private readonly NpgsqlConnection _connection;
public OpenDatabase(NpgsqlConnection connection)
{
_connection = connection;
}
public async Task InvokeAsync(HttpContext context, RequestDelegate next)
{
await _connection.OpenAsync();
try
{
await next(context);
}
finally
{
await _connection.CloseAsync();
}
}
}

6
VoidCat/Services/Paywall/PaywallFactory.cs
View File

@ -25,16 +25,14 @@ public class PaywallFactory : IPaywallFactory
public static class Paywall
{
public static IServiceCollection AddVoidPaywall(this IServiceCollection services)
public static void AddVoidPaywall(this IServiceCollection services)
{
services.AddTransient<IPaywallFactory, PaywallFactory>();
services.AddTransient<IPaywallStore, PaywallStore>();
// strike
services.AddTransient<StrikeApi>();
services.AddTransient<StrikePaywallProvider>();
services.AddTransient<IPaywallProvider>((svc) => svc.GetRequiredService<StrikePaywallProvider>());
return services;
}
}

9
VoidCat/Services/Users/EmailVerification.cs
View File

@ -12,7 +12,8 @@ public class EmailVerification : IEmailVerification
private readonly ILogger<EmailVerification> _logger;
private readonly RazorPartialToStringRenderer _renderer;
public EmailVerification(ICache cache, ILogger<EmailVerification> logger, VoidSettings settings, RazorPartialToStringRenderer renderer)
public EmailVerification(ICache cache, ILogger<EmailVerification> logger, VoidSettings settings,
RazorPartialToStringRenderer renderer)
{
_cache = cache;
_logger = logger;
@ -30,7 +31,7 @@ public class EmailVerification : IEmailVerification
};
await _cache.Set(MapToken(code.Id), code, TimeSpan.FromHours(codeExpire));
_logger.LogInformation("Saved email verification token for User={Id} Token={Token}", user.Id, code.Id);
// send email
try
{
@ -40,7 +41,7 @@ public class EmailVerification : IEmailVerification
sc.Port = conf?.Server?.Port ?? 25;
sc.EnableSsl = conf?.Server?.Scheme == "tls";
sc.Credentials = new NetworkCredential(conf?.Username, conf?.Password);
var msgContent = await _renderer.RenderPartialToStringAsync("~/Pages/EmailCode.cshtml", code);
var msg = new MailMessage();
msg.From = new MailAddress(conf?.Username ?? "no-reply@void.cat");
@ -65,7 +66,7 @@ public class EmailVerification : IEmailVerification
{
var token = await _cache.Get<EmailVerificationCode>(MapToken(code));
if (token == default) return false;
var isValid = user.Id == token.UserId && token.Expires > DateTimeOffset.UtcNow;
if (isValid)
{

103
VoidCat/Services/Users/PostgresUserStore.cs Normal file
View File

@ -0,0 +1,103 @@
using Dapper;
using Npgsql;
using VoidCat.Model;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Users;
public class PostgresUserStore : IUserStore
{
private readonly NpgsqlConnection _connection;
public PostgresUserStore(NpgsqlConnection connection)
{
_connection = connection;
}
public ValueTask<VoidUser?> Get(Guid id)
{
return Get<VoidUser>(id);
}
public async ValueTask Set(Guid id, InternalVoidUser obj)
{
await _connection.ExecuteAsync(
@"insert into
""Users""(""Id"", ""Email"", ""Password"", ""LastLogin"", ""DisplayName"", ""Avatar"", ""Flags"")
values(:id, :email, :password, :lastLogin, :displayName, :avatar, :flags)
on conflict (""Id"") do update set ""LastLogin"" = :lastLogin, ""DisplayName"" = :displayName, ""Avatar"" = :avatar, ""Flags"" = :flags",
new
{
Id = id,
email = obj.Email,
password = obj.PasswordHash,
displayName = obj.DisplayName,
lastLogin = obj.LastLogin,
avatar = obj.Avatar,
flags = (int) obj.Flags
});
}
public async ValueTask Delete(Guid id)
{
await _connection.ExecuteAsync(@"delete from ""Users"" where ""Id"" = :id", new {id});
}
public async ValueTask<T?> Get<T>(Guid id) where T : VoidUser
{
return await _connection.QuerySingleOrDefaultAsync<T?>(@"select * from ""Users"" where ""Id"" = :id", new {id});
}
public async ValueTask<Guid?> LookupUser(string email)
{
return await _connection.QuerySingleOrDefaultAsync<Guid?>(
@"select ""Id"" from ""Users"" where ""Email"" = :email",
new {email});
}
public async ValueTask<PagedResult<PrivateVoidUser>> ListUsers(PagedRequest request)
{
var orderBy = request.SortBy switch
{
PagedSortBy.Date => "Created",
PagedSortBy.Name => "DisplayName",
_ => "Id"
};
var sortBy = request.SortOrder switch
{
PageSortOrder.Dsc => "desc",
_ => "asc"
};
var totalUsers = await _connection.ExecuteScalarAsync<int>(@"select count(*) from ""Users""");
var users = await _connection.QueryAsync<PrivateVoidUser>(
$@"select * from ""Users"" order by ""{orderBy}"" {sortBy} offset :offset limit :limit",
new
{
offset = request.PageSize * request.Page,
limit = request.PageSize
});
async IAsyncEnumerable<PrivateVoidUser> Enumerate()
{
foreach (var u in users ?? Enumerable.Empty<PrivateVoidUser>())
{
yield return u;
}
}
return new()
{
Page = request.Page,
PageSize = request.PageSize,
TotalResults = totalUsers,
Results = Enumerate()
};
}
public async ValueTask UpdateProfile(PublicVoidUser newUser)
{
await _connection.ExecuteAsync(
@"update ""Users"" set ""DisplayName"" = @displayName, ""Avatar"" = @avatar where ""Id"" = :id",
new {id = newUser.Id, displayName = newUser.DisplayName, avatar = newUser.Avatar});
}
}

73
VoidCat/Services/Users/PostgresUserUploadStore.cs Normal file
View File

@ -0,0 +1,73 @@
using Dapper;
using Npgsql;
using VoidCat.Model;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Users;
public class PostgresUserUploadStore : IUserUploadsStore
{
private readonly NpgsqlConnection _connection;
private readonly IFileInfoManager _fileInfoManager;
public PostgresUserUploadStore(NpgsqlConnection connection, IFileInfoManager fileInfoManager)
{
_connection = connection;
_fileInfoManager = fileInfoManager;
}
public async ValueTask<PagedResult<PublicVoidFile>> ListFiles(Guid user, PagedRequest request)
{
var query = @"select {0}
from ""UserFiles"" uf, ""Files"" f
where uf.""User"" = :user
and uf.""File"" = f.""Id""";
var queryOrder = @"order by f.""{1}"" {2} limit :limit offset :offset";
var orderBy = request.SortBy switch
{
PagedSortBy.Name => "Name",
PagedSortBy.Date => "Uploaded",
PagedSortBy.Size => "Size",
_ => "Id"
};
var sortOrder = request.SortOrder switch
{
PageSortOrder.Dsc => "desc",
_ => "asc"
};
var count = await _connection.ExecuteScalarAsync<int>(string.Format(query, "count(*)"), new {user});
var files = await _connection.QueryAsync<Guid>(
string.Format(query + queryOrder, "uf.\"File\"", orderBy, sortOrder),
new {user, offset = request.Page * request.PageSize, limit = request.PageSize});
async IAsyncEnumerable<PublicVoidFile> EnumerateFiles()
{
foreach (var file in files ?? Enumerable.Empty<Guid>())
{
var v = await _fileInfoManager.Get(file);
if (v != default)
{
yield return v;
}
}
}
return new()
{
Page = request.Page,
PageSize = request.PageSize,
TotalResults = count,
Results = EnumerateFiles()
};
}
public async ValueTask AddFile(Guid user, PrivateVoidFile voidFile)
{
await _connection.ExecuteAsync(@"insert into ""UserFiles""(""File"", ""User"") values(:file, :user)", new
{
file = voidFile.Id,
user
});
}
}

6
VoidCat/Services/Users/UserManager.cs
View File

@ -32,7 +32,7 @@ public class UserManager : IUserManager
public async ValueTask<InternalVoidUser> Register(string email, string password)
{
var existingUser = await _store.LookupUser(email);
if (existingUser != Guid.Empty) throw new InvalidOperationException("User already exists");
if (existingUser != Guid.Empty && existingUser != null) throw new InvalidOperationException("User already exists");
var newUser = new InternalVoidUser(Guid.NewGuid(), email, password.HashPassword())
{
@ -50,9 +50,9 @@ public class UserManager : IUserManager
newUser.Roles.Add(Roles.Admin);
}
}
await _store.Set(newUser.Id, newUser);
await _emailVerification.SendNewCode(newUser);
return newUser;
}
}
}

2
VoidCat/Services/Users/UserUploadStore.cs
View File

@ -51,4 +51,4 @@ public class UserUploadStore : IUserUploadsStore
}
private static string MapKey(Guid id) => $"user:{id}:uploads";
}
}

21
VoidCat/Services/Users/UsersStartup.cs Normal file
View File

@ -0,0 +1,21 @@
using VoidCat.Model;
using VoidCat.Services.Abstractions;
namespace VoidCat.Services.Users;
public static class UsersStartup
{
public static void AddUserServices(this IServiceCollection services, VoidSettings settings)
{
services.AddTransient<IUserManager, UserManager>();
services.AddTransient<IEmailVerification, EmailVerification>();
if (settings.Postgres != default)
{
services.AddTransient<IUserStore, PostgresUserStore>();
}
else
{
services.AddTransient<IUserStore, UserStore>();
}
}
}

5
VoidCat/VoidCat.csproj
View File

@ -15,6 +15,10 @@
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.7.8.2" />
<PackageReference Include="Dapper" Version="2.0.123" />
<PackageReference Include="FluentMigrator" Version="3.3.2" />
<PackageReference Include="FluentMigrator.Runner" Version="3.3.2" />
<PackageReference Include="FluentMigrator.Runner.Postgres" Version="3.3.2" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.1" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.1" />
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.14.0" />
@ -24,6 +28,7 @@
</PackageReference>
<PackageReference Include="NBitcoin" Version="6.0.19" />
<PackageReference Include="nClam" Version="7.0.0" />
<PackageReference Include="Npgsql" Version="6.0.4" />
<PackageReference Include="prometheus-net.AspNetCore" Version="5.0.2" />
<PackageReference Include="Seq.Extensions.Logging" Version="6.0.0" />
<PackageReference Include="StackExchange.Redis" Version="2.5.27-prerelease" />

4
VoidCat/spa/src/Admin/UserList.js
View File

@ -18,7 +18,7 @@ export function UserList() {
let pageReq = {
page: page,
pageSize,
sortBy: PagedSortBy.Id,
sortBy: PagedSortBy.Date,
sortOrder: PageSortOrder.Asc
};
let req = await AdminApi.userList(pageReq);
@ -33,7 +33,7 @@ export function UserList() {
function renderUser(u) {
return (
<tr>
<tr key={u.id}>
<td><a href={`/u/${u.id}`}>{u.id.substring(0, 4)}..</a></td>
<td>{moment(u.created).fromNow()}</td>
<td>{moment(u.lastLogin).fromNow()}</td>