highperfocused/void.cat
1
0
Fork 0
mirror of https://git.v0l.io/Kieran/void.cat.git synced 2025-03-29 16:11:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

Segmented upload

Browse Source
This commit is contained in:
Kieran 2022-02-10 22:22:34 +00:00
parent 9d125b22f4
commit c0367ff2e9
No known key found for this signature in database
GPG Key ID: DE71CEB3925BE941
8 changed files with 238 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
VoidCat/Controllers/UploadController.cs
View File

@ -20,16 +20,54 @@ namespace VoidCat.Controllers
[HttpPost]
[DisableRequestSizeLimit]
[DisableFormValueModelBinding]
public Task<InternalVoidFile> UploadFile()
public async Task<UploadResult> UploadFile()
{
var meta = new VoidFileMeta()
try
{
MimeType = Request.ContentType,
Name = Request.Headers
.FirstOrDefault(a => a.Key.Equals("X-Filename", StringComparison.InvariantCultureIgnoreCase)).Value.ToString()
};
return Request.HasFormContentType ?
saveFromForm() : _storage.Ingress(Request.Body, meta, HttpContext.RequestAborted);
var meta = new VoidFileMeta()
{
MimeType = Request.ContentType,
Name = Request.Headers.GetHeader("X-Filename")
};
var digest = Request.Headers.GetHeader("X-Digest");
var vf = await (Request.HasFormContentType ?
saveFromForm() : _storage.Ingress(new(Request.Body, meta, digest!), HttpContext.RequestAborted));
return UploadResult.Success(vf);
}
catch (Exception ex)
{
return UploadResult.Error(ex.Message);
}
}
[HttpPost]
[DisableRequestSizeLimit]
[DisableFormValueModelBinding]
[Route("{id}")]
public async Task<UploadResult> UploadFileAppend([FromRoute] string id)
{
try
{
var gid = id.FromBase58Guid();
var fileInfo = await _storage.Get(gid);
if (fileInfo == default) return null;
var editSecret = Request.Headers.GetHeader("X-EditSecret");
var digest = Request.Headers.GetHeader("X-Digest");
var vf = await _storage.Ingress(new(Request.Body, fileInfo.Metadata, digest!)
{
EditSecret = editSecret?.FromBase58Guid(),
Id = gid
}, HttpContext.RequestAborted);
return UploadResult.Success(vf);
}
catch (Exception ex)
{
return UploadResult.Error(ex.Message);
}
}
[HttpGet]
@ -38,10 +76,10 @@ namespace VoidCat.Controllers
{
return _storage.Get(id.FromBase58Guid());
}
[HttpPatch]
[Route("{id}")]
public Task UpdateFileInfo([FromRoute]string id, [FromBody]UpdateFileInfoRequest request)
public Task UpdateFileInfo([FromRoute] string id, [FromBody] UpdateFileInfoRequest request)
{
return _storage.UpdateInfo(new VoidFile()
{
@ -73,4 +111,13 @@ namespace VoidCat.Controllers
{
}
}
public record UploadResult(bool Ok, InternalVoidFile? File, string? ErrorMessage)
{
public static UploadResult Success(InternalVoidFile vf)
=> new(true, vf, null);
public static UploadResult Error(string message)
=> new(false, null, message);
}
}

6
VoidCat/Model/Extensions.cs
View File

@ -13,4 +13,10 @@ public static class Extensions
var enc = new NBitcoin.DataEncoders.Base58Encoder();
return enc.EncodeData(id.ToByteArray());
}
public static string? GetHeader(this IHeaderDictionary headers, string key)
{
return headers
.FirstOrDefault(a => a.Key.Equals(key, StringComparison.InvariantCultureIgnoreCase)).Value.ToString();
}
}

6
VoidCat/Model/VoidFile.cs
View File

@ -3,7 +3,7 @@ using Newtonsoft.Json;
namespace VoidCat.Model
{
public class VoidFile
public record class VoidFile
{
[JsonConverter(typeof(Base58GuidConverter))]
public Guid Id { get; init; }
@ -15,13 +15,13 @@ namespace VoidCat.Model
public DateTimeOffset Uploaded { get; init; }
}
public class InternalVoidFile : VoidFile
public record class InternalVoidFile : VoidFile
{
[JsonConverter(typeof(Base58GuidConverter))]
public Guid EditSecret { get; init; }
}
public class VoidFileMeta
public record class VoidFileMeta
{
public string? Name { get; init; }

10
VoidCat/Services/IFileStore.cs
View File

@ -6,7 +6,7 @@ namespace VoidCat.Services
{
Task<VoidFile?> Get(Guid id);
Task<InternalVoidFile> Ingress(Stream inStream, VoidFileMeta meta, CancellationToken cts);
Task<InternalVoidFile> Ingress(IngressPayload payload, CancellationToken cts);
Task Egress(EgressRequest request, Stream outStream, CancellationToken cts);
@ -15,6 +15,14 @@ namespace VoidCat.Services
IAsyncEnumerable<VoidFile> ListFiles();
}
public record IngressPayload(Stream InStream, VoidFileMeta Meta, string Hash)
{
public Guid? Id { get; init; }
public Guid? EditSecret { get; init; }
public bool IsAppend => Id.HasValue && EditSecret.HasValue;
}
public record EgressRequest(Guid Id, IEnumerable<RangeRequest> Ranges)
{
}

81
VoidCat/Services/LocalDiskFileStorage.cs
View File

@ -1,4 +1,5 @@
using System.Buffers;
using System.Security.Cryptography;
using VoidCat.Model;
using VoidCat.Model.Exceptions;
@ -44,33 +45,52 @@ public class LocalDiskFileIngressFactory : IFileStore
}
}
public async Task<InternalVoidFile> Ingress(Stream inStream, VoidFileMeta meta, CancellationToken cts)
public async Task<InternalVoidFile> Ingress(IngressPayload payload, CancellationToken cts)
{
var id = Guid.NewGuid();
var id = payload.Id ?? Guid.NewGuid();
var fPath = MapPath(id);
await using var fsTemp = new FileStream(fPath, FileMode.Create, FileAccess.ReadWrite);
using var buffer = MemoryPool<byte>.Shared.Rent();
var total = 0UL;
var readLength = 0;
while ((readLength = await inStream.ReadAsync(buffer.Memory, cts)) > 0)
InternalVoidFile? vf = null;
if (payload.IsAppend)
{
await fsTemp.WriteAsync(buffer.Memory[..readLength], cts);
await _stats.TrackIngress(id, (ulong)readLength);
total += (ulong)readLength;
vf = await _metadataStore.Get(payload.Id!.Value);
if (vf?.EditSecret != null && vf.EditSecret != payload.EditSecret)
{
throw new VoidNotAllowedException("Edit secret incorrect!");
}
}
var fm = new InternalVoidFile()
{
Id = id,
Size = total,
Metadata = meta,
Uploaded = DateTimeOffset.UtcNow,
EditSecret = Guid.NewGuid()
};
// open file
await using var fsTemp = new FileStream(fPath,
payload.IsAppend ? FileMode.Append : FileMode.Create, FileAccess.Write);
await _metadataStore.Set(fm);
return fm;
var (total, hash) = await IngressInternal(id, payload.InStream, fsTemp, cts);
if (!hash.Equals(payload.Hash, StringComparison.InvariantCultureIgnoreCase))
{
throw new CryptographicException("Invalid file hash");
}
if (payload.IsAppend)
{
vf = vf! with
{
Size = vf.Size + total
};
}
else
{
vf = new InternalVoidFile()
{
Id = id,
Metadata = payload.Meta,
Uploaded = DateTimeOffset.UtcNow,
EditSecret = Guid.NewGuid(),
Size = total
};
}
await _metadataStore.Set(vf);
return vf;
}
public Task UpdateInfo(VoidFile patch, Guid editSecret)
@ -93,6 +113,25 @@ public class LocalDiskFileIngressFactory : IFileStore
}
}
private async Task<(ulong, string)> IngressInternal(Guid id, Stream ingress, Stream fs, CancellationToken cts)
{
using var buffer = MemoryPool<byte>.Shared.Rent();
var total = 0UL;
var readLength = 0;
var sha = SHA256.Create();
while ((readLength = await ingress.ReadAsync(buffer.Memory, cts)) > 0)
{
var buf = buffer.Memory[..readLength];
await fs.WriteAsync(buf, cts);
await _stats.TrackIngress(id, (ulong)readLength);
sha.TransformBlock(buf.ToArray(), 0, buf.Length, null, 0);
total += (ulong)readLength;
}
sha.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
return (total, BitConverter.ToString(sha.Hash!).Replace("-", string.Empty));
}
private async Task EgressFull(Guid id, FileStream fileStream, Stream outStream,
CancellationToken cts)
{

3
VoidCat/spa/src/FilePreview.js
View File

@ -26,7 +26,8 @@ export function FilePreview(props) {
case "video/mp4":
case "video/matroksa":
case "video/x-matroska":
case "video/webm": {
case "video/webm":
case "video/quicktime": {
return <video src={link} controls />;
}
case "text/plain":{

124
VoidCat/spa/src/FileUpload.js
View File

@ -1,52 +1,63 @@
import {useEffect, useState} from "react";
import "./FileUpload.css";
import {FormatBytes} from "./Util";
import {buf2hex, ConstName, FormatBytes} from "./Util";
import {RateCalculator} from "./RateCalculator";
import {upload} from "@testing-library/user-event/dist/upload";
const UploadState = {
NotStarted: 0,
Starting: 1,
Hashing: 2,
Uploading: 3,
Done: 4,
Failed: 5
};
export function FileUpload(props) {
let [speed, setSpeed] = useState(0);
let [progress, setProgress] = useState(0);
let [result, setResult] = useState();
let calc = new RateCalculator();
const [speed, setSpeed] = useState(0);
const [progress, setProgress] = useState(0);
const [result, setResult] = useState();
const [uState, setUState] = useState(UploadState.NotStarted);
const calc = new RateCalculator();
function handleProgress(e) {
console.log(e);
if(e instanceof ProgressEvent) {
if (e instanceof ProgressEvent) {
let newProgress = e.loaded / e.total;
calc.ReportLoaded(e.loaded);
setSpeed(calc.RateWindow(5));
setProgress(newProgress);
}
}
async function doStreamUpload() {
let offset = 0;
let rs = new ReadableStream({
start: (controller) => {
},
pull: async (controller) => {
if(offset > props.file.size) {
if (offset > props.file.size) {
controller.cancel();
}
let requestedSize = props.file.size / controller.desiredSize;
console.log(`Reading ${requestedSize} Bytes`);
let end = Math.min(offset + requestedSize, props.file.size);
let blob = props.file.slice(offset, end, props.file.type);
controller.enqueue(await blob.arrayBuffer());
offset += blob.size;
},
cancel: (reason) => {
}
}, {
highWaterMark: 100
});
let req = await fetch("/upload", {
method: "POST",
body: rs,
@ -55,35 +66,76 @@ export function FileUpload(props) {
"X-Filename": props.file.name
}
});
if(req.ok) {
if (req.ok) {
let rsp = await req.json();
console.log(rsp);
setResult(rsp);
}
}
async function doXHRUpload() {
let xhr = await new Promise((resolve, reject) => {
let req = new XMLHttpRequest();
req.onreadystatechange = (ev) => {
if(req.readyState === XMLHttpRequest.DONE && req.status === 200) {
let rsp = JSON.parse(req.responseText);
resolve(rsp);
/**
* Upload a segment of the file
* @param segment {ArrayBuffer}
* @param id {string}
* @param editSecret {string?}
* @returns {Promise<any>}
*/
async function xhrSegment(segment, id, editSecret) {
const DigestAlgo = "SHA-256";
setUState(UploadState.Hashing);
const digest = await crypto.subtle.digest(DigestAlgo, segment);
setUState(UploadState.Uploading);
return await new Promise((resolve, reject) => {
try {
let req = new XMLHttpRequest();
req.onreadystatechange = (ev) => {
if (req.readyState === XMLHttpRequest.DONE && req.status === 200) {
let rsp = JSON.parse(req.responseText);
console.log(rsp);
resolve(rsp);
}
};
req.upload.onprogress = handleProgress;
req.open("POST", typeof(id) === "string" ? `/upload/${id}` : "/upload");
req.setRequestHeader("Content-Type", props.file.type);
req.setRequestHeader("X-Filename", props.file.name);
req.setRequestHeader("X-Digest", buf2hex(digest));
if (typeof(editSecret) === "string") {
req.setRequestHeader("X-EditSecret", editSecret);
}
};
req.upload.onprogress = handleProgress;
req.open("POST", "/upload");
req.setRequestHeader("Content-Type", props.file.type);
req.setRequestHeader("X-Filename", props.file.name);
req.send(props.file);
req.send(segment);
} catch (e) {
reject(e);
}
});
setResult(xhr);
}
async function doXHRUpload() {
const UploadSize = 100_000_000;
// upload file in segments of 100MB
let xhr = null;
const segments = props.file.size / UploadSize;
for (let s = 0; s < segments; s++) {
let offset = s * UploadSize;
let slice = props.file.slice(offset, offset + UploadSize, props.file.type);
xhr = await xhrSegment(await slice.arrayBuffer(), xhr?.file?.id, xhr?.file?.editSecret);
if(!xhr.ok) {
break;
}
}
if(xhr.ok) {
setUState(UploadState.Done);
setResult(xhr.file);
} else {
setUState(UploadState.Failed);
setResult(xhr.errorMessage);
}
}
function renderStatus() {
if(result) {
if (result) {
return (
<dl>
<dt>Link:</dt>
@ -97,6 +149,8 @@ export function FileUpload(props) {
<dd>{FormatBytes(speed)}/s</dd>
<dt>Progress:</dt>
<dd>{(progress * 100).toFixed(0)}%</dd>
<dt>Status:</dt>
<dd>{ConstName(UploadState, uState)}</dd>
</dl>
);
}
@ -104,7 +158,7 @@ export function FileUpload(props) {
useEffect(() => {
console.log(props.file);
doXHRUpload();
doXHRUpload().catch(console.error);
}, []);
return (

12
VoidCat/spa/src/Util.js
View File

@ -24,4 +24,16 @@ export function FormatBytes(b, f) {
if (b >= Const.kiB)
return (b / Const.kiB).toFixed(f) + ' KiB';
return b.toFixed(f) + ' B'
}
export function buf2hex(buffer) {
return [...new Uint8Array(buffer)].map(x => x.toString(16).padStart(2, '0')).join('');
}
export function ConstName(type, val) {
for(let [k, v] of Object.entries(type)) {
if(v === val) {
return k;
}
}
}