From ff9099d33f50131ffb1ff921e5fec1ec2e8c9254 Mon Sep 17 00:00:00 2001 From: Kieran Date: Tue, 8 Feb 2022 23:52:01 +0000 Subject: [PATCH] Add range requests --- VoidCat/Controllers/DownloadController.cs | 85 +++++++++++++++++++++-- VoidCat/Services/IFileStore.cs | 27 ++++++- VoidCat/Services/LocalDiskFileStorage.cs | 63 +++++++++++++---- 3 files changed, 154 insertions(+), 21 deletions(-) diff --git a/VoidCat/Controllers/DownloadController.cs b/VoidCat/Controllers/DownloadController.cs index fd98f3c..ef50c74 100644 --- a/VoidCat/Controllers/DownloadController.cs +++ b/VoidCat/Controllers/DownloadController.cs @@ -1,3 +1,4 @@ +using System.Net; using Microsoft.AspNetCore.Mvc; using VoidCat.Model; using VoidCat.Services; @@ -8,28 +9,100 @@ namespace VoidCat.Controllers; public class DownloadController : Controller { private readonly IFileStore _storage; + private readonly ILogger _logger; - public DownloadController(IFileStore storage) + public DownloadController(IFileStore storage, ILogger logger) { _storage = storage; + _logger = logger; } + [HttpOptions] + [Route("{id}")] + public Task DownloadFileOptions([FromRoute] string id) + { + var gid = id.FromBase58Guid(); + return SetupDownload(gid); + } + [ResponseCache(Location = ResponseCacheLocation.Any, Duration = 86400)] [HttpGet] [Route("{id}")] public async Task DownloadFile([FromRoute] string id) { var gid = id.FromBase58Guid(); - var meta = await _storage.Get(gid); + var meta = await SetupDownload(gid); + + var egressReq = new EgressRequest(gid, GetRanges(Request, (long)meta!.Size)); + if (egressReq.Ranges.Count() > 1) + { + _logger.LogWarning("Multi-range request not supported!"); + // downgrade to full send + egressReq = egressReq with + { + Ranges = Enumerable.Empty() + }; + } + else if(egressReq.Ranges.Count() == 1) + { + Response.StatusCode = (int)HttpStatusCode.PartialContent; + } + else + { + Response.Headers.AcceptRanges = "bytes"; + } + + foreach (var range in egressReq.Ranges) + { + Response.Headers.Add("content-range", range.ToContentRange()); + Response.ContentLength = range.Size; + } + + var cts = HttpContext.RequestAborted; + await Response.StartAsync(cts); + await _storage.Egress(egressReq, Response.Body, cts); + await Response.CompleteAsync(); + } + + private async Task SetupDownload(Guid id) + { + var meta = await _storage.Get(id); if (meta == null) { Response.StatusCode = 404; - return; + return null; } - + Response.Headers.XFrameOptions = "SAMEORIGIN"; Response.Headers.ContentDisposition = $"inline; filename=\"{meta?.Metadata?.Name}\""; Response.ContentType = meta?.Metadata?.MimeType ?? "application/octet-stream"; - await _storage.Egress(gid, Response.Body, HttpContext.RequestAborted); + + return meta; } -} \ No newline at end of file + + private IEnumerable GetRanges(HttpRequest request, long totalSize) + { + foreach (var rangeHeader in request.Headers.Range) + { + if (string.IsNullOrEmpty(rangeHeader)) + { + continue; + } + + var ranges = rangeHeader.Replace("bytes=", string.Empty).Split(","); + foreach (var range in ranges) + { + var rangeValues = range.Split("-"); + + long? endByte = null, startByte = 0; + if (long.TryParse(rangeValues[1], out var endParsed)) + endByte = endParsed; + + if (long.TryParse(rangeValues[0], out var startParsed)) + startByte = startParsed; + + yield return new(totalSize, startByte, endByte); + } + } + } +} diff --git a/VoidCat/Services/IFileStore.cs b/VoidCat/Services/IFileStore.cs index ad2345a..9f48ab8 100644 --- a/VoidCat/Services/IFileStore.cs +++ b/VoidCat/Services/IFileStore.cs @@ -5,13 +5,36 @@ namespace VoidCat.Services public interface IFileStore { Task Get(Guid id); - + Task Ingress(Stream inStream, VoidFileMeta meta, CancellationToken cts); - Task Egress(Guid id, Stream outStream, CancellationToken cts); + Task Egress(EgressRequest request, Stream outStream, CancellationToken cts); Task UpdateInfo(VoidFile patch, Guid editSecret); IAsyncEnumerable ListFiles(); } + + public record EgressRequest(Guid Id, IEnumerable Ranges) + { + } + + public record RangeRequest(long? TotalSize, long? Start, long? End) + { + private const long DefaultBufferSize = 1024L * 512L; + + public long? Size + => Start.HasValue ? + (End ?? Math.Min(TotalSize!.Value, Start.Value + DefaultBufferSize)) - Start.Value : End; + + public bool IsForFullFile + => Start is 0 && !End.HasValue; + + /// + /// Return Content-Range header content for this range + /// + /// + public string ToContentRange() + => $"bytes {Start}-{End ?? (Start + Size - 1L)}/{TotalSize?.ToString() ?? "*"}"; + } } diff --git a/VoidCat/Services/LocalDiskFileStorage.cs b/VoidCat/Services/LocalDiskFileStorage.cs index dde0fa3..7e701f7 100644 --- a/VoidCat/Services/LocalDiskFileStorage.cs +++ b/VoidCat/Services/LocalDiskFileStorage.cs @@ -9,8 +9,8 @@ public class LocalDiskFileIngressFactory : IFileStore private readonly VoidSettings _settings; private readonly IStatsCollector _stats; private readonly IFileMetadataStore _metadataStore; - - public LocalDiskFileIngressFactory(VoidSettings settings, IStatsCollector stats, + + public LocalDiskFileIngressFactory(VoidSettings settings, IStatsCollector stats, IFileMetadataStore metadataStore) { _settings = settings; @@ -28,18 +28,19 @@ public class LocalDiskFileIngressFactory : IFileStore return await _metadataStore.Get(id); } - public async Task Egress(Guid id, Stream outStream, CancellationToken cts) + public async Task Egress(EgressRequest request, Stream outStream, CancellationToken cts) { - var path = MapPath(id); - if (!File.Exists(path)) throw new VoidFileNotFoundException(id); + var path = MapPath(request.Id); + if (!File.Exists(path)) throw new VoidFileNotFoundException(request.Id); await using var fs = new FileStream(path, FileMode.Open, FileAccess.Read); - using var buffer = MemoryPool.Shared.Rent(); - var readLength = 0; - while ((readLength = await fs.ReadAsync(buffer.Memory, cts)) > 0) + if (request.Ranges.Any()) { - await outStream.WriteAsync(buffer.Memory[..readLength], cts); - await _stats.TrackEgress(id, (ulong)readLength); + await EgressRanges(request.Id, request.Ranges, fs, outStream, cts); + } + else + { + await EgressFull(request.Id, fs, outStream, cts); } } @@ -58,7 +59,7 @@ public class LocalDiskFileIngressFactory : IFileStore await _stats.TrackIngress(id, (ulong)readLength); total += (ulong)readLength; } - + var fm = new InternalVoidFile() { Id = id, @@ -76,7 +77,7 @@ public class LocalDiskFileIngressFactory : IFileStore { return _metadataStore.Update(patch, editSecret); } - + public async IAsyncEnumerable ListFiles() { foreach (var fe in Directory.EnumerateFiles(_settings.DataDirectory)) @@ -92,6 +93,42 @@ public class LocalDiskFileIngressFactory : IFileStore } } + private async Task EgressFull(Guid id, FileStream fileStream, Stream outStream, + CancellationToken cts) + { + using var buffer = MemoryPool.Shared.Rent(); + var readLength = 0; + while ((readLength = await fileStream.ReadAsync(buffer.Memory, cts)) > 0) + { + await outStream.WriteAsync(buffer.Memory[..readLength], cts); + await _stats.TrackEgress(id, (ulong)readLength); + await outStream.FlushAsync(cts); + } + } + + private async Task EgressRanges(Guid id, IEnumerable ranges, FileStream fileStream, Stream outStream, + CancellationToken cts) + { + using var buffer = MemoryPool.Shared.Rent(); + foreach (var range in ranges) + { + fileStream.Seek(range.Start ?? range.End ?? 0L, + range.Start.HasValue ? SeekOrigin.Begin : SeekOrigin.End); + + var readLength = 0; + var dataRemaining = range.Size ?? 0L; + while ((readLength = await fileStream.ReadAsync(buffer.Memory, cts)) > 0 + && dataRemaining > 0) + { + var toWrite = Math.Min(readLength, dataRemaining); + await outStream.WriteAsync(buffer.Memory[..(int)toWrite], cts); + await _stats.TrackEgress(id, (ulong)toWrite); + dataRemaining -= toWrite; + await outStream.FlushAsync(cts); + } + } + } + private string MapPath(Guid id) => Path.Join(_settings.DataDirectory, id.ToString()); -} \ No newline at end of file +}