Use CryptoJS to hash files >2GB

2025-03-27 00:31:44 +01:00 · 2022-08-28 13:15:50 +01:00 · 2022-08-28 13:15:50 +01:00 · 5c955712e1
commit 5c955712e1
parent 679fc80eb2
10 changed files with 87 additions and 28 deletions
--- a/VoidCat/Controllers/InfoController.cs
+++ b/VoidCat/Controllers/InfoController.cs
@ -34,12 +34,28 @@ public class InfoController : Controller
        var bw = await _statsReporter.GetBandwidth();
        var storeStats = await _fileMetadata.Stats();

-        return new(bw, storeStats.Size, storeStats.Files, BuildInfo.GetBuildInfo(),
-            _settings.CaptchaSettings?.SiteKey,
-            await _timeSeriesStats.GetBandwidth(DateTime.UtcNow.AddDays(-30), DateTime.UtcNow),
-            _fileStores);
+        return new()
+        {
+            Bandwidth = bw,
+            TotalBytes = storeStats.Size,
+            Count = storeStats.Files,
+            BuildInfo = BuildInfo.GetBuildInfo(),
+            CaptchaSiteKey = _settings.CaptchaSettings?.SiteKey,
+            TimeSeriesMetrics = await _timeSeriesStats.GetBandwidth(DateTime.UtcNow.AddDays(-30), DateTime.UtcNow),
+            FileStores = _fileStores,
+            UploadSegmentSize = _settings.UploadSegmentSize
+        };
    }

-    public sealed record GlobalInfo(Bandwidth Bandwidth, ulong TotalBytes, long Count, BuildInfo BuildInfo,
-        string? CaptchaSiteKey, IEnumerable<BandwidthPoint> TimeSeriesMetrics, IEnumerable<string?> FileStores);
-}
+    public sealed class GlobalInfo
+    {
+        public Bandwidth Bandwidth { get; init; }
+        public ulong TotalBytes { get; init; }
+        public long Count { get; init; }
+        public BuildInfo BuildInfo { get; init; }
+        public string? CaptchaSiteKey { get; init; }
+        public IEnumerable<BandwidthPoint> TimeSeriesMetrics { get; init; }
+        public IEnumerable<string?> FileStores { get; init; }
+        public ulong? UploadSegmentSize { get; init; }
+    }
+}
--- a/VoidCat/Model/VoidSettings.cs
+++ b/VoidCat/Model/VoidSettings.cs
@ -11,7 +11,12 @@ namespace VoidCat.Model
        /// Data directory to store files in
        /// </summary>
        public string DataDirectory { get; init; } = "./data";
-
+        
+        /// <summary>
+        /// Size in bytes to split uploads into chunks
+        /// </summary>
+        public ulong? UploadSegmentSize { get; init; } = null;
+        
        /// <summary>
        /// Tor configuration
        /// </summary>
--- a/VoidCat/Services/Files/FileStorageStartup.cs
+++ b/VoidCat/Services/Files/FileStorageStartup.cs
@ -30,11 +30,11 @@ public static class FileStorageStartup
            }
        }

-        if (!string.IsNullOrEmpty(settings.Postgres))
+        if (settings.HasPostgres())
        {
            services.AddTransient<IUserUploadsStore, PostgresUserUploadStore>();
            services.AddTransient<IFileStore, LocalDiskFileStore>();
-            if (settings.MetadataStore == "postgres")
+            if (settings.MetadataStore is "postgres" or "local-disk")
            {
                services.AddSingleton<IFileMetadataStore, PostgresFileMetadataStore>();
            }
--- a/VoidCat/spa/package.json
+++ b/VoidCat/spa/package.json
@ -6,6 +6,7 @@
  "dependencies": {
    "@hcaptcha/react-hcaptcha": "^1.1.1",
    "@reduxjs/toolkit": "^1.7.2",
+    "crypto-js": "^4.1.1",
    "feather-icons-react": "^0.5.0",
    "moment": "^2.29.4",
    "preval.macro": "^5.0.0",
--- a/VoidCat/spa/src/FileUpload.js
+++ b/VoidCat/spa/src/FileUpload.js
@ -1,6 +1,7 @@
 import {useEffect, useState} from "react";
-import {buf2hex, ConstName, FormatBytes} from "./Util";
+import {ConstName, FormatBytes} from "./Util";
 import {RateCalculator} from "./RateCalculator";
+import * as CryptoJS from 'crypto-js';

 import "./FileUpload.css";
 import {useSelector} from "react-redux";
@ -20,6 +21,7 @@ export const DigestAlgo = "SHA-256";

 export function FileUpload(props) {
    const auth = useSelector(state => state.login.jwt);
+    const info = useSelector(state => state.info.info);
    const [speed, setSpeed] = useState(0);
    const [progress, setProgress] = useState(0);
    const [result, setResult] = useState();
@ -84,14 +86,14 @@ export function FileUpload(props) {
    /**
     * Upload a segment of the file
     * @param segment {ArrayBuffer}
-     * @param id {string}
+     * @param fullDigest {string} Full file hash
+     * @param id {string?}
     * @param editSecret {string?}
-     * @param fullDigest {string?} Full file hash
     * @param part {int?} Segment number
     * @param partOf {int?} Total number of segments
     * @returns {Promise<any>}
     */
-    async function xhrSegment(segment, id, editSecret, fullDigest, part, partOf) {
+    async function xhrSegment(segment, fullDigest, id, editSecret, part, partOf) {
        setUState(UploadState.Uploading);

        return await new Promise((resolve, reject) => {
@ -133,23 +135,36 @@ export function FileUpload(props) {
    }

    async function doXHRUpload() {
-        // upload file in segments of 50MB
-        const UploadSize = 50_000_000;
+        let uploadSize = info.uploadSegmentSize ?? Number.MAX_VALUE;

        setUState(UploadState.Hashing);
-        let digest = await crypto.subtle.digest(DigestAlgo, await props.file.arrayBuffer());
+        let hash = await digest(props.file);
+        if(props.file.size >= uploadSize) {
+            await doSplitXHRUpload(hash, uploadSize);
+        } else {
+            let xhr = await xhrSegment(props.file, hash);
+            handleXHRResult(xhr);
+        }
+    }
+
+    async function doSplitXHRUpload(hash, splitSize) {
        let xhr = null;
-        const segments = Math.ceil(props.file.size / UploadSize);
+        setProgress(0);
+        const segments = Math.ceil(props.file.size / splitSize);
        for (let s = 0; s < segments; s++) {
            calc.ResetLastLoaded();
-            let offset = s * UploadSize;
-            let slice = props.file.slice(offset, offset + UploadSize, props.file.type);
+            let offset = s * splitSize;
+            let slice = props.file.slice(offset, offset + splitSize, props.file.type);
            let segment = await slice.arrayBuffer();
-            xhr = await xhrSegment(segment, xhr?.file?.id, xhr?.file?.metadata?.editSecret, buf2hex(digest), s + 1, segments);
+            xhr = await xhrSegment(segment, xhr?.file?.id, xhr?.file?.metadata?.editSecret, hash, s + 1, segments);
            if (!xhr.ok) {
                break;
            }
        }
+        handleXHRResult(xhr);
+    }
+    
+    function handleXHRResult(xhr) {
        if (xhr.ok) {
            setUState(UploadState.Done);
            setResult(xhr.file);
@ -159,6 +174,19 @@ export function FileUpload(props) {
            setResult(xhr.errorMessage);
        }
    }
+    
+    async function digest(file) {
+        const chunkSize = 10_000_000;
+        let sha = CryptoJS.algo.SHA256.create();
+        for (let x = 0; x < Math.ceil(file.size / chunkSize); x++) {
+            let offset = x * chunkSize;
+            let slice = file.slice(offset, offset + chunkSize, file.type);
+            let data = Uint32Array.from(await slice.arrayBuffer());
+            sha.update(new CryptoJS.lib.WordArray.init(data, slice.length));
+            setProgress(offset / parseFloat(file.size));
+        }
+        return sha.finalize().toString();
+    }

    function renderStatus() {
        if (result) {
--- a/VoidCat/spa/src/GlobalStats.js
+++ b/VoidCat/spa/src/GlobalStats.js
@ -7,7 +7,7 @@ import moment from "moment";
 import {useSelector} from "react-redux";

 export function GlobalStats() {
-    let stats = useSelector(state => state.info.stats);
+    let stats = useSelector(state => state.info.info);

    return (
        <Fragment>
--- a/VoidCat/spa/src/Header.js
+++ b/VoidCat/spa/src/Header.js
@ -5,7 +5,7 @@ import {InlineProfile} from "./InlineProfile";
 import {useApi} from "./Api";
 import {logout, setProfile} from "./LoginState";
 import {useEffect} from "react";
-import {setStats} from "./SiteInfoStore";
+import {setInfo} from "./SiteInfoStore";

 export function Header() {
    const dispatch = useDispatch();
@ -26,7 +26,7 @@ export function Header() {
    async function loadStats() {
        let req = await Api.info();
        if (req.ok) {
-            dispatch(setStats(await req.json()));
+            dispatch(setInfo(await req.json()));
        }
    }

--- a/VoidCat/spa/src/SiteInfoStore.js
+++ b/VoidCat/spa/src/SiteInfoStore.js
@ -3,14 +3,14 @@
 export const SiteInfoState = createSlice({
    name: "SiteInfo",
    initialState: {
-        stats: null
+        info: null
    },
    reducers: {
-        setStats: (state, action) => {
-            state.stats = action.payload;
+        setInfo: (state, action) => {
+            state.info = action.payload;
        },
    }
 });

-export const {setStats} = SiteInfoState.actions;
+export const {setInfo} = SiteInfoState.actions;
 export default SiteInfoState.reducer;
--- a/VoidCat/spa/yarn.lock
+++ b/VoidCat/spa/yarn.lock
@ -3022,6 +3022,11 @@ cross-spawn@^7.0.2, cross-spawn@^7.0.3:
    shebang-command "^2.0.0"
    which "^2.0.1"

+crypto-js@^4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/crypto-js/-/crypto-js-4.1.1.tgz#9e485bcf03521041bd85844786b83fb7619736cf"
+  integrity sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==
+
 crypto-random-string@^2.0.0:
  version "2.0.0"
  resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -11,8 +11,12 @@ services:
      - redis
  redis:
    image: "redis:alpine"
+    ports:
+      - "6379:6379"
  postgres:
    image: "postgres:14.1"
+    ports:
+      - "5432:5432"
    environment:
      - "POSTGRES_DB=void"
      - "POSTGRES_HOST_AUTH_METHOD=trust"