From e3f7f2a59e86f86a27a06d8725f4f6401af7f190 Mon Sep 17 00:00:00 2001 From: Kieran Date: Mon, 25 Mar 2019 17:39:09 +0800 Subject: [PATCH] add VBF2 --- README.md | 25 +++++++------ src/js/FileDownloader.js | 5 ++- src/js/FileUpload.js | 2 +- src/js/VBF.js | 79 +++++++++++++++++++++++++++++++++------- src/php/blobfile.php | 28 +++++++++++--- src/php/filestore.php | 41 +++++++++++++++++++-- src/php/upload.php | 27 +++++++++----- 7 files changed, 159 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 2dd3358..e146b0d 100644 --- a/README.md +++ b/README.md @@ -17,20 +17,23 @@ location ~* "^\/([0-9a-z]{27})$" { } ``` -Void Binary File Format (VBF) +Void Binary File Format (VBF2) === -| Name | Type | Description | +*All numbers are little endian* +| Name | Size | Description | |---|---|---| -| version | uint8_t | Binary file format version | -| hash | SHA256 hash | The hash of the unencrypted file | -| uploaded | uint32_t | Timestamp of when the upload started | -| payload | >EOF | The encrypted payload | +| version | 1 byte unsigned number | Binary file format version | +| magic | 4 bytes | "VOID" encoded to UTF8 string +| uploaded | 4 byte unsigned number | Unix timestamp of when the upload started | +| payload | EOF - 32 bytes | The encrypted payload | +| hash | 32 bytes HMAC-SHA265 | The HMAC of the unencrypted file* | + +*\* Using the encryption key as the HMAC key* ---- VBF Payload Format - -| Name | Type | Description | +==== +| Name | Size | Description | |---|---|---| -| header_length | uint16_t | Length of the header section | -| header | string | The header json | +| header_length | 2 byte unsigned number | Length of the header section | +| header | {header_length} UTF8 String | The header json | | file | >EOF | The file | diff --git a/src/js/FileDownloader.js b/src/js/FileDownloader.js index 5f9a078..84ec6c1 100644 --- a/src/js/FileDownloader.js +++ b/src/js/FileDownloader.js @@ -107,7 +107,7 @@ const FileDownloader = function (fileinfo, key, iv) { let header = VBF.Parse(blob); let hash_text = Utils.ArrayToHex(header.hmac); - Log.I(`${this.fileinfo.FileId} blob header version is ${header.version} and hash is ${hash_text} uploaded on ${header.uploaded}`); + Log.I(`${this.fileinfo.FileId} blob header version is ${header.version} and hash is ${hash_text} uploaded on ${header.uploaded} (Magic: ${Utils.ArrayToHex(header.magic)})`); let key_raw = Utils.HexToArray(this.key); let iv_raw = Utils.HexToArray(this.iv); @@ -116,7 +116,8 @@ const FileDownloader = function (fileinfo, key, iv) { let key = await crypto.subtle.importKey("raw", key_raw, EncryptionKeyDetails, false, ['decrypt']); let keyhmac = await crypto.subtle.importKey("raw", key_raw, HMACKeyDetails, false, ['verify']); - let decrypted_file = await crypto.subtle.decrypt({ name: EncryptionAlgo, iv: iv_raw }, key, blob.slice(VBF.HeaderSize)); + let enc_data = VBF.GetEncryptedPart(header.version, blob); + let decrypted_file = await crypto.subtle.decrypt({ name: EncryptionAlgo, iv: iv_raw }, key, enc_data); //read the header let json_header_length = new Uint16Array(decrypted_file.slice(0, 2))[0]; diff --git a/src/js/FileUpload.js b/src/js/FileUpload.js index f3132f4..1b63ddc 100644 --- a/src/js/FileUpload.js +++ b/src/js/FileUpload.js @@ -245,7 +245,7 @@ const FileUpload = function (file, host) { this.UploadData = async function (fileData) { this.uploadStats.lastProgress = new Date().getTime(); this.HandleProgress('state-upload-start'); - let uploadResult = await XHR("POST", `${window.location.protocol}//${this.host}/upload`, fileData, undefined, function (ev) { + let uploadResult = await XHR("POST", `${window.location.protocol}//${this.host}/upload`, fileData, { "Content-Type": "application/octet-stream" }, function (ev) { let now = new Date().getTime(); let dxLoaded = ev.loaded - this.uploadStats.lastLoaded; let dxTime = now - this.uploadStats.lastProgress; diff --git a/src/js/VBF.js b/src/js/VBF.js index b1dd49c..72f9625 100644 --- a/src/js/VBF.js +++ b/src/js/VBF.js @@ -1,22 +1,59 @@ const VBF = { - Version: 1, - HeaderSize: 37, + Version: 2, - Create: function (hash, encryptedData) { - //upload the encrypted file data - let upload_payload = new Uint8Array(VBF.HeaderSize + encryptedData.byteLength); + Create: function (hash, encryptedData, version) { + version = typeof version === "number" ? version : VBF.Version; + switch (version) { + case 1: + return VBF.CreateV1(hash, encryptedData); + case 2: + return VBF.CreateV2(hash, encryptedData); + } + }, + + CreateV1: function (hash, encryptedData) { + let upload_payload = new Uint8Array(37 + encryptedData.byteLength); let created = new ArrayBuffer(4); new DataView(created).setUint32(0, parseInt(new Date().getTime() / 1000), true); - upload_payload[0] = VBF.Version; //blob version + upload_payload[0] = 1; //blob version upload_payload.set(new Uint8Array(hash), 1); upload_payload.set(new Uint8Array(created), hash.byteLength + 1); - upload_payload.set(new Uint8Array(encryptedData), VBF.HeaderSize); + upload_payload.set(new Uint8Array(encryptedData), 37); return upload_payload; }, + CreateV2: function (hash, encryptedData) { + let upload_payload = new Uint8Array(9 + encryptedData.byteLength + hash.byteLength); + + let created = new ArrayBuffer(4); + new DataView(created).setUint32(0, parseInt(new Date().getTime() / 1000), true); + + upload_payload[0] = 2; //blob version + upload_payload.set(new TextEncoder().encode("VOID"), 1); + upload_payload.set(new Uint8Array(created), 5); + upload_payload.set(new Uint8Array(encryptedData), 9); + upload_payload.set(new Uint8Array(hash), 9 + encryptedData.byteLength); + + return upload_payload; + }, + + /** + * Returns the encrypted part of the VBF blob + * @param {number} version + * @param {ArrayBuffer} blob + */ + GetEncryptedPart: function (version, blob) { + switch (version) { + case 1: + return blob.slice(37); + case 2: + return blob.slice(9, blob.byteLength - 32); + } + }, + /** * Parses the header of the raw file * @param {ArrayBuffer} data - Raw data from the server @@ -24,13 +61,27 @@ const VBF = { */ Parse: function (data) { let version = new Uint8Array(data)[0]; - let hmac = data.slice(1, 33); - let uploaded = new DataView(data.slice(33, 37)).getUint32(0, true); + if (version === 1) { + let hmac = data.slice(1, 33); + let uploaded = new DataView(data.slice(33, 37)).getUint32(0, true); - return { - version, - hmac, - uploaded - }; + return { + version, + hmac, + uploaded, + magic: null + }; + } else if (version === 2) { + let magic = data.slice(1, 5); + let hmac = data.slice(data.byteLength - 32); + let uploaded = new DataView(data.slice(5, 9)).getUint32(0, true); + + return { + version, + hmac, + uploaded, + magic + }; + } } }; \ No newline at end of file diff --git a/src/php/blobfile.php b/src/php/blobfile.php index 79f8f04..cb415b3 100644 --- a/src/php/blobfile.php +++ b/src/php/blobfile.php @@ -6,16 +6,32 @@ public static function LoadHeader($path) : ?BlobFile { $input = fopen($path, "rb"); - $header = fread($input, 37); //1 version byte + 32 byte hash (64 hex digits) + 4 byte timestamp - fclose($input); + $version = ord(fread($input, 1)); + error_log($version); - $header_data = unpack("C1version/H64hash256/Vuploaded", $header); - if($header_data["version"] == 1){ - $bf = new BlobFile(); - $bf->Version = $header_data["version"]; + $bf = new BlobFile(); + if($version == 1) { + $header = fread($input, 36); //+32 byte hash (64 hex digits) + 4 byte timestamp + fclose($input); + $header_data = unpack("H64hash256/Vuploaded", $header); + + $bf->Version = 1; $bf->Hash = $header_data["hash256"]; $bf->Uploaded = $header_data["uploaded"]; return $bf; + } elseif($version == 2) { + $header = fread($input, 8); //+4 magic bytes + 4 byte timestamp + $header_data = unpack("H8magic/Vuploaded", $header); + fclose($input); + + error_log("Magic is: " . $header_data["magic"]); + if($header_data["magic"] == "564f4944") { //VOID as hex (UTF-8) + $bf->Version = 2; + $bf->Uploaded = $header_data["uploaded"]; + return $bf; + } + } else { + fclose($input); } return null; diff --git a/src/php/filestore.php b/src/php/filestore.php index 85b1bfd..02d2b8a 100644 --- a/src/php/filestore.php +++ b/src/php/filestore.php @@ -81,14 +81,47 @@ return file_exists($file_path); } - public function StoreFile($file, $id) { + public function StoreFile($file, $id) : bool { $file_path = $this->GetAbsoluteFilePath($id); + + if(!file_exists($file_path)) { + $fout = fopen($file_path, 'wb+'); + stream_copy_to_stream($file, $fout); + fclose($fout); + return true; + } + return false; + } + + public function StoreV1File($bf, $file) : ?string { + $id = gmp_strval(gmp_init("0x" . hash(Config::$Instance->public_hash_algo, $bf->Hash)), 62); $input = fopen($file, "rb"); - $fout = fopen($file_path, 'wb+'); - stream_copy_to_stream($input, $fout); - fclose($fout); + $res = $this->StoreFile($input, $id); fclose($input); + + return $res ? $id : null; + } + + public function StoreV2File($bf, $file) : ?string { + //we need to seek to the end before finding the id, do that first + $input = fopen($file, "rb"); + $temp_name = tempnam($this->GetUploadDirAbsolute(), "VTMP_"); + $input_temp = fopen($temp_name, "wb+"); + stream_copy_to_stream($input, $input_temp); + fclose($input); + + fseek($input_temp, -32, SEEK_END); + $hash = unpack("H64hash256", fread($input_temp, 32)); + fclose($input_temp); + + $id = gmp_strval(gmp_init("0x" . hash(Config::$Instance->public_hash_algo, $hash["hash256"])), 62); + $file_path = $this->GetAbsoluteFilePath($id); + if(!file_exists($file_path)){ + rename($temp_name, $file_path); + return $id; + } + return null; } public function GetFileSize($id) : int { diff --git a/src/php/upload.php b/src/php/upload.php index 56f23d1..6016b7f 100644 --- a/src/php/upload.php +++ b/src/php/upload.php @@ -66,9 +66,14 @@ $id = $this->SaveUpload($bf); //sync to other servers - $rsp->sync = $this->SyncFileUpload($id); - $rsp->status = 200; - $rsp->id = $id; + if($id == null) { + $rsp->status = 4; + $rsp->msg = "Invalid VBF or file already exists"; + } else { + $rsp->sync = $this->SyncFileUpload($id); + $rsp->status = 200; + $rsp->id = $id; + } } else { $rsp->status = 2; $rsp->msg = "Invalid file header"; @@ -96,13 +101,15 @@ return array(); } - function SaveUpload($bf) : string { - $id = gmp_strval(gmp_init("0x" . hash(Config::$Instance->public_hash_algo, $bf->Hash)), 62); - + function SaveUpload($bf) : ?string { $fs = new FileStore(Config::$Instance->upload_folder); - $fs->StoreFile("php://input", $id); - - return $id; + switch($bf->Version) { + case 1: + return $fs->StoreV1File($bf, "php://input"); + case 2: + return $fs->StoreV2File($bf, "php://input"); + } + return null; } function SaveLegacyUpload() : ?string { @@ -111,7 +118,7 @@ $id = gmp_strval(gmp_init("0x" . hash(Config::$Instance->public_hash_algo, $hash)), 62); $fs = new FileStore(Config::$Instance->upload_folder); - $fs->StoreFile("php://input", $id); + $fs->StoreFile(fopen("php://input", "rb"), $id); $info = new FileInfo(); $info->FileId = $id;