diff --git a/src/exe/cimbar_recv2/recv2.cpp b/src/exe/cimbar_recv2/recv2.cpp index 7c4cef52..34b0a3ca 100644 --- a/src/exe/cimbar_recv2/recv2.cpp +++ b/src/exe/cimbar_recv2/recv2.cpp @@ -150,17 +150,28 @@ int main(int argc, char** argv) // attempt save uint32_t fileId = res; - unsigned size = cimbard_get_filesize(fileId); - std::string filename = fmt::format("0.{}", size); - std::cerr << "Saving file " << filename << " of size " << size << std::endl; + int size = cimbard_get_filesize(fileId); + + std::string filename; + filename.resize(255, '\0'); + int fnsize = cimbard_get_filename(fileId, filename.data(), filename.size()); + if (fnsize > 0) + filename.resize(fnsize); + else // fallback + filename = fmt::format("0.{}", size); + std::cerr << "Saving file " << filename << " of (compressed) size " << size << std::endl; + + std::string file_path = fmt::format("{}/{}", outpath, filename); + std::ofstream outs(file_path, std::ios::binary); std::vector data; - data.resize(size); - int res = cimbard_finish_copy(fileId, data.data(), size); - if (res != 0) - std::cerr << "failed fountain_finish_copy " << res << std::endl; + data.resize(cimbard_get_decompress_bufsize()); - decompress_on_store(outpath, true)(filename, data); + int res = 1; + while ((res = cimbard_decompress_read(fileId, data.data(), data.size())) > 0) + outs.write(reinterpret_cast(data.data()), res); + if (res < 0) + std::cerr << "failed cimbard_decompress_read " << res << std::endl; } } diff --git a/src/lib/cimbar_js/CMakeLists.txt b/src/lib/cimbar_js/CMakeLists.txt index ffce5a5c..dc3f2679 100644 --- a/src/lib/cimbar_js/CMakeLists.txt +++ b/src/lib/cimbar_js/CMakeLists.txt @@ -14,8 +14,6 @@ set(SOURCES cimbar_recv_js.h cimbar_recv_js.cpp - cimbar_zstd_js.h - cimbar_zstd_js.cpp ) endif() @@ -66,16 +64,14 @@ set(LINK_WASM_EXPORTED_FUNCTIONS ${LINK_WASM_EXPORTED_FUNCTIONS} , "_cimbard_get_report" + , "_cimbard_get_bufsize" , "_cimbard_scan_extract_decode" , "_cimbard_fountain_decode" - , "_cimbard_get_bufsize" , "_cimbard_get_filesize" , "_cimbard_get_filename" - , "_cimbard_finish_copy" + , "_cimbard_get_decompress_bufsize" + , "_cimbard_decompress_read" , "_cimbard_configure_decode" - , "_cimbarz_init_decompress" - , "_cimbarz_get_bufsize" - , "_cimbarz_decompress_read" ) endif() diff --git a/src/lib/cimbar_js/cimbar_recv_js.cpp b/src/lib/cimbar_js/cimbar_recv_js.cpp index 570e612c..0564d416 100644 --- a/src/lib/cimbar_js/cimbar_recv_js.cpp +++ b/src/lib/cimbar_js/cimbar_recv_js.cpp @@ -19,10 +19,16 @@ namespace { - + // for decode std::shared_ptr _sink; - std::string _reporting; + // for decompress + // we support only one decompress at a time! + uint32_t _decId = 0; + std::vector _reassembled; + std::unique_ptr> _dec; + + std::string _reporting; cv::Mat _debugFrame; TimeAccumulator _tScanExtract; @@ -31,6 +37,48 @@ namespace { // settings int _modeVal = 68; + // set up stateful decompressor + // for api simplicity, this is coupled to recover_contents() + // ... but we *could* split them up + int init_decompress(uint32_t id) + { + if (id != _decId) + return -11; + if (_dec) + _dec.reset(); + _dec = std::make_unique>(); + if (!_dec) + return -12; + _dec->init_decompress(reinterpret_cast(_reassembled.data()), _reassembled.size()); + return 0; + } + + // recovers file contents into _reassembled, and sets _decId = id if so. + // if the contents cannot be recovered, amounts to a no-op with error + int recover_contents(uint32_t id) + { + if (id != _decId) + { + if (!_sink) + return -1; + if (_sink->is_done(id)) + return -2; // it's gone man + + _reassembled.resize(cimbard_get_filesize(id)); + if (!_sink->recover(id, _reassembled.data(), _reassembled.size())) + return -3; + _decId = id; + + int res = init_decompress(id); + if (res < 0) + return res; + } + if (_reassembled.empty()) + return -5; + + return 0; + } + unsigned fountain_chunks_per_frame() { return cimbar::Config::fountain_chunks_per_frame( @@ -166,14 +214,24 @@ int64_t cimbard_fountain_decode(const unsigned char* buffer, unsigned size) return res; } -int cimbard_get_filesize(uint32_t id) +// mostly for internal use, but also helpful for debugging +unsigned cimbard_get_filesize(uint32_t id) { FountainMetadata md(id); return md.file_size(); } -int cimbard_get_filename(const uchar* finbuffer, unsigned size, char* filename, unsigned fnsize) +// do recover() if does not exist (fail with appropriate neg values), if it does use the bytes. +// stateful against a map (same as cimbard_decompress_read() +int cimbard_get_filename(uint32_t id, char* filename, unsigned fnsize) { + int res = recover_contents(id); + if (res < 0) + return res; + + const uchar* finbuffer = _reassembled.data(); + unsigned size = _reassembled.size(); + std::string fn = cimbar::zstd_header_check::get_filename(finbuffer, size); if (!fn.empty()) fn = File::basename(fn); @@ -186,15 +244,29 @@ int cimbard_get_filename(const uchar* finbuffer, unsigned size, char* filename, return fn.size(); } -// if fountain_decode returned a >0 value, call this to retrieve the reassembled file -// bouth fountain_*() calls should be from the same js webworker/thread -int cimbard_finish_copy(uint32_t id, uchar* buffer, unsigned size) +int cimbard_decompress_read(uint32_t id, unsigned char* buffer, unsigned size) { - if (!_sink) - return -1; - if (!_sink->recover(id, buffer, size)) - return -2; - return 0; + int res = recover_contents(id); + if (res < 0) + return res; + + if (!_dec) + return -13; + if (!_dec->good()) + return -14; + + _dec->str(std::string()); + _dec->write_once(); + std::string temp = _dec->str(); + if (size > temp.size()) + size = temp.size(); + std::copy(temp.data(), temp.data()+size, buffer); + return size; +} + +int cimbard_get_decompress_bufsize() +{ + return ZSTD_DStreamOutSize(); } int cimbard_configure_decode(int mode_val) @@ -215,4 +287,12 @@ int cimbard_configure_decode(int mode_val) return 0; } +// testing +unsigned char* cimbard_get_reassembled_file_buff() +{ + if (_reassembled.empty()) + return nullptr; + return _reassembled.data(); +} + } diff --git a/src/lib/cimbar_js/cimbar_recv_js.h b/src/lib/cimbar_js/cimbar_recv_js.h index eb0e1111..608efe8c 100644 --- a/src/lib/cimbar_js/cimbar_recv_js.h +++ b/src/lib/cimbar_js/cimbar_recv_js.h @@ -20,20 +20,24 @@ int cimbard_scan_extract_decode(const unsigned char* imgdata, unsigned imgw, uns // persists state, the return value (if >0) corresponds to a uint32_t id int64_t cimbard_fountain_decode(const unsigned char* buffer, unsigned size); -// get filesize from id -int cimbard_get_filesize(uint32_t id); - -// if fountain_decode returned a >0 value, call this to retrieve the reassembled file -// wherever a uint32_t id is passed, it should be on the same js thread -// ... or at least in the same js shared memory... -// as the fountain_decode() call -int cimbard_finish_copy(uint32_t id, unsigned char* finbuffer, unsigned size); - -// get filename from reassembled file -int cimbard_get_filename(const unsigned char* finbuffer, unsigned size, char* filename, unsigned fnsize); +// get compressed filesize from id +// you probably don't need to use this. +unsigned cimbard_get_filesize(uint32_t id); + +// if fountain_decode returned a >0 value, +// get filename and (partial) contents from reassembled file +// wherever a uint32_t id is passed, it should be in the +// same js shared memory as the fountain_decode() call +// cimbard_decompress_read() will return 0 when all file contents have been read +int cimbard_get_filename(uint32_t id, char* filename, unsigned fnsize); +int cimbard_get_decompress_bufsize(); +int cimbard_decompress_read(uint32_t id, unsigned char* buffer, unsigned size); int cimbard_configure_decode(int mode_val); +// testing usage only! +unsigned char* cimbard_get_reassembled_file_buff(); + #ifdef __cplusplus } #endif diff --git a/src/lib/cimbar_js/cimbar_zstd_js.cpp b/src/lib/cimbar_js/cimbar_zstd_js.cpp deleted file mode 100644 index 6135c480..00000000 --- a/src/lib/cimbar_js/cimbar_zstd_js.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/* This code is subject to the terms of the Mozilla Public License, v.2.0. http://mozilla.org/MPL/2.0/. */ -#include "cimbar_zstd_js.h" - -#include "compression/zstd_decompressor.h" - -#include -#include - -namespace { - // maybe a map eventually - std::unique_ptr> _dec; - -} - -// zstd_decompressor constructor, reads skippable frame at start? (needs to for ofstream case) -// if we make _dec a map, this'll also take the uin32_t id we got from the decoder sink, probably -int cimbarz_init_decompress(unsigned char* buffer, unsigned size) -{ - if (_dec) - _dec.reset(); - - _dec = std::make_unique>(); - if (!_dec) - return -1; - _dec->init_decompress(reinterpret_cast(buffer), size); - return 0; -} - -int cimbarz_get_bufsize() -{ - return ZSTD_DStreamOutSize(); -} - -// "decompress_chunk()/write()" -int cimbarz_decompress_read(unsigned char* buffer, unsigned size) -{ - if (!_dec) - return -1; - if (!_dec->good()) - return -2; - - _dec->str(std::string()); - _dec->write_once(); - std::string temp = _dec->str(); - if (size > temp.size()) - size = temp.size(); - std::copy(temp.data(), temp.data()+size, buffer); - return size; -} diff --git a/src/lib/cimbar_js/cimbar_zstd_js.h b/src/lib/cimbar_js/cimbar_zstd_js.h deleted file mode 100644 index 371f038f..00000000 --- a/src/lib/cimbar_js/cimbar_zstd_js.h +++ /dev/null @@ -1,20 +0,0 @@ -/* This code is subject to the terms of the Mozilla Public License, v.2.0. http://mozilla.org/MPL/2.0/. */ -#ifndef CIMBAR_ZSTD_JS_API_H -#define CIMBAR_ZSTD_JS_API_H - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -int cimbarz_init_decompress(unsigned char* buffer, unsigned size); -int cimbarz_get_filename(char* buffer, unsigned size); -int cimbarz_get_bufsize(); -int cimbarz_decompress_read(unsigned char* buffer, unsigned size); - -#ifdef __cplusplus -} -#endif - -#endif // CIMBAR_ZSTD_JS_API_H diff --git a/src/lib/cimbar_js/test/cimbar_jsTest.cpp b/src/lib/cimbar_js/test/cimbar_jsTest.cpp index ff91c6d7..06bd5eff 100644 --- a/src/lib/cimbar_js/test/cimbar_jsTest.cpp +++ b/src/lib/cimbar_js/test/cimbar_jsTest.cpp @@ -5,7 +5,6 @@ #include "cimb_translator/Config.h" #include "cimbar_js/cimbar_js.h" #include "cimbar_js/cimbar_recv_js.h" -#include "cimbar_js/cimbar_zstd_js.h" #include "serialize/format.h" #include @@ -55,27 +54,17 @@ TEST_CASE( "cimbar_jsTest/testRoundtrip", "[unit]" ) { uint32_t fileId = res; - unsigned size = cimbard_get_filesize(fileId); - assertEquals( 5282, size ); - - std::vector data; - data.resize(size); - int res = cimbard_finish_copy(fileId, data.data(), size); - assertEquals( 0, res ); - std::string actualFilename; actualFilename.resize(255); - int fnsz = cimbard_get_filename(data.data(), size, actualFilename.data(), actualFilename.size()); + int fnsz = cimbard_get_filename(fileId, actualFilename.data(), actualFilename.size()); assertEquals( 21, fnsz ); actualFilename.resize(fnsz); assertEquals( "foobar-c语言版.txt", actualFilename ); - assertEquals(0, cimbarz_init_decompress(data.data(), data.size())); - std::vector zstdbuff; - zstdbuff.resize(cimbarz_get_bufsize()); + zstdbuff.resize(cimbard_get_decompress_bufsize()); - int outsize = cimbarz_decompress_read(zstdbuff.data(), zstdbuff.size()); + int outsize = cimbard_decompress_read(fileId, zstdbuff.data(), zstdbuff.size()); assertEquals(contents.size(), outsize); std::string_view finalOutput{reinterpret_cast(zstdbuff.data()), contents.size()}; diff --git a/src/lib/cimbar_js/test/cimbar_recv_jsTest.cpp b/src/lib/cimbar_js/test/cimbar_recv_jsTest.cpp index 5bca3f01..d0fc1ae1 100644 --- a/src/lib/cimbar_js/test/cimbar_recv_jsTest.cpp +++ b/src/lib/cimbar_js/test/cimbar_recv_jsTest.cpp @@ -83,9 +83,14 @@ TEST_CASE( "cimbar_recv_jsTest/testWirehairReassemble", "[unit]" ) assertTrue(dec > 0); assertEquals(SIZE, cimbard_get_filesize(dec)); - std::string reassembled(SIZE, '\0'); - assertEquals(0, cimbard_finish_copy(dec, reinterpret_cast(reassembled.data()), SIZE)); - + // first call to cimbard_get_filename or cimbard_decompress_read() reassembles file + std::string actualFilename; + actualFilename.resize(255); + assertEquals(0, cimbard_get_filename(dec, actualFilename.data(), actualFilename.size())); // no filename though + + unsigned char* data = cimbard_get_reassembled_file_buff(); + assertFalse( data == nullptr ); + std::string_view reassembled(reinterpret_cast(data), SIZE); assertEquals(contents, reassembled); } @@ -109,18 +114,26 @@ TEST_CASE( "cimbar_recv_jsTest/testFullDecode", "[unit]" ) { uint32_t fileId = res; - unsigned size = cimbard_get_filesize(fileId); - assertEquals( 7347, size ); + assertEquals( 7347, cimbard_get_filesize(fileId) ); + + unsigned size = cimbard_get_decompress_bufsize(); + assertEquals( 131072, size ); // defined by zstd. sanity check std::vector data; data.resize(size); - int res = cimbard_finish_copy(fileId, data.data(), size); - assertEquals( 0, res ); - cimbar::zstd_decompressor dss; - assertTrue( dss.write(reinterpret_cast(data.data()), data.size()) ); + std::stringstream ss; + int res = 1; + while (res > 0) + { + res = cimbard_decompress_read(fileId, data.data(), data.size()); + if (res == 0) + break; + assertTrue( res > 0 ); + ss.write(reinterpret_cast(data.data()), res); + } - assertEquals( 7538, dss.str().size() ); + assertEquals( 7538, ss.str().size() ); // decompressed } } diff --git a/web/main.js b/web/main.js index c76a5fa6..859d79b9 100644 --- a/web/main.js +++ b/web/main.js @@ -61,6 +61,7 @@ var Main = function () { var height = window.innerHeight - 10; Main.scaleCanvas(canvas, width, height); Main.alignInvisibleClick(canvas); + Main.checkNavButtonOverlap(); }, toggleFullscreen: function () { @@ -210,10 +211,6 @@ var Main = function () { _renderTime += elapsed; Main.setHTML("status", elapsed + " : " + frameCount + " : " + Math.ceil(_renderTime / frameCount)); } - if (!(_counter & 15)) { - Main.resize(); - Main.checkNavButtonOverlap(); - } }, setActive: function (active) { @@ -366,3 +363,7 @@ window.addEventListener("drop", function (e) { Main.dragDrop(e); document.body.style["opacity"] = 1.0; }, false); + +window.addEventListener('resize', () => { + Main.resize(); +}); \ No newline at end of file diff --git a/web/recv.js b/web/recv.js index 05cd7832..62045676 100644 --- a/web/recv.js +++ b/web/recv.js @@ -55,31 +55,22 @@ var Sink = function () { reassemble_file: function (id) { const size = Module._cimbard_get_filesize(id); //alert("we did it!?! " + size); - const dataPtr = Module._malloc(size); - const buff = new Uint8Array(Module.HEAPU8.buffer, dataPtr, size); try { - var res = Module._cimbard_finish_copy(id, buff.byteOffset, buff.length); - if (res < 0) { + var name = id + "." + size; + const fnsize = Module._cimbard_get_filename(id, _errBuff, _errBuffSize); + if (fnsize < 0) { alert("reassemble_file failed :(" + res); console.log("we biffed it. :( " + res); Recv.set_HTML("errorbox", "reassemble_file failed :( " + res); } - else { - var name = id + "." + size; - const fnsize = Module._cimbard_get_filename(buff.byteOffset, buff.length, _errBuff, _errBuffSize); - if (fnsize > 0) { - const temparr = new Uint8Array(Module.HEAPU8.buffer, _errBuff, fnsize); - name = new TextDecoder("utf-8").decode(temparr); - } - //Recv.download_bytes(buff, size + ".zst"); // size -> name, eventually - Zstd.decompress(name, buff); + else if (fnsize > 0) { + const temparr = new Uint8Array(Module.HEAPU8.buffer, _errBuff, fnsize); + name = new TextDecoder("utf-8").decode(temparr); } + Zstd.decompress(name, id); } catch (error) { console.log("failed finish copy or download?? " + error); } - // this needs to happen after decompress() completes - // currently decompress is sync, so it's fine. But... - Module._free(dataPtr); } }; }(); diff --git a/web/test/test_base.js b/web/test/test_base.js index 00b176f9..893988c3 100644 --- a/web/test/test_base.js +++ b/web/test/test_base.js @@ -1,19 +1,13 @@ QUnit.module("base"); QUnit.config.reorder = false; +QUnit.config.testTimeout = 10000; -var Zstd = function () { - - // public interface - return { - - calls: [], - - decompress: function (name, data) { - Zstd.calls.push({ decompress: [name, data.length] }); - } - }; -}(); +let _zstdCalls = []; +Zstd.download_blob = function (name, blob) { + console.log("in fake download blob for " + name); + _zstdCalls.push({ download_blob: [name, blob.size] }); +}; function wait_for(assert, block) { var done = assert.async(); @@ -56,7 +50,7 @@ QUnit.testStart(async function (details) { }); QUnit.testDone(function (details) { - + _zstdCalls = []; }); QUnit.test("stable decode", async function (assert) { @@ -90,7 +84,7 @@ QUnit.test("stable decode", async function (assert) { return document.querySelector(query); }); assert.equal(pro2.style.width, "92.3077%"); - assert.deepEqual(Zstd.calls, []); + assert.deepEqual(_zstdCalls, []); pro2.remove(); @@ -103,7 +97,11 @@ QUnit.test("stable decode", async function (assert) { }); assert.equal(pro3.style.width, "100%"); - assert.deepEqual(Zstd.calls, [ - { decompress: ["576454656.23586", 23586] } + // might be something better to wait on, but for now this is fine. + const numCalls = await wait_for(assert, () => { + return _zstdCalls.length > 0; + }); + assert.deepEqual(_zstdCalls, [ + { download_blob: ["576454656.23586", 23947] } ]); }); diff --git a/web/test_recv.html b/web/test_recv.html index 63d5ef3e..1e93fe49 100644 --- a/web/test_recv.html +++ b/web/test_recv.html @@ -23,6 +23,7 @@ +