Improve encode c api

src/exe/cimbar_send/send.cpp

@@ -103,11 +103,21 @@ int main(int argc, char** argv)
 					continue;
 				}
 
-				// start encode_id is 109. This is mostly unimportant (it only needs to wrap between [0,127]), but useful
-				// for the decoder -- because it gives it a better distribution of colors in the first frame header it sees.
-				if (cimbare_encode(reinterpret_cast<unsigned char*>(contents.data()), contents.size(), filename.data(), filename.size(), static_cast<int>(i+109)) < 0)
+				if (cimbare_init_encode(filename.data(), filename.size(), -1) < 0)
 				{
-					std::cerr << "failed to encode file " << filename << std::endl;
+					std::cerr << "failed to 'init encode' file " << filename << std::endl;
+					continue; // abort??
+				}
+
+				int res = cimbare_encode(reinterpret_cast<unsigned char*>(contents.data()), contents.size());
+				if (res < 0)
+				{
+					std::cerr << "failed to compress file " << filename << std::endl;
+					continue;
+				}
+				else if (res == 1 and cimbare_encode(nullptr, 0) != 0) // fallback finish encode
+				{
+					std::cerr << "failed to encode for file" << filename << std::endl;
 					continue;
 				}
 			}

src/lib/cimbar_js/CMakeLists.txt

@@ -54,7 +54,9 @@ set (LINK_WASM_EXPORTED_FUNCTIONS
 	, "_free"
 	, "_cimbare_render"
 	, "_cimbare_next_frame"
+	, "_cimbare_init_encode"
 	, "_cimbare_encode"
+	, "_cimbare_encode_bufsize"
 	, "_cimbare_configure"
 	, "_cimbare_get_aspect_ratio"
 )

src/lib/cimbar_js/cimbar_js.cpp

@@ -2,6 +2,7 @@
 #include "cimbar_js.h"
 
 #include "cimb_translator/Config.h"
+#include "compression/zstd_compressor.h"
 #include "encoder/Encoder.h"
 #include "gui/window_glfw.h"
 #include "util/byte_istream.h"
@@ -15,7 +16,12 @@ namespace {
 	std::shared_ptr<fountain_encoder_stream> _fes;
 	std::optional<cv::Mat> _next;
 
+	// compressing the file
+	std::unique_ptr<cimbar::zstd_compressor<std::stringstream>> _comp;
+
 	int _frameCount = 0;
+	// start encode_id is 109. This is mostly unimportant (it only needs to wrap between [0,127]), but useful
+	// for the decoder -- because it gives it a better distribution of colors in the first frame header it sees.
 	uint8_t _encodeId = 109;
 
 	// settings, will be overriden by first call to configure()
@@ -108,7 +114,11 @@ int cimbare_next_frame()
 	return ++_frameCount;
 }
 
-int cimbare_encode(const unsigned char* buffer, unsigned size, const char* filename, unsigned fnsize, int encode_id)
+// maybe init_encode w/ filename,size,encode_id,
+// then encode() with buff,size? ... when size < chunksize (or size ==0), we're done
+// return 0 on done, 1 iff work to continue?
+
+int cimbare_init_encode(const char* filename, unsigned fnsize, int encode_id)
 {
 	_frameCount = 0;
 	if (!FountainInit::init())
@@ -117,17 +127,51 @@ int cimbare_encode(const unsigned char* buffer, unsigned size, const char* filen
 		return -5;
 	}
 
-	Encoder enc;
 	if (encode_id < 0)
-		enc.set_encode_id(++_encodeId); // increment _encodeId every time we change files
+		++_encodeId; // increment _encodeId every time we change files
 	else
-		enc.set_encode_id(static_cast<uint8_t>(encode_id));
+		_encodeId = encode_id;
+
+	_comp = std::make_unique<cimbar::zstd_compressor<std::stringstream>>();
+	if (!_comp)
+		return -1;
+
+	_comp->set_compression_level(_compressionLevel);
+
+	if (fnsize > 0 and filename != nullptr)
+		_comp->write_header(filename, fnsize);
+	return 0;
+}
+
+int cimbare_encode_bufsize()
+{
+	return cimbar::zstd_compressor<std::stringstream>::CHUNK_SIZE;
+}
 
-	cimbar::byte_istream bis(reinterpret_cast<const char*>(buffer), size);
-	_fes = enc.create_fountain_encoder(bis, std::string_view(filename, fnsize), _compressionLevel);
+int cimbare_encode(const unsigned char* buffer, unsigned size)
+{
+	if (!_comp)
+		return -1;
 
+	if (size > 0)
+	{
+		if (!_comp->write(reinterpret_cast<const char*>(buffer), size))
+			return -2;
+	}
+	if (size == cimbare_encode_bufsize())
+		return 1; // more to do
+
+	// otherwise, we're ready
+	unsigned fountainChunkSize = cimbar::Config::fountain_chunk_size();
+	size_t compressedSize = _comp->size();
+	if (compressedSize < fountainChunkSize)
+		_comp->pad(fountainChunkSize - compressedSize + 1);
+
+	// create the encoder stream
+	_fes = fountain_encoder_stream::create(*_comp, fountainChunkSize, _encodeId);
+	_comp.reset();
 	if (!_fes)
-		return -1; // return -1 plz
+		return -3;
 
 	_next.reset();
 	return 0;

src/lib/cimbar_js/cimbar_js.h

@@ -9,7 +9,9 @@ extern "C" {
 int cimbare_init_window(int width, int height);
 int cimbare_render();
 int cimbare_next_frame();
-int cimbare_encode(const unsigned char* buffer, unsigned size, const char* filename, unsigned fnsize, int encode_id);  // encode_id == -1 -> auto-increment
+int cimbare_init_encode(const char* filename, unsigned fnsize, int encode_id);
+int cimbare_encode_bufsize();
+int cimbare_encode(const unsigned char* buffer, unsigned size);
 int cimbare_configure(int mode_val, int compression);
 float cimbare_get_aspect_ratio();
 

src/lib/cimbar_js/test/cimbar_jsTest.cpp

@@ -33,7 +33,8 @@ TEST_CASE( "cimbar_jsTest/testRoundtrip", "[unit]" )
 	const int SIZE = 7000;
 	std::string contents = random_string(SIZE);
 	std::string filename = "/tmp/foobar-c语言版.txt";
-	assertEquals( 0, cimbare_encode(reinterpret_cast<unsigned char*>(contents.data()), contents.size(), filename.data(), filename.size(), 100) );
+	assertEquals( 0, cimbare_init_encode(filename.data(), filename.size(), 100) );
+	assertEquals( 0, cimbare_encode(reinterpret_cast<unsigned char*>(contents.data()), contents.size()) );
 
 	assertEquals( 1, cimbare_next_frame() );
 

src/lib/compression/zstd_compressor.h

@@ -6,6 +6,7 @@
 #include <fstream>
 #include <iostream>
 #include <sstream>
+#include <vector>
 
 namespace cimbar {
 
@@ -14,6 +15,7 @@ class zstd_compressor : public STREAM
 {
 public:
 	using STREAM::STREAM; // pull in constructors
+	static const size_t CHUNK_SIZE = 0x4000;
 
 public:
 	~zstd_compressor()
@@ -22,6 +24,8 @@ class zstd_compressor : public STREAM
 			ZSTD_freeCCtx(_cctx);
 	}
 
+	// if you call write directly, len should be a multiple of CHUNK_SIZE
+	// .. or the final bytes of the input
 	bool write(const char* data, size_t len)
 	{
 		size_t writeLen = CHUNK_SIZE;
@@ -102,7 +106,6 @@ class zstd_compressor : public STREAM
 	}
 
 protected:
-	static const size_t CHUNK_SIZE = 0x4000;
 	int _compressionLevel = 16;
 	ZSTD_CCtx* _cctx = ZSTD_createCCtx();
 	std::vector<char> _compBuff = std::vector<char>(ZSTD_compressBound(CHUNK_SIZE));

web/main.js

@@ -9,6 +9,7 @@ var Main = function () {
 
   // cached
   var _idealRatio = 1;
+  var _compressBuff = undefined;
 
   function toggleFullscreen() {
     if (document.fullscreenElement) {
@@ -19,17 +20,45 @@ var Main = function () {
     }
   }
 
-  function importFile(f) {
-    const fileReader = new FileReader();
-    fileReader.onload = (event) => {
-      const fileData = new Uint8Array(event.target.result);
-      Main.encode(f.name, fileData);
-    };
-    fileReader.onerror = () => {
-      console.error('Unable to read file ' + f.name + '.');
+  function importFile(file) {
+    let chunkSize = Module._cimbare_encode_bufsize();
+    if (_compressBuff === undefined) {
+      const dataPtr = Module._malloc(chunkSize);
+      _compressBuff = new Uint8Array(Module.HEAPU8.buffer, dataPtr, chunkSize);
+    }
+    let offset = 0;
+    let reader = new FileReader();
+
+    Main.encode_init(file.name);
+
+    reader.onload = function (event) {
+      const datalen = event.target.result.byteLength;
+      if (datalen > 0) {
+        // copy to wasm buff and write
+        const uint8View = new Uint8Array(event.target.result);
+        _compressBuff.set(uint8View);
+        const buffView = new Uint8Array(Module.HEAPU8.buffer, _compressBuff.byteOffset, datalen);
+        Main.encode_bytes(buffView);
+
+        offset += chunkSize;
+        readNext();
+      } else {
+        // Done reading file
+        console.log("Finished reading file.");
+
+        // this null call is functionally a flush()
+        // so a no-op, unless it isn't
+        const nullBuff = new Uint8Array(Module.HEAPU8.buffer, _compressBuff.byteOffset, 0);
+        Main.encode_bytes(nullBuff);
+      }
     };
 
-    fileReader.readAsArrayBuffer(f);
+    function readNext() {
+      let slice = file.slice(offset, offset + chunkSize);
+      reader.readAsArrayBuffer(slice);
+    }
+
+    readNext();
   }
 
   function copyToWasmHeap(abuff) {
@@ -127,22 +156,27 @@ var Main = function () {
       invisible_click.style.zoom = canvas.style.zoom;
     },
 
-    encode: function (filename, data) {
+    encode_init: function (filename) {
       console.log("encoding " + filename);
-      const wasmData = copyToWasmHeap(data);
       const wasmFn = copyToWasmHeap(new TextEncoder("utf-8").encode(filename));
-
       try {
-        var res = Module._cimbare_encode(wasmData.byteOffset, wasmData.length, wasmFn.byteOffset, wasmFn.length, -1);
-        console.log("encode returned " + res);
+        var res = Module._cimbare_init_encode(wasmFn.byteOffset, wasmFn.length, -1);
+        console.log("init_encode returned " + res);
       } finally {
-        Module._free(wasmData.byteOffset);
         Module._free(wasmFn.byteOffset);
       }
 
       Main.setTitle(filename);
       Main.setHTML("current-file", filename);
-      Main.setActive(true);
+    },
+
+    encode_bytes: function (wasmData) {
+      var res = Module._cimbare_encode(wasmData.byteOffset, wasmData.length);
+      console.log("encode returned " + res);
+
+      if (res == 0) {
+        Main.setActive(true);
+      }
     },
 
     dragDrop: function (event) {
@@ -366,4 +400,4 @@ window.addEventListener("drop", function (e) {
 
 window.addEventListener('resize', () => {
   Main.resize();
-});
+});