fix(binary): update precision for float/uint conversions (both ways)

- add tests

Author: postspectacular

packages/binary/src/float.ts

@@ -1,4 +1,4 @@
-import type { FnN, FnN2 } from "@thi.ng/api";
+import type { FnN, FnN2, FnN3 } from "@thi.ng/api";
 
 const F64 = new Float64Array(1);
 const F32 = new Float32Array(F64.buffer);
@@ -78,43 +78,45 @@ export const floatToSortableInt: FnN = (x) => {
 	return x < 0 ? ~i | (1 << 31) : i;
 };
 
-const clamp11: FnN = (x) => (x < -1 ? -1 : x > 1 ? 1 : x);
+/** @internal */
+const __f2u: FnN3 = (x, n, p) =>
+	x < 0 ? (x < -1 ? n : x * n) : x > 1 ? p : x * p;
 
 /**
  * Converts normalized float ([-1..1] range) to u8.
  *
  * @param x -
  */
-export const f32u8: FnN = (x) => (clamp11(x) * 0x7f) & 0xff;
+export const f32u8: FnN = (x) => __f2u(x, 0x80, 0x7f) & 0xff;
 
 /**
  * Converts normalized float ([-1..1] range) to u16.
  *
  * @param x -
  */
-export const f32u16: FnN = (x) => (clamp11(x) * 0x7fff) & 0xffff;
+export const f32u16: FnN = (x) => __f2u(x, 0x8000, 0x7fff) & 0xffff;
 
 /**
  * Converts normalized float ([-1..1] range) to u24.
  *
  * @param x -
  */
-export const f32u24: FnN = (x) => (clamp11(x) * 0x7fffff) & 0xffffff;
+export const f32u24: FnN = (x) => __f2u(x, 0x80_0000, 0x7f_ffff) & 0xff_ffff;
 
 /**
  * Converts normalized float ([-1..1] range) to u32.
  *
  * @param x -
  */
-export const f32u32: FnN = (x) => (clamp11(x) * 0x7fffffff) >>> 0;
+export const f32u32: FnN = (x) => __f2u(x, 0x8000_0000, 0x7fff_ffff) >>> 0;
 
 /**
  * Reverse op of {@link f32u8}.
  *
  * @param x -
  */
 export const u8f32: FnN = (x) => (
-	(x &= 0xff), (x | ((x >> 7) * 0xffffff00)) / 0x7f
+	(x &= 0xff), (x | ((x >> 7) * 0xffff_ff00)) / (0x7f + (x >> 7))
 );
 
 /**
@@ -123,7 +125,7 @@ export const u8f32: FnN = (x) => (
  * @param x -
  */
 export const u16f32: FnN = (x) => (
-	(x &= 0xffff), (x | ((x >> 15) * 0xffff0000)) / 0x7fff
+	(x &= 0xffff), (x | ((x >> 15) * 0xffff_0000)) / (0x7fff + (x >> 15))
 );
 
 /**
@@ -132,12 +134,12 @@ export const u16f32: FnN = (x) => (
  * @param x -
  */
 export const u24f32: FnN = (x) => (
-	(x &= 0xffffff), (x | ((x >> 23) * 0xff000000)) / 0x7fffff
+	(x &= 0xffffff), (x | ((x >> 23) * 0xff00_0000)) / (0x7f_ffff + (x >> 23))
 );
 
 /**
  * Reverse op of {@link f32u32}.
  *
  * @param x -
  */
-export const u32f32: FnN = (x) => (x | 0) / 0x7fffffff;
+export const u32f32: FnN = (x) => (x | 0) / (0x7fff_ffff + (x >>> 31));

packages/binary/test/float.test.ts

@@ -0,0 +1,44 @@
+import { expect, test } from "bun:test";
+import { f32u16, f32u8, u16f32, u8f32 } from "../src/index.js";
+
+const VALS8 = [
+	[-1.2, 0x80, -1],
+	[-1, 0x80],
+	[-0.5, 0xc0],
+	[0.5, 0x3f],
+	[1, 0x7f],
+	[1.2, 0x7f, 1],
+];
+
+const VALS16 = [
+	[-1.2, 0x8000, -1],
+	[-1, 0x8000],
+	[-0.5, 0xc000],
+	[0.5, 0x3fff],
+	[1, 0x7fff],
+	[1.2, 0x7fff, 1],
+];
+
+test("f32u8", () => {
+	for (let [f, u] of VALS8) {
+		expect(f32u8(f)).toBe(u);
+	}
+});
+
+test("u8f32", () => {
+	for (let [f, u, f2] of VALS8) {
+		expect(u8f32(u)).toBeCloseTo(f2 || f, 2);
+	}
+});
+
+test("f32u16", () => {
+	for (let [f, u] of VALS16) {
+		expect(f32u16(f)).toBe(u);
+	}
+});
+
+test("u16f32", () => {
+	for (let [f, u, f2] of VALS16) {
+		expect(u16f32(u)).toBeCloseTo(f2 || f, 4);
+	}
+});