From ea09fc90b4dfc8e3e5ac92a4666246e921d015cb Mon Sep 17 00:00:00 2001 From: Moritz Heidkamp Date: Sat, 6 Apr 2024 11:52:18 +0200 Subject: [PATCH 1/3] Revert "Merge pull request #705 from clj-commons/add-brotli-and-zstd-deps" This reverts commit 544561d2be2ff3c50dec6e4ff22fe74ef056743e, reversing changes made to b5794d437a6f0f1111f3bcc1bfd483ad2c0dc724. --- deps.edn | 16 +------ project.clj | 24 +++++------ .../aleph/http/AlephCompressionOptions.java | 42 +++++++++++++++++++ src/aleph/http/compression.clj | 37 +++++++--------- 4 files changed, 71 insertions(+), 48 deletions(-) create mode 100644 src-java/aleph/http/AlephCompressionOptions.java diff --git a/deps.edn b/deps.edn index acb5965c..0158b369 100644 --- a/deps.edn +++ b/deps.edn @@ -29,21 +29,7 @@ io.netty/netty-resolver {:mvn/version "4.1.108.Final"}, io.netty/netty-resolver-dns {:mvn/version "4.1.108.Final"}, metosin/malli - {:mvn/version "0.15.0", :exclusions [org.clojure/clojure]}, - com.aayushatharva.brotli4j/brotli4j {:mvn/version "1.16.0"}, - com.aayushatharva.brotli4j/service {:mvn/version "1.16.0"}, - com.aayushatharva.brotli4j/native-linux-aarch64 - {:mvn/version "1.16.0"}, - com.aayushatharva.brotli4j/native-linux-armv7 - {:mvn/version "1.16.0"}, - com.aayushatharva.brotli4j/native-linux-x86_64 - {:mvn/version "1.16.0"}, - com.aayushatharva.brotli4j/native-osx-aarch64 - {:mvn/version "1.16.0"}, - com.aayushatharva.brotli4j/native-osx-x86_64 {:mvn/version "1.16.0"}, - com.aayushatharva.brotli4j/native-windows-x86_64 - {:mvn/version "1.16.0"}, - com.github.luben/zstd-jni {:mvn/version "1.5.6-2"}}, + {:mvn/version "0.15.0", :exclusions [org.clojure/clojure]}}, :aliases {:lein2deps {:deps diff --git a/project.clj b/project.clj index ab228d8c..1803340f 100644 --- a/project.clj +++ b/project.clj @@ -27,17 +27,7 @@ [io.netty/netty-handler-proxy ~netty-version] [io.netty/netty-resolver ~netty-version] [io.netty/netty-resolver-dns ~netty-version] - [metosin/malli "0.15.0" :exclusions [org.clojure/clojure]] - ;;[com.aayushatharva.brotli4j/all ~brotli-version] - [com.aayushatharva.brotli4j/brotli4j ~brotli-version] - [com.aayushatharva.brotli4j/service ~brotli-version] - [com.aayushatharva.brotli4j/native-linux-aarch64 ~brotli-version] - [com.aayushatharva.brotli4j/native-linux-armv7 ~brotli-version] - [com.aayushatharva.brotli4j/native-linux-x86_64 ~brotli-version] - [com.aayushatharva.brotli4j/native-osx-aarch64 ~brotli-version] - [com.aayushatharva.brotli4j/native-osx-x86_64 ~brotli-version] - [com.aayushatharva.brotli4j/native-windows-x86_64 ~brotli-version] - [com.github.luben/zstd-jni "1.5.6-2"]] + [metosin/malli "0.15.0" :exclusions [org.clojure/clojure]]] :profiles {:dev {:dependencies [[criterium "0.4.6"] [cheshire "5.13.0"] [org.slf4j/slf4j-simple "2.0.12"] @@ -47,7 +37,17 @@ [org.bouncycastle/bcprov-jdk18on "1.77"] [org.bouncycastle/bcpkix-jdk18on "1.77"] ;;[org.bouncycastle/bctls-jdk18on "1.75"] - [io.netty/netty-tcnative-boringssl-static "2.0.65.Final"]] + [io.netty/netty-tcnative-boringssl-static "2.0.65.Final"] + ;;[com.aayushatharva.brotli4j/all ~brotli-version] + [com.aayushatharva.brotli4j/brotli4j ~brotli-version] + [com.aayushatharva.brotli4j/service ~brotli-version] + [com.aayushatharva.brotli4j/native-linux-aarch64 ~brotli-version] + [com.aayushatharva.brotli4j/native-linux-armv7 ~brotli-version] + [com.aayushatharva.brotli4j/native-linux-x86_64 ~brotli-version] + [com.aayushatharva.brotli4j/native-osx-aarch64 ~brotli-version] + [com.aayushatharva.brotli4j/native-osx-x86_64 ~brotli-version] + [com.aayushatharva.brotli4j/native-windows-x86_64 ~brotli-version] + [com.github.luben/zstd-jni "1.5.6-2"]] :jvm-opts ["-Dorg.slf4j.simpleLogger.defaultLogLevel=debug" "-Dorg.slf4j.simpleLogger.showThreadName=false" "-Dorg.slf4j.simpleLogger.showThreadId=true" diff --git a/src-java/aleph/http/AlephCompressionOptions.java b/src-java/aleph/http/AlephCompressionOptions.java new file mode 100644 index 00000000..a5f2fe5e --- /dev/null +++ b/src-java/aleph/http/AlephCompressionOptions.java @@ -0,0 +1,42 @@ +package aleph.http; + +import io.netty.handler.codec.compression.BrotliOptions; +import io.netty.handler.codec.compression.DeflateOptions; +import io.netty.handler.codec.compression.GzipOptions; +import io.netty.handler.codec.compression.SnappyOptions; +import io.netty.handler.codec.compression.StandardCompressionOptions; +import io.netty.handler.codec.compression.ZstdOptions; + +/** + * {@link AlephCompressionOptions} exists because the Clojure compiler cannot + * distinguish between static fields and static methods without reflection. + * + * This is a problem when using Netty's StandardCompressionOptions, because + * reflection triggers a load of all the methods referencing optional classes, + * which may not exist in the classpath, resulting in a ClassNotFoundException. + */ +public class AlephCompressionOptions { + private AlephCompressionOptions() { + // Prevent outside initialization + } + + public static BrotliOptions brotli() { + return StandardCompressionOptions.brotli(); + } + + public static ZstdOptions zstd() { + return StandardCompressionOptions.zstd(); + } + + public static SnappyOptions snappy() { + return StandardCompressionOptions.snappy(); + } + + public static GzipOptions gzip() { + return StandardCompressionOptions.gzip(); + } + + public static DeflateOptions deflate() { + return StandardCompressionOptions.deflate(); + } +} diff --git a/src/aleph/http/compression.clj b/src/aleph/http/compression.clj index 6a465105..0029ac83 100644 --- a/src/aleph/http/compression.clj +++ b/src/aleph/http/compression.clj @@ -1,6 +1,6 @@ (ns ^:no-doc aleph.http.compression - "Currently focused on HTTP/2, since Netty offers better support for - compression in its HTTP/1 code. + "Currently only for HTTP/2, since Netty offers better support for + compression in HTTP/1 code. Best supported compression codecs on the web are Brotli, gzip, and deflate. @@ -8,8 +8,8 @@ open-source databases. It's not on track to be a web standard, but is well-supported by Netty. - Zstd is a promising Facebook codec that is registered with IANA, but is not - yet widely available. (See https://caniuse.com/zstd). + Zstd is a Facebook codec that is registered with IANA, but is not yet + widely available. (See https://caniuse.com/zstd). See https://www.iana.org/assignments/http-parameters/http-parameters.xml#content-coding" (:require @@ -17,17 +17,15 @@ [clj-commons.primitive-math :as p] [clojure.tools.logging :as log]) (:import + (aleph.http AlephCompressionOptions) (io.netty.channel ChannelHandler) (io.netty.handler.codec.compression Brotli - BrotliOptions - CompressionOptions + BrotliOptions CompressionOptions DeflateOptions GzipOptions SnappyOptions - StandardCompressionOptions - Zstd - ZstdOptions) + Zstd ZstdOptions) (io.netty.handler.codec.http HttpHeaderNames) (io.netty.handler.codec.http2 Http2HeadersFrame) (io.netty.util AsciiString) @@ -38,10 +36,6 @@ (def ^:private ^AsciiString head-method (AsciiString. "HEAD")) (def ^:private ^AsciiString connect-method (AsciiString. "CONNECT")) -;; From 0.7.0-rc2 on, Brotli and Zstd should be available by default -(Brotli/ensureAvailability) -(Zstd/ensureAvailability) - (defn- contains-class? "Returns true if the class is in the array" [^"[Lio.netty.handler.codec.compression.CompressionOptions;" a ^Class klazz] @@ -57,11 +51,11 @@ available-compressor-options "A Java array of all available compressor options" (into-array CompressionOptions - [(StandardCompressionOptions/brotli) - (StandardCompressionOptions/deflate) - (StandardCompressionOptions/gzip) - (StandardCompressionOptions/zstd) - (StandardCompressionOptions/snappy)])) + (cond-> [(AlephCompressionOptions/deflate) + (AlephCompressionOptions/gzip) + (AlephCompressionOptions/snappy)] + (Brotli/isAvailable) (conj (AlephCompressionOptions/brotli)) + (Zstd/isAvailable) (conj (AlephCompressionOptions/zstd))))) (defn- qvalue @@ -122,10 +116,12 @@ ;; no named encodings were listed, so we'll apply *'s qval to unset ones (p/> star 0.0) - (cond (p/== br -1.0) + (cond (and (p/== br -1.0) + (Brotli/isAvailable)) "br" - (p/== zstd -1.0) + (and (p/== zstd -1.0) + (Zstd/isAvailable)) "zstd" (p/== snappy -1.0) @@ -219,7 +215,6 @@ (p/== 204 status) (p/== 304 status))) (log/debug "Setting content-encoding to:" @encoding) - ;; TODO: add "vary" header (.set headers HttpHeaderNames/CONTENT_ENCODING chosen-encoding)))))) (.write ctx msg promise)))))) From 21848f7da18018d15e1b404f13d6111f22f7d894 Mon Sep 17 00:00:00 2001 From: Moritz Heidkamp Date: Sat, 6 Apr 2024 12:00:50 +0200 Subject: [PATCH 2/3] Work around optional compressor loading issue See code comment --- src/aleph/http/compression.clj | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/aleph/http/compression.clj b/src/aleph/http/compression.clj index 0029ac83..4d08de69 100644 --- a/src/aleph/http/compression.clj +++ b/src/aleph/http/compression.clj @@ -21,7 +21,8 @@ (io.netty.channel ChannelHandler) (io.netty.handler.codec.compression Brotli - BrotliOptions CompressionOptions + BrotliOptions + CompressionOptions DeflateOptions GzipOptions SnappyOptions @@ -76,6 +77,16 @@ (defrecord Qvals [^double star ^double br ^double snappy ^double zstd ^double gzip ^double deflate]) +;; We can't reference these options classes directly in `choose-codec` since the compiler would try +;; to initialize them even when the necessary dependencies aren't available, resulting in an +;; error. Indirectly referencing them like this works, though. See +;; https://github.com/clj-commons/aleph/issues/703 +(def brotli-options-class + BrotliOptions) + +(def zstd-options-class + ZstdOptions) + (defn choose-codec "Based on Netty's algorithm, which only compares with the next-best option. E.g., Brotli's q-value is only compared with zstd, not gzip or deflate. @@ -92,12 +103,12 @@ ;; some encodings were listed (cond (and (p/not== br -1.0) (p/>= br zstd) - (contains-class? compressor-options BrotliOptions)) + (contains-class? compressor-options brotli-options-class)) "br" (and (p/not== zstd -1.0) (p/>= zstd snappy) - (contains-class? compressor-options ZstdOptions)) + (contains-class? compressor-options zstd-options-class)) "zstd" (and (p/not== snappy -1.0) From 5e469074998f62fe4df990f124faa3d913545fdf Mon Sep 17 00:00:00 2001 From: Moritz Heidkamp Date: Tue, 9 Apr 2024 11:48:39 +0200 Subject: [PATCH 3/3] Re-implement compression options check in Java The Clojure implementation needed a hacky workaround[1] which felt too fragile after all. [1] Due to https://clojure.atlassian.net/browse/CLJ-2842 --- .../aleph/http/AlephCompressionOptions.java | 32 +++++++++++++++++++ src/aleph/http/compression.clj | 31 +++--------------- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src-java/aleph/http/AlephCompressionOptions.java b/src-java/aleph/http/AlephCompressionOptions.java index a5f2fe5e..c35f921c 100644 --- a/src-java/aleph/http/AlephCompressionOptions.java +++ b/src-java/aleph/http/AlephCompressionOptions.java @@ -1,6 +1,7 @@ package aleph.http; import io.netty.handler.codec.compression.BrotliOptions; +import io.netty.handler.codec.compression.CompressionOptions; import io.netty.handler.codec.compression.DeflateOptions; import io.netty.handler.codec.compression.GzipOptions; import io.netty.handler.codec.compression.SnappyOptions; @@ -39,4 +40,35 @@ public static GzipOptions gzip() { public static DeflateOptions deflate() { return StandardCompressionOptions.deflate(); } + + private static boolean containsInstanceOf(CompressionOptions[] options, Class optionClass) { + for (CompressionOptions o : options) { + // NOTE: Can't use optionClass.instanceOf(o) because GzipOptions inherits from + // DeflateOptions which yields false positives. + if (optionClass == o.getClass()) { + return true; + } + } + return false; + } + + public static boolean hasBrotli(CompressionOptions[] options) { + return containsInstanceOf(options, BrotliOptions.class); + } + + public static boolean hasZstd(CompressionOptions[] options) { + return containsInstanceOf(options, ZstdOptions.class); + } + + public static boolean hasSnappy(CompressionOptions[] options) { + return containsInstanceOf(options, SnappyOptions.class); + } + + public static boolean hasGzip(CompressionOptions[] options) { + return containsInstanceOf(options, GzipOptions.class); + } + + public static boolean hasDeflate(CompressionOptions[] options) { + return containsInstanceOf(options, DeflateOptions.class); + } } diff --git a/src/aleph/http/compression.clj b/src/aleph/http/compression.clj index 4d08de69..5c6a1a92 100644 --- a/src/aleph/http/compression.clj +++ b/src/aleph/http/compression.clj @@ -37,17 +37,6 @@ (def ^:private ^AsciiString head-method (AsciiString. "HEAD")) (def ^:private ^AsciiString connect-method (AsciiString. "CONNECT")) -(defn- contains-class? - "Returns true if the class is in the array" - [^"[Lio.netty.handler.codec.compression.CompressionOptions;" a ^Class klazz] - (let [len (alength a)] - (loop [i 0] - (if (>= i len) - false - (if (.equals klazz (class (aget ^"[Lio.netty.handler.codec.compression.CompressionOptions;" a i))) - true - (recur (unchecked-inc-int i))))))) - (def ^"[Lio.netty.handler.codec.compression.CompressionOptions;" available-compressor-options "A Java array of all available compressor options" @@ -77,16 +66,6 @@ (defrecord Qvals [^double star ^double br ^double snappy ^double zstd ^double gzip ^double deflate]) -;; We can't reference these options classes directly in `choose-codec` since the compiler would try -;; to initialize them even when the necessary dependencies aren't available, resulting in an -;; error. Indirectly referencing them like this works, though. See -;; https://github.com/clj-commons/aleph/issues/703 -(def brotli-options-class - BrotliOptions) - -(def zstd-options-class - ZstdOptions) - (defn choose-codec "Based on Netty's algorithm, which only compares with the next-best option. E.g., Brotli's q-value is only compared with zstd, not gzip or deflate. @@ -103,26 +82,26 @@ ;; some encodings were listed (cond (and (p/not== br -1.0) (p/>= br zstd) - (contains-class? compressor-options brotli-options-class)) + (AlephCompressionOptions/hasBrotli compressor-options)) "br" (and (p/not== zstd -1.0) (p/>= zstd snappy) - (contains-class? compressor-options zstd-options-class)) + (AlephCompressionOptions/hasZstd compressor-options)) "zstd" (and (p/not== snappy -1.0) (p/>= snappy gzip) - (contains-class? compressor-options SnappyOptions)) + (AlephCompressionOptions/hasSnappy compressor-options)) "snappy" (and (p/not== gzip -1.0) (p/>= gzip deflate) - (contains-class? compressor-options GzipOptions)) + (AlephCompressionOptions/hasGzip compressor-options)) "gzip" (and (p/not== deflate -1.0) - (contains-class? compressor-options DeflateOptions)) + (AlephCompressionOptions/hasDeflate compressor-options)) "deflate") ;; no named encodings were listed, so we'll apply *'s qval to unset ones