enhancement(clickhouse sink): add support for Arrow complex types (#24409)

* refactor: split arrow encoder into smaller files

* enhancement(clickhouse sink): add support for complex types

* refactor: use idiomatic Arrow types instead of Arc

* refactor: unify primitives, decimal, temporal + complex logic

* refactor: use append_option where possible

* refactor: use iterator method for building record batch

* chore: remove duplicated code in arrow tests

* chore: update docs

* chore: add changelog fragment

* chore: add support for named tuples

* chore: improve handling for to_arrow

* refactor: use serde_arrow for Arrow encoding

* refactor: use nom for clickhouse type parsing

* refactor: replace to_arrow() with TryFrom impl

* refactor: use iterator chain instead of loops

* chore: avoid fallible unwrap when fetching schema

* refactor: implement FromStr for ClickHouseType

* refactor: use String directly

* refactor: use try_collect for schema parsing

* refactor: simplify schema parsing with TryFrom for Field

* refactor: use form_urlencoded for query string building

* refactor: return to single file format/arrow.rs

* refactor: defer to serde_arrow error handling

* chore: add recursion warning for try_from

Co-authored-by: Thomas <[email protected]>

* chore: add validation for DataType::Map

* refactor: eliminate unreachable!() in ClickHouseType conversion

* chore: clippy

* chore: update license

* Update licenses

---------

Co-authored-by: Thomas <[email protected]>
Co-authored-by: Thomas <[email protected]>

Author: benjamin-awd

Cargo.lock

@@ -852,7 +852,7 @@ sinks-azure_blob = ["dep:azure_core", "dep:azure_storage_blob"]
 sinks-azure_monitor_logs = []
 sinks-blackhole = []
 sinks-chronicle = []
-sinks-clickhouse = ["dep:rust_decimal", "codecs-arrow"]
+sinks-clickhouse = ["dep:nom", "dep:rust_decimal", "codecs-arrow"]
 sinks-console = []
 sinks-databend = ["dep:databend-client"]
 sinks-datadog_events = []

Cargo.toml

@@ -141,6 +141,7 @@ bytecheck,https://github.com/djkoloski/bytecheck,MIT,David Koloski <djkoloski@gm
 bytecheck_derive,https://github.com/djkoloski/bytecheck,MIT,David Koloski <[email protected]>
 bytecount,https://github.com/llogiq/bytecount,Apache-2.0 OR MIT,"Andre Bogus <[email protected]>, Joshua Landau <[email protected]>"
 bytemuck,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor <[email protected]>
+bytemuck_derive,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor <[email protected]>
 byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant <[email protected]>
 bytes,https://github.com/carllerche/bytes,MIT,Carl Lerche <[email protected]>
 bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche <[email protected]>, Sean McArthur <[email protected]>"
@@ -458,6 +459,7 @@ macro_magic_core,https://github.com/sam0x17/macro_magic,MIT,The macro_magic_core
 macro_magic_core_macros,https://github.com/sam0x17/macro_magic,MIT,The macro_magic_core_macros Authors
 macro_magic_macros,https://github.com/sam0x17/macro_magic,MIT,The macro_magic_macros Authors
 malloc_buf,https://github.com/SSheldon/malloc_buf,MIT,Steven Sheldon
+marrow,https://github.com/chmp/marrow,MIT,Christopher Prohm <[email protected]>
 match_cfg,https://github.com/gnzlbg/match_cfg,MIT OR Apache-2.0,gnzlbg <[email protected]>
 matchers,https://github.com/hawkw/matchers,MIT,Eliza Weisman <[email protected]>
 matchit,https://github.com/ibraheemdev/matchit,MIT AND BSD-3-Clause,Ibraheem Ahmed <[email protected]>
@@ -691,6 +693,7 @@ serde,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar <eric
 serde-aux,https://github.com/iddm/serde-aux,MIT,Victor Polevoy <[email protected]>
 serde-toml-merge,https://github.com/jdrouet/serde-toml-merge,MIT,Jeremie Drouet <[email protected]>
 serde-value,https://github.com/arcnmx/serde-value,MIT,arcnmx
+serde_arrow,https://github.com/chmp/serde_arrow,MIT,Christopher Prohm <[email protected]>
 serde_bytes,https://github.com/serde-rs/bytes,MIT OR Apache-2.0,David Tolnay <[email protected]>
 serde_core,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar <[email protected]>, David Tolnay <[email protected]>"
 serde_derive,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar <[email protected]>, David Tolnay <[email protected]>"

LICENSE-3rdparty.csv

@@ -0,0 +1,3 @@
+The `clickhouse` sink now supports complex data types (Array, Map, and Tuple) when using the `arrow_stream` format.
+
+authors: benjamin-awd

changelog.d/24074_clickhouse_arrow_complex_types.enhancement.md

@@ -14,7 +14,7 @@ path = "tests/bin/generate-avro-fixtures.rs"
 
 [dependencies]
 apache-avro = { version = "0.20.0", default-features = false }
-arrow = { version = "56.2.0", default-features = false, features = ["ipc"] }
+arrow = { version = "56.2.0", default-features = false, features = ["ipc"], optional = true }
 async-trait.workspace = true
 bytes.workspace = true
 chrono.workspace = true
@@ -38,6 +38,7 @@ regex.workspace = true
 serde.workspace = true
 serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros", "chrono_0_4"] }
 serde_json.workspace = true
+serde_arrow = { version = "0.13", features = ["arrow-56"], optional = true }
 serde-aux = { version = "4.5", optional = true }
 smallvec = { version = "1", default-features = false, features = ["union"] }
 snafu.workspace = true
@@ -67,7 +68,7 @@ uuid.workspace = true
 vrl.workspace = true
 
 [features]
-arrow = []
+arrow = ["dep:arrow", "dep:serde_arrow"]
 opentelemetry = ["dep:opentelemetry-proto"]
 syslog = ["dep:syslog_loose", "dep:strum", "dep:derive_more", "dep:serde-aux", "dep:toml"]
 test = []