diff --git a/Cargo.lock b/Cargo.lock index c6294fa..3b21e6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -170,6 +170,39 @@ dependencies = [ "term", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -182,6 +215,53 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -542,6 +622,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -853,6 +948,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -884,12 +985,34 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -910,6 +1033,7 @@ checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -1005,6 +1129,31 @@ dependencies = [ "subtle", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.13.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -1107,6 +1256,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "hyper" version = "1.8.1" @@ -1117,9 +1272,11 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "pin-utils", @@ -1145,6 +1302,19 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -1162,7 +1332,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.2", "tokio", "tower-service", "tracing", @@ -1309,6 +1479,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.13.0" @@ -1523,6 +1703,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md-5" version = "0.10.6" @@ -1554,6 +1740,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c797b9d6bb23aab2fc369c65f871be49214f5c759af65bde26ffaaa2b646b492" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1624,6 +1816,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -1706,6 +1907,84 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "opentelemetry" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c365a63eec4f55b7efeceb724f1336f26a9cf3427b70e59e2cd2a5b947fba96" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror 1.0.69", +] + +[[package]] +name = "opentelemetry-http" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad31e9de44ee3538fb9d64fe3376c1362f406162434609e79aea2a41a0af78ab" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b925a602ffb916fb7421276b86756027b37ee708f9dce2dbdcc51739f07e727" +dependencies = [ + "async-trait", + "futures-core", + "http", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest", + "thiserror 1.0.69", + "tokio", + "tonic", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ee9f20bff9c984511a02f082dc8ede839e4a9bf15cc2487c8d6fea5ad850d9" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692eac490ec80f24a17828d49b40b60f5aeaccdfe6a503f939713afd22bc28df" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "percent-encoding", + "rand 0.8.5", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "p256" version = "0.13.2" @@ -1800,7 +2079,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.13.0", ] [[package]] @@ -1812,6 +2091,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -1927,6 +2226,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pyo3" version = "0.21.2" @@ -2003,7 +2325,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls", - "socket2", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tracing", @@ -2040,7 +2362,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.2", "tracing", "windows-sys 0.60.2", ] @@ -2066,6 +2388,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ + "libc", "rand_chacha 0.3.1", "rand_core 0.6.4", ] @@ -2186,7 +2509,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tower", + "tower 0.5.3", "tower-http", "tower-service", "url", @@ -2457,7 +2780,7 @@ dependencies = [ "sha1collisiondetection", "sha2", "sha3", - "thiserror 1.0.69", + "thiserror 2.0.18", "twofish", "typenum", "x25519-dalek", @@ -2583,6 +2906,8 @@ dependencies = [ "pyo3-build-config", "shadi_memory", "shadi_sandbox", + "shadi_telemetry", + "tracing", ] [[package]] @@ -2591,9 +2916,23 @@ version = "0.1.0" dependencies = [ "libc", "thiserror 1.0.69", + "tracing", "windows-sys 0.61.2", ] +[[package]] +name = "shadi_telemetry" +version = "0.1.0" +dependencies = [ + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", + "tracing", + "tracing-appender", + "tracing-opentelemetry", + "tracing-subscriber", +] + [[package]] name = "shadictl" version = "0.1.0" @@ -2611,7 +2950,18 @@ dependencies = [ "sha2", "shadi_memory", "shadi_sandbox", + "shadi_telemetry", "tempfile", + "tracing", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", ] [[package]] @@ -2665,6 +3015,16 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.2" @@ -2820,6 +3180,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "time" version = "0.3.47" @@ -2886,10 +3255,22 @@ dependencies = [ "libc", "mio", "pin-project-lite", - "socket2", + "socket2 0.6.2", + "tokio-macros", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -2900,6 +3281,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.8.23" @@ -2927,7 +3332,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap", + "indexmap 2.13.0", "serde", "serde_spanned", "toml_datetime", @@ -2941,6 +3346,56 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.3" @@ -2969,7 +3424,7 @@ dependencies = [ "http-body", "iri-string", "pin-project-lite", - "tower", + "tower 0.5.3", "tower-layer", "tower-service", ] @@ -2993,9 +3448,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +dependencies = [ + "crossbeam-channel", + "thiserror 2.0.18", + "time", + "tracing-subscriber", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -3003,6 +3482,63 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9784ed4da7d921bc8df6963f8c80a0e4ce34ba6ba76668acadd3edbd985ff3b" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +dependencies = [ + "nu-ansi-term", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -3084,6 +3620,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -3215,7 +3757,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap", + "indexmap 2.13.0", "wasm-encoder", "wasmparser", ] @@ -3228,7 +3770,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.13.0", "semver", ] @@ -3522,7 +4064,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck 0.5.0", - "indexmap", + "indexmap 2.13.0", "prettyplease", "syn", "wasm-metadata", @@ -3553,7 +4095,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags", - "indexmap", + "indexmap 2.13.0", "log", "serde", "serde_derive", @@ -3572,7 +4114,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap", + "indexmap 2.13.0", "log", "semver", "serde", diff --git a/Cargo.toml b/Cargo.toml index b5b0efb..f1ac806 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/shadi_memory", "crates/shadi_py", "crates/shadi_sandbox", + "crates/shadi_telemetry", "crates/shadictl" ] resolver = "2" diff --git a/agents/secops/telemetry.py b/agents/secops/telemetry.py index 12c8970..723d47a 100644 --- a/agents/secops/telemetry.py +++ b/agents/secops/telemetry.py @@ -27,6 +27,7 @@ _resource = Resource( attributes={ RES_SERVICE_NAME: SERVICE_NAME, + "service.namespace": "shadi", "telemetry.sdk.language": "python", } ) diff --git a/crates/shadi_py/Cargo.toml b/crates/shadi_py/Cargo.toml index a33f4cc..03f1168 100644 --- a/crates/shadi_py/Cargo.toml +++ b/crates/shadi_py/Cargo.toml @@ -13,6 +13,8 @@ doctest = false agent_secrets = { path = "../agent_secrets", features = ["onepassword"] } shadi_memory = { path = "../shadi_memory" } shadi_sandbox = { path = "../shadi_sandbox" } +shadi_telemetry = { path = "../shadi_telemetry" } +tracing = "0.1" [dependencies.pyo3] version = "0.21" diff --git a/crates/shadi_py/src/lib.rs b/crates/shadi_py/src/lib.rs index c75a010..bfc39ab 100644 --- a/crates/shadi_py/src/lib.rs +++ b/crates/shadi_py/src/lib.rs @@ -14,6 +14,7 @@ use pyo3::prelude::*; use pyo3::types::{PyBytes, PyModule}; use shadi_memory::{MemoryEntry as ShadiMemoryEntry, SqlCipherStore}; use shadi_sandbox::{spawn_sandboxed, SandboxError, SandboxPolicy}; +use tracing::{field, info_span}; struct SessionFlagVerifier; @@ -127,6 +128,8 @@ impl ShadiStore { } fn put(&self, session: &PySessionContext, key: &str, secret: &[u8]) -> PyResult<()> { + let span = info_span!("shadi.secret.put", secret.key = %key); + let _guard = span.enter(); let ctx = session.to_context(); let guard = self.store.lock().map_err(|_| PyRuntimeError::new_err("lock poisoned"))?; let access = AgentSecretAccess::new(guard.as_ref(), &self.verifier); @@ -141,6 +144,8 @@ impl ShadiStore { session: &PySessionContext, key: &str, ) -> PyResult> { + let span = info_span!("shadi.secret.get", secret.key = %key); + let _guard = span.enter(); let ctx = session.to_context(); let guard = self.store.lock().map_err(|_| PyRuntimeError::new_err("lock poisoned"))?; let access = AgentSecretAccess::new(guard.as_ref(), &self.verifier); @@ -150,6 +155,8 @@ impl ShadiStore { } fn delete(&self, session: &PySessionContext, key: &str) -> PyResult<()> { + let span = info_span!("shadi.secret.delete", secret.key = %key); + let _guard = span.enter(); let ctx = session.to_context(); let guard = self.store.lock().map_err(|_| PyRuntimeError::new_err("lock poisoned"))?; let access = AgentSecretAccess::new(guard.as_ref(), &self.verifier); @@ -159,6 +166,8 @@ impl ShadiStore { } fn list_keys(&self, session: &PySessionContext) -> PyResult> { + let span = info_span!("shadi.secret.list_keys"); + let _guard = span.enter(); let ctx = session.to_context(); AgentSecretAccess::require_verified(&ctx).map_err(map_secret_error)?; let guard = self.store.lock().map_err(|_| PyRuntimeError::new_err("lock poisoned"))?; @@ -178,12 +187,16 @@ impl SqlCipherMemoryStore { } fn put(&self, scope: &str, entry_key: &str, payload: &str) -> PyResult { + let span = info_span!("shadi.memory.put", memory.scope = %scope, memory.entry_key = %entry_key); + let _guard = span.enter(); self.store .put(scope, entry_key, payload) .map_err(|err| PyRuntimeError::new_err(err.to_string())) } fn get_latest(&self, scope: &str, entry_key: &str) -> PyResult> { + let span = info_span!("shadi.memory.get_latest", memory.scope = %scope, memory.entry_key = %entry_key); + let _guard = span.enter(); let entry = self .store .get_latest(scope, entry_key) @@ -193,6 +206,13 @@ impl SqlCipherMemoryStore { #[pyo3(signature = (query, scope=None, limit=10))] fn search(&self, query: &str, scope: Option, limit: usize) -> PyResult> { + let span = info_span!( + "shadi.memory.search", + memory.query = %query, + memory.scope = %scope.as_deref().unwrap_or(""), + memory.limit = limit as i64, + ); + let _guard = span.enter(); let entries = self .store .search(scope.as_deref(), query, limit) @@ -205,6 +225,12 @@ impl SqlCipherMemoryStore { #[pyo3(signature = (scope=None, limit=50))] fn list(&self, scope: Option, limit: usize) -> PyResult> { + let span = info_span!( + "shadi.memory.list", + memory.scope = %scope.as_deref().unwrap_or(""), + memory.limit = limit as i64, + ); + let _guard = span.enter(); let entries = self .store .list(scope.as_deref(), limit) @@ -216,6 +242,8 @@ impl SqlCipherMemoryStore { } fn delete(&self, scope: &str, entry_key: &str) -> PyResult { + let span = info_span!("shadi.memory.delete", memory.scope = %scope, memory.entry_key = %entry_key); + let _guard = span.enter(); self.store .delete(scope, entry_key) .map_err(|err| PyRuntimeError::new_err(err.to_string())) @@ -286,6 +314,7 @@ impl PySessionContext { #[pymodule] fn shadi(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + shadi_telemetry::init("shadi-runtime"); m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -325,6 +354,8 @@ fn inject_keychain_with_store( command: &mut Command, mappings: &[String], ) -> Result<(), String> { + let span = info_span!("shadi.secrets.inject", secret.count = mappings.len() as i64); + let _guard = span.enter(); for mapping in mappings { let (key, env) = parse_key_env(mapping)?; let secret = store @@ -361,6 +392,15 @@ fn run_sandboxed( return Err(PyRuntimeError::new_err("command must not be empty")); } + let cwd_value = cwd.as_deref().unwrap_or(""); + let span = info_span!( + "shadi.sandbox.run", + command = %command[0], + cwd = %cwd_value, + exit.code = field::Empty, + ); + let _guard = span.enter(); + let mut cmd = Command::new(&command[0]); if command.len() > 1 { cmd.args(&command[1..]); @@ -381,6 +421,7 @@ fn run_sandboxed( let status = child .wait() .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + span.record("exit.code", &status.code().unwrap_or(-1)); Ok(status.code().unwrap_or(1)) } diff --git a/crates/shadi_sandbox/Cargo.toml b/crates/shadi_sandbox/Cargo.toml index d8b4a1d..dce2e79 100644 --- a/crates/shadi_sandbox/Cargo.toml +++ b/crates/shadi_sandbox/Cargo.toml @@ -9,6 +9,7 @@ coverage = [] [dependencies] thiserror = "1.0" +tracing = "0.1" [target.'cfg(target_os = "macos")'.dependencies] libc = "0.2" diff --git a/crates/shadi_sandbox/src/lib.rs b/crates/shadi_sandbox/src/lib.rs index e55f614..1048806 100644 --- a/crates/shadi_sandbox/src/lib.rs +++ b/crates/shadi_sandbox/src/lib.rs @@ -7,8 +7,32 @@ mod platform; pub use policy::SandboxPolicy; use std::process::{Command, ExitStatus}; use std::io; +use tracing::{field, info_span}; pub fn spawn_sandboxed(command: &mut Command, policy: &SandboxPolicy) -> Result { + let program = command.get_program().to_string_lossy().to_string(); + let args = command + .get_args() + .map(|arg| arg.to_string_lossy()) + .collect::>() + .join(" "); + let cwd = command + .get_current_dir() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| "".to_string()); + let allowed_paths = policy.allow_read().len() + policy.allow_write().len(); + let network_mode = if policy.net_blocked() { "blocked" } else { "allowed" }; + + let span = info_span!( + "shadi.sandbox.spawn", + command = %program, + args = %args, + cwd = %cwd, + policy.allowed_paths = allowed_paths as i64, + network.mode = %network_mode, + ); + let _guard = span.enter(); + platform::spawn_sandboxed(command, policy) } @@ -37,14 +61,26 @@ impl SandboxedChild { } pub fn wait(&mut self) -> io::Result { - match &mut self.inner { + let span = info_span!("shadi.sandbox.wait", pid = self.id(), exit.code = field::Empty); + let _guard = span.enter(); + + let status = match &mut self.inner { SandboxedChildInner::Std(child) => child.wait(), #[cfg(target_os = "windows")] SandboxedChildInner::Windows(child) => child.wait(), + }; + + if let Ok(ref status) = status { + span.record("exit.code", &status.code().unwrap_or(-1)); } + + status } pub fn kill(&mut self) -> io::Result<()> { + let span = info_span!("shadi.sandbox.kill", pid = self.id()); + let _guard = span.enter(); + match &mut self.inner { SandboxedChildInner::Std(child) => child.kill(), #[cfg(target_os = "windows")] diff --git a/crates/shadi_telemetry/Cargo.toml b/crates/shadi_telemetry/Cargo.toml new file mode 100644 index 0000000..c880af1 --- /dev/null +++ b/crates/shadi_telemetry/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "shadi_telemetry" +version = "0.1.0" +edition = "2021" + +[dependencies] +opentelemetry = "0.24" +opentelemetry_sdk = "0.24" +opentelemetry-otlp = { version = "0.17", features = ["http-proto", "reqwest-client"] } +tracing = "0.1" +tracing-opentelemetry = "0.25" +tracing-subscriber = { version = "0.3", features = ["json"] } +tracing-appender = "0.2" diff --git a/crates/shadi_telemetry/src/lib.rs b/crates/shadi_telemetry/src/lib.rs new file mode 100644 index 0000000..824275c --- /dev/null +++ b/crates/shadi_telemetry/src/lib.rs @@ -0,0 +1,221 @@ +// Copyright AGNTCY Contributors (https://github.com/agntcy) +// SPDX-License-Identifier: Apache-2.0 + +use std::env; +use std::path::{Path, PathBuf}; +use std::sync::{Once, OnceLock}; + +use opentelemetry::KeyValue; +use opentelemetry::trace::TracerProvider; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::{trace, Resource}; +use tracing_subscriber::layer::SubscriberExt; + +static INIT: Once = Once::new(); +static PROVIDER: OnceLock = OnceLock::new(); +static FILE_GUARD: OnceLock = OnceLock::new(); + +pub fn init(service_name: &str) { + INIT.call_once(|| { + let config = load_config(service_name); + + if !telemetry_enabled( + &config.otlp_endpoint, + config.console_enabled, + config.file_path.as_deref(), + ) { + return; + } + + let resource = Resource::new(vec![ + KeyValue::new("service.name", config.service_name), + KeyValue::new("service.namespace", "shadi"), + KeyValue::new("service.version", env!("CARGO_PKG_VERSION")), + KeyValue::new("telemetry.sdk.language", "rust"), + ]); + + let otel_layer = if !config.otlp_endpoint.is_empty() { + let exporter = opentelemetry_otlp::new_exporter() + .http() + .with_endpoint(config.otlp_endpoint); + let provider = opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter(exporter) + .with_trace_config(trace::Config::default().with_resource(resource)) + .install_simple(); + + provider.ok().map(|provider| { + let _ = PROVIDER.set(provider); + let tracer = PROVIDER + .get() + .expect("telemetry provider") + .tracer("shadi.telemetry"); + tracing_opentelemetry::layer().with_tracer(tracer) + }) + } else { + None + }; + + let file_layer = config.file_path.as_ref().and_then(|path| { + let (dir, file_name) = resolve_trace_path(path)?; + if std::fs::create_dir_all(&dir).is_err() { + return None; + } + + let appender = tracing_appender::rolling::never(dir, file_name); + let (non_blocking, guard) = tracing_appender::non_blocking(appender); + let _ = FILE_GUARD.set(guard); + + Some( + tracing_subscriber::fmt::layer() + .json() + .with_current_span(true) + .with_span_list(true) + .with_ansi(false) + .with_writer(non_blocking), + ) + }); + + let fmt_layer = config.console_enabled.then(|| tracing_subscriber::fmt::layer()); + + let subscriber = tracing_subscriber::registry() + .with(otel_layer) + .with(fmt_layer) + .with(file_layer); + + let _ = tracing::subscriber::set_global_default(subscriber); + }); +} + +fn parse_bool_env(key: &str) -> bool { + let value = env::var(key).unwrap_or_default().trim().to_ascii_lowercase(); + matches!(value.as_str(), "1" | "true" | "yes") +} + +fn normalize_file_path(value: &str) -> Option { + let trimmed = value.trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + } +} + +fn resolve_trace_path(path: &str) -> Option<(PathBuf, String)> { + let trace_path = Path::new(path); + let dir = trace_path.parent().unwrap_or_else(|| Path::new(".")).to_path_buf(); + let file_name = trace_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("traces.jsonl") + .to_string(); + Some((dir, file_name)) +} + +#[derive(Debug, Clone)] +struct TelemetryConfig { + otlp_endpoint: String, + console_enabled: bool, + file_path: Option, + service_name: String, +} + +fn load_config(default_service_name: &str) -> TelemetryConfig { + let otlp_endpoint = env::var("OTEL_EXPORTER_OTLP_ENDPOINT").unwrap_or_default(); + let console_enabled = parse_bool_env("SHADI_OTEL_CONSOLE"); + let file_path = env::var("SHADI_OTEL_FILE") + .ok() + .and_then(|value| normalize_file_path(&value)); + let service_name = resolve_service_name(default_service_name); + + TelemetryConfig { + otlp_endpoint, + console_enabled, + file_path, + service_name, + } +} + +fn resolve_service_name(default_service_name: &str) -> String { + env::var("OTEL_SERVICE_NAME") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| default_service_name.to_string()) +} + +fn telemetry_enabled(otlp_endpoint: &str, console_enabled: bool, file_path: Option<&str>) -> bool { + !otlp_endpoint.is_empty() || console_enabled || file_path.is_some() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + static ENV_LOCK: Mutex<()> = Mutex::new(()); + + #[test] + fn parse_bool_env_accepts_truthy_values() { + let _guard = ENV_LOCK.lock().unwrap(); + std::env::set_var("SHADI_OTEL_CONSOLE", "1"); + assert!(parse_bool_env("SHADI_OTEL_CONSOLE")); + std::env::set_var("SHADI_OTEL_CONSOLE", "true"); + assert!(parse_bool_env("SHADI_OTEL_CONSOLE")); + std::env::set_var("SHADI_OTEL_CONSOLE", "yes"); + assert!(parse_bool_env("SHADI_OTEL_CONSOLE")); + std::env::set_var("SHADI_OTEL_CONSOLE", "no"); + assert!(!parse_bool_env("SHADI_OTEL_CONSOLE")); + std::env::remove_var("SHADI_OTEL_CONSOLE"); + } + + #[test] + fn normalize_file_path_trims_and_rejects_empty() { + assert_eq!(normalize_file_path(""), None); + assert_eq!(normalize_file_path(" "), None); + assert_eq!(normalize_file_path("/tmp/trace.jsonl"), Some("/tmp/trace.jsonl".to_string())); + assert_eq!(normalize_file_path(" ./traces.jsonl "), Some("./traces.jsonl".to_string())); + } + + #[test] + fn resolve_trace_path_builds_dir_and_name() { + let (dir, file) = resolve_trace_path("/tmp/trace.jsonl").expect("path"); + assert_eq!(dir, PathBuf::from("/tmp")); + assert_eq!(file, "trace.jsonl"); + } + + #[test] + fn load_config_reads_env_vars() { + let _guard = ENV_LOCK.lock().unwrap(); + std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318"); + std::env::set_var("SHADI_OTEL_CONSOLE", "true"); + std::env::set_var("SHADI_OTEL_FILE", "./traces.jsonl"); + std::env::set_var("OTEL_SERVICE_NAME", "shadi-test"); + + let config = load_config("default"); + assert_eq!(config.otlp_endpoint, "http://localhost:4318"); + assert!(config.console_enabled); + assert_eq!(config.file_path, Some("./traces.jsonl".to_string())); + assert_eq!(config.service_name, "shadi-test"); + + std::env::remove_var("OTEL_EXPORTER_OTLP_ENDPOINT"); + std::env::remove_var("SHADI_OTEL_CONSOLE"); + std::env::remove_var("SHADI_OTEL_FILE"); + std::env::remove_var("OTEL_SERVICE_NAME"); + } + + #[test] + fn resolve_service_name_defaults() { + let _guard = ENV_LOCK.lock().unwrap(); + std::env::remove_var("OTEL_SERVICE_NAME"); + let name = resolve_service_name("default-service"); + assert_eq!(name, "default-service"); + } + + #[test] + fn telemetry_enabled_requires_any_sink() { + assert!(!telemetry_enabled("", false, None)); + assert!(telemetry_enabled("http://localhost:4318", false, None)); + assert!(telemetry_enabled("", true, None)); + assert!(telemetry_enabled("", false, Some("/tmp/trace.jsonl"))); + } +} diff --git a/crates/shadictl/Cargo.toml b/crates/shadictl/Cargo.toml index 571cedd..b846d44 100644 --- a/crates/shadictl/Cargo.toml +++ b/crates/shadictl/Cargo.toml @@ -8,6 +8,7 @@ clap = { version = "4.5", features = ["derive"] } shadi_sandbox = { path = "../shadi_sandbox" } agent_secrets = { path = "../agent_secrets", features = ["onepassword"] } shadi_memory = { path = "../shadi_memory" } +shadi_telemetry = { path = "../shadi_telemetry" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" bs58 = "0.5" @@ -16,6 +17,7 @@ ed25519-dalek = { version = "2.1", features = ["rand_core"] } hkdf = "0.12" sha2 = "0.10" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } +tracing = "0.1" [target.'cfg(windows)'.dependencies] sequoia-openpgp = { version = "2.2", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto", "compression-deflate", "compression-bzip2"] } diff --git a/crates/shadictl/src/main.rs b/crates/shadictl/src/main.rs index b8716d2..f11aff3 100644 --- a/crates/shadictl/src/main.rs +++ b/crates/shadictl/src/main.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeSet, HashSet}; use std::path::{Path, PathBuf}; use std::process::{Command, ExitCode}; +use std::io::BufRead; #[cfg(test)] use std::collections::HashMap; @@ -25,6 +26,7 @@ use shadi_sandbox::{spawn_sandboxed, SandboxPolicy}; use agent_secrets::{SecretPolicy, SecretStore}; use shadi_memory::{MemoryEntry, SqlCipherStore}; use sequoia_openpgp as openpgp; +use tracing::{field, info_span}; #[derive(Parser, Debug)] #[command(name = "shadi")] @@ -99,6 +101,34 @@ struct MemoryCli { command: MemoryCommand, } +#[derive(Parser, Debug)] +#[command(name = "trace", about = "Inspect local SHADI trace logs")] +struct TraceCli { + #[arg(long, value_name = "PATH")] + file: Option, + + #[command(subcommand)] + command: TraceCommand, +} + +#[derive(Subcommand, Debug)] +enum TraceCommand { + List { + #[arg(long, default_value = "50")] + limit: usize, + #[arg(long)] + name: Option, + #[arg(long)] + command: Option, + #[arg(long)] + exit_code: Option, + }, + Summary { + #[arg(long, default_value = "200")] + limit: usize, + }, +} + #[derive(Subcommand, Debug)] enum MemoryCommand { Init, @@ -404,6 +434,7 @@ fn test_store_get(key: &str) -> Option> { fn main() -> ExitCode { + shadi_telemetry::init("shadi-core"); let cli = Cli::parse(); run_cli(cli) } @@ -412,6 +443,9 @@ fn run_cli(cli: Cli) -> ExitCode { if matches!(cli.command.first().map(|cmd| cmd.as_str()), Some("memory")) { return run_memory_command(&cli.command[1..]); } + if matches!(cli.command.first().map(|cmd| cmd.as_str()), Some("trace")) { + return run_trace_command(&cli.command[1..]); + } if matches!(cli.command.first().map(|cmd| cmd.as_str()), Some("did-from-gpg")) { return run_did_from_gpg_command(&cli.command); } @@ -529,8 +563,189 @@ fn run_cli(cli: Cli) -> ExitCode { run_sandboxed_command(&cli, &resolved, &cwd) } +fn run_trace_command(args: &[String]) -> ExitCode { + let mut argv = Vec::with_capacity(args.len() + 1); + argv.push("shadictl-trace".to_string()); + argv.extend_from_slice(args); + let cli = match TraceCli::try_parse_from(argv) { + Ok(cli) => cli, + Err(err) => { + eprintln!("{}", err); + return ExitCode::from(2); + } + }; + + let path = resolve_trace_file(cli.file); + match &cli.command { + TraceCommand::List { + limit, + name, + command, + exit_code, + } => match trace_list(&path, *limit, name.as_deref(), command.as_deref(), *exit_code) { + Ok(()) => ExitCode::SUCCESS, + Err(err) => { + eprintln!("{}", err); + ExitCode::from(1) + } + }, + TraceCommand::Summary { limit } => match trace_summary(&path, *limit) { + Ok(()) => ExitCode::SUCCESS, + Err(err) => { + eprintln!("{}", err); + ExitCode::from(1) + } + }, + } +} + +fn resolve_trace_file(cli_path: Option) -> PathBuf { + if let Some(path) = cli_path { + return path; + } + if let Ok(path) = std::env::var("SHADI_OTEL_FILE") { + if !path.trim().is_empty() { + return PathBuf::from(path); + } + } + PathBuf::from(".shadi/traces.jsonl") +} + +fn trace_list( + path: &Path, + limit: usize, + name: Option<&str>, + command: Option<&str>, + exit_code: Option, +) -> Result<(), String> { + let lines = read_trace_lines(path, limit)?; + for line in lines { + if let Some(value) = parse_trace_line(&line) { + if !trace_matches(&value, name, command, exit_code) { + continue; + } + } + println!("{}", line); + } + Ok(()) +} + +fn trace_summary(path: &Path, limit: usize) -> Result<(), String> { + let lines = read_trace_lines(path, limit)?; + let mut counts: std::collections::BTreeMap = std::collections::BTreeMap::new(); + for line in lines { + if let Some(value) = parse_trace_line(&line) { + if let Some(name) = trace_span_name(&value) { + *counts.entry(name).or_insert(0) += 1; + } + } + } + + for (name, count) in counts { + println!("{}\t{}", count, name); + } + Ok(()) +} + +fn read_trace_lines(path: &Path, limit: usize) -> Result, String> { + let file = std::fs::File::open(path) + .map_err(|err| format!("failed to open trace file {}: {}", path.display(), err))?; + let reader = std::io::BufReader::new(file); + let mut lines: std::collections::VecDeque = std::collections::VecDeque::new(); + for line in reader.lines() { + let line = line.map_err(|err| format!("failed to read trace file: {}", err))?; + if limit == 0 { + continue; + } + lines.push_back(line); + if lines.len() > limit { + lines.pop_front(); + } + } + Ok(lines.into_iter().collect()) +} + +fn parse_trace_line(line: &str) -> Option { + serde_json::from_str(line).ok() +} + +fn trace_span_name(value: &serde_json::Value) -> Option { + if let Some(name) = value + .get("span") + .and_then(|span| span.get("name")) + .and_then(|name| name.as_str()) + { + return Some(name.to_string()); + } + + if let Some(spans) = value.get("spans").and_then(|spans| spans.as_array()) { + if let Some(name) = spans + .iter() + .filter_map(|span| span.get("name")) + .filter_map(|name| name.as_str()) + .next() + { + return Some(name.to_string()); + } + } + + None +} + +fn trace_matches( + value: &serde_json::Value, + name: Option<&str>, + command: Option<&str>, + exit_code: Option, +) -> bool { + if let Some(expected) = name { + if trace_span_name(value) + .as_deref() + .map(|value| !value.contains(expected)) + .unwrap_or(true) + { + return false; + } + } + + if let Some(expected) = command { + let found = value + .get("fields") + .and_then(|fields| fields.get("command")) + .and_then(|value| value.as_str()) + .map(|value| value.contains(expected)) + .unwrap_or(false); + if !found { + return false; + } + } + + if let Some(expected) = exit_code { + let found = value + .get("fields") + .and_then(|fields| fields.get("exit.code")) + .and_then(|value| value.as_i64()) + .map(|value| value == expected as i64) + .unwrap_or(false); + if !found { + return false; + } + } + + true +} + fn run_sandboxed_command(cli: &Cli, resolved: &ResolvedPolicy, cwd: &Path) -> ExitCode { let cmd_name = cli.command.first().map(|cmd| cmd.as_str()).unwrap_or(""); + let policy_source = cli + .policy_file + .as_ref() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| "default".to_string()); + let mut allowed_paths = BTreeSet::new(); + allowed_paths.extend(resolved.policy.allow_read().iter().cloned()); + allowed_paths.extend(resolved.policy.allow_write().iter().cloned()); + let network_mode = if resolved.policy.net_blocked() { "blocked" } else { "allowed" }; let mut command = Command::new(cmd_name); if cli.command.len() > 1 { @@ -544,21 +759,56 @@ fn run_sandboxed_command(cli: &Cli, resolved: &ResolvedPolicy, cwd: &Path) -> Ex } let mut snapshot = GitSnapshotSession::start(cli, resolved, cwd); + let snapshot_enabled = snapshot.is_some(); + + let span = info_span!( + "shadi.sandbox.run", + command = %cmd_name, + cwd = %cwd.display(), + policy.source = %policy_source, + policy.allowed_paths = allowed_paths.len() as i64, + network.mode = %network_mode, + snapshot.enabled = snapshot_enabled, + exit.code = field::Empty, + snapshot.path = field::Empty, + ); + let _guard = span.enter(); match spawn_sandboxed(&mut command, &resolved.policy) { Ok(mut child) => match child.wait() { Ok(status) => { - finalize_git_snapshot(snapshot.as_mut(), status.code(), None); + let exit_code = status.code().unwrap_or(1); + span.record("exit.code", &exit_code); + let snapshot_path = finalize_git_snapshot(snapshot.as_mut(), status.code(), None); + if let Some(path) = snapshot_path { + span.record("snapshot.path", &path.display().to_string()); + } ExitCode::from(status.code().unwrap_or(1) as u8) } Err(err) => { - finalize_git_snapshot(snapshot.as_mut(), None, Some(format!("failed to wait for child: {}", err))); + span.record("exit.code", &-1); + let snapshot_path = finalize_git_snapshot( + snapshot.as_mut(), + None, + Some(format!("failed to wait for child: {}", err)), + ); + if let Some(path) = snapshot_path { + span.record("snapshot.path", &path.display().to_string()); + } eprintln!("failed to wait for child: {}", err); ExitCode::from(1) } }, Err(err) => { - finalize_git_snapshot(snapshot.as_mut(), None, Some(format!("failed to start sandboxed command: {}", err))); + span.record("exit.code", &-1); + let snapshot_path = finalize_git_snapshot( + snapshot.as_mut(), + None, + Some(format!("failed to start sandboxed command: {}", err)), + ); + if let Some(path) = snapshot_path { + span.record("snapshot.path", &path.display().to_string()); + } eprintln!("failed to start sandboxed command: {}", err); ExitCode::from(1) } @@ -569,11 +819,17 @@ fn finalize_git_snapshot( snapshot: Option<&mut GitSnapshotSession>, exit_code: Option, error: Option, -) { +) -> Option { if let Some(snapshot) = snapshot { - if let Err(err) = snapshot.finish(exit_code, error) { - eprintln!("warning: failed to write git snapshot artifact: {}", err); + match snapshot.finish(exit_code, error) { + Ok(path) => Some(path), + Err(err) => { + eprintln!("warning: failed to write git snapshot artifact: {}", err); + None + } } + } else { + None } } @@ -1303,14 +1559,30 @@ fn run_memory_command(args: &[String]) -> ExitCode { } fn handle_memory_command(cli: &MemoryCli, store: &SqlCipherStore) -> Result { + let span = info_span!( + "shadi.memory.command", + memory.command = field::Empty, + memory.scope = field::Empty, + memory.entry_key = field::Empty, + memory.limit = field::Empty, + memory.query = field::Empty, + ); + let _guard = span.enter(); + match &cli.command { - MemoryCommand::Init => Ok("ok".to_string()), + MemoryCommand::Init => { + span.record("memory.command", &"init"); + Ok("ok".to_string()) + } MemoryCommand::Put { scope, entry_key, payload, payload_file, } => { + span.record("memory.command", &"put"); + span.record("memory.scope", &field::display(scope)); + span.record("memory.entry_key", &field::display(entry_key)); let payload = read_memory_payload(payload.clone(), payload_file.clone())?; let id = store .put(scope, entry_key, &payload) @@ -1318,6 +1590,9 @@ fn handle_memory_command(cli: &MemoryCli, store: &SqlCipherStore) -> Result { + span.record("memory.command", &"get"); + span.record("memory.scope", &field::display(scope)); + span.record("memory.entry_key", &field::display(entry_key)); let entry = store .get_latest(scope, entry_key) .map_err(|err| err.to_string())?; @@ -1331,18 +1606,32 @@ fn handle_memory_command(cli: &MemoryCli, store: &SqlCipherStore) -> Result { + span.record("memory.command", &"search"); + if let Some(scope) = scope.as_ref() { + span.record("memory.scope", &field::display(scope)); + } + span.record("memory.query", &field::display(query)); + span.record("memory.limit", &(*limit as i64)); let entries = store .search(scope.as_deref(), query, *limit) .map_err(|err| err.to_string())?; format_memory_entries(entries) } MemoryCommand::List { scope, limit } => { + span.record("memory.command", &"list"); + if let Some(scope) = scope.as_ref() { + span.record("memory.scope", &field::display(scope)); + } + span.record("memory.limit", &(*limit as i64)); let entries = store .list(scope.as_deref(), *limit) .map_err(|err| err.to_string())?; format_memory_entries(entries) } MemoryCommand::Delete { scope, entry_key } => { + span.record("memory.command", &"delete"); + span.record("memory.scope", &field::display(scope)); + span.record("memory.entry_key", &field::display(entry_key)); let affected = store .delete(scope, entry_key) .map_err(|err| err.to_string())?; @@ -2086,6 +2375,19 @@ fn resolve_policy(cli: &Cli, file_policy: &PolicyFile) -> Result "strict", + LauncherProfile::Balanced => "balanced", + LauncherProfile::Connected => "connected", + }; + let span = info_span!( + "shadi.policy.resolve", + policy.allowed_paths = field::Empty, + network.mode = field::Empty, + policy.profile = %profile_name, + ); + let _guard = span.enter(); + let profile = profile_defaults(cli.profile); let profile_net_block = profile.net_block.unwrap_or(false); let mut policy = SandboxPolicy::new().block_network(cli.net_block || file_policy.net_block.unwrap_or(profile_net_block)); @@ -2102,6 +2404,13 @@ fn resolve_policy(cli: &Cli, file_policy: &PolicyFile) -> Result std::io::Result { } fn load_policy_file(path: &Path) -> std::io::Result { + let span = info_span!("shadi.policy.load", policy.source = %path.display()); + let _guard = span.enter(); let data = std::fs::read_to_string(path)?; serde_json::from_str(&data).map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err)) } @@ -2232,6 +2543,8 @@ fn inject_keychain_secrets(command: &mut Command, mappings: &[String]) -> Result return Ok(()); } + let span = info_span!("shadi.secrets.inject", secret.count = mappings.len() as i64); + let _guard = span.enter(); let store = default_secret_store(); inject_keychain_with_store(store.as_ref(), command, mappings) } @@ -3737,4 +4050,118 @@ mod tests { let err = derive_agent_keypair(b"root-secret", " ").unwrap_err(); assert!(err.contains("agent name")); } + + #[test] + fn resolve_trace_file_prefers_cli_path() { + let dir = temp_dir(); + let cli_path = dir.path().join("trace.jsonl"); + std::env::set_var("SHADI_OTEL_FILE", "/tmp/ignored.jsonl"); + + let resolved = resolve_trace_file(Some(cli_path.clone())); + assert_eq!(resolved, cli_path); + + std::env::remove_var("SHADI_OTEL_FILE"); + } + + #[test] + fn resolve_trace_file_uses_env_var() { + let dir = temp_dir(); + let env_path = dir.path().join("env-trace.jsonl"); + std::env::set_var("SHADI_OTEL_FILE", env_path.to_string_lossy().to_string()); + + let resolved = resolve_trace_file(None); + assert_eq!(resolved, env_path); + + std::env::remove_var("SHADI_OTEL_FILE"); + } + + #[test] + fn trace_span_name_reads_span_name() { + let value = json!({"span": {"name": "shadi.sandbox.run"}}); + assert_eq!(trace_span_name(&value), Some("shadi.sandbox.run".to_string())); + } + + #[test] + fn trace_matches_filters_command_and_exit() { + let value = json!({ + "span": {"name": "shadi.sandbox.run"}, + "fields": {"command": "echo hi", "exit.code": 0} + }); + + assert!(trace_matches(&value, Some("sandbox"), Some("echo"), Some(0))); + assert!(!trace_matches(&value, Some("sandbox"), Some("missing"), Some(0))); + assert!(!trace_matches(&value, Some("sandbox"), Some("echo"), Some(1))); + } + + #[test] + fn read_trace_lines_keeps_tail() { + let dir = temp_dir(); + let path = dir.path().join("traces.jsonl"); + std::fs::write(&path, "one\ntwo\nthree\nfour\n").expect("write traces"); + + let lines = read_trace_lines(&path, 2).expect("read lines"); + assert_eq!(lines, vec!["three".to_string(), "four".to_string()]); + } + + #[test] + fn trace_list_errors_on_missing_file() { + let err = trace_list(Path::new("/tmp/does-not-exist.jsonl"), 5, None, None, None) + .unwrap_err(); + assert!(err.contains("failed to open trace file")); + } + + #[test] + fn trace_summary_counts_span_names() { + let dir = temp_dir(); + let path = dir.path().join("traces.jsonl"); + let lines = vec![ + json!({"span": {"name": "shadi.sandbox.run"}}).to_string(), + json!({"span": {"name": "shadi.sandbox.run"}}).to_string(), + json!({"span": {"name": "shadi.policy.resolve"}}).to_string(), + ]; + std::fs::write(&path, lines.join("\n")).expect("write traces"); + + trace_summary(&path, 10).expect("summary"); + } + + #[test] + fn trace_matches_filters_on_missing_fields() { + let value = json!({"span": {"name": "shadi.sandbox.run"}}); + assert!(!trace_matches(&value, Some("sandbox"), Some("echo"), None)); + assert!(!trace_matches(&value, Some("sandbox"), None, Some(1))); + } + + #[test] + fn trace_span_name_reads_spans_array() { + let value = json!({"spans": [{"name": "shadi.trace"}]}); + assert_eq!(trace_span_name(&value), Some("shadi.trace".to_string())); + } + + #[test] + fn resolve_trace_file_defaults_when_unset() { + std::env::remove_var("SHADI_OTEL_FILE"); + let resolved = resolve_trace_file(None); + assert_eq!(resolved, PathBuf::from(".shadi/traces.jsonl")); + } + + #[test] + fn parse_trace_line_rejects_invalid_json() { + assert!(parse_trace_line("not-json").is_none()); + } + + #[test] + fn trace_list_respects_filters() { + let dir = temp_dir(); + let path = dir.path().join("traces.jsonl"); + let lines = vec![ + json!({"span": {"name": "shadi.sandbox.run"}, "fields": {"command": "echo hi", "exit.code": 0}}) + .to_string(), + json!({"span": {"name": "shadi.policy.resolve"}, "fields": {"command": "cat", "exit.code": 1}}) + .to_string(), + ]; + std::fs::write(&path, lines.join("\n")).expect("write traces"); + + trace_list(&path, 10, Some("sandbox"), Some("echo"), Some(0)).expect("list"); + trace_list(&path, 10, Some("policy"), None, Some(1)).expect("list"); + } } diff --git a/docs/operations.md b/docs/operations.md index a0b50bc..c23f3b8 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -92,6 +92,7 @@ Keep these pages close when running the system: - [Sandbox and Policies](sandbox.md) for snapshot capture, launcher profiles, and enforcement boundaries - [Security Notes](security.md) for threat-model boundaries and backend caveats - [Architecture](architecture.md) for the control-plane versus runtime split +- [Telemetry](telemetry.md) for OpenTelemetry configuration and local collectors ## Scope of This Page diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000..1b3aedc --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,76 @@ +# Telemetry + +SHADI uses OpenTelemetry spans to trace core runtime activity and SecOps workflows. +Telemetry is opt-in and only enabled when an exporter or console output is configured. + +## Environment Variables + +The core runtime (shadictl, shadi_py, and tools) and the SecOps agent respect the +standard OpenTelemetry variables below. + +- `OTEL_EXPORTER_OTLP_ENDPOINT`: OTLP/HTTP endpoint for trace export. + - Example: `http://localhost:4318` +- `OTEL_SERVICE_NAME`: Override the service name reported by SHADI components. + - Defaults: `shadi-core`, `shadi-runtime`, and `shadi-secops`. +- `SHADI_OTEL_CONSOLE`: Set to `1` to print spans to stdout when no OTLP endpoint is set. +- `SHADI_OTEL_FILE`: Write JSON trace logs to a local file (one JSON object per line). + +## Local Collector Setup + +You can run a local OpenTelemetry Collector and point SHADI to it. + +Example `otelcol.yaml`: + +```yaml +receivers: + otlp: + protocols: + http: + +exporters: + logging: + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging] +``` + +Run the collector: + +```bash +otelcol --config otelcol.yaml +``` + +Then set the endpoint and launch SHADI: + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +export OTEL_SERVICE_NAME=shadi-core +shadictl --policy ./policy.json -- echo "hello" +``` + +## Local Trace Files + +To write trace logs directly to disk, set `SHADI_OTEL_FILE`: + +```bash +export SHADI_OTEL_FILE=.shadi/traces.jsonl +shadictl --policy ./policy.json -- echo "hello" +``` + +You can inspect the logs with `shadictl`: + +```bash +shadictl trace list --file .shadi/traces.jsonl --limit 50 +shadictl trace list --file .shadi/traces.jsonl --name shadi.sandbox.run +shadictl trace summary --file .shadi/traces.jsonl +``` + +## Notes + +- When `OTEL_EXPORTER_OTLP_ENDPOINT` is unset and `SHADI_OTEL_CONSOLE` is not enabled, + tracing is a no-op. +- Service naming is standardized under the `service.namespace=shadi` resource attribute + so core runtime and SecOps spans can be correlated. diff --git a/mkdocs.yml b/mkdocs.yml index 5cd7fe8..bb5ab0c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -61,6 +61,7 @@ nav: - Guides: - Getting Started: getting_started.md - Operations: operations.md + - Telemetry: telemetry.md - Sandbox and Policies: sandbox.md - Demo Walkthrough: demo.md - SecOps Demo: secops_agent.md diff --git a/tools/run_sandboxed_agent.py b/tools/run_sandboxed_agent.py index cf5664e..5f5f1c7 100644 --- a/tools/run_sandboxed_agent.py +++ b/tools/run_sandboxed_agent.py @@ -5,16 +5,30 @@ from pathlib import Path from shadi import SandboxPolicyHandle, run_sandboxed +from telemetry import tracer def load_policy(policy_path: str) -> tuple[SandboxPolicyHandle, dict]: policy_file = Path(policy_path) if not policy_file.exists(): raise FileNotFoundError(f"Policy file not found: {policy_file}") - try: - policy_data = json.loads(policy_file.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - raise ValueError(f"Policy file is not valid JSON: {exc}") from exc + with tracer.start_as_current_span("shadi.policy.load") as span: + span.set_attribute("policy.source", str(policy_file)) + try: + policy_data = json.loads(policy_file.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Policy file is not valid JSON: {exc}") from exc + + allowed_paths = set() + for path in policy_data.get("read", []) or []: + allowed_paths.add(str(path)) + for path in policy_data.get("write", []) or []: + allowed_paths.add(str(path)) + for path in policy_data.get("allow", []) or []: + allowed_paths.add(str(path)) + span.set_attribute("policy.allowed_paths.count", len(allowed_paths)) + network_mode = "blocked" if policy_data.get("net_block") else "allowed" + span.set_attribute("network.mode", network_mode) policy = SandboxPolicyHandle() for path in policy_data.get("read", []) or []: @@ -72,8 +86,25 @@ def main() -> int: print(str(exc), file=sys.stderr) return 2 + allowed_paths = set() + for path in policy_data.get("read", []) or []: + allowed_paths.add(str(path)) + for path in policy_data.get("write", []) or []: + allowed_paths.add(str(path)) + for path in policy_data.get("allow", []) or []: + allowed_paths.add(str(path)) + network_mode = "blocked" if policy_data.get("net_block") else "allowed" + env = build_env(policy_data) - return run_sandboxed(command, policy, cwd=args.cwd, env=env) + with tracer.start_as_current_span("shadi.sandbox.run") as span: + span.set_attribute("command", " ".join(command)) + span.set_attribute("cwd", args.cwd or str(Path.cwd())) + span.set_attribute("policy.source", args.policy) + span.set_attribute("policy.allowed_paths.count", len(allowed_paths)) + span.set_attribute("network.mode", network_mode) + exit_code = run_sandboxed(command, policy, cwd=args.cwd, env=env) + span.set_attribute("exit.code", exit_code) + return exit_code if __name__ == "__main__": diff --git a/tools/telemetry.py b/tools/telemetry.py new file mode 100644 index 0000000..cfba3b7 --- /dev/null +++ b/tools/telemetry.py @@ -0,0 +1,88 @@ +# Copyright AGNTCY Contributors (https://github.com/agntcy) +# SPDX-License-Identifier: Apache-2.0 +""" +OpenTelemetry setup for SHADI core tooling. + +Configure export with standard OTel env vars: + OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 (OTLP/HTTP) + OTEL_SERVICE_NAME=shadi-runtime (default) + +Set SHADI_OTEL_CONSOLE=1 to print spans to stdout when no OTLP endpoint is set. + +Without any configuration, all telemetry is a no-op. +""" +import os + +SERVICE_NAME = os.getenv("OTEL_SERVICE_NAME") or "shadi-runtime" + +try: + from opentelemetry import trace + from opentelemetry.sdk.resources import SERVICE_NAME as RES_SERVICE_NAME + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter + + # Use Resource() directly (not Resource.create()) so the SDK's environment + # detectors cannot override our explicit service.name with "unknown_service". + _resource = Resource( + attributes={ + RES_SERVICE_NAME: SERVICE_NAME, + "service.namespace": "shadi", + "telemetry.sdk.language": "python", + } + ) + _provider = TracerProvider(resource=_resource) + + _endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip() + if _endpoint: + try: + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + + _exporter = OTLPSpanExporter(endpoint=_endpoint.rstrip("/") + "/v1/traces") + _provider.add_span_processor(BatchSpanProcessor(_exporter)) + except ImportError: + pass # OTLP exporter package not installed + + if not _endpoint and os.getenv("SHADI_OTEL_CONSOLE", "").strip() in ("1", "true", "yes"): + _provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter())) + + trace.set_tracer_provider(_provider) + tracer = trace.get_tracer("shadi.runtime") + +except ImportError: + # Graceful no-op when opentelemetry packages are not installed. + + class _Span: + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + def add_event(self, *_, **__): + pass + + def set_attribute(self, *_, **__): + pass + + def record_exception(self, *_, **__): + pass + + def set_status(self, *_, **__): + pass + + class _SpanCtx: + def __init__(self, *_, **__): + self._span = _Span() + + def __enter__(self): + return self._span + + def __exit__(self, *_): + pass + + class _NoOpTracer: + def start_as_current_span(self, name, *_, **__): + return _SpanCtx(name) + + tracer = _NoOpTracer()