diff --git a/Cargo.lock b/Cargo.lock index 780620f0db..124c39c43b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,16 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "actix-codec" version = "0.5.2" @@ -139,7 +149,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8720bceaa6797fd8b2deab968d52e1120b2a8c30950939f6c8cdb42a910bc885" dependencies = [ "bytesize", - "darling", + "darling 0.23.0", "proc-macro2", "quote", "syn 2.0.119", @@ -845,6 +855,22 @@ dependencies = [ "wyz", ] +[[package]] +name = "blazesym" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847a0a95b041ad5aae1bdc44f2bd54743f76eb0065c4f86b139f81343c287eaf" +dependencies = [ + "cpp_demangle", + "crc32fast", + "flate2", + "gimli 0.33.0", + "libc", + "memmap2", + "rustc-demangle", + "tempfile", +] + [[package]] name = "blink-alloc" version = "0.4.0" @@ -922,6 +948,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bon" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5" +dependencies = [ + "darling 0.24.1", + "ident_case", + "prettyplease 0.3.0", + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "brotli" version = "8.0.2" @@ -1022,6 +1071,12 @@ dependencies = [ "bytes", ] +[[package]] +name = "c-enum" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd17eb909a8c6a894926bfcc3400a4bb0e732f5a57d37b1f14e8b29e329bace8" + [[package]] name = "camino" version = "1.2.4" @@ -1331,7 +1386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1340,7 +1395,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1547,6 +1602,15 @@ dependencies = [ "siphasher 1.0.3", ] +[[package]] +name = "cpp_demangle" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0667304c32ea56cb4cd6d2d7c0cfe9a2f8041229db8c033af7f8d69492429def" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1671,6 +1735,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1785,8 +1858,18 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", ] [[package]] @@ -1802,17 +1885,41 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 3.0.4", +] + [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core", + "darling_core 0.23.0", "quote", "syn 2.0.119", ] +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.4", +] + [[package]] name = "dary_heap" version = "0.3.8" @@ -1881,6 +1988,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive-where" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1904,6 +2022,136 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "dial9" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b11bce55173f77d9003cce8f129b4af889a4639005c53c3498cef933bc895c" +dependencies = [ + "bon", + "dial9-core", + "dial9-macro", + "dial9-metrique", + "dial9-perf-self-profile", + "dial9-tokio-telemetry", + "dial9-trace-format", + "tokio", + "tracing", +] + +[[package]] +name = "dial9-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8cbbc8955394be626249a3b52ddd6bf664373661eaeae70d87eb12bf6f20b6" +dependencies = [ + "arc-swap", + "bon", + "bytes", + "crossbeam-queue", + "dial9-trace-format", + "flate2", + "futures-util", + "libc", + "metrique", + "metrique-timesource", + "tokio", + "tokio-util", + "tracing", + "ulid", +] + +[[package]] +name = "dial9-macro" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4177b8f8bf95d8e3769c1aad596f5d6fb231953a45c4c3d2a5946fe8e4ec327b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dial9-metrique" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d42fe0e3e14e8780c4ecf9ee14c183dbcb9b1ae3612cc9a689e7dc83b13adef1" +dependencies = [ + "dial9-core", + "dial9-trace-format", + "metrique", + "metrique-writer", + "tokio", + "tracing", +] + +[[package]] +name = "dial9-perf-self-profile" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f65948455c504bf08576c7b5cfe93bfcaa486d661dc4ee85c2a6309ab89629" +dependencies = [ + "blazesym", + "bytes", + "crossbeam-utils", + "dial9-core", + "dial9-trace-format", + "libc", + "perf-event-data", + "perf-event-open-sys2", + "tokio", + "tracing", +] + +[[package]] +name = "dial9-tokio-telemetry" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1244091367c805b5d98a6a590f8ab992efda06e6d2560a6967ef898ffd30a2" +dependencies = [ + "bon", + "bytes", + "dial9-core", + "dial9-perf-self-profile", + "dial9-trace-format", + "flate2", + "futures-util", + "hostname", + "libc", + "metrique-timesource", + "pin-project-lite", + "serde", + "serde_json", + "smallvec", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "dial9-trace-format" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86083b7240114b2d0da4a7e9d041571d80ca3b1f92ae0ec794f1be21709daa6a" +dependencies = [ + "dial9-trace-format-derive", + "serde", + "typeid", +] + +[[package]] +name = "dial9-trace-format-derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9309248f12e414d88bcc9505f78b0c9ed47f79c5b62db611493e19a612cdc9d6" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -2166,6 +2414,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "env_filter" version = "2.0.0" @@ -2252,7 +2506,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2842,6 +3096,9 @@ version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" dependencies = [ + "fnv", + "hashbrown 0.16.1", + "indexmap 2.14.0", "stable_deref_trait", ] @@ -3109,6 +3366,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + [[package]] name = "http" version = "0.2.12" @@ -3654,7 +3922,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4177,6 +4445,141 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-util" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" +dependencies = [ + "aho-corasick", + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "indexmap 2.14.0", + "metrics", + "ordered-float 5.5.0", + "quanta", + "radix_trie", + "rand 0.9.2", + "rand_xoshiro", + "rapidhash", + "sketches-ddsketch", +] + +[[package]] +name = "metrique" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b55bfa39e6f5e44a37a59a794915ce36d04371471cf62ae365cd9703f58a5e0" +dependencies = [ + "itoa", + "metrique-core", + "metrique-macro", + "metrique-service-metrics", + "metrique-timesource", + "metrique-writer", + "metrique-writer-core", + "metrique-writer-macro", + "ryu", + "tokio", +] + +[[package]] +name = "metrique-core" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bd60ab3f5d8e13e36aa1d1dd9cc4920012dd21a25bd7311ae3c39810648e068" +dependencies = [ + "itertools 0.14.0", + "metrique-writer-core", +] + +[[package]] +name = "metrique-macro" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7605c49951713cce25af6bee77ed4432dde0839c5d9935bcb6f9643e6e82c7c3" +dependencies = [ + "Inflector", + "darling 0.23.0", + "metrique-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "metrique-service-metrics" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ee17caa35f83b730a83c8fa8f7a9aaa61a0becc7a71feb5bfb854bc75440d7" +dependencies = [ + "metrique-writer", +] + +[[package]] +name = "metrique-timesource" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2faca4e4480069ff02b1763b3b79f5cec7e8628e24d9dc5b6073f53d2577a4d9" + +[[package]] +name = "metrique-writer" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb97854dd4e2a92fe54b30e6e19976dcc0b2b7d39da3e84e86e2ce1e2ed8ce0d" +dependencies = [ + "ahash", + "crossbeam-queue", + "crossbeam-utils", + "metrics", + "metrics-util", + "metrique-core", + "metrique-writer-core", + "metrique-writer-macro", + "rand 0.9.2", + "smallvec", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "metrique-writer-core" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c9fc241cff4eedc2b6c3e2edece7f3edab2920e7531f62aac864ce933cdbb6" +dependencies = [ + "derive-where", + "itertools 0.14.0", + "serde", + "smallvec", +] + +[[package]] +name = "metrique-writer-macro" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1eef190f855405b9df4166ee0495925162fb2a7985bac62c6ae44f40061fac00" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "str_inflector", + "syn 2.0.119", + "synstructure", +] + [[package]] name = "mime" version = "0.3.17" @@ -4310,6 +4713,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab250442c86f1850815b5d268639dff018c0627022bc1940eb2d642ca1ce12f0" +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nix" version = "0.23.2" @@ -4791,6 +5203,27 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "perf-event-data" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575828d9d7d205188048eb1508560607a03d21eafdbba47b8cade1736c1c28e1" +dependencies = [ + "bitflags 2.11.0", + "c-enum", + "perf-event-open-sys2", +] + +[[package]] +name = "perf-event-open-sys2" +version = "5.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c25955321465255e437600b54296983fab1feac2cd0c38958adeb26dbae49e" +dependencies = [ + "libc", + "memoffset 0.9.1", +] + [[package]] name = "pest" version = "2.8.6" @@ -5129,6 +5562,15 @@ dependencies = [ "elliptic-curve", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error-attr3" version = "3.0.2" @@ -5290,7 +5732,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "petgraph 0.8.3", @@ -5337,7 +5779,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -5701,6 +6143,21 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quantization" version = "0.1.0" @@ -5828,7 +6285,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5858,6 +6315,16 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "raft" version = "0.7.0" @@ -5987,6 +6454,33 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.11.0", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -6322,7 +6816,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6335,7 +6829,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6403,7 +6897,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6901,7 +7395,7 @@ version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" dependencies = [ - "darling", + "darling 0.23.0", "proc-macro2", "quote", "syn 2.0.119", @@ -7102,6 +7596,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -7269,6 +7769,7 @@ dependencies = [ "collection", "common", "dashmap", + "dial9", "env_logger", "fs-err", "futures", @@ -7307,6 +7808,16 @@ dependencies = [ "wal", ] +[[package]] +name = "str_inflector" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0b848d5a7695b33ad1be00f84a3c079fe85c9278a325ff9159e6c99cef4ef7" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "str_stack" version = "0.1.0" @@ -7533,7 +8044,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7788,7 +8299,7 @@ dependencies = [ "indexmap 2.14.0", "serde_core", "serde_spanned", - "toml_datetime", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", "winnow 0.7.15", @@ -7804,12 +8315,33 @@ dependencies = [ ] [[package]] -name = "toml_parser" -version = "1.0.9+spec-1.1.0" +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ - "winnow 0.7.15", + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.0", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.0", ] [[package]] @@ -8162,6 +8694,16 @@ dependencies = [ "tonic-prost-build", ] +[[package]] +name = "ulid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +dependencies = [ + "rand 0.9.2", + "web-time", +] + [[package]] name = "unarray" version = "0.1.4" @@ -8459,7 +9001,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240e4b81c20a1d6d50d1d7265c658dfbd204e8b9ac4d80f3c931f39462196335" dependencies = [ - "darling", + "darling 0.23.0", "proc-macro-error3", "proc-macro2", "quote", @@ -8768,7 +9310,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d3c8554f80..2d1ff3d967 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ console-subscriber = ["tracing", "dep:console-subscriber"] tracy = ["tracing-tracy"] tracing-tracy = ["tracing", "dep:tracing-tracy"] tokio-tracing = ["tokio/tracing"] +dial9 = ["storage/dial9"] stacktrace = ["rstack-self"] chaos-testing = [] data-consistency-check = ["collection/data-consistency-check"] diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 25e9acc331..1245cffe1c 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -245,6 +245,8 @@ that can be enabled with optional features. - note, that you'll also have to [pass `--cfg tokio_unstable` arguments to `rustc`][tokio-tracing] to enable this feature - this is required (and enabled automatically) by the `console` feature - but you can enable it explicitly with the `tracy` feature, to see Tokio traces in [`Tracy`] profiler +- `dial9` feature enables [dial9](https://github.com/dial9-rs/dial9) Tokio runtime telemetry, + see [its own section](#dial9-tokio-runtime-telemetry) below Qdrant code is **not** instrumented by default, so you'll have to manually add `#[tracing::instrument]` attributes on functions and methods that you want to profile. @@ -279,6 +281,51 @@ fn some_other_function() { [`log`]: https://docs.rs/log/latest/log/ [tracing-log-warning]: https://docs.rs/tracing-log/latest/tracing_log/#caution-mixing-both-conversions +### dial9 (Tokio runtime telemetry) + +The `dial9` feature attaches [dial9](https://github.com/dial9-rs/dial9) hooks to Qdrant's storage +Tokio runtimes (`general`, `update`, `search-cpu`, `search-io`) and records worker, task and poll +events, with optional CPU and schedule profiling. Unlike [`tracing`] it needs no `#[instrument]` +attributes: it observes the runtime rather than your code, and it does *not* pick up [`tracing`] +spans. + +Recording needs both the feature at build time and `DIAL9_ENABLED=true` at run time. With the +feature off, runtime construction is unchanged. + +Three rustflags matter, and none of them is set anywhere in the repo: + +- `--cfg tokio_unstable` is **required for any task data at all**, not just for fuller coverage. + dial9's poll, spawn and terminate hooks are all `#[cfg(tokio_unstable)]`, so without it a trace + carries worker park/unpark and runtime metrics only, with no task timeline, and + `DIAL9_TASK_TRACKING_ENABLED` is a silent no-op. +- `-C force-frame-pointers=yes` lets the CPU and schedule profilers unwind. +- `-C debuginfo=2` lets those frames symbolize. `[profile.perf]` inherits `release` and sets no + `debug` key, so without it the symbolizer falls back to the ELF symtab: inlined callees collapse + into their caller and frames carry no file or line. + +```console +# Build with dial9 support (perf profile: release opts, no LTO) +$ RUSTFLAGS='--cfg tokio_unstable -C force-frame-pointers=yes -C debuginfo=2' \ + cargo build --profile perf --features dial9 + +# Run with telemetry + CPU/schedule profiling +$ DIAL9_ENABLED=true DIAL9_TRACE_DIR=/tmp/qdrant-dial9 ./target/perf/qdrant +``` + +Env knobs: `DIAL9_TRACE_DIR` (default `/tmp/dial9-traces`), `DIAL9_MAX_DISK_USAGE_MB`, +`DIAL9_ROTATION_SECS`, `DIAL9_TASK_TRACKING_ENABLED`, `DIAL9_CPU_PROFILE_ENABLED`, +`DIAL9_SCHEDULE_PROFILE_ENABLED`, `DIAL9_CPU_SAMPLE_HZ`. Booleans accept `1/true/yes/on` and +`0/false/no/off`; blank or unrecognized values log a warning and keep the default. + +Use a fresh `DIAL9_TRACE_DIR` per run: segments from separate runs in one directory are merged into +a single trace spanning the gap between them. + +View traces with the hosted viewer at , or install the +CLI with `cargo install --locked dial9 --features cli` and run +`dial9 serve --local-dir $DIAL9_TRACE_DIR`. In the browser, the **Browse** tab is built for dial9's +S3 key layout and stays empty for a local directory; use the **Raw** tab with an empty prefix, or +open a segment directly at `viewer.html?trace=trace.0.bin.gz`. + ### Pyroscope (continuous CPU profiling) Qdrant has built-in [Pyroscope](https://grafana.com/oss/pyroscope/) integration for continuous CPU profiling. Linux only. diff --git a/lib/storage/Cargo.toml b/lib/storage/Cargo.toml index 497590a45b..6de99d9a3d 100644 --- a/lib/storage/Cargo.toml +++ b/lib/storage/Cargo.toml @@ -14,6 +14,7 @@ workspace = true [features] tracing = ["dep:tracing", "api/tracing", "collection/tracing", "segment/tracing"] staging = ["collection/staging"] +dial9 = ["dep:dial9"] [dev-dependencies] collection = { path = "../collection", features = ["testing"] } @@ -70,3 +71,9 @@ async-trait = { workspace = true } tracing = { workspace = true, optional = true } tracing-appender = "0.2" + +# Optional Tokio telemetry (https://github.com/dial9-rs/dial9) +dial9 = { version = "0.5.0", optional = true, default-features = false, features = [ + "tokio", + "cpu-profiling", +] } diff --git a/lib/storage/src/content_manager/toc/runtimes.rs b/lib/storage/src/content_manager/toc/runtimes.rs index dd5ad73b07..309daef720 100644 --- a/lib/storage/src/content_manager/toc/runtimes.rs +++ b/lib/storage/src/content_manager/toc/runtimes.rs @@ -19,13 +19,26 @@ use tokio::runtime::{self, Runtime}; /// heavy work runs on the blocking pool, not the async workers. const SEARCH_ASYNC_WORKERS: usize = 1; +/// Finish building a runtime, optionally attaching dial9 telemetry hooks. +fn finish_runtime(builder: runtime::Builder, runtime_name: &str) -> io::Result { + cfg_select! { + feature = "dial9" => crate::dial9_telemetry::build_runtime(builder, runtime_name), + not(feature = "dial9") => { + let _ = runtime_name; + let mut builder = builder; + builder.build() + } + } +} + /// Build the CPU-friendly search runtime. Sized for steady-state CPU /// saturation: one blocking thread per CPU, so concurrent searches don't /// thrash. pub(super) fn create_high_cpu_search_runtime(max_search_threads: usize) -> io::Result { let blocking_threads = high_cpu_blocking_threads(max_search_threads); let async_workers = SEARCH_ASYNC_WORKERS.min(blocking_threads.max(1)); - runtime::Builder::new_multi_thread() + let mut builder = runtime::Builder::new_multi_thread(); + builder .worker_threads(async_workers) .max_blocking_threads(blocking_threads) .enable_all() @@ -33,8 +46,8 @@ pub(super) fn create_high_cpu_search_runtime(max_search_threads: usize) -> io::R static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); format!("search-cpu-{id}") - }) - .build() + }); + finish_runtime(builder, "search-cpu") } /// Build the IO-friendly search runtime. Blocking pool size comes from @@ -43,7 +56,8 @@ pub(super) fn create_high_cpu_search_runtime(max_search_threads: usize) -> io::R pub(super) fn create_high_io_search_runtime(max_search_threads: usize) -> io::Result { let blocking_threads = high_io_blocking_threads(max_search_threads); let async_workers = SEARCH_ASYNC_WORKERS.min(blocking_threads.max(1)); - runtime::Builder::new_multi_thread() + let mut builder = runtime::Builder::new_multi_thread(); + builder .worker_threads(async_workers) .max_blocking_threads(blocking_threads) .enable_all() @@ -51,8 +65,8 @@ pub(super) fn create_high_io_search_runtime(max_search_threads: usize) -> io::Re static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); format!("search-io-{id}") - }) - .build() + }); + finish_runtime(builder, "search-io") } pub(super) fn create_update_runtime(max_optimization_threads: usize) -> io::Result { @@ -70,11 +84,12 @@ pub(super) fn create_update_runtime(max_optimization_threads: usize) -> io::Resu if max_optimization_threads > 0 { builder.max_blocking_threads(max_optimization_threads); } - builder.build() + finish_runtime(builder, "update") } pub(super) fn create_general_purpose_runtime() -> io::Result { - runtime::Builder::new_multi_thread() + let mut builder = runtime::Builder::new_multi_thread(); + builder .enable_time() .enable_io() .worker_threads(max(common::cpu::get_num_cpus(), 2)) @@ -82,8 +97,8 @@ pub(super) fn create_general_purpose_runtime() -> io::Result { static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0); let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst); format!("general-{id}") - }) - .build() + }); + finish_runtime(builder, "general") } /// Blocking-thread count for the high-CPU search pool. diff --git a/lib/storage/src/dial9_telemetry.rs b/lib/storage/src/dial9_telemetry.rs new file mode 100644 index 0000000000..b0d571e35e --- /dev/null +++ b/lib/storage/src/dial9_telemetry.rs @@ -0,0 +1,216 @@ +//! Optional [dial9](https://github.com/dial9-rs/dial9) Tokio telemetry. +//! +//! Enabled at compile time with the `dial9` feature and at runtime with +//! `DIAL9_ENABLED=true`. When disabled (feature off, or env unset/false), +//! runtime construction is unchanged and recording is a no-op. + +use std::sync::OnceLock; +use std::time::Duration; +use std::{env, io}; + +use dial9::cpu::{CpuProfilingConfig, SchedEventConfig}; +use dial9::{ + Dial9Handle, Dial9HandleTokioExt, DiskBuffer, Recorder, RecorderPerfExt, TokioAttachOptions, + recorder_disabled, recorder_or_disabled, +}; +use tokio::runtime::{Builder, Runtime}; + +/// Process-wide dial9 handle for attaching Tokio runtimes. +static HANDLE: OnceLock = OnceLock::new(); + +/// Holds the dial9 recorder for the process lifetime. +/// +/// Constructed by [`init`]. Dropping it (at process shutdown) flushes and +/// stops the recorder. +#[derive(Debug)] +pub struct Dial9Guard { + recorder: Option, +} + +impl Drop for Dial9Guard { + fn drop(&mut self) { + if let Some(recorder) = self.recorder.take() { + recorder.graceful_shutdown(Duration::from_secs(5)); + } + } +} + +/// Read a boolean `DIAL9_*` variable. +/// +/// Matches dial9's own env parsing: trimmed, case-insensitive, and accepting +/// the usual spellings rather than only `str::parse::`. A blank or +/// unrecognized value warns and falls back to `default`, so an operator who +/// writes `DIAL9_CPU_PROFILE_ENABLED=0` is not silently left with profiling on. +fn env_bool(name: &str, default: bool) -> bool { + let Ok(raw) = env::var(name) else { + return default; + }; + + match raw.trim().to_ascii_lowercase().as_str() { + "" => default, + "1" | "t" | "true" | "y" | "yes" | "on" => true, + "0" | "f" | "false" | "n" | "no" | "off" => false, + other => { + log::warn!("dial9: ignoring invalid {name}={other:?}, using {default}"); + default + } + } +} + +/// Read a `DIAL9_*` variable that dial9 requires to be non-zero. +/// +/// Zero is rejected rather than passed through: dial9 treats a zero disk budget +/// as "evict everything" (killing recording seconds after it starts) and fails +/// outright on a zero rotation period. +fn env_positive_u64(name: &str, default: u64) -> u64 { + let Ok(raw) = env::var(name) else { + return default; + }; + + let raw = raw.trim(); + if raw.is_empty() { + return default; + } + + match raw.parse::() { + Ok(value) if value > 0 => value, + _ => { + log::warn!("dial9: ignoring invalid {name}={raw:?}, using {default}"); + default + } + } +} + +/// Read a path-valued `DIAL9_*` variable, treating a blank value as unset. +/// +/// `env::var` returns `Ok("")` for a variable that is set but empty (a common +/// shape from `docker run -e VAR` and unset compose interpolations), which dial9 +/// would otherwise resolve to relative paths in the working directory. +fn env_path(name: &str, default: &str) -> String { + match env::var(name) { + Ok(raw) if !raw.trim().is_empty() => raw.trim().to_string(), + _ => default.to_string(), + } +} + +/// Initialize dial9 from `DIAL9_*` environment variables. +/// +/// Safe to call once at process start, before any Tokio runtimes are built. +/// Returns a guard that must be kept alive until shutdown. +/// +/// `application_version` is recorded into every trace segment (e.g. the qdrant +/// binary version). +/// +/// Recognized variables (subset of dial9's env surface). Booleans accept +/// `1/t/true/y/yes/on` and `0/f/false/n/no/off`; blank or unrecognized values +/// warn and keep the default: +/// +/// - `DIAL9_ENABLED`: record at all (default: off) +/// - `DIAL9_TRACE_DIR`: trace directory (default: `/tmp/dial9-traces`) +/// - `DIAL9_MAX_DISK_USAGE_MB`: total on-disk budget, must be > 0 (default: `1024`) +/// - `DIAL9_ROTATION_SECS`: segment rotation period, must be > 0 (default: `60`) +/// - `DIAL9_CPU_PROFILE_ENABLED`: CPU stack sampling (default: on, Linux) +/// - `DIAL9_CPU_SAMPLE_HZ`: sampling frequency, must be > 0 (default: `99`) +/// - `DIAL9_SCHEDULE_PROFILE_ENABLED`: sched-switch capture (default: on, Linux) +pub fn init(application_version: &str) -> Dial9Guard { + let enabled = env_bool("DIAL9_ENABLED", false); + + if !enabled { + log::info!("dial9 telemetry disabled (set DIAL9_ENABLED=true to enable)"); + let recorder = recorder_disabled(); + let _ = HANDLE.set(recorder.handle().clone()); + return Dial9Guard { + recorder: Some(recorder), + }; + } + + let trace_dir = env_path("DIAL9_TRACE_DIR", "/tmp/dial9-traces"); + let max_disk_mb = env_positive_u64("DIAL9_MAX_DISK_USAGE_MB", 1024); + let rotation_secs = env_positive_u64("DIAL9_ROTATION_SECS", 60); + let cpu_profile_enabled = env_bool("DIAL9_CPU_PROFILE_ENABLED", cfg!(target_os = "linux")); + let schedule_profile_enabled = + env_bool("DIAL9_SCHEDULE_PROFILE_ENABLED", cfg!(target_os = "linux")); + let cpu_sample_hz = env_positive_u64("DIAL9_CPU_SAMPLE_HZ", 99); + + let writer = DiskBuffer::builder() + .base_path(&trace_dir) + .max_total_size(max_disk_mb.saturating_mul(1024 * 1024)) + .rotation_period(Duration::from_secs(rotation_secs)) + .build(); + + if let Err(ref err) = writer { + // Return before the profiler setup below. `with_cpu_profiling` and + // `with_sched_events` start their profilers immediately (opening a perf + // event per thread, and installing a process-global signal handler on + // the fallback path), and `build()` would only discard them. + log::error!( + "dial9: failed to create trace writer at {trace_dir}: {err}; telemetry disabled" + ); + let recorder = recorder_disabled(); + let _ = HANDLE.set(recorder.handle().clone()); + return Dial9Guard { + recorder: Some(recorder), + }; + } + + let mut builder = recorder_or_disabled(writer).segment_metadata([ + ("service".to_string(), "qdrant".to_string()), + ( + "application.version".to_string(), + application_version.to_string(), + ), + ]); + + if cpu_profile_enabled { + builder = + builder.with_cpu_profiling(CpuProfilingConfig::default().frequency_hz(cpu_sample_hz)); + } + if schedule_profile_enabled { + builder = builder.with_sched_events(SchedEventConfig::default()); + } + + let recorder = builder.build(); + + if recorder.handle().is_connected() { + if let Err(err) = recorder.install_global_handle() { + log::warn!("dial9: could not install global handle: {err}"); + } + log::info!( + "dial9 telemetry enabled; writing traces to {trace_dir} \ + (max {max_disk_mb} MiB, rotate every {rotation_secs}s, \ + cpu_profile={cpu_profile_enabled}, sched_profile={schedule_profile_enabled}, \ + sample_hz={cpu_sample_hz})" + ); + } else { + log::error!( + "dial9: recorder not connected after init (trace_dir={trace_dir}); \ + check DiskBuffer / cpu-profiling permissions; telemetry inactive" + ); + } + + let _ = HANDLE.set(recorder.handle().clone()); + Dial9Guard { + recorder: Some(recorder), + } +} + +fn handle() -> Dial9Handle { + HANDLE.get().cloned().unwrap_or_else(Dial9Handle::disabled) +} + +/// Build a Tokio runtime, attaching dial9 hooks when telemetry is active. +pub fn build_runtime(mut builder: Builder, runtime_name: &str) -> io::Result { + let handle = handle(); + if !handle.is_connected() { + return builder.build(); + } + + let task_tracking = env_bool("DIAL9_TASK_TRACKING_ENABLED", true); + + let options = TokioAttachOptions::builder() + .runtime_name(runtime_name.to_string()) + .task_tracking_enabled(task_tracking) + .build(); + + handle.attach_tokio_runtime(builder, options) +} diff --git a/lib/storage/src/lib.rs b/lib/storage/src/lib.rs index b1613814e8..9204fc2d87 100644 --- a/lib/storage/src/lib.rs +++ b/lib/storage/src/lib.rs @@ -14,6 +14,8 @@ pub mod audit; pub mod audit_reader; mod common; pub mod content_manager; +#[cfg(feature = "dial9")] +pub mod dial9_telemetry; pub mod dispatcher; pub mod issues_subscribers; pub mod quota; diff --git a/src/main.rs b/src/main.rs index 9d884c8169..e13214d15f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -426,6 +426,14 @@ fn main() -> anyhow::Result<()> { welcome(&settings); + // + // Optional dial9 Tokio telemetry (compile with `--features dial9`, + // enable at runtime with DIAL9_ENABLED=true). Must run before any + // TableOfContent Tokio runtimes are constructed. + // + #[cfg(feature = "dial9")] + let _dial9_guard = storage::dial9_telemetry::init(env!("CARGO_PKG_VERSION")); + // // Audit logging //