diff --git a/Cargo.lock b/Cargo.lock index 172a67806..24aa7399a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,7 +80,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -217,7 +217,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -241,17 +241,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.7.8" @@ -408,7 +397,7 @@ dependencies = [ "rayon", "roaring", "tempfile", - "thiserror 1.0.69", + "thiserror 1.0.61", ] [[package]] @@ -427,7 +416,7 @@ dependencies = [ "rayon", "roaring", "tempfile", - "thiserror 1.0.69", + "thiserror 1.0.61", ] [[package]] @@ -448,7 +437,7 @@ checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -562,7 +551,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -656,7 +645,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "syn_derive", ] @@ -781,7 +770,7 @@ checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -805,27 +794,6 @@ dependencies = [ "bytes", ] -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "camino" version = "1.1.6" @@ -853,7 +821,7 @@ dependencies = [ "rand_distr", "rayon", "safetensors", - "thiserror 1.0.69", + "thiserror 1.0.61", "ug", "ug-cuda", "yoke", @@ -881,7 +849,7 @@ dependencies = [ "rayon", "safetensors", "serde", - "thiserror 1.0.69", + "thiserror 1.0.61", ] [[package]] @@ -923,7 +891,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", ] [[package]] @@ -944,9 +912,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.104" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" +checksum = "eaff6f8ce506b9773fa786672d63fc7a191ffea1be33f72bbd4aeacefca9ffc8" dependencies = [ "jobserver", "libc", @@ -1042,16 +1010,6 @@ dependencies = [ "half 1.8.2", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clang-sys" version = "1.7.0" @@ -1065,9 +1023,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.24" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9560b07a799281c7e0958b9296854d6fafd4c5f31444a7e5bb1ad6dde5ccf1bd" +checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" dependencies = [ "clap_builder", "clap_derive", @@ -1075,9 +1033,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.24" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874e0dd3eb68bf99058751ac9712f622e61e6f393a94f7128fa26e3f02f5c7cd" +checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ "anstream", "anstyle", @@ -1094,7 +1052,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1165,12 +1123,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "constant_time_eq" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" - [[package]] name = "convert_case" version = "0.4.0" @@ -1212,21 +1164,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32fast" version = "1.4.2" @@ -1407,7 +1344,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1429,7 +1366,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1459,12 +1396,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "deflate64" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" - [[package]] name = "deranged" version = "0.3.11" @@ -1483,7 +1414,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1525,7 +1456,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1545,7 +1476,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core 0.20.2", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1587,7 +1518,7 @@ dependencies = [ "convert_case 0.6.0", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1651,7 +1582,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1687,7 +1618,7 @@ dependencies = [ "serde_json", "tar", "tempfile", - "thiserror 2.0.9", + "thiserror 2.0.11", "time", "tracing", "uuid", @@ -1809,7 +1740,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1829,7 +1760,7 @@ checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1840,9 +1771,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1876,7 +1807,7 @@ name = "file-store" version = "1.12.2" dependencies = [ "tempfile", - "thiserror 2.0.9", + "thiserror 2.0.11", "tracing", "uuid", ] @@ -1910,7 +1841,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", - "miniz_oxide 0.8.2", + "miniz_oxide 0.8.3", ] [[package]] @@ -2019,7 +1950,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -2245,9 +2176,9 @@ checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" [[package]] name = "git2" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" +checksum = "3fda788993cc341f69012feba8bf45c0ba4f3291fcc08e214b4d5a7332d88aff" dependencies = [ "bitflags 2.6.0", "libc", @@ -2296,9 +2227,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ "atomic-waker", "bytes", @@ -2462,7 +2393,7 @@ dependencies = [ "rand", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 1.0.61", "ureq", ] @@ -2541,7 +2472,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.5", + "h2 0.4.7", "http 1.2.0", "http-body", "httparse", @@ -2705,7 +2636,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -2772,7 +2703,7 @@ dependencies = [ "serde_json", "synchronoise", "tempfile", - "thiserror 2.0.9", + "thiserror 2.0.11", "time", "tracing", "ureq", @@ -2803,15 +2734,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "generic-array", -] - [[package]] name = "insta" version = "1.39.0" @@ -3013,15 +2935,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.169" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libgit2-sys" -version = "0.17.0+1.8.1" +version = "0.18.0+1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224" +checksum = "e1a117465e7e1597e8febea8bb0c410f1c7fb93b1e1cddf34363f8390367ffec" dependencies = [ "cc", "libc", @@ -3036,7 +2958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.1", ] [[package]] @@ -3114,7 +3036,7 @@ dependencies = [ "regex", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 1.0.61", "unicode-blocks", "unicode-normalization", "unicode-segmentation", @@ -3184,7 +3106,7 @@ dependencies = [ "log", "once_cell", "serde", - "thiserror 1.0.69", + "thiserror 1.0.61", "yada", ] @@ -3442,7 +3364,7 @@ checksum = "3b51f1d220e3fa869e24cfd75915efe3164bd09bb11b3165db3f37f57bf673e3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3517,16 +3439,6 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" -[[package]] -name = "lzma-rs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" -dependencies = [ - "byteorder", - "crc", -] - [[package]] name = "macro_rules_attribute" version = "0.2.0" @@ -3552,7 +3464,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3650,7 +3562,7 @@ dependencies = [ "temp-env", "tempfile", "termcolor", - "thiserror 2.0.9", + "thiserror 2.0.11", "time", "tokio", "toml", @@ -3682,7 +3594,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "thiserror 2.0.9", + "thiserror 2.0.11", "time", "uuid", ] @@ -3714,7 +3626,7 @@ dependencies = [ "serde_json", "tar", "tempfile", - "thiserror 2.0.9", + "thiserror 2.0.11", "time", "tokio", "utoipa", @@ -3820,7 +3732,7 @@ dependencies = [ "smallvec", "smartstring", "tempfile", - "thiserror 2.0.9", + "thiserror 2.0.11", "thread_local", "tiktoken-rs", "time", @@ -3831,6 +3743,7 @@ dependencies = [ "url", "utoipa", "uuid", + "zstd", ] [[package]] @@ -3875,9 +3788,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" dependencies = [ "adler2", ] @@ -3923,7 +3836,7 @@ checksum = "371717c0a5543d6a800cac822eac735aa7d2d2fbb41002e9856a4089532dbdce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -4093,7 +4006,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -4241,16 +4154,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498a099351efa4becc6a19c72aa9270598e8fd274ca47052e37455241c88b696" -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest", - "hmac", -] - [[package]] name = "pem" version = "3.0.3" @@ -4281,7 +4184,7 @@ version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" dependencies = [ - "thiserror 1.0.69", + "thiserror 1.0.61", "ucd-trie", ] @@ -4305,7 +4208,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -4359,7 +4262,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -4388,7 +4291,7 @@ checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -4504,9 +4407,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -4548,7 +4451,7 @@ dependencies = [ "parking_lot", "procfs", "protobuf", - "thiserror 1.0.69", + "thiserror 1.0.61", ] [[package]] @@ -4601,24 +4504,24 @@ dependencies = [ "quinn-udp", "rustc-hash 1.1.0", "rustls", - "thiserror 1.0.69", + "thiserror 1.0.61", "tokio", "tracing", ] [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" dependencies = [ "bytes", "rand", "ring", - "rustc-hash 2.1.0", + "rustc-hash 1.1.0", "rustls", "slab", - "thiserror 1.0.69", + "thiserror 1.0.61", "tinyvec", "tracing", ] @@ -4772,7 +4675,7 @@ checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", "redox_syscall 0.2.16", - "thiserror 1.0.69", + "thiserror 1.0.61", ] [[package]] @@ -4800,9 +4703,9 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" [[package]] name = "regex-syntax" @@ -4889,7 +4792,7 @@ source = "git+https://github.com/rhaiscript/rhai?rev=ef3df63121d27aacd838f366f2b dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -5004,9 +4907,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags 2.6.0", "errno", @@ -5017,9 +4920,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" dependencies = [ "log", "once_cell", @@ -5109,7 +5012,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror 2.0.9", + "thiserror 2.0.11", "time", ] @@ -5154,7 +5057,7 @@ checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -5283,7 +5186,7 @@ checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" dependencies = [ "num-bigint", "num-traits", - "thiserror 1.0.69", + "thiserror 1.0.61", "time", ] @@ -5451,7 +5354,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -5473,9 +5376,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.87" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -5491,7 +5394,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -5520,7 +5423,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -5533,7 +5436,7 @@ dependencies = [ "byteorder", "enum-as-inner", "libc", - "thiserror 1.0.69", + "thiserror 1.0.61", "walkdir", ] @@ -5611,42 +5514,42 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.69" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ - "thiserror-impl 1.0.69", + "thiserror-impl 1.0.61", ] [[package]] name = "thiserror" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.9", + "thiserror-impl 2.0.11", ] [[package]] name = "thiserror-impl" -version = "1.0.69" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] name = "thiserror-impl" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -5776,7 +5679,7 @@ dependencies = [ "serde", "serde_json", "spm_precompiled", - "thiserror 1.0.69", + "thiserror 1.0.61", "unicode-normalization-alignments", "unicode-segmentation", "unicode_categories", @@ -5784,9 +5687,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.42.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -5802,13 +5705,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -5877,7 +5780,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.22", + "winnow 0.6.24", ] [[package]] @@ -5940,7 +5843,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -6054,7 +5957,7 @@ dependencies = [ "num", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 1.0.61", ] [[package]] @@ -6067,7 +5970,7 @@ dependencies = [ "half 2.4.1", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 1.0.61", "ug", ] @@ -6077,7 +5980,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c878a167baa8afd137494101a688ef8c67125089ff2249284bd2b5f9bfedb815" dependencies = [ - "thiserror 1.0.69", + "thiserror 1.0.61", ] [[package]] @@ -6225,7 +6128,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.87", + "syn 2.0.90", "uuid", ] @@ -6243,9 +6146,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4" dependencies = [ "getrandom", "serde", @@ -6265,9 +6168,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "9.0.2" +version = "9.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f25fc8f8f05df455c7941e87f093ad22522a9ff33d7a027774815acf6f0639" +checksum = "e0d2f179f8075b805a43a2a21728a46f0cc2921b3c58695b28fa8817e103cd9a" dependencies = [ "anyhow", "derive_builder 0.20.2", @@ -6277,9 +6180,9 @@ dependencies = [ [[package]] name = "vergen-git2" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e63e069d8749fead1e3bab7a9d79e8fb90516b2ec66fc2243a798ecdc1a31d7" +checksum = "d86bae87104cb2790cdee615c2bb54729804d307191732ab27b1c5357ea6ddc5" dependencies = [ "anyhow", "derive_builder 0.20.2", @@ -6292,9 +6195,9 @@ dependencies = [ [[package]] name = "vergen-lib" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c767e6751c09fc85cde58722cf2f1007e80e4c8d5a4321fc90d83dc54ca147" +checksum = "9b07e6010c0f3e59fcb164e0163834597da68d1f864e2b8ca49f74de01e9c166" dependencies = [ "anyhow", "derive_builder 0.20.2", @@ -6364,7 +6267,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "wasm-bindgen-shared", ] @@ -6398,7 +6301,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6512,7 +6415,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -6523,7 +6426,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -6781,9 +6684,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.22" +version = "0.6.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39281189af81c07ec09db316b302a3e67bf9bd7cbf6c820b50e35fee9c2fa980" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" dependencies = [ "memchr", ] @@ -6881,7 +6784,7 @@ checksum = "b0144f1a16a199846cb21024da74edd930b43443463292f536b7110b4855b5c6" dependencies = [ "form_urlencoded", "serde", - "thiserror 1.0.69", + "thiserror 1.0.61", ] [[package]] @@ -6904,7 +6807,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "synstructure", ] @@ -6925,7 +6828,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -6945,7 +6848,7 @@ checksum = "e6a647510471d372f2e6c2e6b7219e44d8c574d24fdc11c610a61455782f18c3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "synstructure", ] @@ -6954,20 +6857,6 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.87", -] [[package]] name = "zerovec" @@ -6988,7 +6877,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -7003,7 +6892,7 @@ dependencies = [ "displaydoc", "indexmap", "num_enum", - "thiserror 1.0.69", + "thiserror 1.0.61", ] [[package]] @@ -7012,27 +6901,15 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae9c1ea7b3a5e1f4b922ff856a129881167511563dc219869afe3787fc0c1a45" dependencies = [ - "aes", "arbitrary", - "bzip2", - "constant_time_eq", "crc32fast", "crossbeam-utils", - "deflate64", "displaydoc", "flate2", - "hmac", "indexmap", - "lzma-rs", "memchr", - "pbkdf2", - "rand", - "sha1", - "thiserror 2.0.9", - "time", - "zeroize", + "thiserror 2.0.11", "zopfli", - "zstd", ] [[package]] @@ -7051,18 +6928,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.2.0" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" dependencies = [ "zstd-sys", ] diff --git a/crates/fuzzers/src/bin/fuzz-indexing.rs b/crates/fuzzers/src/bin/fuzz-indexing.rs index 08711e5e3..36b54fe10 100644 --- a/crates/fuzzers/src/bin/fuzz-indexing.rs +++ b/crates/fuzzers/src/bin/fuzz-indexing.rs @@ -150,7 +150,7 @@ fn main() { // after executing a batch we check if the database is corrupted let res = index.search(&wtxn).execute().unwrap(); - index.documents(&wtxn, res.documents_ids).unwrap(); + index.compressed_documents(&wtxn, res.documents_ids).unwrap(); progression.fetch_add(1, Ordering::Relaxed); } wtxn.abort(); diff --git a/crates/meilisearch-types/src/lib.rs b/crates/meilisearch-types/src/lib.rs index a1a57b7e6..f57429c85 100644 --- a/crates/meilisearch-types/src/lib.rs +++ b/crates/meilisearch-types/src/lib.rs @@ -15,7 +15,7 @@ pub mod star_or; pub mod task_view; pub mod tasks; pub mod versioning; -pub use milli::{heed, Index}; +pub use milli::{heed, zstd, Index}; use uuid::Uuid; pub use versioning::VERSION_FILE_NAME; pub use {milli, serde_cs}; diff --git a/crates/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml index 1d458af34..9324957f7 100644 --- a/crates/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -132,7 +132,7 @@ reqwest = { version = "0.12.12", features = [ sha-1 = { version = "0.10.1", optional = true } static-files = { version = "0.2.4", optional = true } tempfile = { version = "3.15.0", optional = true } -zip = { version = "2.2.2", optional = true } +zip = { version = "2.2.2", default-features = false, features = ["deflate"], optional = true } [features] default = ["meilisearch-types/all-tokenizations", "mini-dashboard"] diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 54f01b4d6..47d2e9e07 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -1411,43 +1411,50 @@ fn some_documents<'a, 't: 'a>( retrieve_vectors: RetrieveVectors, ) -> Result> + 'a, ResponseError> { let fields_ids_map = index.fields_ids_map(rtxn)?; + let dictionary = index.document_decompression_dictionary(rtxn)?; let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); let embedding_configs = index.embedding_configs(rtxn)?; + let mut buffer = Vec::new(); - Ok(index.iter_documents(rtxn, doc_ids)?.map(move |ret| { - ret.map_err(ResponseError::from).and_then(|(key, document)| -> Result<_, ResponseError> { - let mut document = milli::obkv_to_json(&all_fields, &fields_ids_map, document)?; - match retrieve_vectors { - RetrieveVectors::Hide => { - document.remove("_vectors"); - } - RetrieveVectors::Retrieve => { - // Clippy is simply wrong - #[allow(clippy::manual_unwrap_or_default)] - let mut vectors = match document.remove("_vectors") { - Some(Value::Object(map)) => map, - _ => Default::default(), - }; - for (name, vector) in index.embeddings(rtxn, key)? { - let user_provided = embedding_configs - .iter() - .find(|conf| conf.name == name) - .is_some_and(|conf| conf.user_provided.contains(key)); - let embeddings = ExplicitVectors { - embeddings: Some(vector.into()), - regenerate: !user_provided, - }; - vectors.insert( - name, - serde_json::to_value(embeddings).map_err(MeilisearchHttpError::from)?, - ); + Ok(index.iter_compressed_documents(rtxn, doc_ids)?.map(move |ret| { + ret.map_err(ResponseError::from).and_then( + |(key, compressed_document)| -> Result<_, ResponseError> { + let document = compressed_document + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())?; + let mut document = milli::obkv_to_json(&all_fields, &fields_ids_map, document)?; + match retrieve_vectors { + RetrieveVectors::Hide => { + document.remove("_vectors"); + } + RetrieveVectors::Retrieve => { + // Clippy is simply wrong + #[allow(clippy::manual_unwrap_or_default)] + let mut vectors = match document.remove("_vectors") { + Some(Value::Object(map)) => map, + _ => Default::default(), + }; + for (name, vector) in index.embeddings(rtxn, key)? { + let user_provided = embedding_configs + .iter() + .find(|conf| conf.name == name) + .is_some_and(|conf| conf.user_provided.contains(key)); + let embeddings = ExplicitVectors { + embeddings: Some(vector.into()), + regenerate: !user_provided, + }; + vectors.insert( + name, + serde_json::to_value(embeddings) + .map_err(MeilisearchHttpError::from)?, + ); + } + document.insert("_vectors".into(), vectors.into()); } - document.insert("_vectors".into(), vectors.into()); } - } - Ok(document) - }) + Ok(document) + }, + ) })) } diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index aab8ae919..c3eb3ef08 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -1330,6 +1330,20 @@ impl<'a> HitMaker<'a> { let (_, obkv) = self.index.iter_documents(self.rtxn, std::iter::once(id))?.next().unwrap()?; + // let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer_builder.build()); + // formatter_builder.crop_marker(format.crop_marker); + // formatter_builder.highlight_prefix(format.highlight_pre_tag); + // formatter_builder.highlight_suffix(format.highlight_post_tag); + // let decompression_dictionary = index.document_decompression_dictionary(rtxn)?; + // let mut buffer = Vec::new(); + // let mut documents = Vec::new(); + // let embedding_configs = index.embedding_configs(rtxn)?; + // let documents_iter = index.compressed_documents(rtxn, documents_ids)?; + // for ((id, compressed), score) in documents_iter.into_iter().zip(document_scores.into_iter()) { + // let obkv = compressed + // .decompress_with_optional_dictionary(&mut buffer, decompression_dictionary.as_ref()) + // // TODO use a better error? + // .map_err(|e| MeilisearchHttpError::HeedError(e.into()))?; // First generate a document with all the displayed fields let displayed_document = make_document(&self.displayed_ids, &self.fields_ids_map, obkv)?; diff --git a/crates/meilitool/src/main.rs b/crates/meilitool/src/main.rs index 44eb4960e..4938e03eb 100644 --- a/crates/meilitool/src/main.rs +++ b/crates/meilitool/src/main.rs @@ -280,6 +280,7 @@ fn export_a_dump( // 4. Dump the indexes let mut count = 0; + let mut buffer = Vec::new(); for result in index_mapping.iter(&rtxn)? { let (uid, uuid) = result?; let index_path = db_path.join("indexes").join(uuid.to_string()); @@ -288,6 +289,7 @@ fn export_a_dump( })?; let rtxn = index.read_txn()?; + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); let metadata = IndexMetadata { uid: uid.to_owned(), primary_key: index.primary_key(&rtxn)?.map(String::from), @@ -300,8 +302,11 @@ fn export_a_dump( let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect(); // 4.1. Dump the documents - for ret in index.all_documents(&rtxn)? { - let (_id, doc) = ret?; + for ret in index.all_compressed_documents(&rtxn)? { + let (_id, compressed_doc) = ret?; + let doc = compressed_doc + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); let document = obkv_to_json(&all_fields, &fields_ids_map, doc)?; index_dumper.push_document(&document)?; } diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index d22829045..30150429d 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -36,6 +36,7 @@ heed = { version = "0.20.5", default-features = false, features = [ indexmap = { version = "2.7.0", features = ["serde"] } json-depth-checker = { path = "../json-depth-checker" } levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] } +zstd = { version = "0.13.1", features = ["zdict_builder", "experimental"] } memchr = "2.7.4" memmap2 = "0.9.5" obkv = "0.3.0" diff --git a/crates/milli/src/heed_codec/compressed_obkv_codec.rs b/crates/milli/src/heed_codec/compressed_obkv_codec.rs new file mode 100644 index 000000000..8ddd94d92 --- /dev/null +++ b/crates/milli/src/heed_codec/compressed_obkv_codec.rs @@ -0,0 +1,98 @@ +use std::borrow::Cow; +use std::io; +use std::io::ErrorKind; + +use heed::BoxedError; +use obkv::KvReaderU16; +use zstd::bulk::{Compressor, Decompressor}; +use zstd::dict::{DecoderDictionary, EncoderDictionary}; + +pub struct CompressedObkvCodec; + +impl<'a> heed::BytesDecode<'a> for CompressedObkvCodec { + type DItem = CompressedKvReaderU16<'a>; + + fn bytes_decode(bytes: &'a [u8]) -> Result { + Ok(CompressedKvReaderU16(bytes)) + } +} + +impl heed::BytesEncode<'_> for CompressedObkvCodec { + type EItem = CompressedKvWriterU16; + + fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { + Ok(Cow::Borrowed(&item.0)) + } +} + +// TODO Make this an unsized slice wrapper instead? +// &'a CompressedKvReaderU16([u8]) +pub struct CompressedKvReaderU16<'a>(&'a [u8]); + +impl<'a> CompressedKvReaderU16<'a> { + /// Decompresses the KvReader into the buffer using the provided dictionnary. + pub fn decompress_with<'b>( + &self, + buffer: &'b mut Vec, + dictionary: &DecoderDictionary, + ) -> io::Result<&'b KvReaderU16> { + const TWO_GIGABYTES: usize = 2 * 1024 * 1024 * 1024; + + let mut decompressor = Decompressor::with_prepared_dictionary(dictionary)?; + let mut max_size = self.0.len() * 4; + let size = loop { + buffer.resize(max_size, 0); + match decompressor.decompress_to_buffer(self.0, &mut buffer[..max_size]) { + Ok(size) => break size, + // TODO don't do that !!! But what should I do? + Err(e) if e.kind() == ErrorKind::Other && max_size <= TWO_GIGABYTES => { + max_size *= 2 + } + Err(e) => return Err(e), + } + }; + Ok(KvReaderU16::from_slice(&buffer[..size])) + } + + /// Returns the KvReader like it is not compressed. + /// Happends when there is no dictionary yet. + pub fn as_non_compressed(&self) -> &'a KvReaderU16 { + KvReaderU16::from_slice(self.0) + } + + /// Decompresses this KvReader if necessary. + pub fn decompress_with_optional_dictionary<'b>( + &self, + buffer: &'b mut Vec, + dictionary: Option<&DecoderDictionary>, + ) -> io::Result<&'b KvReaderU16> + where + 'a: 'b, + { + match dictionary { + Some(dict) => self.decompress_with(buffer, dict), + None => Ok(self.as_non_compressed()), + } + } + + pub fn decompress_as_owned_with_optinal_dictionary( + &self, + dictionary: Option<&DecoderDictionary>, + ) -> io::Result> { + todo!("Impl owned version of KvReader") + } +} + +pub struct CompressedKvWriterU16(Vec); + +impl CompressedKvWriterU16 { + // TODO ask for a KvReaderU16 here + pub fn new_with_dictionary(input: &[u8], dictionary: &EncoderDictionary) -> io::Result { + let mut compressor = Compressor::with_prepared_dictionary(dictionary)?; + compressor.compress(input).map(CompressedKvWriterU16) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} diff --git a/crates/milli/src/heed_codec/mod.rs b/crates/milli/src/heed_codec/mod.rs index 575b886bd..3ce9306dd 100644 --- a/crates/milli/src/heed_codec/mod.rs +++ b/crates/milli/src/heed_codec/mod.rs @@ -1,6 +1,7 @@ mod beu16_str_codec; mod beu32_str_codec; mod byte_slice_ref; +mod compressed_obkv_codec; pub mod facet; mod field_id_word_count_codec; mod fst_set_codec; @@ -18,6 +19,9 @@ use thiserror::Error; pub use self::beu16_str_codec::BEU16StrCodec; pub use self::beu32_str_codec::BEU32StrCodec; +pub use self::compressed_obkv_codec::{ + CompressedKvReaderU16, CompressedKvWriterU16, CompressedObkvCodec, +}; pub use self::field_id_word_count_codec::FieldIdWordCountCodec; pub use self::fst_set_codec::FstSetCodec; pub use self::obkv_codec::ObkvCodec; diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 9829df2ee..7e8b30a34 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -9,6 +9,7 @@ use heed::{CompactionOption, Database, RoTxn, RwTxn, Unspecified}; use roaring::RoaringBitmap; use rstar::RTree; use serde::{Deserialize, Serialize}; +use zstd::dict::{DecoderDictionary, EncoderDictionary}; use crate::constants::RESERVED_VECTORS_FIELD_NAME; use crate::documents::PrimaryKey; @@ -18,14 +19,17 @@ use crate::heed_codec::facet::{ FacetGroupKeyCodec, FacetGroupValueCodec, FieldDocIdFacetF64Codec, FieldDocIdFacetStringCodec, FieldIdCodec, OrderedF64Codec, }; -use crate::heed_codec::{BEU16StrCodec, FstSetCodec, StrBEU16Codec, StrRefCodec}; +use crate::heed_codec::{ + BEU16StrCodec, CompressedKvReaderU16, CompressedObkvCodec, FstSetCodec, StrBEU16Codec, + StrRefCodec, +}; use crate::order_by_map::OrderByMap; use crate::proximity::ProximityPrecision; use crate::vector::{ArroyWrapper, Embedding, EmbeddingConfig}; use crate::{ default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds, FacetDistribution, FieldDistribution, FieldId, FieldIdMapMissingEntry, FieldIdWordCountCodec, - FieldidsWeightsMap, GeoPoint, LocalizedAttributesRule, ObkvCodec, Result, RoaringBitmapCodec, + FieldidsWeightsMap, GeoPoint, LocalizedAttributesRule, Result, RoaringBitmapCodec, RoaringBitmapLenCodec, Search, U8StrStrCodec, Weight, BEU16, BEU32, BEU64, }; @@ -69,6 +73,7 @@ pub mod main_key { pub const PROXIMITY_PRECISION: &str = "proximity-precision"; pub const EMBEDDING_CONFIGS: &str = "embedding_configs"; pub const SEARCH_CUTOFF: &str = "search_cutoff"; + pub const DOCUMENT_COMPRESSION_DICTIONARY: &str = "document-compression-dictionary"; pub const LOCALIZED_ATTRIBUTES_RULES: &str = "localized_attributes_rules"; pub const FACET_SEARCH: &str = "facet_search"; pub const PREFIX_SEARCH: &str = "prefix_search"; @@ -167,7 +172,7 @@ pub struct Index { pub vector_arroy: arroy::Database, /// Maps the document id to the document as an obkv store. - pub(crate) documents: Database, + pub(crate) documents: Database, } impl Index { @@ -331,6 +336,50 @@ impl Index { self.env.prepare_for_closing() } + /* document compression dictionary */ + + /// Writes the dictionnary that will further be used to compress the documents. + pub fn put_document_compression_dictionary( + &self, + wtxn: &mut RwTxn, + dictionary: &[u8], + ) -> heed::Result<()> { + self.main.remap_types::().put( + wtxn, + main_key::DOCUMENT_COMPRESSION_DICTIONARY, + dictionary, + ) + } + + /// Deletes the document compression dictionary. + pub fn delete_document_compression_dictionary(&self, wtxn: &mut RwTxn) -> heed::Result { + self.main.remap_key_type::().delete(wtxn, main_key::DOCUMENT_COMPRESSION_DICTIONARY) + } + + /// Returns the optional raw bytes dictionary to be used when reading or writing the OBKV documents. + pub fn document_compression_raw_dictionary<'t>( + &self, + rtxn: &'t RoTxn, + ) -> heed::Result> { + self.main.remap_types::().get(rtxn, main_key::DOCUMENT_COMPRESSION_DICTIONARY) + } + + pub fn document_decompression_dictionary<'t>( + &self, + rtxn: &'t RoTxn, + ) -> heed::Result>> { + self.document_compression_raw_dictionary(rtxn).map(|opt| opt.map(DecoderDictionary::new)) + } + + pub fn document_compression_dictionary( + &self, + rtxn: &RoTxn, + ) -> heed::Result>> { + const COMPRESSION_LEVEL: i32 = 19; + self.document_compression_raw_dictionary(rtxn) + .map(|opt| opt.map(|bytes| EncoderDictionary::copy(bytes, COMPRESSION_LEVEL))) + } + /* documents ids */ /// Writes the documents ids that corresponds to the user-ids-documents-ids FST. @@ -1258,43 +1307,43 @@ impl Index { /* documents */ /// Returns a document by using the document id. - pub fn document<'t>(&self, rtxn: &'t RoTxn, id: DocumentId) -> Result<&'t obkv::KvReaderU16> { + pub fn compressed_document<'t>( + &self, + rtxn: &'t RoTxn, + id: DocumentId, + ) -> Result> { self.documents .get(rtxn, &id)? .ok_or(UserError::UnknownInternalDocumentId { document_id: id }) .map_err(Into::into) } - /// Returns an iterator over the requested documents. The next item will be an error if a document is missing. - pub fn iter_documents<'a, 't: 'a>( + /// Returns an iterator over the requested compressed documents. The next item will be an error if a document is missing. + pub fn iter_compressed_documents<'a, 't: 'a>( &'a self, rtxn: &'t RoTxn<'t>, ids: impl IntoIterator + 'a, - ) -> Result> + 'a> { - Ok(ids.into_iter().map(move |id| { - let kv = self - .documents - .get(rtxn, &id)? - .ok_or(UserError::UnknownInternalDocumentId { document_id: id })?; - Ok((id, kv)) - })) + ) -> Result)>> + 'a> { + Ok(ids + .into_iter() + .map(move |id| self.compressed_document(rtxn, id).map(|compressed| (id, compressed)))) } /// Returns a [`Vec`] of the requested documents. Returns an error if a document is missing. - pub fn documents<'t>( + pub fn compressed_documents<'t>( &self, rtxn: &'t RoTxn<'t>, ids: impl IntoIterator, - ) -> Result> { - self.iter_documents(rtxn, ids)?.collect() + ) -> Result)>> { + self.iter_compressed_documents(rtxn, ids)?.collect() } /// Returns an iterator over all the documents in the index. - pub fn all_documents<'a, 't: 'a>( + pub fn all_compressed_documents<'a, 't: 'a>( &'a self, rtxn: &'t RoTxn<'t>, - ) -> Result> + 'a> { - self.iter_documents(rtxn, self.documents_ids(rtxn)?) + ) -> Result)>> + 'a> { + self.iter_compressed_documents(rtxn, self.documents_ids(rtxn)?) } pub fn external_id_of<'a, 't: 'a>( @@ -1315,9 +1364,14 @@ impl Index { process: "external_id_of", }) })?; - Ok(self.iter_documents(rtxn, ids)?.map(move |entry| -> Result<_> { - let (_docid, obkv) = entry?; - match primary_key.document_id(obkv, &fields)? { + let dictionary = + self.document_compression_raw_dictionary(rtxn)?.map(DecoderDictionary::copy); + let mut buffer = Vec::new(); + Ok(self.iter_compressed_documents(rtxn, ids)?.map(move |entry| -> Result<_> { + let (_docid, compressed_obkv) = entry?; + let obkv = compressed_obkv + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())?; + match primary_key.document_id(&obkv, &fields)? { Ok(document_id) => Ok(document_id), Err(_) => Err(InternalError::DocumentsError( crate::documents::Error::InvalidDocumentFormat, @@ -2625,7 +2679,12 @@ pub(crate) mod tests { "###); let rtxn = index.read_txn().unwrap(); - let (_docid, obkv) = index.documents(&rtxn, [0]).unwrap()[0]; + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); + let (_docid, compressed_obkv) = index.compressed_documents(&rtxn, [0]).unwrap().remove(0); + let mut buffer = Vec::new(); + let obkv = compressed_obkv + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); let json = obkv_to_json(&[0, 1, 2], &index.fields_ids_map(&rtxn).unwrap(), obkv).unwrap(); insta::assert_debug_snapshot!(json, @r###" { @@ -2634,7 +2693,10 @@ pub(crate) mod tests { "###); // Furthermore, when we retrieve document 34, it is not the result of merging 35 with 34 - let (_docid, obkv) = index.documents(&rtxn, [2]).unwrap()[0]; + let (_docid, compressed_obkv) = index.compressed_documents(&rtxn, [2]).unwrap().remove(0); + let obkv = compressed_obkv + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); let json = obkv_to_json(&[0, 1, 2], &index.fields_ids_map(&rtxn).unwrap(), obkv).unwrap(); insta::assert_debug_snapshot!(json, @r###" { @@ -2643,6 +2705,7 @@ pub(crate) mod tests { } "###); + drop(dictionary); drop(rtxn); // Add new documents again @@ -2841,11 +2904,16 @@ pub(crate) mod tests { } = search.execute().unwrap(); let primary_key_id = index.fields_ids_map(&rtxn).unwrap().id("primary_key").unwrap(); documents_ids.sort_unstable(); - let docs = index.documents(&rtxn, documents_ids).unwrap(); + let compressed_docs = index.compressed_documents(&rtxn, documents_ids).unwrap(); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); + let mut buffer = Vec::new(); let mut all_ids = HashSet::new(); - for (_docid, obkv) in docs { - let id = obkv.get(primary_key_id).unwrap(); - assert!(all_ids.insert(id)); + for (_docid, compressed) in compressed_docs { + let doc = compressed + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); + let id = doc.get(primary_key_id).unwrap(); + assert!(all_ids.insert(id.to_vec())); } } diff --git a/crates/milli/src/lib.rs b/crates/milli/src/lib.rs index ea88d2b78..9bc8786b0 100644 --- a/crates/milli/src/lib.rs +++ b/crates/milli/src/lib.rs @@ -48,7 +48,7 @@ pub use search::new::{ }; use serde_json::Value; pub use thread_pool_no_abort::{PanicCatched, ThreadPoolNoAbort, ThreadPoolNoAbortBuilder}; -pub use {charabia as tokenizer, heed, rhai}; +pub use {charabia as tokenizer, heed, rhai, zstd}; pub use self::asc_desc::{AscDesc, AscDescError, Member, SortError}; pub use self::criterion::{default_criteria, Criterion, CriterionError}; diff --git a/crates/milli/src/search/new/tests/mod.rs b/crates/milli/src/search/new/tests/mod.rs index 37bca7597..484418af9 100644 --- a/crates/milli/src/search/new/tests/mod.rs +++ b/crates/milli/src/search/new/tests/mod.rs @@ -25,8 +25,13 @@ fn collect_field_values( ) -> Vec { let mut values = vec![]; let fid = index.fields_ids_map(txn).unwrap().id(fid).unwrap(); - for doc in index.documents(txn, docids.iter().copied()).unwrap() { - if let Some(v) = doc.1.get(fid) { + let mut buffer = Vec::new(); + let dictionary = index.document_decompression_dictionary(txn).unwrap(); + for (_id, compressed_doc) in index.compressed_documents(txn, docids.iter().copied()).unwrap() { + let doc = compressed_doc + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); + if let Some(v) = doc.get(fid) { let v: serde_json::Value = serde_json::from_slice(v).unwrap(); let v = v.to_string(); values.push(v); diff --git a/crates/milli/src/snapshot_tests.rs b/crates/milli/src/snapshot_tests.rs index 6635ab2f4..a01ad1b1d 100644 --- a/crates/milli/src/snapshot_tests.rs +++ b/crates/milli/src/snapshot_tests.rs @@ -407,9 +407,15 @@ pub fn snap_documents(index: &Index) -> String { let rtxn = index.read_txn().unwrap(); let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); let display = fields_ids_map.ids().collect::>(); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); + let mut buffer = Vec::new(); - for document in index.all_documents(&rtxn).unwrap() { - let doc = obkv_to_json(&display, &fields_ids_map, document.unwrap().1).unwrap(); + for result in index.all_compressed_documents(&rtxn).unwrap() { + let (_id, compressed_document) = result.unwrap(); + let document = compressed_document + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); + let doc = obkv_to_json(&display, &fields_ids_map, document).unwrap(); snap.push_str(&serde_json::to_string(&doc).unwrap()); snap.push('\n'); } diff --git a/crates/milli/src/update/clear_documents.rs b/crates/milli/src/update/clear_documents.rs index b0ae070de..885dc3fb6 100644 --- a/crates/milli/src/update/clear_documents.rs +++ b/crates/milli/src/update/clear_documents.rs @@ -62,6 +62,7 @@ impl<'t, 'i> ClearDocuments<'t, 'i> { self.index.put_field_distribution(self.wtxn, &FieldDistribution::default())?; self.index.delete_geo_rtree(self.wtxn)?; self.index.delete_geo_faceted_documents_ids(self.wtxn)?; + self.index.delete_document_compression_dictionary(self.wtxn)?; // Remove all user-provided bits from the configs let mut configs = self.index.embedding_configs(self.wtxn)?; diff --git a/crates/milli/src/update/index_documents/mod.rs b/crates/milli/src/update/index_documents/mod.rs index 154db7875..2c736878d 100644 --- a/crates/milli/src/update/index_documents/mod.rs +++ b/crates/milli/src/update/index_documents/mod.rs @@ -4,8 +4,8 @@ mod helpers; mod transform; mod typed_chunk; -use std::collections::HashSet; -use std::io::{Read, Seek}; +use std::collections::{HashMap, HashSet}; +use std::io::{BufWriter, Read, Seek, Write}; use std::iter; use std::num::NonZeroU32; use std::sync::Arc; @@ -13,9 +13,8 @@ use std::sync::Arc; use crossbeam_channel::{Receiver, Sender}; use enrich::enrich_documents_batch; use grenad::{Merger, MergerBuilder}; -use hashbrown::HashMap; -use heed::types::Str; -use heed::Database; +use heed::types::{Bytes, Str}; +use heed::{Database, PutFlags}; use rand::SeedableRng as _; use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; @@ -28,7 +27,8 @@ pub use self::helpers::*; pub use self::transform::{Transform, TransformOutput}; use super::new::StdResult; use crate::documents::{obkv_to_object, DocumentsBatchReader}; -use crate::error::{Error, InternalError}; +use crate::error::{Error, InternalError, UserError}; +use crate::heed_codec::{CompressedKvWriterU16, CompressedObkvCodec}; use crate::index::{PrefixSearch, PrefixSettings}; use crate::thread_pool_no_abort::ThreadPoolNoAbortBuilder; pub use crate::update::index_documents::helpers::CursorClonableMmap; @@ -36,7 +36,7 @@ use crate::update::{ IndexerConfig, UpdateIndexingStep, WordPrefixDocids, WordPrefixIntegerDocids, WordsPrefixesFst, }; use crate::vector::{ArroyWrapper, EmbeddingConfigs}; -use crate::{CboRoaringBitmapCodec, Index, Result, UserError}; +use crate::{CboRoaringBitmapCodec, Index, Result, BEU32}; static MERGED_DATABASE_COUNT: usize = 7; static PREFIX_DATABASE_COUNT: usize = 4; @@ -201,7 +201,7 @@ where target = "indexing::details", name = "index_documents_raw" )] - pub fn execute_raw(self, output: TransformOutput) -> Result + pub fn execute_raw(mut self, output: TransformOutput) -> Result where FP: Fn(UpdateIndexingStep) + Sync, FA: Fn() -> bool + Sync, @@ -523,6 +523,10 @@ where word_fid_docids.map(MergerBuilder::build), )?; + // This call contains an internal condition to ensure we do not always + // generate compression dictionaries and always compress documents. + self.manage_compression_dictionary()?; + Ok(number_of_documents) } @@ -533,7 +537,7 @@ where name = "index_documents_prefix_databases" )] pub fn execute_prefix_databases( - self, + &mut self, word_docids: Option>, exact_word_docids: Option>, word_position_docids: Option>, @@ -723,6 +727,64 @@ where Ok(()) } + + /// Computes a new dictionay and compress the documents with it in the database. + /// + /// Documents still need to be directly compressed when being written in the database and a dictionary exists. + #[tracing::instrument( + level = "trace", + skip_all, + target = "indexing::compression", + name = "compress_documents_database" + )] + pub fn manage_compression_dictionary(&mut self) -> Result<()> { + /// The size of the dictionary generated from a sample of the documents already + /// in the database. It will be used when compressing and decompressing documents. + const COMPRESSION_DICTIONARY_SIZE: usize = 64_000; + /// The minimum number of documents to trigger the generation of the compression dictionary. + const COMPRESSION_ON_NUMBER_OF_DOCUMENTS: usize = 10_000; + + if self.index.number_of_documents(self.wtxn)? < COMPRESSION_ON_NUMBER_OF_DOCUMENTS as u64 + || self.index.document_compression_dictionary(self.wtxn)?.is_some() + { + return Ok(()); + } + + let mut sample_file = tempfile::tempfile().map(BufWriter::new)?; + let mut sample_sizes = Vec::new(); + // TODO make this 1_000 be 10k and const + let documents = self.index.documents.remap_types::(); + for result in documents.iter(self.wtxn)?.take(COMPRESSION_ON_NUMBER_OF_DOCUMENTS) { + let (_id, bytes) = result?; + sample_file.write_all(bytes)?; + sample_sizes.push(bytes.len()); + } + + let sample_file = sample_file.into_inner().map_err(|ie| ie.into_error())?; + let sample_data = unsafe { memmap2::Mmap::map(&sample_file)? }; + let dictionary = + zstd::dict::from_continuous(&sample_data, &sample_sizes, COMPRESSION_DICTIONARY_SIZE)?; + self.index.put_document_compression_dictionary(self.wtxn, &dictionary)?; + // safety: We just set the dictionary above. It must be there when we get it back. + let dictionary = self.index.document_compression_dictionary(self.wtxn)?.unwrap(); + + let mut iter = self.index.documents.iter_mut(self.wtxn)?; + while let Some(result) = iter.next() { + let (docid, document) = result?; + let document = document.as_non_compressed().as_bytes(); + let compressed = CompressedKvWriterU16::new_with_dictionary(document, &dictionary)?; + // safety: the compressed document is entirely owned + unsafe { + iter.put_current_with_options::( + PutFlags::empty(), + &docid, + &compressed, + )?; + } + } + + Ok(()) + } } /// Run the word prefix docids update operation. @@ -814,7 +876,7 @@ mod tests { let rtxn = index.read_txn().unwrap(); let count = index.number_of_documents(&rtxn).unwrap(); assert_eq!(count, 3); - let count = index.all_documents(&rtxn).unwrap().count(); + let count = index.all_compressed_documents(&rtxn).unwrap().count(); assert_eq!(count, 3); drop(rtxn); @@ -823,6 +885,7 @@ mod tests { #[test] fn simple_document_merge() { let mut index = TempIndex::new(); + let mut buffer = Vec::new(); index.index_documents_config.update_method = IndexDocumentsMethod::UpdateDocuments; // First we send 3 documents with duplicate ids and @@ -841,16 +904,21 @@ mod tests { assert_eq!(count, 1); // Check that we get only one document from the database. - let docs = index.documents(&rtxn, Some(0)).unwrap(); - assert_eq!(docs.len(), 1); - let (id, doc) = docs[0]; + let mut compressed_docs = index.compressed_documents(&rtxn, Some(0)).unwrap(); + assert_eq!(compressed_docs.len(), 1); + let (id, compressed_doc) = compressed_docs.remove(0); assert_eq!(id, 0); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); + let doc = compressed_doc + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); // Check that this document is equal to the last one sent. let mut doc_iter = doc.iter(); assert_eq!(doc_iter.next(), Some((0, &b"1"[..]))); assert_eq!(doc_iter.next(), Some((1, &br#""benoit""#[..]))); assert_eq!(doc_iter.next(), None); + drop(dictionary); drop(rtxn); // Second we send 1 document with id 1, to force it to be merged with the previous one. @@ -862,10 +930,14 @@ mod tests { assert_eq!(count, 1); // Check that we get only one document from the database. - let docs = index.documents(&rtxn, Some(0)).unwrap(); - assert_eq!(docs.len(), 1); - let (id, doc) = docs[0]; + let mut compressed_docs = index.compressed_documents(&rtxn, Some(0)).unwrap(); + assert_eq!(compressed_docs.len(), 1); + let (id, compressed_doc) = compressed_docs.remove(0); assert_eq!(id, 0); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); + let doc = compressed_doc + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); // Check that this document is equal to the last one sent. let mut doc_iter = doc.iter(); @@ -873,6 +945,129 @@ mod tests { assert_eq!(doc_iter.next(), Some((1, &br#""benoit""#[..]))); assert_eq!(doc_iter.next(), Some((2, &b"25"[..]))); assert_eq!(doc_iter.next(), None); + drop(dictionary); + drop(rtxn); + } + + #[test] + fn not_auto_generated_documents_ids() { + let index = TempIndex::new(); + + let result = index.add_documents(documents!([ + { "name": "kevin" }, + { "name": "kevina" }, + { "name": "benoit" } + ])); + assert!(result.is_err()); + + // Check that there is no document. + let rtxn = index.read_txn().unwrap(); + let count = index.number_of_documents(&rtxn).unwrap(); + assert_eq!(count, 0); + drop(rtxn); + } + + #[test] + fn simple_auto_generated_documents_ids() { + let mut index = TempIndex::new(); + let mut buffer = Vec::new(); + index.index_documents_config.autogenerate_docids = true; + // First we send 3 documents with ids from 1 to 3. + index + .add_documents(documents!([ + { "name": "kevin" }, + { "name": "kevina" }, + { "name": "benoit" } + ])) + .unwrap(); + + // Check that there is 3 documents now. + let rtxn = index.read_txn().unwrap(); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); + let count = index.number_of_documents(&rtxn).unwrap(); + assert_eq!(count, 3); + + let compressed_docs = index.compressed_documents(&rtxn, vec![0, 1, 2]).unwrap(); + let (_id, compressed_obkv) = compressed_docs + .iter() + .find(|(_id, compressed_doc)| { + let doc = compressed_doc + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); + doc.get(0) == Some(br#""kevin""#) + }) + .unwrap(); + + let obkv = compressed_obkv + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); + let kevin_uuid: String = serde_json::from_slice(obkv.get(1).unwrap()).unwrap(); + drop(dictionary); + drop(rtxn); + + // Second we send 1 document with the generated uuid, to erase the previous ones. + index.add_documents(documents!([ { "name": "updated kevin", "id": kevin_uuid } ])).unwrap(); + + // Check that there is **always** 3 documents. + let rtxn = index.read_txn().unwrap(); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); + let count = index.number_of_documents(&rtxn).unwrap(); + assert_eq!(count, 3); + + // the document 0 has been deleted and reinserted with the id 3 + let mut compressed_docs = index.compressed_documents(&rtxn, vec![1, 2, 0]).unwrap(); + let kevin_position = compressed_docs + .iter() + .position(|(_, compressed_doc)| { + let doc = compressed_doc + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); + + doc.get(0).unwrap() == br#""updated kevin""# + }) + .unwrap(); + assert_eq!(kevin_position, 2); + let (_, compressed_doc) = compressed_docs.remove(kevin_position); + let doc = compressed_doc + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); + + // Check that this document is equal to the last + // one sent and that an UUID has been generated. + assert_eq!(doc.get(0), Some(&br#""updated kevin""#[..])); + // This is an UUID, it must be 36 bytes long plus the 2 surrounding string quotes ("). + assert_eq!(doc.get(1).unwrap().len(), 36 + 2); + drop(dictionary); + drop(rtxn); + } + + #[test] + fn reordered_auto_generated_documents_ids() { + let mut index = TempIndex::new(); + + // First we send 3 documents with ids from 1 to 3. + index + .add_documents(documents!([ + { "id": 1, "name": "kevin" }, + { "id": 2, "name": "kevina" }, + { "id": 3, "name": "benoit" } + ])) + .unwrap(); + + // Check that there is 3 documents now. + let rtxn = index.read_txn().unwrap(); + let count = index.number_of_documents(&rtxn).unwrap(); + assert_eq!(count, 3); + drop(rtxn); + + // Second we send 1 document without specifying the id. + index.index_documents_config.autogenerate_docids = true; + index.add_documents(documents!([ { "name": "new kevin" } ])).unwrap(); + + // Check that there is 4 documents now. + let rtxn = index.read_txn().unwrap(); + let count = index.number_of_documents(&rtxn).unwrap(); + assert_eq!(count, 4); drop(rtxn); } @@ -974,7 +1169,7 @@ mod tests { let rtxn = index.read_txn().unwrap(); let count = index.number_of_documents(&rtxn).unwrap(); assert_eq!(count, 6); - let count = index.all_documents(&rtxn).unwrap().count(); + let count = index.all_compressed_documents(&rtxn).unwrap().count(); assert_eq!(count, 6); db_snap!(index, word_docids, "updated"); @@ -1392,7 +1587,7 @@ mod tests { index.add_documents(documents!({ "a" : { "b" : { "c" : 1 }}})).unwrap(); let rtxn = index.read_txn().unwrap(); - let all_documents_count = index.all_documents(&rtxn).unwrap().count(); + let all_documents_count = index.all_compressed_documents(&rtxn).unwrap().count(); assert_eq!(all_documents_count, 1); let external_documents_ids = index.external_documents_ids(); assert!(external_documents_ids.get(&rtxn, "1").unwrap().is_some()); @@ -2844,7 +3039,7 @@ mod tests { // Ensuring all the returned IDs actually exists let rtxn = index.read_txn().unwrap(); let res = index.search(&rtxn).execute().unwrap(); - index.documents(&rtxn, res.documents_ids).unwrap(); + index.compressed_documents(&rtxn, res.documents_ids).unwrap(); } fn delete_documents<'t>( @@ -3223,7 +3418,7 @@ mod tests { let rtxn = index.read_txn().unwrap(); // list all documents - let results = index.all_documents(&rtxn).unwrap(); + let results = index.all_compressed_documents(&rtxn).unwrap(); for result in results { let (id, _) = result.unwrap(); assert!( diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index d87524a34..d664110cf 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -174,10 +174,12 @@ impl<'a, 'i> Transform<'a, 'i> { let external_documents_ids = self.index.external_documents_ids(); let mapping = create_fields_mapping(&mut self.fields_ids_map, &fields_index)?; + let dictionary = self.index.document_decompression_dictionary(wtxn)?; let primary_key = cursor.primary_key().to_string(); let primary_key_id = self.fields_ids_map.insert(&primary_key).ok_or(UserError::AttributeLimitReached)?; + let mut decompression_buffer = Vec::new(); let mut obkv_buffer = Vec::new(); let mut document_sorter_value_buffer = Vec::new(); let mut document_sorter_key_buffer = Vec::new(); @@ -253,18 +255,17 @@ impl<'a, 'i> Transform<'a, 'i> { let mut skip_insertion = false; if let Some(original_docid) = original_docid { let original_key = original_docid; - let base_obkv = self - .index - .documents - .remap_data_type::() - .get(wtxn, &original_key)? - .ok_or(InternalError::DatabaseMissingEntry { - db_name: db_name::DOCUMENTS, - key: None, - })?; + let base_compressed_obkv = self.index.documents.get(wtxn, &original_key)?.ok_or( + InternalError::DatabaseMissingEntry { db_name: db_name::DOCUMENTS, key: None }, + )?; + + let base_obkv = base_compressed_obkv.decompress_with_optional_dictionary( + &mut decompression_buffer, + dictionary.as_ref(), + )?; // we check if the two documents are exactly equal. If it's the case we can skip this document entirely - if base_obkv == obkv_buffer { + if base_obkv.as_bytes() == obkv_buffer { // we're not replacing anything self.replaced_documents_ids.remove(original_docid); // and we need to put back the original id as it was before @@ -284,13 +285,12 @@ impl<'a, 'i> Transform<'a, 'i> { document_sorter_value_buffer.clear(); document_sorter_value_buffer.push(Operation::Addition as u8); into_del_add_obkv( - KvReaderU16::from_slice(base_obkv), + base_obkv, deladd_operation, &mut document_sorter_value_buffer, )?; self.original_sorter .insert(&document_sorter_key_buffer, &document_sorter_value_buffer)?; - let base_obkv = KvReader::from_slice(base_obkv); if let Some(flattened_obkv) = Self::flatten_from_fields_ids_map(base_obkv, &mut self.fields_ids_map)? { @@ -354,9 +354,12 @@ impl<'a, 'i> Transform<'a, 'i> { documents_seen: documents_count, }); + drop(dictionary); + self.index.put_fields_ids_map(wtxn, &self.fields_ids_map)?; self.index.put_primary_key(wtxn, &primary_key)?; self.documents_count += documents_count; + // Now that we have a valid sorter that contains the user id and the obkv we // give it to the last transforming function which returns the TransformOutput. Ok(documents_count) @@ -857,15 +860,21 @@ impl<'a, 'i> Transform<'a, 'i> { if original_sorter.is_some() || flattened_sorter.is_some() { let modified_faceted_fields = settings_diff.modified_faceted_fields(); + let dictionary = self.index.document_decompression_dictionary(wtxn)?; + let mut original_obkv_buffer = Vec::new(); let mut flattened_obkv_buffer = Vec::new(); let mut document_sorter_key_buffer = Vec::new(); + let mut buffer = Vec::new(); for result in self.index.external_documents_ids().iter(wtxn)? { let (external_id, docid) = result?; - let old_obkv = self.index.documents.get(wtxn, &docid)?.ok_or( + let old_compressed_obkv = self.index.documents.get(wtxn, &docid)?.ok_or( InternalError::DatabaseMissingEntry { db_name: db_name::DOCUMENTS, key: None }, )?; + let old_obkv = old_compressed_obkv + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())?; + let injected_vectors: std::result::Result< serde_json::Map, arroy::Error, diff --git a/crates/milli/src/update/index_documents/typed_chunk.rs b/crates/milli/src/update/index_documents/typed_chunk.rs index d5c250e2d..d10e14692 100644 --- a/crates/milli/src/update/index_documents/typed_chunk.rs +++ b/crates/milli/src/update/index_documents/typed_chunk.rs @@ -17,6 +17,7 @@ use super::helpers::{ }; use crate::external_documents_ids::{DocumentOperation, DocumentOperationKind}; use crate::facet::FacetType; +use crate::heed_codec::CompressedKvWriterU16; use crate::index::db_name::DOCUMENTS; use crate::index::IndexEmbeddingConfig; use crate::proximity::MAX_DISTANCE; @@ -158,6 +159,7 @@ pub(crate) fn write_typed_chunk_into_index( .into_iter() .map(|IndexEmbeddingConfig { name, .. }| name) .collect(); + let dictionary = index.document_compression_dictionary(wtxn)?; let mut vectors_buffer = Vec::new(); while let Some((key, reader)) = iter.next()? { let mut writer: KvWriter<_, FieldId> = KvWriter::memory(); @@ -207,7 +209,17 @@ pub(crate) fn write_typed_chunk_into_index( let db = index.documents.remap_data_type::(); if !writer.is_empty() { - db.put(wtxn, &docid, &writer.into_inner().unwrap())?; + let uncompressed_document_bytes = writer.into_inner().unwrap(); + match dictionary.as_ref() { + Some(dictionary) => { + let compressed = CompressedKvWriterU16::new_with_dictionary( + &uncompressed_document_bytes, + dictionary, + )?; + db.put(wtxn, &docid, compressed.as_bytes())? + } + None => db.put(wtxn, &docid, &uncompressed_document_bytes)?, + } operations.push(DocumentOperation { external_id: external_id.to_string(), internal_id: docid, diff --git a/crates/milli/src/update/new/document.rs b/crates/milli/src/update/new/document.rs index ffcf93312..a18d80a33 100644 --- a/crates/milli/src/update/new/document.rs +++ b/crates/milli/src/update/new/document.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; use bumparaw_collections::RawMap; @@ -47,23 +48,15 @@ pub trait Document<'doc> { fn geo_field(&self) -> Result>; } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct DocumentFromDb<'t, Mapper: FieldIdMapper> where Mapper: FieldIdMapper, { fields_ids_map: &'t Mapper, - content: &'t KvReaderFieldId, + content: Cow<'t, KvReaderFieldId>, } -impl<'t, Mapper: FieldIdMapper> Clone for DocumentFromDb<'t, Mapper> { - #[inline] - fn clone(&self) -> Self { - *self - } -} -impl<'t, Mapper: FieldIdMapper> Copy for DocumentFromDb<'t, Mapper> {} - impl<'t, Mapper: FieldIdMapper> Document<'t> for DocumentFromDb<'t, Mapper> { fn iter_top_level_fields(&self) -> impl Iterator> { let mut it = self.content.iter(); diff --git a/crates/milli/src/update/new/extract/documents.rs b/crates/milli/src/update/new/extract/documents.rs index 01041af42..9427f135e 100644 --- a/crates/milli/src/update/new/extract/documents.rs +++ b/crates/milli/src/update/new/extract/documents.rs @@ -54,6 +54,8 @@ impl<'a, 'b, 'extractor> Extractor<'extractor> for DocumentsExtractor<'a, 'b> { let external_docid = change.external_docid().to_owned(); + todo!("manage documents compression"); + // document but we need to create a function that collects and compresses documents. match change { DocumentChange::Deletion(deletion) => { diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 85259c2d0..d73d135b3 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -1944,6 +1944,8 @@ mod tests { // Check that the searchable field is correctly set to "name" only. let rtxn = index.read_txn().unwrap(); + let mut buffer = Vec::new(); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); // When we search for something that is not in // the searchable fields it must not return any document. let result = index.search(&rtxn).query("23").execute().unwrap(); @@ -1952,10 +1954,17 @@ mod tests { // When we search for something that is in the searchable fields // we must find the appropriate document. let result = index.search(&rtxn).query(r#""kevin""#).execute().unwrap(); - let documents = index.documents(&rtxn, result.documents_ids).unwrap(); + let mut compressed_documents = + index.compressed_documents(&rtxn, result.documents_ids).unwrap(); let fid_map = index.fields_ids_map(&rtxn).unwrap(); - assert_eq!(documents.len(), 1); - assert_eq!(documents[0].1.get(fid_map.id("name").unwrap()), Some(&br#""kevin""#[..])); + assert_eq!(compressed_documents.len(), 1); + let (_id, compressed_document) = compressed_documents.remove(0); + let document = compressed_document + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); + + assert_eq!(document.get(fid_map.id("name").unwrap()), Some(&br#""kevin""#[..])); + drop(dictionary); drop(rtxn); // We change the searchable fields to be the "name" field only. @@ -1980,6 +1989,7 @@ mod tests { // Check that the searchable field have been reset and documents are found now. let rtxn = index.read_txn().unwrap(); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); let fid_map = index.fields_ids_map(&rtxn).unwrap(); let user_defined_searchable_fields = index.user_defined_searchable_fields(&rtxn).unwrap(); snapshot!(format!("{user_defined_searchable_fields:?}"), @"None"); @@ -1988,8 +1998,13 @@ mod tests { snapshot!(format!("{searchable_fields:?}"), @r###"["id", "name", "age"]"###); let result = index.search(&rtxn).query("23").execute().unwrap(); assert_eq!(result.documents_ids.len(), 1); - let documents = index.documents(&rtxn, result.documents_ids).unwrap(); - assert_eq!(documents[0].1.get(fid_map.id("name").unwrap()), Some(&br#""kevin""#[..])); + let mut compressed_documents = + index.compressed_documents(&rtxn, result.documents_ids).unwrap(); + let (_id, compressed_document) = compressed_documents.remove(0); + let document = compressed_document + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); + assert_eq!(document.get(fid_map.id("name").unwrap()), Some(&br#""kevin""#[..])); } #[test] @@ -2120,15 +2135,20 @@ mod tests { // Check that the displayed fields are correctly set. let rtxn = index.read_txn().unwrap(); + let mut buffer = Vec::new(); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); let fields_ids = index.filterable_fields(&rtxn).unwrap(); assert_eq!(fields_ids, hashset! { S("age") }); // Only count the field_id 0 and level 0 facet values. // TODO we must support typed CSVs for numbers to be understood. let fidmap = index.fields_ids_map(&rtxn).unwrap(); - for document in index.all_documents(&rtxn).unwrap() { - let document = document.unwrap(); - let json = crate::obkv_to_json(&fidmap.ids().collect::>(), &fidmap, document.1) + for result in index.all_compressed_documents(&rtxn).unwrap() { + let (_id, compressed_document) = result.unwrap(); + let document = compressed_document + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) .unwrap(); + let json = + crate::obkv_to_json(&fidmap.ids().collect::>(), &fidmap, document).unwrap(); println!("json: {:?}", json); } let count = index @@ -2139,6 +2159,7 @@ mod tests { .unwrap() .count(); assert_eq!(count, 3); + drop(dictionary); drop(rtxn); // Index a little more documents with new and current facets values. @@ -2228,6 +2249,7 @@ mod tests { #[test] fn set_asc_desc_field() { let index = TempIndex::new(); + let mut buffer = Vec::new(); // Set the filterable fields to be the age. index @@ -2248,12 +2270,16 @@ mod tests { // Run an empty query just to ensure that the search results are ordered. let rtxn = index.read_txn().unwrap(); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); let SearchResult { documents_ids, .. } = index.search(&rtxn).execute().unwrap(); - let documents = index.documents(&rtxn, documents_ids).unwrap(); + let compressed_documents = index.compressed_documents(&rtxn, documents_ids).unwrap(); // Fetch the documents "age" field in the ordre in which the documents appear. let age_field_id = index.fields_ids_map(&rtxn).unwrap().id("age").unwrap(); - let iter = documents.into_iter().map(|(_, doc)| { + let iter = compressed_documents.into_iter().map(|(_, compressed_doc)| { + let doc = compressed_doc + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); let bytes = doc.get(age_field_id).unwrap(); let string = std::str::from_utf8(bytes).unwrap(); string.parse::().unwrap() @@ -2645,6 +2671,7 @@ mod tests { #[test] fn setting_impact_relevancy() { let index = TempIndex::new(); + let mut buffer = Vec::new(); // Set the genres setting index @@ -2676,8 +2703,12 @@ mod tests { let rtxn = index.read_txn().unwrap(); let SearchResult { documents_ids, .. } = index.search(&rtxn).query("S").execute().unwrap(); let first_id = documents_ids[0]; - let documents = index.documents(&rtxn, documents_ids).unwrap(); - let (_, content) = documents.iter().find(|(id, _)| *id == first_id).unwrap(); + let documents = index.compressed_documents(&rtxn, documents_ids).unwrap(); + let (_, compressed_content) = documents.iter().find(|(id, _)| *id == first_id).unwrap(); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); + let content = compressed_content + .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref()) + .unwrap(); let fid = index.fields_ids_map(&rtxn).unwrap().id("title").unwrap(); let line = std::str::from_utf8(content.get(fid).unwrap()).unwrap(); @@ -2851,7 +2882,7 @@ mod tests { wtxn.commit().unwrap(); let rtxn = index.write_txn().unwrap(); - let docs: StdResult, _> = index.all_documents(&rtxn).unwrap().collect(); + let docs: StdResult, _> = index.all_compressed_documents(&rtxn).unwrap().collect(); let docs = docs.unwrap(); assert_eq!(docs.len(), 5); } diff --git a/crates/milli/tests/search/query_criteria.rs b/crates/milli/tests/search/query_criteria.rs index 304059915..f3434fdc6 100644 --- a/crates/milli/tests/search/query_criteria.rs +++ b/crates/milli/tests/search/query_criteria.rs @@ -349,7 +349,20 @@ fn criteria_ascdesc() { wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); - let documents = index.all_documents(&rtxn).unwrap().map(|doc| doc.unwrap()).collect::>(); + let dictionary = index.document_decompression_dictionary(&rtxn).unwrap(); + let mut buffers = vec![Vec::new(); index.number_of_documents(&rtxn).unwrap() as usize]; + let documents = index + .all_compressed_documents(&rtxn) + .unwrap() + .zip(buffers.iter_mut()) + .map(|(compressed, buffer)| { + let (id, compressed) = compressed.unwrap(); + let doc = compressed + .decompress_with_optional_dictionary(buffer, dictionary.as_ref()) + .unwrap(); + (id, doc) + }) + .collect::>(); for criterion in [Asc(S("name")), Desc(S("name")), Asc(S("age")), Desc(S("age"))] { eprintln!("Testing with criterion: {:?}", &criterion); diff --git a/milli/examples/search.rs b/milli/examples/search.rs new file mode 100644 index 000000000..e69de29bb