From c573261ac42cb48079ed1281d9597d3c867183f2 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 31 Oct 2023 13:04:32 +0100 Subject: [PATCH] WIP: start updating the zookeeper client => leader election is missing --- Cargo.lock | 491 +++++++++++++++++----------------- index-scheduler/Cargo.toml | 2 +- index-scheduler/src/lib.rs | 169 ++++++------ meilisearch-auth/Cargo.toml | 2 +- meilisearch-auth/src/error.rs | 2 +- meilisearch-auth/src/lib.rs | 43 ++- meilisearch/Cargo.toml | 2 +- 7 files changed, 351 insertions(+), 360 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85d92244d..41766a857 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ "bitflags 1.3.2", - "bytes 1.4.0", + "bytes", "futures-core", "futures-sink", "memchr", @@ -49,7 +49,7 @@ dependencies = [ "base64 0.21.2", "bitflags 1.3.2", "brotli", - "bytes 1.4.0", + "bytes", "bytestring", "derive_more", "encoding_rs", @@ -119,9 +119,9 @@ dependencies = [ "actix-utils", "futures-core", "futures-util", - "mio 0.8.7", + "mio", "num_cpus", - "socket2", + "socket2 0.4.9", "tokio", "tracing", ] @@ -182,9 +182,9 @@ dependencies = [ "actix-utils", "actix-web-codegen", "ahash 0.7.6", - "bytes 1.4.0", + "bytes", "bytestring", - "cfg-if 1.0.0", + "cfg-if", "cookie", "derive_more", "encoding_rs", @@ -202,7 +202,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2", + "socket2 0.4.9", "time", "url", ] @@ -251,7 +251,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cipher", "cpufeatures", ] @@ -273,7 +273,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "getrandom", "once_cell", "version_check", @@ -303,6 +303,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "anes" version = "0.1.6" @@ -442,7 +448,7 @@ checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" dependencies = [ "addr2line", "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "miniz_oxide 0.6.2", "object", @@ -472,7 +478,7 @@ name = "benchmarks" version = "1.3.0" dependencies = [ "anyhow", - "bytes 1.4.0", + "bytes", "convert_case 0.6.0", "criterion", "csv", @@ -612,12 +618,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - [[package]] name = "bytes" version = "1.4.0" @@ -630,7 +630,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" dependencies = [ - "bytes 1.4.0", + "bytes", ] [[package]] @@ -670,6 +670,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "castaway" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.0.79" @@ -688,12 +697,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -803,7 +806,7 @@ version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.28", @@ -821,6 +824,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "compact_str" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b5c3ee2b4ffa00ac2b00d1645cd9229ade668139bccf95f15fadcf374127b" +dependencies = [ + "castaway", + "itoa", + "ryu", +] + [[package]] name = "concat-arrays" version = "0.1.2" @@ -844,6 +858,26 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "const_format" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a214c7af3d04997541b18d432afaff4c455e79e2029079647e72fc2bd27673" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f6ff08fd20f4f299298a28e2dfa8a8ba1036e6cd2460ac1de7b425d76f2500" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -909,7 +943,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -960,7 +994,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -974,7 +1008,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", ] @@ -984,7 +1018,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] @@ -996,7 +1030,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", "memoffset", "scopeguard", @@ -1008,7 +1042,7 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", ] @@ -1018,7 +1052,7 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -1194,7 +1228,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf36e65a80337bea855cd4ef9b8401ffce06a7baedf2e85ec467b1ac3f6e82b6" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "dirs-sys-next", ] @@ -1206,7 +1240,7 @@ checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", "redox_users", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1319,7 +1353,7 @@ version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -1439,7 +1473,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall 0.2.16", "windows-sys 0.48.0", @@ -1493,27 +1527,11 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags 1.3.2", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - [[package]] name = "futures" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" dependencies = [ "futures-channel", "futures-core", @@ -1526,9 +1544,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", "futures-sink", @@ -1536,15 +1554,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" dependencies = [ "futures-core", "futures-task", @@ -1553,15 +1571,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", @@ -1570,21 +1588,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -1642,7 +1660,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi", ] @@ -1701,7 +1719,7 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ - "bytes 1.4.0", + "bytes", "fnv", "futures-core", "futures-sink", @@ -1743,6 +1761,19 @@ name = "hashbrown" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +dependencies = [ + "ahash 0.8.3", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.0", +] [[package]] name = "heapless" @@ -1757,6 +1788,15 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.4.1" @@ -1833,7 +1873,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.4.0", + "bytes", "fnv", "itoa", ] @@ -1844,7 +1884,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.4.0", + "bytes", "http", "pin-project-lite", ] @@ -1873,7 +1913,7 @@ version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ - "bytes 1.4.0", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -1884,7 +1924,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -1920,6 +1960,12 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "ignore-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "665ff4dce8edd10d490641ccb78949832f1ddbff02c584fb1f85ab888fe0e50c" + [[package]] name = "index-scheduler" version = "1.3.0" @@ -1952,7 +1998,7 @@ dependencies = [ "time", "tokio", "uuid 1.4.1", - "zookeeper", + "zookeeper-client-sync", ] [[package]] @@ -2007,7 +2053,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -2036,15 +2082,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "ipnet" version = "2.7.2" @@ -2143,16 +2180,6 @@ dependencies = [ "simple_asn1", ] -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "language-tags" version = "0.3.2" @@ -2165,12 +2192,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "levenshtein_automata" version = "0.2.1" @@ -2182,9 +2203,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.147" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libgit2-sys" @@ -2555,7 +2576,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "digest", ] @@ -2591,7 +2612,7 @@ dependencies = [ "brotli", "bstr", "byte-unit", - "bytes 1.4.0", + "bytes", "cargo_toml", "clap", "crossbeam-channel", @@ -2664,7 +2685,7 @@ dependencies = [ "walkdir", "yaup", "zip", - "zookeeper", + "zookeeper-client-sync", ] [[package]] @@ -2685,7 +2706,7 @@ dependencies = [ "thiserror", "time", "uuid 1.4.1", - "zookeeper", + "zookeeper-client-sync", ] [[package]] @@ -2847,25 +2868,6 @@ dependencies = [ "adler", ] -[[package]] -name = "mio" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" -dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", - "libc", - "log", - "miow", - "net2", - "slab", - "winapi 0.2.8", -] - [[package]] name = "mio" version = "0.8.7" @@ -2878,46 +2880,11 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "mio-extras" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" -dependencies = [ - "lazycell", - "log", - "mio 0.6.23", - "slab", -] - -[[package]] -name = "miow" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", -] - [[package]] name = "nelson" version = "0.1.0" source = "git+https://github.com/meilisearch/nelson.git?rev=675f13885548fb415ead8fbb447e9e6d9314000a#675f13885548fb415ead8fbb447e9e6d9314000a" -[[package]] -name = "net2" -version = "0.2.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi 0.3.9", -] - [[package]] name = "nom" version = "7.1.3" @@ -2945,7 +2912,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2989,6 +2956,27 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "object" version = "0.30.3" @@ -3032,7 +3020,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3042,7 +3030,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3061,7 +3049,7 @@ version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall 0.2.16", "smallvec", @@ -3219,9 +3207,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -3284,6 +3272,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3336,7 +3334,7 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fnv", "lazy_static", "libc", @@ -3362,7 +3360,7 @@ dependencies = [ "anyhow", "bincode", "byteorder", - "cfg-if 1.0.0", + "cfg-if", "instant", "lz4_flex", "once_cell", @@ -3522,7 +3520,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ "base64 0.21.2", - "bytes 1.4.0", + "bytes", "encoding_rs", "futures-core", "futures-util", @@ -3572,7 +3570,7 @@ dependencies = [ "spin 0.5.2", "untrusted", "web-sys", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3850,7 +3848,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest", ] @@ -3861,7 +3859,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest", ] @@ -3872,7 +3870,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest", ] @@ -3952,12 +3950,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "snowflake" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27207bb65232eda1f588cf46db2fee75c0808d557f6b3cf19a75f5d6d7c94df1" - [[package]] name = "socket2" version = "0.4.9" @@ -3965,7 +3957,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys 0.48.0", ] [[package]] @@ -4027,6 +4029,28 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + [[package]] name = "subtle" version = "2.5.0" @@ -4082,13 +4106,13 @@ version = "0.28.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c2f3ca6693feb29a89724516f016488e9aafc7f37264f898593ee4b942f31b" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "core-foundation-sys", "libc", "ntapi", "once_cell", "rayon", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4117,7 +4141,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fastrand", "redox_syscall 0.3.5", "rustix 0.37.19", @@ -4135,18 +4159,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", @@ -4207,19 +4231,19 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.2" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ - "autocfg", - "bytes 1.4.0", + "backtrace", + "bytes", "libc", - "mio 0.8.7", + "mio", "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", ] @@ -4273,7 +4297,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ - "bytes 1.4.0", + "bytes", "futures-core", "futures-sink", "pin-project-lite", @@ -4327,7 +4351,7 @@ version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "log", "pin-project-lite", "tracing-core", @@ -4487,7 +4511,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f21b881cd6636ece9735721cf03c1fe1e774fe258683d084bb2812ab67435749" dependencies = [ "anyhow", - "cfg-if 1.0.0", + "cfg-if", "enum-iterator", "getset", "git2", @@ -4545,7 +4569,7 @@ version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] @@ -4570,7 +4594,7 @@ version = "0.4.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -4653,12 +4677,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -4669,12 +4687,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -4687,7 +4699,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4843,17 +4855,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", + "winapi", ] [[package]] @@ -4938,29 +4940,36 @@ dependencies = [ ] [[package]] -name = "zookeeper" -version = "0.8.0" +name = "zookeeper-client" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2312b424380193701a7341cec0551b80d2e3afd827ea0d3440af67899156ce10" +checksum = "d39d162869ce27d6b4b548f39c93db8de67860def306a818726bae1e87a0c942" dependencies = [ - "byteorder", - "bytes 0.5.6", - "lazy_static", + "bytes", + "compact_str", + "const_format", + "either", + "hashbrown 0.12.3", + "hashlink", + "ignore-result", "log", - "mio 0.6.23", - "mio-extras", - "snowflake", - "zookeeper_derive", + "num_enum", + "static_assertions", + "strum", + "thiserror", + "tokio", + "uuid 1.4.1", ] [[package]] -name = "zookeeper_derive" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42307291e3c8b2e4082e5647572da863f0470511d0ecb1618a4cd0a361549723" +name = "zookeeper-client-sync" +version = "0.1.0" dependencies = [ - "quote", - "syn 1.0.109", + "futures", + "once_cell", + "thiserror", + "tokio", + "zookeeper-client", ] [[package]] diff --git a/index-scheduler/Cargo.toml b/index-scheduler/Cargo.toml index ecd0e2323..4eef89a3b 100644 --- a/index-scheduler/Cargo.toml +++ b/index-scheduler/Cargo.toml @@ -32,7 +32,7 @@ thiserror = "1.0.40" time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] } uuid = { version = "1.3.1", features = ["serde", "v4"] } tokio = { version = "1.27.0", features = ["full"] } -zookeeper = "0.8.0" +zookeeper-client-sync = { path = "../../zookeeper-client-sync" } parking_lot = "0.12.1" strois = "0.0.4" diff --git a/index-scheduler/src/lib.rs b/index-scheduler/src/lib.rs index 265919ac5..4ec0b064c 100644 --- a/index-scheduler/src/lib.rs +++ b/index-scheduler/src/lib.rs @@ -63,9 +63,8 @@ use time::format_description::well_known::Rfc3339; use time::OffsetDateTime; use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, map_bound}; use uuid::Uuid; -use zookeeper::recipes::leader::LeaderLatch; -use zookeeper::{ - Acl, AddWatchMode, CreateMode, WatchedEvent, WatchedEventType, ZkError, ZooKeeper, +use zookeeper_client_sync::{ + Acls, AddWatchMode, CreateMode, Error as ZkError, EventType, WatchedEvent, Zookeeper, }; use crate::index_mapper::IndexMapper; @@ -281,7 +280,7 @@ pub struct IndexSchedulerOptions { /// The experimental features enabled for this instance. pub instance_features: InstanceTogglableFeatures, /// zookeeper client - pub zookeeper: Option>, + pub zookeeper: Option>, /// S3 bucket pub s3: Option>, } @@ -291,7 +290,7 @@ pub struct IndexSchedulerOptions { #[derive(Clone)] pub struct IndexScheduler { inner: Arc>>, - zookeeper: Option>, + zookeeper: Option>, pub s3: Option>, wake_up: Arc, } @@ -317,9 +316,8 @@ impl IndexScheduler { // Create all the required directories in zookeeper match zookeeper.create( "/election", - vec![], - Acl::open_unsafe().clone(), - CreateMode::Persistent, + &vec![], + &CreateMode::Persistent.with_acls(Acls::anyone_all()), ) { Ok(_) | Err(ZkError::NodeExists) => (), Err(e) => panic!("{e}"), @@ -327,9 +325,8 @@ impl IndexScheduler { match zookeeper.create( "/snapshots", - vec![], - Acl::open_unsafe().clone(), - CreateMode::Persistent, + &[], + &CreateMode::Persistent.with_acls(Acls::anyone_all()), ) { Ok(_) | Err(ZkError::NodeExists) => (), Err(e) => panic!("{e}"), @@ -395,37 +392,35 @@ impl IndexScheduler { // See https://zookeeper.apache.org/doc/current/recipes.html#sc_leaderElection let latchc = latch.clone(); let this = self.clone(); - zookeeper - .add_watch("/snapshots", AddWatchMode::PersistentRecursive, move |event| { - if !latchc.has_leadership() { - let WatchedEvent { event_type, path, keeper_state: _ } = event; - match event_type { - WatchedEventType::NodeCreated => { - let path = path.unwrap(); - log::info!("The snapshot {} is in preparation", path); - } - WatchedEventType::NodeDataChanged => { - let path = path.unwrap(); - let snapshot_id = - path.strip_prefix("/snapshots/snapshot-").unwrap(); - let snapshot_dir = format!("snapshots/{}", snapshot_id); - load_snapshot(&this, &snapshot_dir).unwrap(); - } - otherwise => panic!("{otherwise:?}"), + let watcher = + zookeeper.watch("/snapshots", AddWatchMode::PersistentRecursive).unwrap(); + watcher.run_on_change(move |event| { + if !latchc.has_leadership() { + let WatchedEvent { event_type, path, .. } = event; + match event_type { + EventType::NodeCreated => { + log::info!("The snapshot {} is in preparation", path); } + EventType::NodeDataChanged => { + let snapshot_id = + path.strip_prefix("/snapshots/snapshot-").unwrap(); + let snapshot_dir = format!("snapshots/{}", snapshot_id); + load_snapshot(&this, &snapshot_dir).unwrap(); + } + otherwise => panic!("{otherwise:?}"), } - }) - .unwrap(); + } + }); { // TODO we must lock the IndexSchedulerInner here to make sure that we don't // load this snapshot after the upper watcher find a more recent one. - let mut snapshots = zookeeper.get_children("/snapshots", false).unwrap(); + let (mut snapshots, _) = zookeeper.get_children("/snapshots").unwrap(); snapshots.sort_unstable(); for snapshot_name in dbg!(snapshots).iter().rev() { let (_, snapshot_id) = snapshot_name.rsplit_once('-').unwrap(); let snapshot_path = format!("/snapshots/{snapshot_name}"); - match zookeeper.get_data(&snapshot_path, false) { + match zookeeper.get_data(&snapshot_path) { Ok((data, _stat)) => { if data == b"ok" { eprintln!("Loading snapshot {snapshot_path}"); @@ -442,16 +437,15 @@ impl IndexScheduler { match zookeeper.create( "/tasks", - vec![], - Acl::open_unsafe().clone(), - CreateMode::Persistent, + &[], + &CreateMode::Persistent.with_acls(Acls::anyone_all()), ) { Ok(_) => (), Err(ZkError::NodeExists) => { log::warn!("Tasks directory already exists, we're going to import all the tasks on the zk without altering the tasks already on disk."); - let children = zookeeper - .get_children("/tasks", false) + let (children, _) = zookeeper + .get_children("/tasks") .expect("Internal, the /tasks directory was deleted during execution."); // TODO change me log::info!("Importing {} tasks", children.len()); @@ -459,7 +453,7 @@ impl IndexScheduler { let mut wtxn = inner.env.write_txn().unwrap(); for path in children { log::info!(" Importing {}", path); - match zookeeper.get_data(&format!("/tasks/{}", &path), false) { + match zookeeper.get_data(&format!("/tasks/{}", &path)) { Ok((data, _stat)) => { if data != b"ok" { log::info!(" Task {} was not \"ok\", skipping.", path); @@ -490,40 +484,34 @@ impl IndexScheduler { // TODO: fix unwrap by returning a clear error. let this = self.clone(); let zookeeperc = zookeeper.clone(); - zookeeper - .add_watch("/tasks", AddWatchMode::PersistentRecursive, move |event| { - let WatchedEvent { event_type, path, keeper_state: _ } = event; - match event_type { - WatchedEventType::NodeDataChanged => { - let path = path.unwrap(); - // Add raw task content in local DB - log::info!("Received a new task from the cluster at {}", path); - let inner = this.inner(); - let (data, _stat) = zookeeperc.get_data(&path, false).unwrap(); - if data == b"ok" { - let mut wtxn = inner.env.write_txn().unwrap(); - let id = path - .rsplit_once('-') - .map(|(_, id)| id.parse::().unwrap()) - .unwrap(); - let s3 = inner.options.s3.as_ref().unwrap(); - let task = - s3.get_object_json(format!("tasks/{id:0>10}")).unwrap(); - inner.register_raw_task(&mut wtxn, &task).unwrap(); - wtxn.commit().unwrap(); - } + let watcher = zookeeper.watch("/tasks", AddWatchMode::PersistentRecursive).unwrap(); + watcher.run_on_change(move |event| { + let WatchedEvent { event_type, path, .. } = event; + match event_type { + EventType::NodeDataChanged => { + // Add raw task content in local DB + log::info!("Received a new task from the cluster at {}", path); + let inner = this.inner(); + let (data, _stat) = zookeeperc.get_data(&path).unwrap(); + if data == b"ok" { + let mut wtxn = inner.env.write_txn().unwrap(); + let id = path + .rsplit_once('-') + .map(|(_, id)| id.parse::().unwrap()) + .unwrap(); + let s3 = inner.options.s3.as_ref().unwrap(); + let task = s3.get_object_json(format!("tasks/{id:0>10}")).unwrap(); + inner.register_raw_task(&mut wtxn, &task).unwrap(); + wtxn.commit().unwrap(); } - WatchedEventType::None - | WatchedEventType::NodeCreated - | WatchedEventType::NodeDeleted => (), - WatchedEventType::NodeChildrenChanged - | WatchedEventType::DataWatchRemoved - | WatchedEventType::ChildWatchRemoved => panic!("{event_type:?}"), } + EventType::NodeCreated | EventType::NodeDeleted => (), + EventType::NodeChildrenChanged => panic!("Node children changed"), + EventType::Session => panic!("Session error"), + } - this.wake_up.signal(); - }) - .unwrap(); + this.wake_up.signal(); + }); Some(latch) } @@ -553,20 +541,14 @@ impl IndexScheduler { } let rtxn = inner.env.read_txn().unwrap(); - let snapshot_path = zookeeper + let (_, snapshot_id) = zookeeper .create( "/snapshots/snapshot-", - vec![], - Acl::open_unsafe().clone(), - CreateMode::PersistentSequential, + &[], + &CreateMode::Persistent.with_acls(Acls::anyone_all()), ) .unwrap(); - let snapshot_id = snapshot_path - .rsplit_once('-') - .map(|(_, id)| id.parse::().unwrap()) - .unwrap(); - let zk_snapshots = format!("snapshots"); let snapshot_dir = format!("{zk_snapshots}/{snapshot_id:0>10?}"); @@ -621,8 +603,8 @@ impl IndexScheduler { // we must notify everyone that we dropped a new snapshot on the s3 let _stat = zookeeper.set_data( - &format!("/snapshots/snapshot-{:0>10?}", snapshot_id), - b"ok".to_vec(), + &format!("/snapshots/snapshot-{snapshot_id}"), + b"ok", None, ); log::info!( @@ -1018,8 +1000,8 @@ pub struct IndexSchedulerInner { /// The path to the version file of Meilisearch. pub(crate) version_file_path: PathBuf, - /// The URL to the ZooKeeper cluster - pub(crate) zookeeper: Option>, + /// The URL to the Zookeeper cluster + pub(crate) zookeeper: Option>, // ================= test // The next entry is dedicated to the tests. @@ -1435,16 +1417,14 @@ impl IndexSchedulerInner { /// /// If it fails and data was associated with the task, it tries to delete the associated data. pub fn register(&self, kind: KindWithContent) -> Result { - let id = match &self.zookeeper { + let zk_id = match &self.zookeeper { Some(zookeeper) => { - // Reserve uniq ID on zookeeper. And give it to the spawn blocking. match zookeeper.create( "/tasks/task-", - vec![], - Acl::open_unsafe().clone(), - CreateMode::PersistentSequential, + &[], + &CreateMode::Persistent.with_acls(Acls::anyone_all()), ) { - Ok(path) => path.rsplit_once('-').map(|(_, id)| id.parse::().unwrap()), + Ok((_, id)) => Some(id), Err(e) => panic!("{e}"), } } @@ -1461,8 +1441,11 @@ impl IndexSchedulerInner { } // Retrieve the id generated by zookeeper or generate a local id. - let id = match id { - Some(id) => id as u32, + let id = match zk_id { + Some(id) => id + .into_i64() + .try_into() + .expect("Can't convert zookeeper task id to meilisearch task id"), None => self.next_task_id(&wtxn)?, }; @@ -1509,12 +1492,14 @@ impl IndexSchedulerInner { // TODO: send task to ZK in raw json. if let Some(zookeeper) = &self.zookeeper { + // safe because if we had a zookeeper at the beginning we must have a zk_id + let zk_id = zk_id.unwrap(); let s3 = self.options.s3.as_ref().unwrap(); - s3.put_object(format!("tasks/{id:0>10?}"), &serde_json::to_vec_pretty(&task).unwrap()) + s3.put_object(format!("tasks/{zk_id}"), &serde_json::to_vec_pretty(&task).unwrap()) .unwrap(); // TODO: ugly unwrap - zookeeper.set_data(&format!("/tasks/task-{:0>10?}", id), b"ok".to_vec(), None).unwrap(); + zookeeper.set_data(&format!("/tasks/task-{zk_id}"), b"ok", None).unwrap(); } Ok(task) diff --git a/meilisearch-auth/Cargo.toml b/meilisearch-auth/Cargo.toml index 06071c8bd..509368efb 100644 --- a/meilisearch-auth/Cargo.toml +++ b/meilisearch-auth/Cargo.toml @@ -25,4 +25,4 @@ sha2 = "0.10.6" thiserror = "1.0.40" time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] } uuid = { version = "1.3.1", features = ["serde", "v4"] } -zookeeper = "0.8.0" +zookeeper-client-sync = { path = "../../zookeeper-client-sync" } diff --git a/meilisearch-auth/src/error.rs b/meilisearch-auth/src/error.rs index d5b91cae5..883ec43be 100644 --- a/meilisearch-auth/src/error.rs +++ b/meilisearch-auth/src/error.rs @@ -19,7 +19,7 @@ internal_error!( AuthControllerError: meilisearch_types::milli::heed::Error, std::io::Error, serde_json::Error, - zookeeper::ZkError, + zookeeper_client_sync::Error, std::str::Utf8Error ); diff --git a/meilisearch-auth/src/lib.rs b/meilisearch-auth/src/lib.rs index 4bc45a1d4..08872450e 100644 --- a/meilisearch-auth/src/lib.rs +++ b/meilisearch-auth/src/lib.rs @@ -16,22 +16,22 @@ pub use store::open_auth_store_env; use store::{generate_key_as_hexa, HeedAuthStore}; use time::OffsetDateTime; use uuid::Uuid; -use zookeeper::{ - Acl, AddWatchMode, CreateMode, WatchedEvent, WatchedEventType, ZkError, ZooKeeper, +use zookeeper_client_sync::{ + Acls, AddWatchMode, CreateMode, Error as ZkError, EventType, WatchedEvent, Zookeeper, }; #[derive(Clone)] pub struct AuthController { store: Arc, master_key: Option, - zookeeper: Option>, + zookeeper: Option>, } impl AuthController { pub fn new( db_path: impl AsRef, master_key: &Option, - zookeeper: Option>, + zookeeper: Option>, ) -> Result { let store = HeedAuthStore::new(db_path)?; let controller = Self { store: Arc::new(store), master_key: master_key.clone(), zookeeper }; @@ -42,22 +42,21 @@ impl AuthController { // Zookeeper Event listener loop let controller_clone = controller.clone(); let zkk = zookeeper.clone(); - zookeeper.add_watch("/auth", AddWatchMode::PersistentRecursive, move |event| { - let WatchedEvent { event_type, path, keeper_state: _ } = dbg!(event); + let watcher = zookeeper.watch("/auth", AddWatchMode::PersistentRecursive)?; + watcher.run_on_change(move |event| { + let WatchedEvent { event_type, path, .. } = dbg!(event); match event_type { - WatchedEventType::NodeDeleted => { + EventType::NodeDeleted => { // TODO: ugly unwraps - let path = path.unwrap(); let uuid = path.strip_prefix("/auth/").unwrap(); let uuid = Uuid::parse_str(&uuid).unwrap(); log::info!("The key {} has been deleted", uuid); controller_clone.store.delete_api_key(uuid).unwrap(); } - WatchedEventType::NodeCreated | WatchedEventType::NodeDataChanged => { - let path = path.unwrap(); + EventType::NodeCreated | EventType::NodeDataChanged => { if path.strip_prefix("/auth/").map_or(false, |s| !s.is_empty()) { - let (key, _stat) = zkk.get_data(&path, false).unwrap(); + let (key, _stat) = zkk.get_data(&path).unwrap(); let key: Key = serde_json::from_slice(&key).unwrap(); log::info!("The key {} has been deleted", key.uid); controller_clone.store.put_api_key(key).unwrap(); @@ -65,15 +64,14 @@ impl AuthController { } otherwise => panic!("Got the unexpected `{otherwise:?}` event!"), } - })?; + }); // TODO: we should catch the potential unexpected errors here https://docs.rs/zookeeper-client/latest/zookeeper_client/struct.Client.html#method.create // for the moment we consider that `create` only returns Error::NodeExists. match zookeeper.create( "/auth", - vec![], - Acl::open_unsafe().clone(), - CreateMode::Persistent, + &[], + &CreateMode::Persistent.with_acls(Acls::anyone_all()), ) { // If the store is empty, we must generate and push the default api-keys. Ok(_) => generate_default_keys(&controller)?, @@ -83,14 +81,14 @@ impl AuthController { let store = controller.store.clone(); store.delete_all_keys()?; - let children = zookeeper - .get_children("/auth", false) + let (children, _) = zookeeper + .get_children("/auth") .expect("Internal, the auth directory was deleted during execution."); log::info!("Importing {} api-keys", children.len()); for path in children { log::info!(" Importing {}", path); - match zookeeper.get_data(&format!("/auth/{}", &path), false) { + match zookeeper.get_data(&format!("/auth/{}", &path)) { Ok((key, _stat)) => { let key = serde_json::from_slice(&key).unwrap(); let store = controller.store.clone(); @@ -104,7 +102,7 @@ impl AuthController { } } e @ Err( - ZkError::NoNode | ZkError::NoChildrenForEphemerals | ZkError::InvalidACL, + ZkError::NoNode | ZkError::NoChildrenForEphemerals | ZkError::InvalidAcl, ) => unreachable!("{e:?}"), Err(e) => panic!("{e}"), } @@ -154,9 +152,8 @@ impl AuthController { Some(zookeeper) => { zookeeper.create( &format!("/auth/{}", key.uid), - serde_json::to_vec_pretty(&key)?, - Acl::open_unsafe().clone(), - CreateMode::Persistent, + &serde_json::to_vec_pretty(&key)?, + &CreateMode::Persistent.with_acls(Acls::anyone_all()), )?; Ok(key) @@ -182,7 +179,7 @@ impl AuthController { Some(zookeeper) => { zookeeper.set_data( &format!("/auth/{}", key.uid), - serde_json::to_vec_pretty(&key)?, + &serde_json::to_vec_pretty(&key)?, None, )?; diff --git a/meilisearch/Cargo.toml b/meilisearch/Cargo.toml index f8787f4e4..c6b43a792 100644 --- a/meilisearch/Cargo.toml +++ b/meilisearch/Cargo.toml @@ -106,7 +106,7 @@ walkdir = "2.3.3" yaup = "0.2.1" serde_urlencoded = "0.7.1" termcolor = "1.2.0" -zookeeper = "0.8.0" +zookeeper-client-sync = { path = "../../zookeeper-client-sync" } [dev-dependencies] actix-rt = "2.8.0"