diff --git a/Cargo.lock b/Cargo.lock index 962311cbc..ea6fb55d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -749,6 +749,25 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" +dependencies = [ + "crossbeam-utils 0.6.6", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if 0.1.10", + "lazy_static", +] + [[package]] name = "crossbeam-utils" version = "0.7.2" @@ -968,6 +987,16 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00" +dependencies = [ + "matches", + "percent-encoding", +] + [[package]] name = "fs_extra" version = "1.2.0" @@ -1188,9 +1217,9 @@ dependencies = [ [[package]] name = "heed" -version = "0.8.1" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c9d6f92f402a1452d658038a279d882c77fc5c1f50f29eb491b5fac537f479" +checksum = "2eaba3b0edee6a9cd551f24caca2027922b03259f7203a15f0b86af4c1348fcc" dependencies = [ "byteorder", "heed-traits", @@ -1199,6 +1228,7 @@ dependencies = [ "lmdb-rkv-sys", "once_cell", "page_size", + "synchronoise", "url", "zerocopy", ] @@ -1211,9 +1241,9 @@ checksum = "b328f6260a7e51bdb0ca6b68e6ea27ee3d11fba5dee930896ee7ff6ad5fc072c" [[package]] name = "heed-types" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e100387815256b00dbb4f48db990f7fa03e9b88b4a89c2a1661b7d9d77b77c46" +checksum = "e628efb08beaee58355f80dc4adba79d644940ea9eef60175ea17dc218aab405" dependencies = [ "bincode", "heed-traits", @@ -1489,9 +1519,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.79" +version = "0.2.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" [[package]] name = "linked-hash-map" @@ -2744,6 +2774,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synchronoise" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d717ed0efc9d39ab3b642a096bc369a3e02a38a51c41845d7fe31bdad1d6eaeb" +dependencies = [ + "crossbeam-queue", +] + [[package]] name = "synstructure" version = "0.12.4" @@ -3152,10 +3191,11 @@ dependencies = [ [[package]] name = "url" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e" dependencies = [ + "form_urlencoded", "idna", "matches", "percent-encoding", diff --git a/meilisearch-core/Cargo.toml b/meilisearch-core/Cargo.toml index d3f143d8a..416ba6897 100644 --- a/meilisearch-core/Cargo.toml +++ b/meilisearch-core/Cargo.toml @@ -18,7 +18,7 @@ either = "1.5.3" env_logger = "0.8.2" fst = "0.4.4" hashbrown = { version = "0.9.0", features = ["serde"] } -heed = "0.8.0" +heed = "0.10.5" indexmap = { version = "1.3.2", features = ["serde-1"] } intervaltree = "0.2.5" itertools = "0.9.0" diff --git a/meilisearch-core/src/criterion/mod.rs b/meilisearch-core/src/criterion/mod.rs index a2f68c28e..3fe77115a 100644 --- a/meilisearch-core/src/criterion/mod.rs +++ b/meilisearch-core/src/criterion/mod.rs @@ -61,7 +61,7 @@ pub trait Criterion { } pub struct ContextMut<'h, 'p, 'tag, 'txn, 'q> { - pub reader: &'h heed::RoTxn, + pub reader: &'h heed::RoTxn<'h, MainT>, pub postings_lists: &'p mut SmallArena<'tag, PostingsListView<'txn>>, pub query_mapping: &'q HashMap>, pub documents_fields_counts_store: store::DocumentsFieldsCounts, diff --git a/meilisearch-core/src/database.rs b/meilisearch-core/src/database.rs index 82f5fa89d..d93c81acb 100644 --- a/meilisearch-core/src/database.rs +++ b/meilisearch-core/src/database.rs @@ -22,11 +22,11 @@ type ArcSwapFn = arc_swap::ArcSwapOption; type SerdeDatetime = SerdeBincode>; -pub type MainWriter<'a> = heed::RwTxn<'a, MainT>; -pub type MainReader = heed::RoTxn; +pub type MainWriter<'a, 'b> = heed::RwTxn<'a, 'b, MainT>; +pub type MainReader<'a, 'b> = heed::RoTxn<'a, MainT>; -pub type UpdateWriter<'a> = heed::RwTxn<'a, UpdateT>; -pub type UpdateReader = heed::RoTxn; +pub type UpdateWriter<'a, 'b> = heed::RwTxn<'a, 'b, UpdateT>; +pub type UpdateReader<'a> = heed::RoTxn<'a, UpdateT>; const LAST_UPDATE_KEY: &str = "last-update"; diff --git a/meilisearch-core/src/serde/deserializer.rs b/meilisearch-core/src/serde/deserializer.rs index e5e02a4d6..0d091951e 100644 --- a/meilisearch-core/src/serde/deserializer.rs +++ b/meilisearch-core/src/serde/deserializer.rs @@ -51,7 +51,7 @@ impl From for DeserializerError { pub struct Deserializer<'a> { pub document_id: DocumentId, - pub reader: &'a heed::RoTxn, + pub reader: &'a heed::RoTxn<'a, MainT>, pub documents_fields: DocumentsFields, pub schema: &'a Schema, pub fields: Option<&'a HashSet>, diff --git a/meilisearch-core/src/store/docs_words.rs b/meilisearch-core/src/store/docs_words.rs index 11e81f673..6b412d584 100644 --- a/meilisearch-core/src/store/docs_words.rs +++ b/meilisearch-core/src/store/docs_words.rs @@ -33,7 +33,7 @@ impl DocsWords { self.docs_words.clear(writer) } - pub fn doc_words(self, reader: &heed::RoTxn, document_id: DocumentId) -> ZResult { + pub fn doc_words<'a>(self, reader: &'a heed::RoTxn<'a, MainT>, document_id: DocumentId) -> ZResult { let document_id = BEU32::new(document_id.0); match self.docs_words.get(reader, &document_id)? { Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), diff --git a/meilisearch-core/src/store/facets.rs b/meilisearch-core/src/store/facets.rs index 7436f3d65..1a7579099 100644 --- a/meilisearch-core/src/store/facets.rs +++ b/meilisearch-core/src/store/facets.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use std::collections::HashMap; use std::mem; -use heed::{RwTxn, RoTxn, RoRange, types::Str, BytesEncode, BytesDecode}; +use heed::{RwTxn, RoTxn, RoPrefix, types::Str, BytesEncode, BytesDecode}; use sdset::{SetBuf, Set, SetOperation}; use meilisearch_types::DocumentId; @@ -62,7 +62,7 @@ impl Facets { Ok(self.facets.put(writer, &facet_key, &(facet_value, doc_ids))?) } - pub fn field_document_ids<'txn>(&self, reader: &'txn RoTxn, field_id: FieldId) -> MResult> { + pub fn field_document_ids<'txn>(&self, reader: &'txn RoTxn, field_id: FieldId) -> MResult> { Ok(self.facets.prefix_iter(reader, &FacetKey::new(field_id, String::new()))?) } diff --git a/meilisearch-core/src/store/main.rs b/meilisearch-core/src/store/main.rs index 5accdfbc7..2b60a5680 100644 --- a/meilisearch-core/src/store/main.rs +++ b/meilisearch-core/src/store/main.rs @@ -143,7 +143,7 @@ impl Main { self.put_external_docids(writer, &external_docids) } - pub fn external_docids(self, reader: &heed::RoTxn) -> MResult { + pub fn external_docids<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult { match self.main.get::<_, Str, ByteSlice>(reader, EXTERNAL_DOCIDS_KEY)? { Some(bytes) => Ok(fst::Map::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), None => Ok(fst::Map::default().map_data(Cow::Owned).unwrap()), @@ -155,7 +155,7 @@ impl Main { Ok(external_ids.get(external_docid).map(|id| DocumentId(id as u32))) } - pub fn words_fst(self, reader: &heed::RoTxn) -> MResult { + pub fn words_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult { match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? { Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()), @@ -170,7 +170,7 @@ impl Main { Ok(self.main.put::<_, Str, CowSlice>(writer, SORTED_DOCUMENT_IDS_CACHE_KEY, documents_ids)?) } - pub fn sorted_document_ids_cache(self, reader: &heed::RoTxn) -> MResult>> { + pub fn sorted_document_ids_cache<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult>> { Ok(self.main.get::<_, Str, CowSlice>(reader, SORTED_DOCUMENT_IDS_CACHE_KEY)?) } @@ -199,7 +199,7 @@ impl Main { Ok(self.main.put::<_, Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)?) } - pub(crate) fn synonyms_fst(self, reader: &heed::RoTxn) -> MResult { + pub(crate) fn synonyms_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult { match self.main.get::<_, Str, ByteSlice>(reader, SYNONYMS_KEY)? { Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()), @@ -219,7 +219,7 @@ impl Main { Ok(self.main.put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)?) } - pub(crate) fn stop_words_fst(self, reader: &heed::RoTxn) -> MResult { + pub(crate) fn stop_words_fst<'a>(self, reader: &'a heed::RoTxn<'a, MainT>) -> MResult { match self.main.get::<_, Str, ByteSlice>(reader, STOP_WORDS_KEY)? { Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()), diff --git a/meilisearch-core/src/update/documents_addition.rs b/meilisearch-core/src/update/documents_addition.rs index 5603966b0..d46aeb7fe 100644 --- a/meilisearch-core/src/update/documents_addition.rs +++ b/meilisearch-core/src/update/documents_addition.rs @@ -146,8 +146,8 @@ where A: AsRef<[u8]>, Ok(()) } -pub fn apply_addition<'a, 'b>( - writer: &'a mut heed::RwTxn<'b, MainT>, +pub fn apply_addition<'a, 'b, 'c>( + writer: &'a mut heed::RwTxn<'b, 'c, MainT>, index: &store::Index, new_documents: Vec>, partial: bool @@ -272,16 +272,16 @@ pub fn apply_addition<'a, 'b>( Ok(()) } -pub fn apply_documents_partial_addition<'a, 'b>( - writer: &'a mut heed::RwTxn<'b, MainT>, +pub fn apply_documents_partial_addition<'a, 'b, 'c>( + writer: &'a mut heed::RwTxn<'b, 'c, MainT>, index: &store::Index, new_documents: Vec>, ) -> MResult<()> { apply_addition(writer, index, new_documents, true) } -pub fn apply_documents_addition<'a, 'b>( - writer: &'a mut heed::RwTxn<'b, MainT>, +pub fn apply_documents_addition<'a, 'b, 'c>( + writer: &'a mut heed::RwTxn<'b, 'c, MainT>, index: &store::Index, new_documents: Vec>, ) -> MResult<()> { diff --git a/meilisearch-core/src/update/mod.rs b/meilisearch-core/src/update/mod.rs index 2f3d7bd8c..d1ea83108 100644 --- a/meilisearch-core/src/update/mod.rs +++ b/meilisearch-core/src/update/mod.rs @@ -212,8 +212,8 @@ pub fn next_update_id( Ok(new_update_id) } -pub fn update_task<'a, 'b>( - writer: &'a mut heed::RwTxn<'b, MainT>, +pub fn update_task<'a, 'b, 'c>( + writer: &'a mut heed::RwTxn<'b, 'c, MainT>, index: &store::Index, update_id: u64, update: Update,