From 79a4bc8129349d63ed505a1e4a7a79aab8b2828a Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 11 Mar 2021 19:40:18 +0100 Subject: [PATCH] use meta from milli --- Cargo.lock | 228 ++++++++++++------ meilisearch-http/Cargo.toml | 3 +- meilisearch-http/src/index/search.rs | 12 +- meilisearch-http/src/index/updates.rs | 2 +- .../src/index_controller/index_actor.rs | 172 ++++--------- 5 files changed, 207 insertions(+), 210 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 84c9b6854..575f03d98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,8 +159,8 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ca8ce00b267af8ccebbd647de0d61e0674b6e61185cc7a592ff88772bed655" dependencies = [ - "quote", - "syn", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -169,8 +169,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef" dependencies = [ - "quote", - "syn", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -363,9 +363,9 @@ version = "0.5.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -462,9 +462,9 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -473,9 +473,9 @@ version = "0.1.48" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -925,9 +925,9 @@ version = "0.99.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -982,9 +982,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" dependencies = [ "heck", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -1029,9 +1029,9 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "synstructure", ] @@ -1170,9 +1170,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea405816a5139fb39af82c2beb921d52143f556038378d6db21183a5c37fbfb7" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -1721,6 +1721,28 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "logging_timer" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d0c249955c17c2f8f86b5f501b16d2509ebbe775f7b1d1d2b1ba85ade2a793" +dependencies = [ + "log", + "logging_timer_proc_macros", +] + +[[package]] +name = "logging_timer_proc_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "482c2c28e6bcfe7c4274f82f701774d755e6aa873edfd619460fcd0966e0eb07" +dependencies = [ + "log", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -1866,11 +1888,11 @@ dependencies = [ [[package]] name = "milli" version = "0.1.0" -source = "git+https://github.com/meilisearch/milli.git?rev=794fce7#794fce7bff3e3461a7f3954fd97f58f8232e5a8e" dependencies = [ "anyhow", "bstr", "byteorder", + "chrono", "crossbeam-channel", "csv", "either", @@ -1884,6 +1906,7 @@ dependencies = [ "levenshtein_automata", "linked-hash-map", "log", + "logging_timer", "meilisearch-tokenizer", "memmap", "num-traits", @@ -1897,9 +1920,11 @@ dependencies = [ "roaring", "serde", "serde_json", + "slice-group-by", "smallstr", "smallvec", "tempfile", + "tinytemplate", "uuid", ] @@ -2170,9 +2195,9 @@ checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" dependencies = [ "pest 2.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "pest_meta", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2248,9 +2273,9 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2259,9 +2284,9 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2301,9 +2326,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "version_check", ] @@ -2313,8 +2338,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "version_check", ] @@ -2330,13 +2355,22 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" dependencies = [ - "unicode-xid", + "unicode-xid 0.2.1", ] [[package]] @@ -2345,13 +2379,22 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.24", ] [[package]] @@ -2799,9 +2842,9 @@ version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -2987,11 +3030,11 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "serde", "serde_derive", - "syn", + "syn 1.0.63", ] [[package]] @@ -3001,13 +3044,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" dependencies = [ "base-x", - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "serde", "serde_derive", "serde_json", "sha1", - "syn", + "syn 1.0.63", ] [[package]] @@ -3041,9 +3084,20 @@ checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" dependencies = [ "heck", "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", +] + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", ] [[package]] @@ -3052,9 +3106,9 @@ version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fd9bc7ccc2688b3344c2f48b9b546648b25ce0b20fc717ee7fa7981a8ca9717" dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", + "proc-macro2 1.0.24", + "quote 1.0.9", + "unicode-xid 0.2.1", ] [[package]] @@ -3072,10 +3126,10 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", + "unicode-xid 0.2.1", ] [[package]] @@ -3146,9 +3200,9 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -3212,10 +3266,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.9", "standback", - "syn", + "syn 1.0.63", +] + +[[package]] +name = "tinytemplate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f" +dependencies = [ + "serde", + "serde_json", ] [[package]] @@ -3281,9 +3345,9 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", ] [[package]] @@ -3479,6 +3543,12 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.1" @@ -3592,9 +3662,9 @@ dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "wasm-bindgen-shared", ] @@ -3616,7 +3686,7 @@ version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ac38da8ef716661f0f36c0d8320b89028efe10c7c0afde65baffb496ce0d3b" dependencies = [ - "quote", + "quote 1.0.9", "wasm-bindgen-macro-support", ] @@ -3626,9 +3696,9 @@ version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc053ec74d454df287b9374ee8abb36ffd5acb95ba87da3ba5b7d3fe20eb401e" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.63", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3788,8 +3858,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ - "proc-macro2", - "syn", + "proc-macro2 1.0.24", + "syn 1.0.63", "synstructure", ] diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index b09c52b87..15319dea6 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -38,7 +38,8 @@ main_error = "0.1.0" meilisearch-error = { path = "../meilisearch-error" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" } memmap = "0.7.0" -milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } +milli = { path = "../../milli/milli" } +#milli = { git = "https://github.com/meilisearch/milli.git", rev = "794fce7" } mime = "0.3.16" once_cell = "1.5.2" rand = "0.7.3" diff --git a/meilisearch-http/src/index/search.rs b/meilisearch-http/src/index/search.rs index 1264d49d6..fb830ec60 100644 --- a/meilisearch-http/src/index/search.rs +++ b/meilisearch-http/src/index/search.rs @@ -6,7 +6,7 @@ use either::Either; use anyhow::bail; use heed::RoTxn; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; -use milli::{FacetCondition, facet::FacetValue}; +use milli::{FacetCondition, MatchingWords, facet::FacetValue}; use serde::{Serialize, Deserialize}; use serde_json::{Value, Map}; @@ -71,7 +71,7 @@ impl Index { let milli::SearchResult { documents_ids, - found_words, + matching_words, candidates, .. } = search.execute()?; @@ -102,7 +102,7 @@ impl Index { for (_id, obkv) in self.documents(&rtxn, documents_ids)? { let mut object = milli::obkv_to_json(&displayed_fields_ids, &fields_ids_map, obkv).unwrap(); if let Some(ref attributes_to_highlight) = query.attributes_to_highlight { - highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + highlighter.highlight_record(&mut object, &matching_words, attributes_to_highlight); } documents.push(object); } @@ -173,7 +173,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { Self { analyzer } } - pub fn highlight_value(&self, value: Value, words_to_highlight: &HashSet) -> Value { + pub fn highlight_value(&self, value: Value, words_to_highlight: &MatchingWords) -> Value { match value { Value::Null => Value::Null, Value::Bool(boolean) => Value::Bool(boolean), @@ -183,7 +183,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { let analyzed = self.analyzer.analyze(&old_string); for (word, token) in analyzed.reconstruct() { if token.is_word() { - let to_highlight = words_to_highlight.contains(token.text()); + let to_highlight = words_to_highlight.matches(token.text()); if to_highlight { string.push_str("") } @@ -215,7 +215,7 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { pub fn highlight_record( &self, object: &mut Map, - words_to_highlight: &HashSet, + words_to_highlight: &MatchingWords, attributes_to_highlight: &HashSet, ) { // TODO do we need to create a string for element that are not and needs to be highlight? diff --git a/meilisearch-http/src/index/updates.rs b/meilisearch-http/src/index/updates.rs index d339406f7..129a340a7 100644 --- a/meilisearch-http/src/index/updates.rs +++ b/meilisearch-http/src/index/updates.rs @@ -12,7 +12,7 @@ use super::Index; #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UpdateResult { DocumentsAddition(DocumentAdditionResult), - DocumentDeletion { deleted: usize }, + DocumentDeletion { deleted: u64 }, Other, } diff --git a/meilisearch-http/src/index_controller/index_actor.rs b/meilisearch-http/src/index_controller/index_actor.rs index d8dc5f014..b5d5792b2 100644 --- a/meilisearch-http/src/index_controller/index_actor.rs +++ b/meilisearch-http/src/index_controller/index_actor.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::fs::{create_dir_all, remove_dir_all, File}; +use std::fs::{create_dir_all, File}; use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -8,15 +8,13 @@ use async_stream::stream; use chrono::{DateTime, Utc}; use futures::pin_mut; use futures::stream::StreamExt; -use heed::{ - types::{ByteSlice, SerdeBincode}, - Database, Env, EnvOpenOptions, -}; +use heed::EnvOpenOptions; use log::debug; use serde::{Deserialize, Serialize}; use thiserror::Error; -use tokio::{sync::{mpsc, oneshot, RwLock}}; +use tokio::sync::{mpsc, oneshot, RwLock}; use tokio::task::spawn_blocking; +use tokio::fs::remove_dir_all; use uuid::Uuid; use super::get_arc_ownership_blocking; @@ -36,12 +34,21 @@ type UpdateResult = std::result::Result, Failed, updated_at: DateTime, primary_key: Option, } +impl IndexMeta { + fn new(index: &Index) -> Result { + let txn = index.read_txn()?; + let created_at = index.created_at(&txn)?; + let updated_at = index.updated_at(&txn)?; + let primary_key = index.primary_key(&txn)?.map(String::from); + Ok(Self { primary_key, updated_at, created_at }) + } +} + enum IndexMsg { CreateIndex { uuid: Uuid, @@ -106,14 +113,9 @@ pub enum IndexError { #[async_trait::async_trait] trait IndexStore { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result; - async fn update_index(&self, uuid: Uuid, f: F) -> Result - where - F: FnOnce(Index) -> Result + Send + Sync + 'static, - R: Sync + Send + 'static; + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result; async fn get(&self, uuid: Uuid) -> Result>; async fn delete(&self, uuid: Uuid) -> Result>; - async fn get_meta(&self, uuid: Uuid) -> Result>; } impl IndexActor { @@ -245,7 +247,11 @@ impl IndexActor { uuid: Uuid, primary_key: Option, ) -> Result { - self.store.create_index(uuid, primary_key).await + let index = self.store.create(uuid, primary_key).await?; + let meta = spawn_blocking(move || IndexMeta::new(&index)) + .await + .map_err(|e| IndexError::Error(e.into()))??; + Ok(meta) } async fn handle_update( @@ -256,16 +262,13 @@ impl IndexActor { debug!("Processing update {}", meta.id()); let uuid = meta.index_uuid().clone(); let update_handler = self.update_handler.clone(); - let handle = self - .store - .update_index(uuid, |index| { - let handle = - spawn_blocking(move || update_handler.handle_update(meta, data, index)); - Ok(handle) - }) - .await?; - - handle.await.map_err(|e| IndexError::Error(e.into())) + let index = match self.store.get(uuid.clone()).await? { + Some(index) => index, + None => self.store.create(uuid, None).await?, + }; + spawn_blocking(move || update_handler.handle_update(meta, data, index)) + .await + .map_err(|e| IndexError::Error(e.into())) } async fn handle_settings(&self, uuid: Uuid) -> Result { @@ -338,8 +341,15 @@ impl IndexActor { } async fn handle_get_meta(&self, uuid: Uuid) -> Result> { - let result = self.store.get_meta(uuid).await?; - Ok(result) + match self.store.get(uuid).await? { + Some(index) => { + let meta = spawn_blocking(move || IndexMeta::new(&index)) + .await + .map_err(|e| IndexError::Error(e.into()))??; + Ok(Some(meta)) + } + None => Ok(None), + } } } @@ -451,24 +461,15 @@ impl IndexActorHandle { } struct HeedIndexStore { - env: Env, - db: Database>, index_store: AsyncMap, path: PathBuf, } impl HeedIndexStore { fn new(path: impl AsRef) -> anyhow::Result { - let mut options = EnvOpenOptions::new(); - options.map_size(1_073_741_824); //1GB let path = path.as_ref().join("indexes/"); - create_dir_all(&path)?; - let env = options.open(&path)?; - let db = env.create_database(None)?; let index_store = Arc::new(RwLock::new(HashMap::new())); Ok(Self { - env, - db, index_store, path, }) @@ -477,76 +478,22 @@ impl HeedIndexStore { #[async_trait::async_trait] impl IndexStore for HeedIndexStore { - async fn create_index(&self, uuid: Uuid, primary_key: Option) -> Result { + async fn create(&self, uuid: Uuid, primary_key: Option) -> Result { let path = self.path.join(format!("index-{}", uuid)); - if path.exists() { return Err(IndexError::IndexAlreadyExists); } - let env = self.env.clone(); - let db = self.db.clone(); - let (index, meta) = spawn_blocking(move || -> Result<(Index, IndexMeta)> { - let now = Utc::now(); - let meta = IndexMeta { - uuid: uuid.clone(), - created_at: now.clone(), - updated_at: now, - primary_key, - }; - let mut txn = env.write_txn()?; - db.put(&mut txn, uuid.as_bytes(), &meta)?; - txn.commit()?; - + let index = spawn_blocking(move || -> Result { let index = open_index(&path, 4096 * 100_000)?; - - Ok((index, meta)) + Ok(index) }) .await - .expect("thread died")?; + .map_err(|e| IndexError::Error(e.into()))??; - self.index_store.write().await.insert(uuid.clone(), index); + self.index_store.write().await.insert(uuid.clone(), index.clone()); - Ok(meta) - } - - async fn update_index(&self, uuid: Uuid, f: F) -> Result - where - F: FnOnce(Index) -> Result + Send + Sync + 'static, - R: Sync + Send + 'static, - { - let guard = self.index_store.read().await; - let index = match guard.get(&uuid) { - Some(index) => index.clone(), - None => { - drop(guard); - self.create_index(uuid.clone(), None).await?; - self.index_store - .read() - .await - .get(&uuid) - .expect("Index should exist") - .clone() - } - }; - - let env = self.env.clone(); - let db = self.db.clone(); - spawn_blocking(move || { - let mut txn = env.write_txn()?; - let mut meta = db.get(&txn, uuid.as_bytes())?.expect("unexisting index"); - match f(index) { - Ok(r) => { - meta.updated_at = Utc::now(); - db.put(&mut txn, uuid.as_bytes(), &meta)?; - txn.commit()?; - Ok(r) - } - Err(e) => Err(e), - } - }) - .await - .expect("thread died") + Ok(index) } async fn get(&self, uuid: Uuid) -> Result> { @@ -561,54 +508,33 @@ impl IndexStore for HeedIndexStore { return Ok(None); } - // TODO: set this info from the database let index = spawn_blocking(|| open_index(path, 4096 * 100_000)) .await - .expect("thread died")?; + .map_err(|e| IndexError::Error(e.into()))??; self.index_store .write() .await .insert(uuid.clone(), index.clone()); - println!("here"); Ok(Some(index)) } } } async fn delete(&self, uuid: Uuid) -> Result> { - let env = self.env.clone(); - let db = self.db.clone(); let db_path = self.path.join(format!("index-{}", uuid)); - spawn_blocking(move || -> Result<()> { - let mut txn = env.write_txn()?; - db.delete(&mut txn, uuid.as_bytes())?; - txn.commit()?; - remove_dir_all(db_path).unwrap(); - Ok(()) - }) - .await - .expect("thread died")?; + remove_dir_all(db_path).await + .map_err(|e| IndexError::Error(e.into()))?; let index = self.index_store.write().await.remove(&uuid); Ok(index) } - - async fn get_meta(&self, uuid: Uuid) -> Result> { - let env = self.env.clone(); - let db = self.db.clone(); - spawn_blocking(move || { - let txn = env.read_txn()?; - let meta = db.get(&txn, uuid.as_bytes())?; - Ok(meta) - }) - .await - .expect("thread died") - } } fn open_index(path: impl AsRef, size: usize) -> Result { - create_dir_all(&path).expect("can't create db"); + create_dir_all(&path) + .map_err(|e| IndexError::Error(e.into()))?; let mut options = EnvOpenOptions::new(); options.map_size(size); - let index = milli::Index::new(options, &path).map_err(|e| IndexError::Error(e))?; + let index = milli::Index::new(options, &path) + .map_err(|e| IndexError::Error(e))?; Ok(Index(Arc::new(index))) }