mirror of
https://github.com/meilisearch/MeiliSearch
synced 2025-07-04 20:37:15 +02:00
implement index mock
This commit is contained in:
parent
607e28749a
commit
4835d82a0b
10 changed files with 386 additions and 377 deletions
|
@ -17,6 +17,7 @@ use crate::options::IndexerOpts;
|
|||
type AsyncMap<K, V> = Arc<RwLock<HashMap<K, V>>>;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
#[cfg_attr(test, mockall::automock)]
|
||||
pub trait IndexStore {
|
||||
async fn create(&self, uuid: Uuid, primary_key: Option<String>) -> Result<Index>;
|
||||
async fn get(&self, uuid: Uuid) -> Result<Option<Index>>;
|
||||
|
@ -72,9 +73,10 @@ impl IndexStore for MapIndexStore {
|
|||
let index = spawn_blocking(move || -> Result<Index> {
|
||||
let index = Index::open(path, index_size, file_store, uuid, update_handler)?;
|
||||
if let Some(primary_key) = primary_key {
|
||||
let mut txn = index.write_txn()?;
|
||||
let inner = index.inner();
|
||||
let mut txn = inner.write_txn()?;
|
||||
|
||||
let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index);
|
||||
let mut builder = UpdateBuilder::new(0).settings(&mut txn, index.inner());
|
||||
builder.set_primary_key(primary_key);
|
||||
builder.execute(|_, _| ())?;
|
||||
|
||||
|
|
|
@ -22,6 +22,7 @@ struct DumpEntry {
|
|||
const UUIDS_DB_PATH: &str = "index_uuids";
|
||||
|
||||
#[async_trait::async_trait]
|
||||
#[cfg_attr(test, mockall::automock)]
|
||||
pub trait UuidStore: Sized {
|
||||
// Create a new entry for `name`. Return an error if `err` and the entry already exists, return
|
||||
// the uuid otherwise.
|
||||
|
|
|
@ -314,7 +314,7 @@ impl IndexController {
|
|||
for (uid, index) in indexes {
|
||||
let meta = index.meta()?;
|
||||
let meta = IndexMetadata {
|
||||
uuid: index.uuid,
|
||||
uuid: index.uuid(),
|
||||
name: uid.clone(),
|
||||
uid,
|
||||
meta,
|
||||
|
@ -366,7 +366,7 @@ impl IndexController {
|
|||
index_settings.uid.take();
|
||||
|
||||
let index = self.index_resolver.get_index(uid.clone()).await?;
|
||||
let uuid = index.uuid;
|
||||
let uuid = index.uuid();
|
||||
let meta =
|
||||
spawn_blocking(move || index.update_primary_key(index_settings.primary_key)).await??;
|
||||
let meta = IndexMetadata {
|
||||
|
@ -386,7 +386,7 @@ impl IndexController {
|
|||
|
||||
pub async fn get_index(&self, uid: String) -> Result<IndexMetadata> {
|
||||
let index = self.index_resolver.get_index(uid.clone()).await?;
|
||||
let uuid = index.uuid;
|
||||
let uuid = index.uuid();
|
||||
let meta = spawn_blocking(move || index.meta()).await??;
|
||||
let meta = IndexMetadata {
|
||||
uuid,
|
||||
|
@ -400,7 +400,7 @@ impl IndexController {
|
|||
pub async fn get_index_stats(&self, uid: String) -> Result<IndexStats> {
|
||||
let update_infos = UpdateMsg::get_info(&self.update_sender).await?;
|
||||
let index = self.index_resolver.get_index(uid).await?;
|
||||
let uuid = index.uuid;
|
||||
let uuid = index.uuid();
|
||||
let mut stats = spawn_blocking(move || index.stats()).await??;
|
||||
// Check if the currently indexing update is from our index.
|
||||
stats.is_indexing = Some(Some(uuid) == update_infos.processing);
|
||||
|
@ -414,7 +414,7 @@ impl IndexController {
|
|||
let mut indexes = BTreeMap::new();
|
||||
|
||||
for (index_uid, index) in self.index_resolver.list().await? {
|
||||
let uuid = index.uuid;
|
||||
let uuid = index.uuid();
|
||||
let (mut stats, meta) = spawn_blocking::<_, IndexResult<_>>(move || {
|
||||
let stats = index.stats()?;
|
||||
let meta = index.meta()?;
|
||||
|
@ -461,7 +461,7 @@ impl IndexController {
|
|||
let meta = spawn_blocking(move || -> IndexResult<_> {
|
||||
let meta = index.meta()?;
|
||||
let meta = IndexMetadata {
|
||||
uuid: index.uuid,
|
||||
uuid: index.uuid(),
|
||||
uid: uid.clone(),
|
||||
name: uid,
|
||||
meta,
|
||||
|
|
|
@ -125,133 +125,133 @@ pub fn load_snapshot(
|
|||
}
|
||||
}
|
||||
|
||||
//#[cfg(test)]
|
||||
//mod test {
|
||||
//use std::iter::FromIterator;
|
||||
//use std::{collections::HashSet, sync::Arc};
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
//use std::iter::FromIterator;
|
||||
//use std::{collections::HashSet, sync::Arc};
|
||||
|
||||
//use futures::future::{err, ok};
|
||||
//use rand::Rng;
|
||||
//use tokio::time::timeout;
|
||||
//use uuid::Uuid;
|
||||
//use futures::future::{err, ok};
|
||||
//use rand::Rng;
|
||||
//use tokio::time::timeout;
|
||||
//use uuid::Uuid;
|
||||
|
||||
//use super::*;
|
||||
//use super::*;
|
||||
|
||||
//#[actix_rt::test]
|
||||
//async fn test_normal() {
|
||||
//let mut rng = rand::thread_rng();
|
||||
//let uuids_num: usize = rng.gen_range(5..10);
|
||||
//let uuids = (0..uuids_num)
|
||||
//.map(|_| Uuid::new_v4())
|
||||
//.collect::<HashSet<_>>();
|
||||
//#[actix_rt::test]
|
||||
//async fn test_normal() {
|
||||
//let mut rng = rand::thread_rng();
|
||||
//let uuids_num: usize = rng.gen_range(5..10);
|
||||
//let uuids = (0..uuids_num)
|
||||
//.map(|_| Uuid::new_v4())
|
||||
//.collect::<HashSet<_>>();
|
||||
|
||||
//let mut uuid_resolver = MockUuidResolverHandle::new();
|
||||
//let uuids_clone = uuids.clone();
|
||||
//uuid_resolver
|
||||
//.expect_snapshot()
|
||||
//.times(1)
|
||||
//.returning(move |_| Box::pin(ok(uuids_clone.clone())));
|
||||
//let mut uuid_resolver = MockUuidResolverHandle::new();
|
||||
//let uuids_clone = uuids.clone();
|
||||
//uuid_resolver
|
||||
//.expect_snapshot()
|
||||
//.times(1)
|
||||
//.returning(move |_| Box::pin(ok(uuids_clone.clone())));
|
||||
|
||||
//let uuids_clone = uuids.clone();
|
||||
//let mut index_handle = MockIndexActorHandle::new();
|
||||
//index_handle
|
||||
//.expect_snapshot()
|
||||
//.withf(move |uuid, _path| uuids_clone.contains(uuid))
|
||||
//.times(uuids_num)
|
||||
//.returning(move |_, _| Box::pin(ok(())));
|
||||
//let uuids_clone = uuids.clone();
|
||||
//let mut index_handle = MockIndexActorHandle::new();
|
||||
//index_handle
|
||||
//.expect_snapshot()
|
||||
//.withf(move |uuid, _path| uuids_clone.contains(uuid))
|
||||
//.times(uuids_num)
|
||||
//.returning(move |_, _| Box::pin(ok(())));
|
||||
|
||||
//let dir = tempfile::tempdir_in(".").unwrap();
|
||||
//let handle = Arc::new(index_handle);
|
||||
//let update_handle =
|
||||
//UpdateActorHandleImpl::<Vec<u8>>::new(handle.clone(), dir.path(), 4096 * 100).unwrap();
|
||||
//let dir = tempfile::tempdir_in(".").unwrap();
|
||||
//let handle = Arc::new(index_handle);
|
||||
//let update_handle =
|
||||
//UpdateActorHandleImpl::<Vec<u8>>::new(handle.clone(), dir.path(), 4096 * 100).unwrap();
|
||||
|
||||
//let snapshot_path = tempfile::tempdir_in(".").unwrap();
|
||||
//let snapshot_service = SnapshotService::new(
|
||||
//uuid_resolver,
|
||||
//update_handle,
|
||||
//Duration::from_millis(100),
|
||||
//snapshot_path.path().to_owned(),
|
||||
//"data.ms".to_string(),
|
||||
//);
|
||||
//let snapshot_path = tempfile::tempdir_in(".").unwrap();
|
||||
//let snapshot_service = SnapshotService::new(
|
||||
//uuid_resolver,
|
||||
//update_handle,
|
||||
//Duration::from_millis(100),
|
||||
//snapshot_path.path().to_owned(),
|
||||
//"data.ms".to_string(),
|
||||
//);
|
||||
|
||||
//snapshot_service.perform_snapshot().await.unwrap();
|
||||
//}
|
||||
//snapshot_service.perform_snapshot().await.unwrap();
|
||||
//}
|
||||
|
||||
//#[actix_rt::test]
|
||||
//async fn error_performing_uuid_snapshot() {
|
||||
//let mut uuid_resolver = MockUuidResolverHandle::new();
|
||||
//uuid_resolver
|
||||
//.expect_snapshot()
|
||||
//.times(1)
|
||||
////abitrary error
|
||||
//.returning(|_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
|
||||
//#[actix_rt::test]
|
||||
//async fn error_performing_uuid_snapshot() {
|
||||
//let mut uuid_resolver = MockUuidResolverHandle::new();
|
||||
//uuid_resolver
|
||||
//.expect_snapshot()
|
||||
//.times(1)
|
||||
////abitrary error
|
||||
//.returning(|_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
|
||||
|
||||
//let update_handle = MockUpdateActorHandle::new();
|
||||
//let update_handle = MockUpdateActorHandle::new();
|
||||
|
||||
//let snapshot_path = tempfile::tempdir_in(".").unwrap();
|
||||
//let snapshot_service = SnapshotService::new(
|
||||
//uuid_resolver,
|
||||
//update_handle,
|
||||
//Duration::from_millis(100),
|
||||
//snapshot_path.path().to_owned(),
|
||||
//"data.ms".to_string(),
|
||||
//);
|
||||
//let snapshot_path = tempfile::tempdir_in(".").unwrap();
|
||||
//let snapshot_service = SnapshotService::new(
|
||||
//uuid_resolver,
|
||||
//update_handle,
|
||||
//Duration::from_millis(100),
|
||||
//snapshot_path.path().to_owned(),
|
||||
//"data.ms".to_string(),
|
||||
//);
|
||||
|
||||
//assert!(snapshot_service.perform_snapshot().await.is_err());
|
||||
////Nothing was written to the file
|
||||
//assert!(!snapshot_path.path().join("data.ms.snapshot").exists());
|
||||
//}
|
||||
//assert!(snapshot_service.perform_snapshot().await.is_err());
|
||||
////Nothing was written to the file
|
||||
//assert!(!snapshot_path.path().join("data.ms.snapshot").exists());
|
||||
//}
|
||||
|
||||
//#[actix_rt::test]
|
||||
//async fn error_performing_index_snapshot() {
|
||||
//let uuid = Uuid::new_v4();
|
||||
//let mut uuid_resolver = MockUuidResolverHandle::new();
|
||||
//uuid_resolver
|
||||
//.expect_snapshot()
|
||||
//.times(1)
|
||||
//.returning(move |_| Box::pin(ok(HashSet::from_iter(Some(uuid)))));
|
||||
//#[actix_rt::test]
|
||||
//async fn error_performing_index_snapshot() {
|
||||
//let uuid = Uuid::new_v4();
|
||||
//let mut uuid_resolver = MockUuidResolverHandle::new();
|
||||
//uuid_resolver
|
||||
//.expect_snapshot()
|
||||
//.times(1)
|
||||
//.returning(move |_| Box::pin(ok(HashSet::from_iter(Some(uuid)))));
|
||||
|
||||
//let mut update_handle = MockUpdateActorHandle::new();
|
||||
//update_handle
|
||||
//.expect_snapshot()
|
||||
////abitrary error
|
||||
//.returning(|_, _| Box::pin(err(UpdateActorError::UnexistingUpdate(0))));
|
||||
//let mut update_handle = MockUpdateActorHandle::new();
|
||||
//update_handle
|
||||
//.expect_snapshot()
|
||||
////abitrary error
|
||||
//.returning(|_, _| Box::pin(err(UpdateActorError::UnexistingUpdate(0))));
|
||||
|
||||
//let snapshot_path = tempfile::tempdir_in(".").unwrap();
|
||||
//let snapshot_service = SnapshotService::new(
|
||||
//uuid_resolver,
|
||||
//update_handle,
|
||||
//Duration::from_millis(100),
|
||||
//snapshot_path.path().to_owned(),
|
||||
//"data.ms".to_string(),
|
||||
//);
|
||||
//let snapshot_path = tempfile::tempdir_in(".").unwrap();
|
||||
//let snapshot_service = SnapshotService::new(
|
||||
//uuid_resolver,
|
||||
//update_handle,
|
||||
//Duration::from_millis(100),
|
||||
//snapshot_path.path().to_owned(),
|
||||
//"data.ms".to_string(),
|
||||
//);
|
||||
|
||||
//assert!(snapshot_service.perform_snapshot().await.is_err());
|
||||
////Nothing was written to the file
|
||||
//assert!(!snapshot_path.path().join("data.ms.snapshot").exists());
|
||||
//}
|
||||
//assert!(snapshot_service.perform_snapshot().await.is_err());
|
||||
////Nothing was written to the file
|
||||
//assert!(!snapshot_path.path().join("data.ms.snapshot").exists());
|
||||
//}
|
||||
|
||||
//#[actix_rt::test]
|
||||
//async fn test_loop() {
|
||||
//let mut uuid_resolver = MockUuidResolverHandle::new();
|
||||
//uuid_resolver
|
||||
//.expect_snapshot()
|
||||
////we expect the funtion to be called between 2 and 3 time in the given interval.
|
||||
//.times(2..4)
|
||||
////abitrary error, to short-circuit the function
|
||||
//.returning(move |_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
|
||||
//#[actix_rt::test]
|
||||
//async fn test_loop() {
|
||||
//let mut uuid_resolver = MockUuidResolverHandle::new();
|
||||
//uuid_resolver
|
||||
//.expect_snapshot()
|
||||
////we expect the funtion to be called between 2 and 3 time in the given interval.
|
||||
//.times(2..4)
|
||||
////abitrary error, to short-circuit the function
|
||||
//.returning(move |_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
|
||||
|
||||
//let update_handle = MockUpdateActorHandle::new();
|
||||
//let update_handle = MockUpdateActorHandle::new();
|
||||
|
||||
//let snapshot_path = tempfile::tempdir_in(".").unwrap();
|
||||
//let snapshot_service = SnapshotService::new(
|
||||
//uuid_resolver,
|
||||
//update_handle,
|
||||
//Duration::from_millis(100),
|
||||
//snapshot_path.path().to_owned(),
|
||||
//"data.ms".to_string(),
|
||||
//);
|
||||
//let snapshot_path = tempfile::tempdir_in(".").unwrap();
|
||||
//let snapshot_service = SnapshotService::new(
|
||||
//uuid_resolver,
|
||||
//update_handle,
|
||||
//Duration::from_millis(100),
|
||||
//snapshot_path.path().to_owned(),
|
||||
//"data.ms".to_string(),
|
||||
//);
|
||||
|
||||
//let _ = timeout(Duration::from_millis(300), snapshot_service.run()).await;
|
||||
//}
|
||||
//}
|
||||
//let _ = timeout(Duration::from_millis(300), snapshot_service.run()).await;
|
||||
//}
|
||||
}
|
||||
|
|
|
@ -34,7 +34,7 @@ impl UpdateStore {
|
|||
// txn must *always* be acquired after state lock, or it will dead lock.
|
||||
let txn = self.env.write_txn()?;
|
||||
|
||||
let uuids = indexes.iter().map(|i| i.uuid).collect();
|
||||
let uuids = indexes.iter().map(|i| i.uuid()).collect();
|
||||
|
||||
self.dump_updates(&txn, &uuids, &path)?;
|
||||
|
||||
|
|
|
@ -509,7 +509,7 @@ impl UpdateStore {
|
|||
|
||||
let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data();
|
||||
|
||||
let uuids: HashSet<_> = indexes.iter().map(|i| i.uuid).collect();
|
||||
let uuids: HashSet<_> = indexes.iter().map(|i| i.uuid()).collect();
|
||||
for entry in pendings {
|
||||
let ((_, uuid, _), pending) = entry?;
|
||||
if uuids.contains(&uuid) {
|
||||
|
@ -528,7 +528,7 @@ impl UpdateStore {
|
|||
let path = path.as_ref().to_owned();
|
||||
indexes
|
||||
.par_iter()
|
||||
.try_for_each(|index| index.snapshot(path.clone()))
|
||||
.try_for_each(|index| index.snapshot(&path))
|
||||
.unwrap();
|
||||
|
||||
Ok(())
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue