implement list indexes

This commit is contained in:
mpostma 2021-02-03 17:44:20 +01:00
parent 482f734e53
commit d43dc4824c
No known key found for this signature in database
GPG Key ID: CBC8A7C1D7A28C3A
6 changed files with 120 additions and 26 deletions

View File

@ -9,8 +9,8 @@ use std::sync::Arc;
use sha2::Digest;
use crate::index_controller::{IndexController, LocalIndexController};
use crate::{option::Opt, index_controller::Settings};
use crate::index_controller::{IndexController, LocalIndexController, IndexMetadata, Settings};
use crate::option::Opt;
#[derive(Clone)]
pub struct Data {
@ -114,6 +114,10 @@ impl Data {
})
}
pub fn list_indexes(&self) -> anyhow::Result<Vec<IndexMetadata>> {
self.index_controller.list_indexes()
}
#[inline]
pub fn http_payload_size_limit(&self) -> usize {
self.options.http_payload_size_limit.get_bytes() as usize

View File

@ -2,8 +2,10 @@ use std::fs::{create_dir_all, remove_dir_all};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use chrono::{DateTime, Utc};
use dashmap::{DashMap, mapref::entry::Entry};
use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn};
use log::error;
use milli::Index;
use rayon::ThreadPool;
use serde::{Serialize, Deserialize};
@ -16,10 +18,11 @@ use super::{UpdateMeta, UpdateResult};
type UpdateStore = super::update_store::UpdateStore<UpdateMeta, UpdateResult, String>;
#[derive(Serialize, Deserialize, Debug, PartialEq)]
struct IndexMeta {
update_store_size: u64,
index_store_size: u64,
uuid: Uuid,
pub struct IndexMeta {
update_size: u64,
index_size: u64,
pub uuid: Uuid,
pub created_at: DateTime<Utc>,
}
impl IndexMeta {
@ -168,7 +171,8 @@ impl IndexStore {
update_size: u64,
index_size: u64,
) -> anyhow::Result<(Arc<Index>, Arc<UpdateStore>)> {
let meta = IndexMeta { update_store_size: update_size, index_store_size: index_size, uuid: uuid.clone() };
let created_at = Utc::now();
let meta = IndexMeta { update_size, index_size, uuid: uuid.clone(), created_at };
self.name_to_uuid_meta.put(txn, name.as_ref(), uuid.as_bytes())?;
self.uuid_to_index_db.put(txn, uuid.as_bytes(), &meta)?;
@ -186,6 +190,29 @@ impl IndexStore {
Ok((index, update_store))
}
/// Returns each index associated with it's metadata;
pub fn list_indexes(&self) -> anyhow::Result<Vec<(String, IndexMeta)>> {
let txn = self.env.read_txn()?;
let indexes = self.name_to_uuid_db
.iter(&txn)?
.filter_map(|entry| entry
.map_err(|e| {
error!("error decoding entry while listing indexes: {}", e);
e
})
.ok())
.map(|(name, uuid)| {
let meta = self.uuid_to_index_db
.get(&txn, &uuid)
.ok()
.flatten()
.unwrap_or_else(|| panic!("corrupted database, index {} should exist.", name));
(name.to_owned(), meta)
})
.collect();
Ok(indexes)
}
}
fn open_or_create_database<K: 'static, V: 'static>(env: &Env, name: Option<&str>) -> anyhow::Result<Database<K, V>> {
@ -264,7 +291,12 @@ mod test {
let txn = store.env.read_txn().unwrap();
assert!(store.retrieve_index(&txn, uuid).unwrap().is_none());
let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() };
let meta = IndexMeta {
update_size: 4096 * 100,
index_size: 4096 * 100,
uuid: uuid.clone(),
created_at: Utc::now(),
};
let mut txn = store.env.write_txn().unwrap();
store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap();
txn.commit().unwrap();
@ -286,7 +318,12 @@ mod test {
assert!(store.index(&name).unwrap().is_none());
let uuid = Uuid::new_v4();
let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() };
let meta = IndexMeta {
update_size: 4096 * 100,
index_size: 4096 * 100,
uuid: uuid.clone(),
created_at: Utc::now(),
};
let mut txn = store.env.write_txn().unwrap();
store.name_to_uuid_meta.put(&mut txn, &name, uuid.as_bytes()).unwrap();
store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap();
@ -301,14 +338,18 @@ mod test {
let store = IndexStore::new(temp, IndexerOpts::default()).unwrap();
let name = "foobar";
store.get_or_create_index(&name, 4096 * 100, 4096 * 100).unwrap();
let update_size = 4096 * 100;
let index_size = 4096 * 100;
store.get_or_create_index(&name, update_size, index_size).unwrap();
let txn = store.env.read_txn().unwrap();
let uuid = store.name_to_uuid_meta.get(&txn, &name).unwrap();
assert_eq!(store.uuid_to_index.len(), 1);
assert!(uuid.is_some());
let uuid = Uuid::from_slice(uuid.unwrap()).unwrap();
let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() };
assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta));
let meta = store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap().unwrap();
assert_eq!(meta.update_size, update_size);
assert_eq!(meta.index_size, index_size);
assert_eq!(meta.uuid, uuid);
}
#[test]
@ -326,8 +367,10 @@ mod test {
assert_eq!(store.uuid_to_index.len(), 1);
assert!(uuid.is_some());
let uuid = Uuid::from_slice(uuid.unwrap()).unwrap();
let meta = IndexMeta { update_store_size: update_size , index_store_size: index_size, uuid: uuid.clone() };
assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta));
let meta = store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap().unwrap();
assert_eq!(meta.update_size, update_size);
assert_eq!(meta.index_size, index_size);
assert_eq!(meta.uuid, uuid);
}
}
}

View File

@ -13,7 +13,7 @@ use crate::option::IndexerOpts;
use index_store::IndexStore;
use super::IndexController;
use super::updates::UpdateStatus;
use super::{UpdateMeta, UpdateResult};
use super::{UpdateMeta, UpdateResult, IndexMetadata};
pub struct LocalIndexController {
indexes: IndexStore,
@ -102,4 +102,29 @@ impl IndexController for LocalIndexController {
}
}
fn list_indexes(&self) -> anyhow::Result<Vec<IndexMetadata>> {
let metas = self.indexes.list_indexes()?;
let mut output_meta = Vec::new();
for (name, meta) in metas {
let created_at = meta.created_at;
let uuid = meta.uuid;
let updated_at = self
.all_update_status(&name)?
.iter()
.filter_map(|u| u.processed().map(|u| u.processed_at))
.max()
.unwrap_or(created_at);
let index_meta = IndexMetadata {
name,
created_at,
updated_at,
uuid,
primary_key: None,
};
output_meta.push(index_meta);
}
Ok(output_meta)
}
}

View File

@ -8,14 +8,26 @@ use std::num::NonZeroUsize;
use std::sync::Arc;
use anyhow::Result;
use chrono::{DateTime, Utc};
use milli::Index;
use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult};
use serde::{Serialize, Deserialize, de::Deserializer};
use uuid::Uuid;
pub use updates::{Processed, Processing, Failed};
pub type UpdateStatus = updates::UpdateStatus<UpdateMeta, UpdateResult, String>;
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "camelCase")]
pub struct IndexMetadata {
pub name: String,
uuid: Uuid,
created_at: DateTime<Utc>,
updated_at: DateTime<Utc>,
pub primary_key: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum UpdateMeta {
@ -140,4 +152,6 @@ pub trait IndexController {
fn update_status(&self, index: impl AsRef<str>, id: u64) -> anyhow::Result<Option<UpdateStatus>>;
fn all_update_status(&self, index: impl AsRef<str>) -> anyhow::Result<Vec<UpdateStatus>>;
fn list_indexes(&self) -> anyhow::Result<Vec<IndexMetadata>>;
}

View File

@ -134,6 +134,13 @@ impl<M, N, E> UpdateStatus<M, N, E> {
UpdateStatus::Failed(u) => u.id(),
}
}
pub fn processed(&self) -> Option<&Processed<M, N>> {
match self {
UpdateStatus::Processed(p) => Some(p),
_ => None,
}
}
}
impl<M, N, E> From<Pending<M>> for UpdateStatus<M, N, E> {

View File

@ -19,19 +19,20 @@ pub fn services(cfg: &mut web::ServiceConfig) {
.service(get_all_updates_status);
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "camelCase")]
pub struct IndexResponse {
pub name: String,
pub uid: String,
created_at: DateTime<Utc>,
updated_at: DateTime<Utc>,
pub primary_key: Option<String>,
}
#[get("/indexes", wrap = "Authentication::Private")]
async fn list_indexes(_data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
todo!()
async fn list_indexes(data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
match data.list_indexes() {
Ok(indexes) => {
let json = serde_json::to_string(&indexes).unwrap();
Ok(HttpResponse::Ok().body(&json))
}
Err(e) => {
error!("error listing indexes: {}", e);
unimplemented!()
}
}
}
#[get("/indexes/{index_uid}", wrap = "Authentication::Private")]