implement list indexes

src/data/mod.rs

@@ -9,8 +9,8 @@ use std::sync::Arc;
 
 use sha2::Digest;
 
-use crate::index_controller::{IndexController, LocalIndexController};
+use crate::index_controller::{IndexController, LocalIndexController, IndexMetadata, Settings};
-use crate::{option::Opt, index_controller::Settings};
+use crate::option::Opt;
 
 #[derive(Clone)]
 pub struct Data {
@@ -114,6 +114,10 @@ impl Data {
         })
     }
 
+    pub fn list_indexes(&self) -> anyhow::Result<Vec<IndexMetadata>> {
+        self.index_controller.list_indexes()
+    }
+
     #[inline]
     pub fn http_payload_size_limit(&self) -> usize {
         self.options.http_payload_size_limit.get_bytes() as usize

src/index_controller/local_index_controller/index_store.rs

@@ -2,8 +2,10 @@ use std::fs::{create_dir_all, remove_dir_all};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
+use chrono::{DateTime, Utc};
 use dashmap::{DashMap, mapref::entry::Entry};
 use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn};
+use log::error;
 use milli::Index;
 use rayon::ThreadPool;
 use serde::{Serialize, Deserialize};
@@ -16,10 +18,11 @@ use super::{UpdateMeta, UpdateResult};
 type UpdateStore = super::update_store::UpdateStore<UpdateMeta, UpdateResult, String>;
 
 #[derive(Serialize, Deserialize, Debug, PartialEq)]
-struct IndexMeta {
+pub struct IndexMeta {
-    update_store_size: u64,
+    update_size: u64,
-    index_store_size: u64,
+    index_size: u64,
-    uuid: Uuid,
+    pub uuid: Uuid,
+    pub created_at: DateTime<Utc>,
 }
 
 impl IndexMeta {
@@ -168,7 +171,8 @@ impl IndexStore {
         update_size: u64,
         index_size: u64,
     ) -> anyhow::Result<(Arc<Index>, Arc<UpdateStore>)> {
-        let meta = IndexMeta { update_store_size: update_size, index_store_size: index_size, uuid: uuid.clone() };
+        let created_at = Utc::now();
+        let meta = IndexMeta { update_size, index_size, uuid: uuid.clone(), created_at };
 
         self.name_to_uuid_meta.put(txn, name.as_ref(), uuid.as_bytes())?;
         self.uuid_to_index_db.put(txn, uuid.as_bytes(), &meta)?;
@@ -186,6 +190,29 @@ impl IndexStore {
 
         Ok((index, update_store))
     }
+
+    /// Returns each index associated with it's metadata;
+    pub fn list_indexes(&self) -> anyhow::Result<Vec<(String, IndexMeta)>> {
+        let txn = self.env.read_txn()?;
+        let indexes = self.name_to_uuid_db
+            .iter(&txn)?
+            .filter_map(|entry| entry
+                .map_err(|e| {
+                    error!("error decoding entry while listing indexes: {}", e);
+                    e
+                })
+                .ok())
+            .map(|(name, uuid)| {
+                let meta = self.uuid_to_index_db
+                    .get(&txn, &uuid)
+                    .ok()
+                    .flatten()
+                    .unwrap_or_else(|| panic!("corrupted database, index {} should exist.", name));
+                (name.to_owned(), meta)
+            })
+            .collect();
+        Ok(indexes)
+    }
 }
 
 fn open_or_create_database<K: 'static, V: 'static>(env: &Env, name: Option<&str>) -> anyhow::Result<Database<K, V>> {
@@ -264,7 +291,12 @@ mod test {
             let txn = store.env.read_txn().unwrap();
             assert!(store.retrieve_index(&txn, uuid).unwrap().is_none());
 
-            let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() };
+            let meta = IndexMeta {
+                update_size: 4096 * 100,
+                index_size: 4096 * 100,
+                uuid: uuid.clone(),
+                created_at: Utc::now(),
+            };
             let mut txn = store.env.write_txn().unwrap();
             store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap();
             txn.commit().unwrap();
@@ -286,7 +318,12 @@ mod test {
             assert!(store.index(&name).unwrap().is_none());
 
             let uuid = Uuid::new_v4();
-            let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() };
+            let meta = IndexMeta {
+                update_size: 4096 * 100,
+                index_size: 4096 * 100,
+                uuid: uuid.clone(),
+                created_at: Utc::now(),
+            };
             let mut txn = store.env.write_txn().unwrap();
             store.name_to_uuid_meta.put(&mut txn, &name, uuid.as_bytes()).unwrap();
             store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap();
@@ -301,14 +338,18 @@ mod test {
             let store = IndexStore::new(temp, IndexerOpts::default()).unwrap();
             let name = "foobar";
 
-            store.get_or_create_index(&name, 4096 * 100, 4096 * 100).unwrap();
+            let update_size = 4096 * 100;
+            let index_size = 4096 * 100;
+            store.get_or_create_index(&name, update_size, index_size).unwrap();
             let txn = store.env.read_txn().unwrap();
             let  uuid = store.name_to_uuid_meta.get(&txn, &name).unwrap();
             assert_eq!(store.uuid_to_index.len(), 1);
             assert!(uuid.is_some());
             let uuid = Uuid::from_slice(uuid.unwrap()).unwrap();
-            let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() };
+            let meta = store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap().unwrap();
-            assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta));
+            assert_eq!(meta.update_size, update_size);
+            assert_eq!(meta.index_size, index_size);
+            assert_eq!(meta.uuid, uuid);
         }
 
         #[test]
@@ -326,8 +367,10 @@ mod test {
             assert_eq!(store.uuid_to_index.len(), 1);
             assert!(uuid.is_some());
             let uuid = Uuid::from_slice(uuid.unwrap()).unwrap();
-            let meta = IndexMeta { update_store_size: update_size , index_store_size: index_size, uuid: uuid.clone() };
+            let meta = store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap().unwrap();
-            assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta));
+            assert_eq!(meta.update_size, update_size);
+            assert_eq!(meta.index_size, index_size);
+            assert_eq!(meta.uuid, uuid);
         }
     }
 }

src/index_controller/local_index_controller/mod.rs

@@ -13,7 +13,7 @@ use crate::option::IndexerOpts;
 use index_store::IndexStore;
 use super::IndexController;
 use super::updates::UpdateStatus;
-use super::{UpdateMeta, UpdateResult};
+use super::{UpdateMeta, UpdateResult, IndexMetadata};
 
 pub struct LocalIndexController {
     indexes: IndexStore,
@@ -102,4 +102,29 @@ impl IndexController for LocalIndexController {
         }
 
     }
+
+    fn list_indexes(&self) -> anyhow::Result<Vec<IndexMetadata>> {
+        let metas = self.indexes.list_indexes()?;
+        let mut output_meta = Vec::new();
+        for (name, meta) in metas {
+            let created_at = meta.created_at;
+            let uuid = meta.uuid;
+            let updated_at = self
+                .all_update_status(&name)?
+                .iter()
+                .filter_map(|u| u.processed().map(|u| u.processed_at))
+                .max()
+                .unwrap_or(created_at);
+
+            let index_meta = IndexMetadata {
+                name,
+                created_at,
+                updated_at,
+                uuid,
+                primary_key: None,
+            };
+            output_meta.push(index_meta);
+        }
+        Ok(output_meta)
+    }
 }

src/index_controller/mod.rs

@@ -8,14 +8,26 @@ use std::num::NonZeroUsize;
 use std::sync::Arc;
 
 use anyhow::Result;
+use chrono::{DateTime, Utc};
 use milli::Index;
 use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult};
 use serde::{Serialize, Deserialize, de::Deserializer};
+use uuid::Uuid;
 
 pub use updates::{Processed, Processing, Failed};
 
 pub type UpdateStatus = updates::UpdateStatus<UpdateMeta, UpdateResult, String>;
 
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct IndexMetadata {
+    pub name: String,
+    uuid: Uuid,
+    created_at: DateTime<Utc>,
+    updated_at: DateTime<Utc>,
+    pub primary_key: Option<String>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(tag = "type")]
 pub enum UpdateMeta {
@@ -140,4 +152,6 @@ pub trait IndexController {
 
     fn update_status(&self, index: impl AsRef<str>, id: u64) -> anyhow::Result<Option<UpdateStatus>>;
     fn all_update_status(&self, index: impl AsRef<str>) -> anyhow::Result<Vec<UpdateStatus>>;
+
+    fn list_indexes(&self) -> anyhow::Result<Vec<IndexMetadata>>;
 }

src/index_controller/updates.rs

@@ -134,6 +134,13 @@ impl<M, N, E> UpdateStatus<M, N, E> {
             UpdateStatus::Failed(u) => u.id(),
         }
     }
+
+    pub fn processed(&self) -> Option<&Processed<M, N>> {
+        match self {
+            UpdateStatus::Processed(p) => Some(p),
+            _ => None,
+        }
+    }
 }
 
 impl<M, N, E> From<Pending<M>> for UpdateStatus<M, N, E> {

src/routes/index.rs

@@ -19,19 +19,20 @@ pub fn services(cfg: &mut web::ServiceConfig) {
         .service(get_all_updates_status);
 }
 
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct IndexResponse {
-    pub name: String,
-    pub uid: String,
-    created_at: DateTime<Utc>,
-    updated_at: DateTime<Utc>,
-    pub primary_key: Option<String>,
-}
 
 #[get("/indexes", wrap = "Authentication::Private")]
-async fn list_indexes(_data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
+async fn list_indexes(data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
-    todo!()
+    match data.list_indexes() {
+        Ok(indexes) => {
+            let json = serde_json::to_string(&indexes).unwrap();
+            Ok(HttpResponse::Ok().body(&json))
+        }
+        Err(e) => {
+            error!("error listing indexes: {}", e);
+            unimplemented!()
+        }
+    }
+
 }
 
 #[get("/indexes/{index_uid}", wrap = "Authentication::Private")]