restore settings updates

This commit is contained in:
mpostma 2021-09-24 14:55:57 +02:00
parent dfce44fa3b
commit c32012c44a
8 changed files with 264 additions and 280 deletions

View File

@ -170,7 +170,7 @@ pub async fn add_documents(
format: DocumentAdditionFormat::Json, format: DocumentAdditionFormat::Json,
}; };
let update_status = meilisearch let update_status = meilisearch
.register_update(path.index_uid.as_str(), update) .register_update(path.into_inner().index_uid, update)
.await?; .await?;
debug!("returns: {:?}", update_status); debug!("returns: {:?}", update_status);
@ -193,7 +193,7 @@ pub async fn update_documents(
format: DocumentAdditionFormat::Json, format: DocumentAdditionFormat::Json,
}; };
let update_status = meilisearch let update_status = meilisearch
.register_update(path.index_uid.as_str(), update) .register_update(path.into_inner().index_uid, update)
.await?; .await?;
debug!("returns: {:?}", update_status); debug!("returns: {:?}", update_status);

View File

@ -32,7 +32,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
.service(web::scope("/documents").configure(documents::configure)) .service(web::scope("/documents").configure(documents::configure))
.service(web::scope("/search").configure(search::configure)) .service(web::scope("/search").configure(search::configure))
.service(web::scope("/updates").configure(updates::configure)) .service(web::scope("/updates").configure(updates::configure))
//.service(web::scope("/settings").configure(settings::configure)), .service(web::scope("/settings").configure(settings::configure)),
); );
} }

View File

@ -1,184 +1,191 @@
//use log::debug; use log::debug;
//use crate::extractors::authentication::{policies::*, GuardedData}; use actix_web::{HttpResponse, web};
//use crate::index::Settings; use meilisearch_lib::MeiliSearch;
//use crate::Data; use meilisearch_lib::index::{Settings, Unchecked};
//use crate::error::ResponseError; use meilisearch_lib::index_controller::Update;
//#[macro_export] use crate::extractors::authentication::{policies::*, GuardedData};
//macro_rules! make_setting_route { use crate::error::ResponseError;
//($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => {
//pub mod $attr {
//use log::debug;
//use actix_web::{web, HttpResponse, Resource};
//use milli::update::Setting; #[macro_export]
macro_rules! make_setting_route {
($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => {
pub mod $attr {
use log::debug;
use actix_web::{web, HttpResponse, Resource};
//use crate::data; use milli::update::Setting;
//use crate::error::ResponseError; use meilisearch_lib::{MeiliSearch, index::Settings, index_controller::Update};
//use crate::index::Settings;
//use crate::extractors::authentication::{GuardedData, policies::*};
//pub async fn delete( use crate::error::ResponseError;
//data: GuardedData<Private, data::Data>, use crate::extractors::authentication::{GuardedData, policies::*};
//index_uid: web::Path<String>,
//) -> Result<HttpResponse, ResponseError> {
//use crate::index::Settings;
//let settings = Settings {
//$attr: Setting::Reset,
//..Default::default()
//};
//let update_status = data.update_settings(index_uid.into_inner(), settings, false).await?;
//debug!("returns: {:?}", update_status);
//Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
//}
//pub async fn update( pub async fn delete(
//data: GuardedData<Private, data::Data>, meilisearch: GuardedData<Private, MeiliSearch>,
//index_uid: actix_web::web::Path<String>, index_uid: web::Path<String>,
//body: actix_web::web::Json<Option<$type>>, ) -> Result<HttpResponse, ResponseError> {
//) -> std::result::Result<HttpResponse, ResponseError> { let settings = Settings {
//let settings = Settings { $attr: Setting::Reset,
//$attr: match body.into_inner() { ..Default::default()
//Some(inner_body) => Setting::Set(inner_body), };
//None => Setting::Reset let update = Update::Settings(settings);
//}, let update_status = meilisearch.register_update(index_uid.into_inner(), update).await?;
//..Default::default() debug!("returns: {:?}", update_status);
//}; Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
//let update_status = data.update_settings(index_uid.into_inner(), settings, true).await?; pub async fn update(
//debug!("returns: {:?}", update_status); meilisearch: GuardedData<Private, MeiliSearch>,
//Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() }))) index_uid: actix_web::web::Path<String>,
//} body: actix_web::web::Json<Option<$type>>,
) -> std::result::Result<HttpResponse, ResponseError> {
let settings = Settings {
$attr: match body.into_inner() {
Some(inner_body) => Setting::Set(inner_body),
None => Setting::Reset
},
..Default::default()
};
//pub async fn get( let update = Update::Settings(settings);
//data: GuardedData<Private, data::Data>, let update_status = meilisearch.register_update(index_uid.into_inner(), update).await?;
//index_uid: actix_web::web::Path<String>, debug!("returns: {:?}", update_status);
//) -> std::result::Result<HttpResponse, ResponseError> { Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
//let settings = data.settings(index_uid.into_inner()).await?; }
//debug!("returns: {:?}", settings);
//let mut json = serde_json::json!(&settings);
//let val = json[$camelcase_attr].take();
//Ok(HttpResponse::Ok().json(val))
//}
//pub fn resources() -> Resource { pub async fn get(
//Resource::new($route) meilisearch: GuardedData<Private, MeiliSearch>,
//.route(web::get().to(get)) index_uid: actix_web::web::Path<String>,
//.route(web::post().to(update)) ) -> std::result::Result<HttpResponse, ResponseError> {
//.route(web::delete().to(delete)) let settings = meilisearch.settings(index_uid.into_inner()).await?;
//} debug!("returns: {:?}", settings);
//} let mut json = serde_json::json!(&settings);
//}; let val = json[$camelcase_attr].take();
//} Ok(HttpResponse::Ok().json(val))
}
//make_setting_route!( pub fn resources() -> Resource {
//"/filterable-attributes", Resource::new($route)
//std::collections::BTreeSet<String>, .route(web::get().to(get))
//filterable_attributes, .route(web::post().to(update))
//"filterableAttributes" .route(web::delete().to(delete))
//); }
}
};
}
//make_setting_route!( make_setting_route!(
//"/sortable-attributes", "/filterable-attributes",
//std::collections::BTreeSet<String>, std::collections::BTreeSet<String>,
//sortable_attributes, filterable_attributes,
//"sortableAttributes" "filterableAttributes"
//); );
//make_setting_route!( make_setting_route!(
//"/displayed-attributes", "/sortable-attributes",
//Vec<String>, std::collections::BTreeSet<String>,
//displayed_attributes, sortable_attributes,
//"displayedAttributes" "sortableAttributes"
//); );
//make_setting_route!( make_setting_route!(
//"/searchable-attributes", "/displayed-attributes",
//Vec<String>, Vec<String>,
//searchable_attributes, displayed_attributes,
//"searchableAttributes" "displayedAttributes"
//); );
//make_setting_route!( make_setting_route!(
//"/stop-words", "/searchable-attributes",
//std::collections::BTreeSet<String>, Vec<String>,
//stop_words, searchable_attributes,
//"stopWords" "searchableAttributes"
//); );
//make_setting_route!( make_setting_route!(
//"/synonyms", "/stop-words",
//std::collections::BTreeMap<String, Vec<String>>, std::collections::BTreeSet<String>,
//synonyms, stop_words,
//"synonyms" "stopWords"
//); );
//make_setting_route!( make_setting_route!(
//"/distinct-attribute", "/synonyms",
//String, std::collections::BTreeMap<String, Vec<String>>,
//distinct_attribute, synonyms,
//"distinctAttribute" "synonyms"
//); );
//make_setting_route!("/ranking-rules", Vec<String>, ranking_rules, "rankingRules"); make_setting_route!(
"/distinct-attribute",
String,
distinct_attribute,
"distinctAttribute"
);
//macro_rules! generate_configure { make_setting_route!("/ranking-rules", Vec<String>, ranking_rules, "rankingRules");
//($($mod:ident),*) => {
//pub fn configure(cfg: &mut web::ServiceConfig) {
//cfg.service(
//web::resource("")
////.route(web::post().to(update_all))
//.route(web::get().to(get_all))
////.route(web::delete().to(delete_all)))
//$(.service($mod::resources()))*;
//}
//};
//}
//generate_configure!( macro_rules! generate_configure {
//filterable_attributes, ($($mod:ident),*) => {
//sortable_attributes, pub fn configure(cfg: &mut web::ServiceConfig) {
//displayed_attributes, cfg.service(
//searchable_attributes, web::resource("")
//distinct_attribute, .route(web::post().to(update_all))
//stop_words, .route(web::get().to(get_all))
//synonyms, .route(web::delete().to(delete_all)))
//ranking_rules $(.service($mod::resources()))*;
//); }
};
}
//pub async fn update_all( generate_configure!(
//data: GuardedData<Private, MeiliSearch>, filterable_attributes,
//index_uid: web::Path<String>, sortable_attributes,
//body: web::Json<Settings<Unchecked>>, displayed_attributes,
//) -> Result<HttpResponse, ResponseError> { searchable_attributes,
//let settings = body.into_inner().check(); distinct_attribute,
//let update_result = data stop_words,
//.update_settings(index_uid.into_inner(), settings, true) synonyms,
//.await?; ranking_rules
//let json = serde_json::json!({ "updateId": update_result.id() }); );
//debug!("returns: {:?}", json);
//Ok(HttpResponse::Accepted().json(json))
//}
//pub async fn get_all( pub async fn update_all(
//data: GuardedData<Private, MeiliSearch>, meilisearch: GuardedData<Private, MeiliSearch>,
//index_uid: web::Path<String>, index_uid: web::Path<String>,
//) -> Result<HttpResponse, ResponseError> { body: web::Json<Settings<Unchecked>>,
//let settings = data.settings(index_uid.into_inner()).await?; ) -> Result<HttpResponse, ResponseError> {
//debug!("returns: {:?}", settings); let settings = body.into_inner();
//Ok(HttpResponse::Ok().json(settings))
//}
//pub async fn delete_all( let update = Update::Settings(settings);
//data: GuardedData<Private, MeiliSearch>, let update_result = meilisearch
//index_uid: web::Path<String>, .register_update(index_uid.into_inner(), update)
//) -> Result<HttpResponse, ResponseError> { .await?;
//let settings = Settings::cleared(); let json = serde_json::json!({ "updateId": update_result.id() });
//let update_result = data debug!("returns: {:?}", json);
//.update_settings(index_uid.into_inner(), settings, false) Ok(HttpResponse::Accepted().json(json))
//.await?; }
//let json = serde_json::json!({ "updateId": update_result.id() });
//debug!("returns: {:?}", json); pub async fn get_all(
//Ok(HttpResponse::Accepted().json(json)) data: GuardedData<Private, MeiliSearch>,
//} index_uid: web::Path<String>,
) -> Result<HttpResponse, ResponseError> {
let settings = data.settings(index_uid.into_inner()).await?;
debug!("returns: {:?}", settings);
Ok(HttpResponse::Ok().json(settings))
}
pub async fn delete_all(
data: GuardedData<Private, MeiliSearch>,
index_uid: web::Path<String>,
) -> Result<HttpResponse, ResponseError> {
let settings = Settings::cleared();
let update = Update::Settings(settings.into_unchecked());
let update_result = data
.register_update(index_uid.into_inner(), update)
.await?;
let json = serde_json::json!({ "updateId": update_result.id() });
debug!("returns: {:?}", json);
Ok(HttpResponse::Accepted().json(json))
}

View File

@ -69,13 +69,13 @@ impl From<&UpdateStatus> for UpdateType {
_ => unreachable!(), _ => unreachable!(),
} }
} }
RegisterUpdate::Settings(settings) => UpdateType::Settings {
settings: settings.clone(),
},
//UpdateMeta::ClearDocuments => UpdateType::ClearAll, //UpdateMeta::ClearDocuments => UpdateType::ClearAll,
//UpdateMeta::DeleteDocuments { ids } => UpdateType::DocumentsDeletion { //UpdateMeta::DeleteDocuments { ids } => UpdateType::DocumentsDeletion {
//number: Some(ids.len()), //number: Some(ids.len()),
//}, //},
//UpdateMeta::Settings(settings) => UpdateType::Settings {
//settings: settings.clone(),
//},
} }
} }
} }

View File

@ -1,10 +1,7 @@
use crate::index::Index;
use milli::update::UpdateBuilder; use milli::update::UpdateBuilder;
use milli::CompressionType; use milli::CompressionType;
use rayon::ThreadPool; use rayon::ThreadPool;
use crate::index_controller::updates::RegisterUpdate;
use crate::index_controller::updates::status::{Failed, Processed, Processing};
use crate::options::IndexerOpts; use crate::options::IndexerOpts;
pub struct UpdateHandler { pub struct UpdateHandler {
@ -49,24 +46,4 @@ impl UpdateHandler {
update_builder.chunk_compression_type(self.chunk_compression_type); update_builder.chunk_compression_type(self.chunk_compression_type);
update_builder update_builder
} }
pub fn handle_update(
&self,
index: &Index,
meta: Processing,
) -> Result<Processed, Failed> {
let update_id = meta.id();
let update_builder = self.update_builder(update_id);
let result = match meta.meta() {
RegisterUpdate::DocumentAddition { primary_key, content_uuid, method } => {
index.update_documents(*method, *content_uuid, update_builder, primary_key.as_deref())
}
};
match result {
Ok(result) => Ok(meta.process(result)),
Err(e) => Err(meta.fail(e)),
}
}
} }

View File

@ -8,6 +8,7 @@ use milli::update::{IndexDocumentsMethod, Setting, UpdateBuilder};
use serde::{Deserialize, Serialize, Serializer}; use serde::{Deserialize, Serialize, Serializer};
use uuid::Uuid; use uuid::Uuid;
use crate::RegisterUpdate;
use crate::index_controller::updates::status::{Failed, Processed, Processing, UpdateResult}; use crate::index_controller::updates::status::{Failed, Processed, Processing, UpdateResult};
use super::{Index, IndexMeta}; use super::{Index, IndexMeta};
@ -164,7 +165,27 @@ pub struct Facets {
impl Index { impl Index {
pub fn handle_update(&self, update: Processing) -> std::result::Result<Processed, Failed> { pub fn handle_update(&self, update: Processing) -> std::result::Result<Processed, Failed> {
self.update_handler.handle_update(self, update) let update_id = update.id();
let update_builder = self.update_handler.update_builder(update_id);
let result = (|| {
let mut txn = self.write_txn()?;
let result = match update.meta() {
RegisterUpdate::DocumentAddition { primary_key, content_uuid, method } => {
self.update_documents(&mut txn, *method, *content_uuid, update_builder, primary_key.as_deref())
}
RegisterUpdate::Settings(settings) => {
let settings = settings.clone().check();
self.update_settings(&mut txn, &settings, update_builder)
},
};
txn.commit()?;
result
})();
match result {
Ok(result) => Ok(update.process(result)),
Err(e) => Err(update.fail(e)),
}
} }
pub fn update_primary_key(&self, primary_key: Option<String>) -> Result<IndexMeta> { pub fn update_primary_key(&self, primary_key: Option<String>) -> Result<IndexMeta> {
@ -188,21 +209,7 @@ impl Index {
} }
} }
pub fn update_documents( fn update_documents<'a, 'b>(
&self,
method: IndexDocumentsMethod,
content_uuid: Uuid,
update_builder: UpdateBuilder,
primary_key: Option<&str>,
) -> Result<UpdateResult> {
let mut txn = self.write_txn()?;
let result = self.update_documents_txn(&mut txn, method, content_uuid, update_builder, primary_key)?;
txn.commit()?;
Ok(result)
}
pub fn update_documents_txn<'a, 'b>(
&'a self, &'a self,
txn: &mut heed::RwTxn<'a, 'b>, txn: &mut heed::RwTxn<'a, 'b>,
method: IndexDocumentsMethod, method: IndexDocumentsMethod,
@ -246,86 +253,75 @@ impl Index {
//.map_err(Into::into) //.map_err(Into::into)
//} //}
//pub fn update_settings_txn<'a, 'b>( fn update_settings<'a, 'b>(
//&'a self, &'a self,
//txn: &mut heed::RwTxn<'a, 'b>, txn: &mut heed::RwTxn<'a, 'b>,
//settings: &Settings<Checked>, settings: &Settings<Checked>,
//update_builder: UpdateBuilder, update_builder: UpdateBuilder,
//) -> Result<UpdateResult> { ) -> Result<UpdateResult> {
//// We must use the write transaction of the update here. // We must use the write transaction of the update here.
//let mut builder = update_builder.settings(txn, self); let mut builder = update_builder.settings(txn, self);
//match settings.searchable_attributes { match settings.searchable_attributes {
//Setting::Set(ref names) => builder.set_searchable_fields(names.clone()), Setting::Set(ref names) => builder.set_searchable_fields(names.clone()),
//Setting::Reset => builder.reset_searchable_fields(), Setting::Reset => builder.reset_searchable_fields(),
//Setting::NotSet => (), Setting::NotSet => (),
//} }
//match settings.displayed_attributes { match settings.displayed_attributes {
//Setting::Set(ref names) => builder.set_displayed_fields(names.clone()), Setting::Set(ref names) => builder.set_displayed_fields(names.clone()),
//Setting::Reset => builder.reset_displayed_fields(), Setting::Reset => builder.reset_displayed_fields(),
//Setting::NotSet => (), Setting::NotSet => (),
//} }
//match settings.filterable_attributes { match settings.filterable_attributes {
//Setting::Set(ref facets) => { Setting::Set(ref facets) => {
//builder.set_filterable_fields(facets.clone().into_iter().collect()) builder.set_filterable_fields(facets.clone().into_iter().collect())
//} }
//Setting::Reset => builder.reset_filterable_fields(), Setting::Reset => builder.reset_filterable_fields(),
//Setting::NotSet => (), Setting::NotSet => (),
//} }
//match settings.sortable_attributes { match settings.sortable_attributes {
//Setting::Set(ref fields) => { Setting::Set(ref fields) => {
//builder.set_sortable_fields(fields.iter().cloned().collect()) builder.set_sortable_fields(fields.iter().cloned().collect())
//} }
//Setting::Reset => builder.reset_sortable_fields(), Setting::Reset => builder.reset_sortable_fields(),
//Setting::NotSet => (), Setting::NotSet => (),
//} }
//match settings.ranking_rules { match settings.ranking_rules {
//Setting::Set(ref criteria) => builder.set_criteria(criteria.clone()), Setting::Set(ref criteria) => builder.set_criteria(criteria.clone()),
//Setting::Reset => builder.reset_criteria(), Setting::Reset => builder.reset_criteria(),
//Setting::NotSet => (), Setting::NotSet => (),
//} }
//match settings.stop_words { match settings.stop_words {
//Setting::Set(ref stop_words) => builder.set_stop_words(stop_words.clone()), Setting::Set(ref stop_words) => builder.set_stop_words(stop_words.clone()),
//Setting::Reset => builder.reset_stop_words(), Setting::Reset => builder.reset_stop_words(),
//Setting::NotSet => (), Setting::NotSet => (),
//} }
//match settings.synonyms { match settings.synonyms {
//Setting::Set(ref synonyms) => { Setting::Set(ref synonyms) => {
//builder.set_synonyms(synonyms.clone().into_iter().collect()) builder.set_synonyms(synonyms.clone().into_iter().collect())
//} }
//Setting::Reset => builder.reset_synonyms(), Setting::Reset => builder.reset_synonyms(),
//Setting::NotSet => (), Setting::NotSet => (),
//} }
//match settings.distinct_attribute { match settings.distinct_attribute {
//Setting::Set(ref attr) => builder.set_distinct_field(attr.clone()), Setting::Set(ref attr) => builder.set_distinct_field(attr.clone()),
//Setting::Reset => builder.reset_distinct_field(), Setting::Reset => builder.reset_distinct_field(),
//Setting::NotSet => (), Setting::NotSet => (),
//} }
//builder.execute(|indexing_step, update_id| { builder.execute(|indexing_step, update_id| {
//debug!("update {}: {:?}", update_id, indexing_step) debug!("update {}: {:?}", update_id, indexing_step)
//})?; })?;
//Ok(UpdateResult::Other) Ok(UpdateResult::Other)
//} }
//pub fn update_settings(
//&self,
//settings: &Settings<Checked>,
//update_builder: UpdateBuilder,
//) -> Result<UpdateResult> {
//let mut txn = self.write_txn()?;
//let result = self.update_settings_txn(&mut txn, settings, update_builder)?;
//txn.commit()?;
//Ok(result)
//}
//pub fn delete_documents( //pub fn delete_documents(
//&self, //&self,

View File

@ -18,7 +18,7 @@ use dump_actor::DumpActorHandle;
pub use dump_actor::{DumpInfo, DumpStatus}; pub use dump_actor::{DumpInfo, DumpStatus};
use snapshot::load_snapshot; use snapshot::load_snapshot;
use crate::index::{Checked, Document, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings}; use crate::index::{Checked, Document, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings, Unchecked};
use crate::index_controller::index_resolver::create_index_resolver; use crate::index_controller::index_resolver::create_index_resolver;
use crate::options::IndexerOpts; use crate::options::IndexerOpts;
use error::Result; use error::Result;
@ -95,6 +95,7 @@ pub struct Stats {
#[derive(derivative::Derivative)] #[derive(derivative::Derivative)]
#[derivative(Debug)] #[derivative(Debug)]
pub enum Update { pub enum Update {
Settings(Settings<Unchecked>),
DocumentAddition { DocumentAddition {
#[derivative(Debug="ignore")] #[derivative(Debug="ignore")]
payload: Payload, payload: Payload,
@ -242,8 +243,8 @@ impl IndexController {
IndexControllerBuilder::default() IndexControllerBuilder::default()
} }
pub async fn register_update(&self, uid: &str, update: Update) -> Result<UpdateStatus> { pub async fn register_update(&self, uid: String, update: Update) -> Result<UpdateStatus> {
match self.index_resolver.get_uuid(uid.to_string()).await { match self.index_resolver.get_uuid(uid).await {
Ok(uuid) => { Ok(uuid) => {
let update_result = UpdateMsg::update(&self.update_handle, uuid, update).await?; let update_result = UpdateMsg::update(&self.update_handle, uuid, update).await?;
Ok(update_result) Ok(update_result)

View File

@ -24,6 +24,7 @@ use uuid::Uuid;
use self::error::{Result, UpdateLoopError}; use self::error::{Result, UpdateLoopError};
pub use self::message::UpdateMsg; pub use self::message::UpdateMsg;
use self::store::{UpdateStore, UpdateStoreInfo}; use self::store::{UpdateStore, UpdateStoreInfo};
use crate::index::{Settings, Unchecked};
use crate::index_controller::update_file_store::UpdateFileStore; use crate::index_controller::update_file_store::UpdateFileStore;
use status::UpdateStatus; use status::UpdateStatus;
@ -53,6 +54,7 @@ pub enum RegisterUpdate {
method: IndexDocumentsMethod, method: IndexDocumentsMethod,
content_uuid: Uuid, content_uuid: Uuid,
}, },
Settings(Settings<Unchecked>),
} }
/// A wrapper type to implement read on a `Stream<Result<Bytes, Error>>`. /// A wrapper type to implement read on a `Stream<Result<Bytes, Error>>`.
@ -207,6 +209,7 @@ impl UpdateLoop {
content_uuid, content_uuid,
} }
} }
Update::Settings(settings) => RegisterUpdate::Settings(settings),
}; };
let store = self.store.clone(); let store = self.store.clone();