searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 04:17:10 +02:00
Code Issues Releases Wiki Activity

Merge #914

Browse source 
914: lazily create an index on documents push r=LegendreM a=qdequele

Create an index if it's possible when a user trying to send data to a non-existing index. https://github.com/meilisearch/MeiliSearch/issues/918

Co-authored-by: qdequele <quentin@meilisearch.com>
Co-authored-by: qdequele <quentin@dequelen.me>
This commit is contained in:
bors[bot] 2020-10-15 09:37:15 +00:00 • committed by GitHub
commit 29b8810db8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 741 additions and 177 deletions
Download patch file Download diff file Expand all files Collapse all files

59
meilisearch-http/src/data.rs
View file

@ -3,9 +3,10 @@ use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
use meilisearch_core::{Database, DatabaseOptions};
use meilisearch_core::{Database, DatabaseOptions, Index};
use sha2::Digest;
use crate::error::{Error as MSError, ResponseError};
use crate::index_update_callback;
use crate::option::Opt;
@ -102,4 +103,60 @@ impl Data {
Ok(data)
}
fn create_index(&self, uid: &str) -> Result<Index, ResponseError> {
if !uid
.chars()
.all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
{
return Err(MSError::InvalidIndexUid.into());
}
let created_index = self.db.create_index(&uid).map_err(|e| match e {
meilisearch_core::Error::IndexAlreadyExists => e.into(),
_ => ResponseError::from(MSError::create_index(e)),
})?;
self.db.main_write::<_, _, ResponseError>(|mut writer| {
created_index.main.put_name(&mut writer, uid)?;
created_index
.main
.created_at(&writer)?
.ok_or(MSError::internal("Impossible to read created at"))?;
created_index
.main
.updated_at(&writer)?
.ok_or(MSError::internal("Impossible to read updated at"))?;
Ok(())
})?;
Ok(created_index)
}
pub fn get_or_create_index<F, R>(&self, uid: &str, f: F) -> Result<R, ResponseError>
where
F: FnOnce(&Index) -> Result<R, ResponseError>,
{
let mut index_has_been_created = false;
let index = match self.db.open_index(&uid) {
Some(index) => index,
None => {
index_has_been_created = true;
self.create_index(&uid)?
}
};
match f(&index) {
Ok(r) => Ok(r),
Err(err) => {
if index_has_been_created {
let _ = self.db.delete_index(&uid);
}
Err(err)
}
}
}
}

68
meilisearch-http/src/routes/document.rs
View file

@ -45,7 +45,8 @@ async fn get_document(
let reader = data.db.main_read_txn()?;
let internal_id = index.main
let internal_id = index
.main
.external_to_internal_docid(&reader, &path.document_id)?
.ok_or(Error::document_not_found(&path.document_id))?;
@ -166,47 +167,41 @@ async fn update_multiple_documents(
body: web::Json<Vec<Document>>,
is_partial: bool,
) -> Result<HttpResponse, ResponseError> {
let index = data
.db
.open_index(&path.index_uid)
.ok_or(Error::index_not_found(&path.index_uid))?;
let update_id = data.get_or_create_index(&path.index_uid, |index| {
let reader = data.db.main_read_txn()?;
let reader = data.db.main_read_txn()?;
let mut schema = index
.main
.schema(&reader)?
.ok_or(meilisearch_core::Error::SchemaMissing)?;
let mut schema = index
.main
.schema(&reader)?
.ok_or(meilisearch_core::Error::SchemaMissing)?;
if schema.primary_key().is_none() {
let id = match &params.primary_key {
Some(id) => id.to_string(),
None => body
.first()
.and_then(find_primary_key)
.ok_or(meilisearch_core::Error::MissingPrimaryKey)?,
};
if schema.primary_key().is_none() {
let id = match &params.primary_key {
Some(id) => id.to_string(),
None => body
.first()
.and_then(find_primary_key)
.ok_or(meilisearch_core::Error::MissingPrimaryKey)?
schema.set_primary_key(&id).map_err(Error::bad_request)?;
data.db.main_write(|w| index.main.put_schema(w, &schema))?;
}
let mut document_addition = if is_partial {
index.documents_partial_addition()
} else {
index.documents_addition()
};
schema
.set_primary_key(&id)
.map_err(Error::bad_request)?;
for document in body.into_inner() {
document_addition.update_document(document);
}
data.db.main_write(|w| index.main.put_schema(w, &schema))?;
}
let mut document_addition = if is_partial {
index.documents_partial_addition()
} else {
index.documents_addition()
};
for document in body.into_inner() {
document_addition.update_document(document);
}
let update_id = data.db.update_write(|w| document_addition.finalize(w))?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
Ok(data.db.update_write(|w| document_addition.finalize(w))?)
})?;
return Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)));
}
#[post("/indexes/{index_uid}/documents", wrap = "Authentication::Private")]
@ -243,7 +238,6 @@ async fn delete_documents(
.open_index(&path.index_uid)
.ok_or(Error::index_not_found(&path.index_uid))?;
let mut documents_deletion = index.documents_deletion();
for document_id in body.into_inner() {

192
meilisearch-http/src/routes/setting.rs
View file

@ -53,13 +53,12 @@ async fn update_all(
path: web::Path<IndexParam>,
body: web::Json<Settings>,
) -> Result<HttpResponse, ResponseError> {
let settings = body
.into_inner()
.to_update()
.map_err(Error::bad_request)?;
let update_id = data.db.update_write::<_, _, Error>(|writer| {
update_all_settings_txn(&data, settings, &path.index_uid, writer)
let update_id = data.get_or_create_index(&path.index_uid, |index| {
Ok(data.db.update_write::<_, _, ResponseError>(|writer| {
let settings = body.into_inner().to_update().map_err(Error::bad_request)?;
let update_id = index.settings_update(writer, settings)?;
Ok(update_id)
})?)
})?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
@ -71,11 +70,7 @@ pub fn get_all_sync(data: &web::Data<Data>, reader: &MainReader, index_uid: &str
.open_index(index_uid)
.ok_or(Error::index_not_found(index_uid))?;
let stop_words: BTreeSet<String> = index
.main
.stop_words(reader)?
.into_iter()
.collect();
let stop_words: BTreeSet<String> = index.main.stop_words(&reader)?.into_iter().collect();
let synonyms_list = index.main.synonyms(reader)?;
@ -94,22 +89,19 @@ pub fn get_all_sync(data: &web::Data<Data>, reader: &MainReader, index_uid: &str
.map(|r| r.to_string())
.collect();
let schema = index.main.schema(reader)?;
let schema = index.main.schema(&reader)?;
let distinct_attribute = match (index.main.distinct_attribute(reader)?, &schema) {
(Some(id), Some(schema)) => schema.name(id).map(str::to_string),
_ => None,
};
let attributes_for_faceting = match (&schema, &index.main.attributes_for_faceting(reader)?) {
(Some(schema), Some(attrs)) => {
attrs
.iter()
.filter_map(|&id| schema.name(id))
.map(str::to_string)
.collect()
}
let attributes_for_faceting = match (&schema, &index.main.attributes_for_faceting(&reader)?) {
(Some(schema), Some(attrs)) => attrs
.iter()
.filter_map(|&id| schema.name(id))
.map(str::to_string)
.collect(),
_ => vec![],
};
@ -159,7 +151,9 @@ async fn delete_all(
attributes_for_faceting: UpdateState::Clear,
};
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let update_id = data
.db
.update_write(|w| index.settings_update(w, settings))?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -198,18 +192,17 @@ async fn update_rules(
path: web::Path<IndexParam>,
body: web::Json<Option<Vec<String>>>,
) -> Result<HttpResponse, ResponseError> {
let index = data
.db
.open_index(&path.index_uid)
.ok_or(Error::index_not_found(&path.index_uid))?;
let update_id = data.get_or_create_index(&path.index_uid, |index| {
let settings = Settings {
ranking_rules: Some(body.into_inner()),
..Settings::default()
};
let settings = Settings {
ranking_rules: Some(body.into_inner()),
..Settings::default()
};
let settings = settings.to_update().map_err(Error::bad_request)?;
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let settings = settings.to_update().map_err(Error::bad_request)?;
Ok(data
.db
.update_write(|w| index.settings_update(w, settings))?)
})?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -232,7 +225,9 @@ async fn delete_rules(
..SettingsUpdate::default()
};
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let update_id = data
.db
.update_write(|w| index.settings_update(w, settings))?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -269,18 +264,17 @@ async fn update_distinct(
path: web::Path<IndexParam>,
body: web::Json<Option<String>>,
) -> Result<HttpResponse, ResponseError> {
let index = data
.db
.open_index(&path.index_uid)
.ok_or(Error::index_not_found(&path.index_uid))?;
let update_id = data.get_or_create_index(&path.index_uid, |index| {
let settings = Settings {
distinct_attribute: Some(body.into_inner()),
..Settings::default()
};
let settings = Settings {
distinct_attribute: Some(body.into_inner()),
..Settings::default()
};
let settings = settings.to_update().map_err(Error::bad_request)?;
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let settings = settings.to_update().map_err(Error::bad_request)?;
Ok(data
.db
.update_write(|w| index.settings_update(w, settings))?)
})?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -303,7 +297,9 @@ async fn delete_distinct(
..SettingsUpdate::default()
};
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let update_id = data
.db
.update_write(|w| index.settings_update(w, settings))?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -322,8 +318,7 @@ async fn get_searchable(
.ok_or(Error::index_not_found(&path.index_uid))?;
let reader = data.db.main_read_txn()?;
let schema = index.main.schema(&reader)?;
let searchable_attributes: Option<Vec<String>> =
schema.as_ref().map(get_indexed_attributes);
let searchable_attributes: Option<Vec<String>> = schema.as_ref().map(get_indexed_attributes);
Ok(HttpResponse::Ok().json(searchable_attributes))
}
@ -337,19 +332,18 @@ async fn update_searchable(
path: web::Path<IndexParam>,
body: web::Json<Option<Vec<String>>>,
) -> Result<HttpResponse, ResponseError> {
let index = data
.db
.open_index(&path.index_uid)
.ok_or(Error::index_not_found(&path.index_uid))?;
let update_id = data.get_or_create_index(&path.index_uid, |index| {
let settings = Settings {
searchable_attributes: Some(body.into_inner()),
..Settings::default()
};
let settings = Settings {
searchable_attributes: Some(body.into_inner()),
..Settings::default()
};
let settings = settings.to_update().map_err(Error::bad_request)?;
let settings = settings.to_update().map_err(Error::bad_request)?;
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
Ok(data
.db
.update_write(|w| index.settings_update(w, settings))?)
})?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -372,7 +366,9 @@ async fn delete_searchable(
..SettingsUpdate::default()
};
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let update_id = data
.db
.update_write(|w| index.settings_update(w, settings))?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -407,18 +403,17 @@ async fn update_displayed(
path: web::Path<IndexParam>,
body: web::Json<Option<BTreeSet<String>>>,
) -> Result<HttpResponse, ResponseError> {
let index = data
.db
.open_index(&path.index_uid)
.ok_or(Error::index_not_found(&path.index_uid))?;
let update_id = data.get_or_create_index(&path.index_uid, |index| {
let settings = Settings {
displayed_attributes: Some(body.into_inner()),
..Settings::default()
};
let settings = Settings {
displayed_attributes: Some(body.into_inner()),
..Settings::default()
};
let settings = settings.to_update().map_err(Error::bad_request)?;
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let settings = settings.to_update().map_err(Error::bad_request)?;
Ok(data
.db
.update_write(|w| index.settings_update(w, settings))?)
})?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -441,7 +436,9 @@ async fn delete_displayed(
..SettingsUpdate::default()
};
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let update_id = data
.db
.update_write(|w| index.settings_update(w, settings))?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -459,20 +456,16 @@ async fn get_attributes_for_faceting(
.open_index(&path.index_uid)
.ok_or(Error::index_not_found(&path.index_uid))?;
let attributes_for_faceting = data
.db
.main_read::<_, _, ResponseError>(|reader| {
let attributes_for_faceting = data.db.main_read::<_, _, ResponseError>(|reader| {
let schema = index.main.schema(reader)?;
let attrs = index.main.attributes_for_faceting(reader)?;
let attr_names = match (&schema, &attrs) {
(Some(schema), Some(attrs)) => {
attrs
.iter()
.filter_map(|&id| schema.name(id))
.map(str::to_string)
.collect()
}
_ => vec![]
(Some(schema), Some(attrs)) => attrs
.iter()
.filter_map(|&id| schema.name(id))
.map(str::to_string)
.collect(),
_ => vec![],
};
Ok(attr_names)
})?;
@ -489,18 +482,17 @@ async fn update_attributes_for_faceting(
path: web::Path<IndexParam>,
body: web::Json<Option<Vec<String>>>,
) -> Result<HttpResponse, ResponseError> {
let index = data
.db
.open_index(&path.index_uid)
.ok_or(Error::index_not_found(&path.index_uid))?;
let update_id = data.get_or_create_index(&path.index_uid, |index| {
let settings = Settings {
attributes_for_faceting: Some(body.into_inner()),
..Settings::default()
};
let settings = Settings {
attributes_for_faceting: Some(body.into_inner()),
..Settings::default()
};
let settings = settings.to_update().map_err(Error::bad_request)?;
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let settings = settings.to_update().map_err(Error::bad_request)?;
Ok(data
.db
.update_write(|w| index.settings_update(w, settings))?)
})?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -523,7 +515,9 @@ async fn delete_attributes_for_faceting(
..SettingsUpdate::default()
};
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let update_id = data
.db
.update_write(|w| index.settings_update(w, settings))?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -532,7 +526,8 @@ fn get_indexed_attributes(schema: &Schema) -> Vec<String> {
if schema.is_indexed_all() {
["*"].iter().map(|s| s.to_string()).collect()
} else {
schema.indexed_name()
schema
.indexed_name()
.iter()
.map(|s| s.to_string())
.collect()
@ -543,7 +538,8 @@ fn get_displayed_attributes(schema: &Schema) -> BTreeSet<String> {
if schema.is_displayed_all() {
["*"].iter().map(|s| s.to_string()).collect()
} else {
schema.displayed_name()
schema
.displayed_name()
.iter()
.map(|s| s.to_string())
.collect()

23
meilisearch-http/src/routes/stop_words.rs
View file

@ -39,17 +39,16 @@ async fn update(
path: web::Path<IndexParam>,
body: web::Json<BTreeSet<String>>,
) -> Result<HttpResponse, ResponseError> {
let index = data
.db
.open_index(&path.index_uid)
.ok_or(Error::index_not_found(&path.index_uid))?;
let update_id = data.get_or_create_index(&path.index_uid, |index| {
let settings = SettingsUpdate {
stop_words: UpdateState::Update(body.into_inner()),
..SettingsUpdate::default()
};
let settings = SettingsUpdate {
stop_words: UpdateState::Update(body.into_inner()),
..SettingsUpdate::default()
};
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
Ok(data
.db
.update_write(|w| index.settings_update(w, settings))?)
})?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -72,7 +71,9 @@ async fn delete(
..SettingsUpdate::default()
};
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let update_id = data
.db
.update_write(|w| index.settings_update(w, settings))?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}

23
meilisearch-http/src/routes/synonym.rs
View file

@ -50,17 +50,16 @@ async fn update(
path: web::Path<IndexParam>,
body: web::Json<BTreeMap<String, Vec<String>>>,
) -> Result<HttpResponse, ResponseError> {
let index = data
.db
.open_index(&path.index_uid)
.ok_or(Error::index_not_found(&path.index_uid))?;
let update_id = data.get_or_create_index(&path.index_uid, |index| {
let settings = SettingsUpdate {
synonyms: UpdateState::Update(body.into_inner()),
..SettingsUpdate::default()
};
let settings = SettingsUpdate {
synonyms: UpdateState::Update(body.into_inner()),
..SettingsUpdate::default()
};
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
Ok(data
.db
.update_write(|w| index.settings_update(w, settings))?)
})?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}
@ -83,7 +82,9 @@ async fn delete(
..SettingsUpdate::default()
};
let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
let update_id = data
.db
.update_write(|w| index.settings_update(w, settings))?;
Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
}