searchengines-web/MeiliSearch
searchengines-web
/
MeiliSearch
mirror of https://github.com/meilisearch/MeiliSearch
1
0
Fork 0
Code Issues Releases Wiki Activity

Fix for review

This commit is contained in:
Quentin de Quelen 2019-11-20 17:28:46 +01:00
parent 7f2e5d091a
commit 055368acd8
9 changed files with 140 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
meilidb-core/src/store/main.rs
View File

@ -6,16 +6,16 @@ use meilidb_schema::Schema;
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
const CREATED_AT: &str = "created-at"; const CREATED_AT_KEY: &str = "created-at";
const CUSTOMS_KEY: &str = "customs-key"; const CUSTOMS_KEY: &str = "customs-key";
const FIELDS_FREQUENCY: &str = "fields-frequency"; const FIELDS_FREQUENCY_KEY: &str = "fields-frequency";
const NAME: &str = "name"; const NAME_KEY: &str = "name";
const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents"; const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents";
const RANKED_MAP_KEY: &str = "ranked-map"; const RANKED_MAP_KEY: &str = "ranked-map";
const SCHEMA_KEY: &str = "schema"; const SCHEMA_KEY: &str = "schema";
const STOP_WORDS_KEY: &str = "stop-words"; const STOP_WORDS_KEY: &str = "stop-words";
const SYNONYMS_KEY: &str = "synonyms"; const SYNONYMS_KEY: &str = "synonyms";
const UPDATED_AT: &str = "updated-at"; const UPDATED_AT_KEY: &str = "updated-at";
const WORDS_KEY: &str = "words"; const WORDS_KEY: &str = "words";
pub type FreqsMap = HashMap<String, usize>; pub type FreqsMap = HashMap<String, usize>;
@ -33,32 +33,32 @@ impl Main {
} }
pub fn put_name(self, writer: &mut heed::RwTxn, name: &str) -> ZResult<()> { pub fn put_name(self, writer: &mut heed::RwTxn, name: &str) -> ZResult<()> {
self.main.put::<Str, Str>(writer, NAME, name) self.main.put::<Str, Str>(writer, NAME_KEY, name)
} }
pub fn name(self, reader: &heed::RoTxn) -> ZResult<Option<String>> { pub fn name(self, reader: &heed::RoTxn) -> ZResult<Option<String>> {
Ok(self Ok(self
.main .main
.get::<Str, Str>(reader, NAME)? .get::<Str, Str>(reader, NAME_KEY)?
.map(|name| name.to_owned())) .map(|name| name.to_owned()))
} }
pub fn put_created_at(self, writer: &mut heed::RwTxn) -> ZResult<()> { pub fn put_created_at(self, writer: &mut heed::RwTxn) -> ZResult<()> {
self.main self.main
.put::<Str, SerdeDatetime>(writer, CREATED_AT, &Utc::now()) .put::<Str, SerdeDatetime>(writer, CREATED_AT_KEY, &Utc::now())
} }
pub fn created_at(self, reader: &heed::RoTxn) -> ZResult<Option<DateTime<Utc>>> { pub fn created_at(self, reader: &heed::RoTxn) -> ZResult<Option<DateTime<Utc>>> {
self.main.get::<Str, SerdeDatetime>(reader, CREATED_AT) self.main.get::<Str, SerdeDatetime>(reader, CREATED_AT_KEY)
} }
pub fn put_updated_at(self, writer: &mut heed::RwTxn) -> ZResult<()> { pub fn put_updated_at(self, writer: &mut heed::RwTxn) -> ZResult<()> {
self.main self.main
.put::<Str, SerdeDatetime>(writer, UPDATED_AT, &Utc::now()) .put::<Str, SerdeDatetime>(writer, UPDATED_AT_KEY, &Utc::now())
} }
pub fn updated_at(self, reader: &heed::RoTxn) -> ZResult<Option<DateTime<Utc>>> { pub fn updated_at(self, reader: &heed::RoTxn) -> ZResult<Option<DateTime<Utc>>> {
self.main.get::<Str, SerdeDatetime>(reader, UPDATED_AT) self.main.get::<Str, SerdeDatetime>(reader, UPDATED_AT_KEY)
} }
pub fn put_words_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> { pub fn put_words_fst(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> ZResult<()> {
@ -159,13 +159,13 @@ impl Main {
fields_frequency: &FreqsMap, fields_frequency: &FreqsMap,
) -> ZResult<()> { ) -> ZResult<()> {
self.main self.main
.put::<Str, SerdeFreqsMap>(writer, FIELDS_FREQUENCY, fields_frequency) .put::<Str, SerdeFreqsMap>(writer, FIELDS_FREQUENCY_KEY, fields_frequency)
} }
pub fn fields_frequency(&self, reader: &heed::RoTxn) -> ZResult<Option<FreqsMap>> { pub fn fields_frequency(&self, reader: &heed::RoTxn) -> ZResult<Option<FreqsMap>> {
match self match self
.main .main
.get::<Str, SerdeFreqsMap>(&reader, FIELDS_FREQUENCY)? .get::<Str, SerdeFreqsMap>(reader, FIELDS_FREQUENCY_KEY)?
{ {
Some(freqs) => Ok(Some(freqs)), Some(freqs) => Ok(Some(freqs)),
None => Ok(None), None => Ok(None),

12
meilidb-http/src/data.rs
View File

@ -4,13 +4,15 @@ use std::sync::Arc;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use heed::types::{SerdeBincode, Str}; use heed::types::{SerdeBincode, Str};
use log::*; use log::error;
use meilidb_core::{Database, Error as MError, MResult}; use meilidb_core::{Database, Error as MError, MResult};
use sysinfo::Pid; use sysinfo::Pid;
use crate::option::Opt; use crate::option::Opt;
use crate::routes::index::index_update_callback; use crate::routes::index::index_update_callback;
const LAST_UPDATE_KEY: &str = "last-update";
type SerdeDatetime = SerdeBincode<DateTime<Utc>>; type SerdeDatetime = SerdeBincode<DateTime<Utc>>;
#[derive(Clone)] #[derive(Clone)]
@ -46,7 +48,7 @@ impl DataInner {
match self match self
.db .db
.common_store() .common_store()
.get::<Str, SerdeDatetime>(&reader, "last-update")? .get::<Str, SerdeDatetime>(reader, LAST_UPDATE_KEY)?
{ {
Some(datetime) => Ok(Some(datetime)), Some(datetime) => Ok(Some(datetime)),
None => Ok(None), None => Ok(None),
@ -56,11 +58,11 @@ impl DataInner {
pub fn set_last_update(&self, writer: &mut heed::RwTxn) -> MResult<()> { pub fn set_last_update(&self, writer: &mut heed::RwTxn) -> MResult<()> {
self.db self.db
.common_store() .common_store()
.put::<Str, SerdeDatetime>(writer, "last-update", &Utc::now()) .put::<Str, SerdeDatetime>(writer, LAST_UPDATE_KEY, &Utc::now())
.map_err(Into::into) .map_err(Into::into)
} }
pub fn compute_stats(&self, mut writer: &mut heed::RwTxn, index_uid: &str) -> MResult<()> { pub fn compute_stats(&self, writer: &mut heed::RwTxn, index_uid: &str) -> MResult<()> {
let index = match self.db.open_index(&index_uid) { let index = match self.db.open_index(&index_uid) {
Some(index) => index, Some(index) => index,
None => { None => {
@ -93,7 +95,7 @@ impl DataInner {
index index
.main .main
.put_fields_frequency(&mut writer, &frequency) .put_fields_frequency(writer, &frequency)
.map_err(MError::Zlmdb) .map_err(MError::Zlmdb)
} }
} }

2
meilidb-http/src/helpers/meilidb.rs
View File

@ -1,6 +1,6 @@
use crate::routes::setting::{RankingOrdering, SettingBody}; use crate::routes::setting::{RankingOrdering, SettingBody};
use indexmap::IndexMap; use indexmap::IndexMap;
use log::*; use log::error;
use meilidb_core::criterion::*; use meilidb_core::criterion::*;
use meilidb_core::Highlight; use meilidb_core::Highlight;
use meilidb_core::{Index, RankedMap}; use meilidb_core::{Index, RankedMap};

2
meilidb-http/src/helpers/tide.rs
View File

@ -39,7 +39,7 @@ impl ContextExt for Context<Data> {
.get::<Str, SerdeBincode<Token>>(&reader, &token_key) .get::<Str, SerdeBincode<Token>>(&reader, &token_key)
.map_err(ResponseError::internal)? .map_err(ResponseError::internal)?
.ok_or(ResponseError::invalid_token(format!( .ok_or(ResponseError::invalid_token(format!(
"token key does not exist: {}", "Api key does not exist: {}",
user_api_key user_api_key
)))?; )))?;

4
meilidb-http/src/routes/document.rs
View File

@ -114,9 +114,7 @@ pub async fn get_all_documents(ctx: Context<Data>) -> SResult<Response> {
} }
} }
Ok(tide::response::json(response_body) Ok(tide::response::json(response_body))
.with_status(StatusCode::OK)
.into_response())
} }
fn infered_schema(document: &IndexMap<String, Value>) -> Option<meilidb_schema::Schema> { fn infered_schema(document: &IndexMap<String, Value>) -> Option<meilidb_schema::Schema> {

148
meilidb-http/src/routes/index.rs
View File

@ -1,6 +1,6 @@
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use http::StatusCode; use http::StatusCode;
use log::*; use log::error;
use meilidb_core::ProcessedUpdateResult; use meilidb_core::ProcessedUpdateResult;
use meilidb_schema::{Schema, SchemaBuilder}; use meilidb_schema::{Schema, SchemaBuilder};
use rand::seq::SliceRandom; use rand::seq::SliceRandom;
@ -32,46 +32,51 @@ pub async fn list_indexes(ctx: Context<Data>) -> SResult<Response> {
let indexes_uids = ctx.state().db.indexes_uids(); let indexes_uids = ctx.state().db.indexes_uids();
let env = &ctx.state().db.env; let env = &ctx.state().db.env;
let mut reader = env.read_txn().map_err(ResponseError::internal)?; let reader = env.read_txn().map_err(ResponseError::internal)?;
let mut response_body = Vec::new(); let mut response_body = Vec::new();
for index_uid in indexes_uids { for index_uid in indexes_uids {
let index = ctx let index = ctx.state().db.open_index(&index_uid);
.state()
.db
.open_index(&index_uid)
.ok_or(ResponseError::internal(&index_uid))?;
let name = index
.main
.name(&mut reader)
.map_err(ResponseError::internal)?
.ok_or(ResponseError::internal("Name not found"))?;
let created_at = index
.main
.created_at(&mut reader)
.map_err(ResponseError::internal)?
.ok_or(ResponseError::internal("Created date not found"))?;
let updated_at = index
.main
.updated_at(&mut reader)
.map_err(ResponseError::internal)?
.ok_or(ResponseError::internal("Updated date not found"))?;
let index_reponse = IndexResponse { match index {
name, Some(index) => {
uid: index_uid, let name = index
created_at, .main
updated_at, .name(&reader)
}; .map_err(ResponseError::internal)?
response_body.push(index_reponse); .ok_or(ResponseError::internal("'name' not found"))?;
let created_at = index
.main
.created_at(&reader)
.map_err(ResponseError::internal)?
.ok_or(ResponseError::internal("'created_at' date not found"))?;
let updated_at = index
.main
.updated_at(&reader)
.map_err(ResponseError::internal)?
.ok_or(ResponseError::internal("'updated_at' date not found"))?;
let index_reponse = IndexResponse {
name,
uid: index_uid,
created_at,
updated_at,
};
response_body.push(index_reponse);
}
None => error!(
"Index {} is referenced in the indexes list but cannot be found",
index_uid
),
}
} }
Ok(tide::response::json(response_body)) Ok(tide::response::json(response_body))
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)] #[serde(rename_all = "camelCase")]
struct IndexResponse { struct IndexResponse {
name: String, name: String,
uid: String, uid: String,
@ -85,24 +90,24 @@ pub async fn get_index(ctx: Context<Data>) -> SResult<Response> {
let index = ctx.index()?; let index = ctx.index()?;
let env = &ctx.state().db.env; let env = &ctx.state().db.env;
let mut reader = env.read_txn().map_err(ResponseError::internal)?; let reader = env.read_txn().map_err(ResponseError::internal)?;
let uid = ctx.url_param("index")?.to_string(); let uid = ctx.url_param("index")?;
let name = index let name = index
.main .main
.name(&mut reader) .name(&reader)
.map_err(ResponseError::internal)? .map_err(ResponseError::internal)?
.ok_or(ResponseError::internal("Name not found"))?; .ok_or(ResponseError::internal("'name' not found"))?;
let created_at = index let created_at = index
.main .main
.created_at(&mut reader) .created_at(&reader)
.map_err(ResponseError::internal)? .map_err(ResponseError::internal)?
.ok_or(ResponseError::internal("Created date not found"))?; .ok_or(ResponseError::internal("'created_at' date not found"))?;
let updated_at = index let updated_at = index
.main .main
.updated_at(&mut reader) .updated_at(&reader)
.map_err(ResponseError::internal)? .map_err(ResponseError::internal)?
.ok_or(ResponseError::internal("Updated date not found"))?; .ok_or(ResponseError::internal("'updated_at' date not found"))?;
let response_body = IndexResponse { let response_body = IndexResponse {
name, name,
@ -114,15 +119,15 @@ pub async fn get_index(ctx: Context<Data>) -> SResult<Response> {
Ok(tide::response::json(response_body)) Ok(tide::response::json(response_body))
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)] #[serde(rename_all = "camelCase", deny_unknown_fields)]
struct IndexCreateRequest { struct IndexCreateRequest {
name: String, name: String,
schema: Option<SchemaBody>, schema: Option<SchemaBody>,
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)] #[serde(rename_all = "camelCase")]
struct IndexCreateResponse { struct IndexCreateResponse {
name: String, name: String,
uid: String, uid: String,
@ -166,11 +171,11 @@ pub async fn create_index(mut ctx: Context<Data>) -> SResult<Response> {
.put_updated_at(&mut writer) .put_updated_at(&mut writer)
.map_err(ResponseError::internal)?; .map_err(ResponseError::internal)?;
let schema: Option<Schema> = body.schema.clone().map(|s| s.into()); let schema: Option<Schema> = body.schema.clone().map(Into::into);
let mut response_update_id = None; let mut response_update_id = None;
if let Some(schema) = schema { if let Some(schema) = schema {
let update_id = created_index let update_id = created_index
.schema_update(&mut writer, schema.clone()) .schema_update(&mut writer, schema)
.map_err(ResponseError::internal)?; .map_err(ResponseError::internal)?;
response_update_id = Some(update_id) response_update_id = Some(update_id)
} }
@ -191,14 +196,14 @@ pub async fn create_index(mut ctx: Context<Data>) -> SResult<Response> {
.into_response()) .into_response())
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)] #[serde(rename_all = "camelCase", deny_unknown_fields)]
struct UpdateIndexRequest { struct UpdateIndexRequest {
name: String, name: String,
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)] #[serde(rename_all = "camelCase")]
struct UpdateIndexResponse { struct UpdateIndexResponse {
name: String, name: String,
uid: String, uid: String,
@ -239,12 +244,12 @@ pub async fn update_index(mut ctx: Context<Data>) -> SResult<Response> {
.main .main
.created_at(&reader) .created_at(&reader)
.map_err(ResponseError::internal)? .map_err(ResponseError::internal)?
.ok_or(ResponseError::internal("Created date not found"))?; .ok_or(ResponseError::internal("'created_at' date not found"))?;
let updated_at = index let updated_at = index
.main .main
.updated_at(&reader) .updated_at(&reader)
.map_err(ResponseError::internal)? .map_err(ResponseError::internal)?
.ok_or(ResponseError::internal("Updated date not found"))?; .ok_or(ResponseError::internal("'updated_at' date not found"))?;
let response_body = UpdateIndexResponse { let response_body = UpdateIndexResponse {
name: body.name, name: body.name,
@ -259,7 +264,7 @@ pub async fn update_index(mut ctx: Context<Data>) -> SResult<Response> {
} }
#[derive(Default, Deserialize)] #[derive(Default, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase", deny_unknown_fields)]
struct SchemaParams { struct SchemaParams {
raw: bool, raw: bool,
} }
@ -283,16 +288,12 @@ pub async fn get_index_schema(ctx: Context<Data>) -> SResult<Response> {
match schema { match schema {
Some(schema) => { Some(schema) => {
if params.raw { if params.raw {
Ok(tide::response::json(schema.to_builder())) Ok(tide::response::json(schema))
} else { } else {
Ok(tide::response::json(SchemaBody::from(schema))) Ok(tide::response::json(SchemaBody::from(schema)))
} }
} }
None => Ok( None => Err(ResponseError::not_found("missing index schema")),
tide::response::json(json!({ "message": "missing index schema" }))
.with_status(StatusCode::NOT_FOUND)
.into_response(),
),
} }
} }
@ -303,7 +304,7 @@ pub async fn update_schema(mut ctx: Context<Data>) -> SResult<Response> {
let params: SchemaParams = ctx.url_query().unwrap_or_default(); let params: SchemaParams = ctx.url_query().unwrap_or_default();
let schema: Schema = if params.raw { let schema = if params.raw {
ctx.body_json::<SchemaBuilder>() ctx.body_json::<SchemaBuilder>()
.await .await
.map_err(ResponseError::bad_request)? .map_err(ResponseError::bad_request)?
@ -397,18 +398,35 @@ pub async fn delete_index(ctx: Context<Data>) -> SResult<StatusCode> {
} }
} }
pub fn index_update_callback(index_uid: &str, data: &Data, _status: ProcessedUpdateResult) { pub fn index_update_callback(index_uid: &str, data: &Data, status: ProcessedUpdateResult) {
let env = &data.db.env; if status.error.is_some() {
let mut writer = env.write_txn().unwrap(); return;
}
data.compute_stats(&mut writer, &index_uid).unwrap();
data.set_last_update(&mut writer).unwrap();
if let Some(index) = data.db.open_index(&index_uid) { if let Some(index) = data.db.open_index(&index_uid) {
let env = &data.db.env;
let mut writer = match env.write_txn() {
Ok(writer) => writer,
Err(e) => {
error!("Impossible to get write_txn; {}", e);
return;
}
};
if let Err(e) = data.compute_stats(&mut writer, &index_uid) {
error!("Impossible to compute stats; {}", e)
}
if let Err(e) = data.set_last_update(&mut writer) {
error!("Impossible to update last_update; {}", e)
}
if let Err(e) = index.main.put_updated_at(&mut writer) { if let Err(e) = index.main.put_updated_at(&mut writer) {
error!("Impossible to update updated_at; {}", e) error!("Impossible to update updated_at; {}", e)
} }
}
writer.commit().unwrap(); if let Err(e) = writer.commit() {
error!("Impossible to get write_txn; {}", e);
}
}
} }

1
meilidb-http/src/routes/search.rs
View File

@ -156,6 +156,7 @@ pub async fn search_multi_index(mut ctx: Context<Data>) -> SResult<Response> {
for index in index_list.clone() { for index in index_list.clone() {
if index == "*" { if index == "*" {
index_list = ctx.state().db.indexes_uids().into_iter().collect(); index_list = ctx.state().db.indexes_uids().into_iter().collect();
break;
} }
} }

56
meilidb-http/src/routes/stats.rs
View File

@ -1,7 +1,9 @@
use std::collections::HashMap;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use log::error;
use pretty_bytes::converter::convert; use pretty_bytes::converter::convert;
use serde::Serialize; use serde::Serialize;
use std::collections::HashMap;
use sysinfo::{NetworkExt, Pid, ProcessExt, ProcessorExt, System, SystemExt}; use sysinfo::{NetworkExt, Pid, ProcessExt, ProcessorExt, System, SystemExt};
use tide::{Context, Response}; use tide::{Context, Response};
use walkdir::WalkDir; use walkdir::WalkDir;
@ -42,7 +44,7 @@ pub async fn index_stat(ctx: Context<Data>) -> SResult<Response> {
.state() .state()
.is_indexing(&reader, &index_uid) .is_indexing(&reader, &index_uid)
.map_err(ResponseError::internal)? .map_err(ResponseError::internal)?
.ok_or(ResponseError::not_found("Index not found"))?; .ok_or(ResponseError::internal("'is_indexing' date not found"))?;
let response = IndexStatsResponse { let response = IndexStatsResponse {
number_of_documents, number_of_documents,
@ -71,31 +73,39 @@ pub async fn get_stats(ctx: Context<Data>) -> SResult<Response> {
let indexes_set = ctx.state().db.indexes_uids(); let indexes_set = ctx.state().db.indexes_uids();
for index_uid in indexes_set { for index_uid in indexes_set {
let index = db.open_index(&index_uid).unwrap(); let index = ctx.state().db.open_index(&index_uid);
let number_of_documents = index match index {
.main Some(index) => {
.number_of_documents(&reader) let number_of_documents = index
.map_err(ResponseError::internal)?; .main
.number_of_documents(&reader)
.map_err(ResponseError::internal)?;
let fields_frequency = index let fields_frequency = index
.main .main
.fields_frequency(&reader) .fields_frequency(&reader)
.map_err(ResponseError::internal)? .map_err(ResponseError::internal)?
.unwrap_or_default(); .unwrap_or_default();
let is_indexing = ctx let is_indexing = ctx
.state() .state()
.is_indexing(&reader, &index_uid) .is_indexing(&reader, &index_uid)
.map_err(ResponseError::internal)? .map_err(ResponseError::internal)?
.ok_or(ResponseError::not_found("Index not found"))?; .ok_or(ResponseError::internal("'is_indexing' date not found"))?;
let response = IndexStatsResponse { let response = IndexStatsResponse {
number_of_documents, number_of_documents,
is_indexing, is_indexing,
fields_frequency, fields_frequency,
}; };
index_list.insert(index_uid, response); index_list.insert(index_uid, response);
}
None => error!(
"Index {:?} is referenced in the indexes list but cannot be found",
index_uid
),
}
} }
let database_size = WalkDir::new(ctx.state().db_path.clone()) let database_size = WalkDir::new(ctx.state().db_path.clone())

2
meilidb-schema/src/lib.rs
View File

@ -145,7 +145,7 @@ struct InnerSchema {
} }
impl Schema { impl Schema {
pub fn to_builder(&self) -> SchemaBuilder { fn to_builder(&self) -> SchemaBuilder {
let identifier = self.inner.identifier.clone(); let identifier = self.inner.identifier.clone();
let attributes = self.attributes_ordered(); let attributes = self.attributes_ordered();
SchemaBuilder { SchemaBuilder {