searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 06:44:27 +01:00
Code Issues Releases Wiki Activity

clippy, fmt & tests

Browse Source
This commit is contained in:
Marin Postma 2021-05-31 16:03:39 +02:00
parent 10fc870684
commit 1c4f0b2ccf
No known key found for this signature in database
GPG Key ID: D5241F0C0C865F30
31 changed files with 196 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
meilisearch-http/build.rs
View File

@ -50,7 +50,7 @@ mod mini_dashboard {
sha1_file.read_to_string(&mut sha1)?; sha1_file.read_to_string(&mut sha1)?;
if sha1 == meta["sha1"].as_str().unwrap() { if sha1 == meta["sha1"].as_str().unwrap() {
// Nothing to do. // Nothing to do.
return Ok(()) return Ok(());
} }
} }
@ -62,7 +62,11 @@ mod mini_dashboard {
hasher.update(&dashboard_assets_bytes); hasher.update(&dashboard_assets_bytes);
let sha1 = hex::encode(hasher.finalize()); let sha1 = hex::encode(hasher.finalize());
assert_eq!(meta["sha1"].as_str().unwrap(), sha1, "Downloaded mini-dashboard shasum differs from the one specified in the Cargo.toml"); assert_eq!(
meta["sha1"].as_str().unwrap(),
sha1,
"Downloaded mini-dashboard shasum differs from the one specified in the Cargo.toml"
);
create_dir_all(&dashboard_dir)?; create_dir_all(&dashboard_dir)?;
let cursor = Cursor::new(&dashboard_assets_bytes); let cursor = Cursor::new(&dashboard_assets_bytes);

10
meilisearch-http/src/data/mod.rs
View File

@ -4,7 +4,9 @@ use std::sync::Arc;
use sha2::Digest; use sha2::Digest;
use crate::index::{Checked, Settings}; use crate::index::{Checked, Settings};
use crate::index_controller::{IndexController, IndexStats, Stats, DumpInfo, IndexMetadata, IndexSettings}; use crate::index_controller::{
DumpInfo, IndexController, IndexMetadata, IndexSettings, IndexStats, Stats,
};
use crate::option::Opt; use crate::option::Opt;
pub mod search; pub mod search;
@ -67,7 +69,11 @@ impl Data {
api_keys.generate_missing_api_keys(); api_keys.generate_missing_api_keys();
let inner = DataInner { index_controller, api_keys, options }; let inner = DataInner {
index_controller,
api_keys,
options,
};
let inner = Arc::new(inner); let inner = Arc::new(inner);
Ok(Data { inner }) Ok(Data { inner })

4
meilisearch-http/src/error.rs
View File

@ -299,7 +299,7 @@ impl From<JsonPayloadError> for Error {
JsonPayloadError::Payload(err) => { JsonPayloadError::Payload(err) => {
Error::BadRequest(format!("Problem while decoding the request: {}", err)) Error::BadRequest(format!("Problem while decoding the request: {}", err))
} }
e => Error::Internal(format!("Unexpected Json error: {}", e)) e => Error::Internal(format!("Unexpected Json error: {}", e)),
} }
} }
} }
@ -310,7 +310,7 @@ impl From<QueryPayloadError> for Error {
QueryPayloadError::Deserialize(err) => { QueryPayloadError::Deserialize(err) => {
Error::BadRequest(format!("Invalid query parameters: {}", err)) Error::BadRequest(format!("Invalid query parameters: {}", err))
} }
e => Error::Internal(format!("Unexpected query payload error: {}", e)) e => Error::Internal(format!("Unexpected query payload error: {}", e)),
} }
} }
} }

33
meilisearch-http/src/helpers/authentication.rs
View File

@ -1,16 +1,16 @@
use std::pin::Pin; use std::pin::Pin;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use actix_web::body::Body;
use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform}; use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform};
use actix_web::web; use actix_web::web;
use actix_web::body::Body;
use futures::ready;
use futures::future::{ok, Future, Ready};
use actix_web::ResponseError as _; use actix_web::ResponseError as _;
use futures::future::{ok, Future, Ready};
use futures::ready;
use pin_project::pin_project; use pin_project::pin_project;
use crate::Data;
use crate::error::{Error, ResponseError}; use crate::error::{Error, ResponseError};
use crate::Data;
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
pub enum Authentication { pub enum Authentication {
@ -59,19 +59,15 @@ where
let data = req.app_data::<web::Data<Data>>().unwrap(); let data = req.app_data::<web::Data<Data>>().unwrap();
if data.api_keys().master.is_none() { if data.api_keys().master.is_none() {
return AuthenticationFuture::Authenticated(self.service.call(req)) return AuthenticationFuture::Authenticated(self.service.call(req));
} }
let auth_header = match req.headers().get("X-Meili-API-Key") { let auth_header = match req.headers().get("X-Meili-API-Key") {
Some(auth) => match auth.to_str() { Some(auth) => match auth.to_str() {
Ok(auth) => auth, Ok(auth) => auth,
Err(_) => { Err(_) => return AuthenticationFuture::NoHeader(Some(req)),
return AuthenticationFuture::NoHeader(Some(req))
}
}, },
None => { None => return AuthenticationFuture::NoHeader(Some(req)),
return AuthenticationFuture::NoHeader(Some(req))
}
}; };
let authenticated = match self.acl { let authenticated = match self.acl {
@ -111,15 +107,13 @@ where
{ {
type Output = Result<ServiceResponse<Body>, actix_web::Error>; type Output = Result<ServiceResponse<Body>, actix_web::Error>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) ->Poll<Self::Output> { fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.project(); let this = self.project();
match this { match this {
AuthProj::Authenticated(fut) => { AuthProj::Authenticated(fut) => match ready!(fut.poll(cx)) {
match ready!(fut.poll(cx)) { Ok(resp) => Poll::Ready(Ok(resp)),
Ok(resp) => Poll::Ready(Ok(resp)), Err(e) => Poll::Ready(Err(e)),
Err(e) => Poll::Ready(Err(e)), },
}
}
AuthProj::NoHeader(req) => { AuthProj::NoHeader(req) => {
match req.take() { match req.take() {
Some(req) => { Some(req) => {
@ -135,7 +129,8 @@ where
AuthProj::Refused(req) => { AuthProj::Refused(req) => {
match req.take() { match req.take() {
Some(req) => { Some(req) => {
let bad_token = req.headers() let bad_token = req
.headers()
.get("X-Meili-API-Key") .get("X-Meili-API-Key")
.map(|h| h.to_str().map(String::from).unwrap_or_default()) .map(|h| h.to_str().map(String::from).unwrap_or_default())
.unwrap_or_default(); .unwrap_or_default();

11
meilisearch-http/src/index/dump.rs
View File

@ -1,4 +1,9 @@
use std::{fs::{create_dir_all, File}, io::{BufRead, BufReader}, path::Path, sync::Arc}; use std::{
fs::{create_dir_all, File},
io::{BufRead, BufReader},
path::Path,
sync::Arc,
};
use anyhow::bail; use anyhow::bail;
use anyhow::Context; use anyhow::Context;
@ -17,8 +22,8 @@ struct DumpMeta {
primary_key: Option<String>, primary_key: Option<String>,
} }
const META_FILE_NAME: &'static str = "meta.json"; const META_FILE_NAME: &str = "meta.json";
const DATA_FILE_NAME: &'static str = "documents.jsonl"; const DATA_FILE_NAME: &str = "documents.jsonl";
impl Index { impl Index {
pub fn dump(&self, path: impl AsRef<Path>) -> anyhow::Result<()> { pub fn dump(&self, path: impl AsRef<Path>) -> anyhow::Result<()> {

12
meilisearch-http/src/index/mod.rs
View File

@ -1,6 +1,10 @@
use std::{collections::{BTreeSet, HashSet}, marker::PhantomData, path::Path};
use std::ops::Deref; use std::ops::Deref;
use std::sync::Arc; use std::sync::Arc;
use std::{
collections::{BTreeSet, HashSet},
marker::PhantomData,
path::Path,
};
use anyhow::{bail, Context}; use anyhow::{bail, Context};
use heed::{EnvOpenOptions, RoTxn}; use heed::{EnvOpenOptions, RoTxn};
@ -9,13 +13,13 @@ use serde_json::{Map, Value};
use crate::helpers::EnvSizer; use crate::helpers::EnvSizer;
pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT}; pub use search::{SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT};
pub use updates::{Facets, Settings, Checked, Unchecked};
use serde::{de::Deserializer, Deserialize}; use serde::{de::Deserializer, Deserialize};
pub use updates::{Checked, Facets, Settings, Unchecked};
mod search;
mod updates;
mod dump; mod dump;
mod search;
pub mod update_handler; pub mod update_handler;
mod updates;
pub type Document = Map<String, Value>; pub type Document = Map<String, Value>;

51
meilisearch-http/src/index/search.rs
View File

@ -90,7 +90,8 @@ impl Index {
let mut documents = Vec::new(); let mut documents = Vec::new();
let fields_ids_map = self.fields_ids_map(&rtxn).unwrap(); let fields_ids_map = self.fields_ids_map(&rtxn).unwrap();
let displayed_ids = self.displayed_fields_ids(&rtxn)? let displayed_ids = self
.displayed_fields_ids(&rtxn)?
.map(|fields| fields.into_iter().collect::<HashSet<_>>()) .map(|fields| fields.into_iter().collect::<HashSet<_>>())
.unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect()); .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect());
@ -156,10 +157,8 @@ impl Index {
}; };
let stop_words = fst::Set::default(); let stop_words = fst::Set::default();
let highlighter = Highlighter::new( let highlighter =
&stop_words, Highlighter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
(String::from("<em>"), String::from("</em>")),
);
for (_id, obkv) in self.documents(&rtxn, documents_ids)? { for (_id, obkv) in self.documents(&rtxn, documents_ids)? {
let document = make_document(&all_attributes, &fields_ids_map, obkv)?; let document = make_document(&all_attributes, &fields_ids_map, obkv)?;
@ -384,17 +383,16 @@ mod test {
#[test] #[test]
fn no_formatted() { fn no_formatted() {
let stop_words = fst::Set::default(); let stop_words = fst::Set::default();
let highlighter = Highlighter::new( let highlighter =
&stop_words, Highlighter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
(String::from("<em>"), String::from("</em>")),
);
let mut fields = FieldsIdsMap::new(); let mut fields = FieldsIdsMap::new();
let id = fields.insert("test").unwrap(); let id = fields.insert("test").unwrap();
let mut buf = Vec::new(); let mut buf = Vec::new();
let mut obkv = obkv::KvWriter::new(&mut buf); let mut obkv = obkv::KvWriter::new(&mut buf);
obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()).unwrap(); obkv.insert(id, Value::String("hello".into()).to_string().as_bytes())
.unwrap();
obkv.finish().unwrap(); obkv.finish().unwrap();
let obkv = obkv::KvReader::new(&buf); let obkv = obkv::KvReader::new(&buf);
@ -410,8 +408,9 @@ mod test {
&highlighter, &highlighter,
&matching_words, &matching_words,
&all_formatted, &all_formatted,
&to_highlight_ids &to_highlight_ids,
).unwrap(); )
.unwrap();
assert!(value.is_empty()); assert!(value.is_empty());
} }
@ -419,17 +418,16 @@ mod test {
#[test] #[test]
fn formatted_no_highlight() { fn formatted_no_highlight() {
let stop_words = fst::Set::default(); let stop_words = fst::Set::default();
let highlighter = Highlighter::new( let highlighter =
&stop_words, Highlighter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
(String::from("<em>"), String::from("</em>")),
);
let mut fields = FieldsIdsMap::new(); let mut fields = FieldsIdsMap::new();
let id = fields.insert("test").unwrap(); let id = fields.insert("test").unwrap();
let mut buf = Vec::new(); let mut buf = Vec::new();
let mut obkv = obkv::KvWriter::new(&mut buf); let mut obkv = obkv::KvWriter::new(&mut buf);
obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()).unwrap(); obkv.insert(id, Value::String("hello".into()).to_string().as_bytes())
.unwrap();
obkv.finish().unwrap(); obkv.finish().unwrap();
let obkv = obkv::KvReader::new(&buf); let obkv = obkv::KvReader::new(&buf);
@ -445,8 +443,9 @@ mod test {
&highlighter, &highlighter,
&matching_words, &matching_words,
&all_formatted, &all_formatted,
&to_highlight_ids &to_highlight_ids,
).unwrap(); )
.unwrap();
assert_eq!(value["test"], "hello"); assert_eq!(value["test"], "hello");
} }
@ -454,17 +453,16 @@ mod test {
#[test] #[test]
fn formatted_with_highlight() { fn formatted_with_highlight() {
let stop_words = fst::Set::default(); let stop_words = fst::Set::default();
let highlighter = Highlighter::new( let highlighter =
&stop_words, Highlighter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
(String::from("<em>"), String::from("</em>")),
);
let mut fields = FieldsIdsMap::new(); let mut fields = FieldsIdsMap::new();
let id = fields.insert("test").unwrap(); let id = fields.insert("test").unwrap();
let mut buf = Vec::new(); let mut buf = Vec::new();
let mut obkv = obkv::KvWriter::new(&mut buf); let mut obkv = obkv::KvWriter::new(&mut buf);
obkv.insert(id, Value::String("hello".into()).to_string().as_bytes()).unwrap(); obkv.insert(id, Value::String("hello".into()).to_string().as_bytes())
.unwrap();
obkv.finish().unwrap(); obkv.finish().unwrap();
let obkv = obkv::KvReader::new(&buf); let obkv = obkv::KvReader::new(&buf);
@ -480,8 +478,9 @@ mod test {
&highlighter, &highlighter,
&matching_words, &matching_words,
&all_formatted, &all_formatted,
&to_highlight_ids &to_highlight_ids,
).unwrap(); )
.unwrap();
assert_eq!(value["test"], "<em>hello</em>"); assert_eq!(value["test"], "<em>hello</em>");
} }

2
meilisearch-http/src/index/updates.rs
View File

@ -198,7 +198,7 @@ impl Index {
builder.index_documents_method(method); builder.index_documents_method(method);
//let indexing_callback = //let indexing_callback =
//|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step); //|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step);
let indexing_callback = |_, _| (); let indexing_callback = |_, _| ();

19
meilisearch-http/src/index_controller/dump_actor/actor.rs
View File

@ -1,13 +1,16 @@
use std::{collections::HashMap, path::{Path, PathBuf}};
use std::sync::Arc; use std::sync::Arc;
use std::{
collections::HashMap,
path::{Path, PathBuf},
};
use async_stream::stream; use async_stream::stream;
use chrono::Utc; use chrono::Utc;
use futures::{lock::Mutex, stream::StreamExt}; use futures::{lock::Mutex, stream::StreamExt};
use log::{error, info}; use log::{error, info};
use tokio::sync::{mpsc, oneshot, RwLock};
use update_actor::UpdateActorHandle; use update_actor::UpdateActorHandle;
use uuid_resolver::UuidResolverHandle; use uuid_resolver::UuidResolverHandle;
use tokio::sync::{mpsc, oneshot, RwLock};
use super::{DumpError, DumpInfo, DumpMsg, DumpResult, DumpStatus, DumpTask}; use super::{DumpError, DumpInfo, DumpMsg, DumpResult, DumpStatus, DumpTask};
use crate::index_controller::{update_actor, uuid_resolver}; use crate::index_controller::{update_actor, uuid_resolver};
@ -107,7 +110,10 @@ where
} }
}; };
self.dump_infos.write().await.insert(uid.clone(), info.clone()); self.dump_infos
.write()
.await
.insert(uid.clone(), info.clone());
ret.send(Ok(info)).expect("Dump actor is dead"); ret.send(Ok(info)).expect("Dump actor is dead");
@ -122,11 +128,8 @@ where
let task_result = tokio::task::spawn(task.run()).await; let task_result = tokio::task::spawn(task.run()).await;
let mut dump_infos = self.dump_infos let mut dump_infos = self.dump_infos.write().await;
.write() let dump_infos = dump_infos
.await;
let dump_infos =
dump_infos
.get_mut(&uid) .get_mut(&uid)
.expect("dump entry deleted while lock was acquired"); .expect("dump entry deleted while lock was acquired");

15
meilisearch-http/src/index_controller/dump_actor/handle_impl.rs
View File

@ -1,7 +1,7 @@
use std::path::Path;
use actix_web::web::Bytes;
use tokio::sync::{mpsc, oneshot};
use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg, DumpResult}; use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg, DumpResult};
use actix_web::web::Bytes;
use std::path::Path;
use tokio::sync::{mpsc, oneshot};
#[derive(Clone)] #[derive(Clone)]
pub struct DumpActorHandleImpl { pub struct DumpActorHandleImpl {
@ -34,7 +34,14 @@ impl DumpActorHandleImpl {
update_db_size: u64, update_db_size: u64,
) -> anyhow::Result<Self> { ) -> anyhow::Result<Self> {
let (sender, receiver) = mpsc::channel(10); let (sender, receiver) = mpsc::channel(10);
let actor = DumpActor::new(receiver, uuid_resolver, update, path, index_db_size, update_db_size); let actor = DumpActor::new(
receiver,
uuid_resolver,
update,
path,
index_db_size,
update_db_size,
);
tokio::task::spawn(actor.run()); tokio::task::spawn(actor.run());

9
meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs
View File

@ -1,4 +1,11 @@
use std::{collections::{BTreeMap, BTreeSet}, fs::File, io::BufRead, marker::PhantomData, path::Path, sync::Arc}; use std::{
collections::{BTreeMap, BTreeSet},
fs::File,
io::BufRead,
marker::PhantomData,
path::Path,
sync::Arc,
};
use heed::EnvOpenOptions; use heed::EnvOpenOptions;
use log::{error, info, warn}; use log::{error, info, warn};

6
meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs
View File

@ -4,7 +4,11 @@ use chrono::{DateTime, Utc};
use log::info; use log::info;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::{index::Index, index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore}, option::IndexerOpts}; use crate::{
index::Index,
index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore},
option::IndexerOpts,
};
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]

4
meilisearch-http/src/index_controller/dump_actor/message.rs
View File

@ -1,7 +1,6 @@
use tokio::sync::oneshot; use tokio::sync::oneshot;
use super::{DumpResult, DumpInfo}; use super::{DumpInfo, DumpResult};
pub enum DumpMsg { pub enum DumpMsg {
CreateDump { CreateDump {
@ -12,4 +11,3 @@ pub enum DumpMsg {
ret: oneshot::Sender<DumpResult<DumpInfo>>, ret: oneshot::Sender<DumpResult<DumpInfo>>,
}, },
} }

8
meilisearch-http/src/index_controller/dump_actor/mod.rs
View File

@ -1,13 +1,13 @@
use std::fs::File; use std::fs::File;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use anyhow::Context;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use log::{error, info, warn}; use log::{error, info, warn};
#[cfg(test)] #[cfg(test)]
use mockall::automock; use mockall::automock;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use thiserror::Error; use thiserror::Error;
use anyhow::Context;
use loaders::v1::MetadataV1; use loaders::v1::MetadataV1;
use loaders::v2::MetadataV2; use loaders::v2::MetadataV2;
@ -25,7 +25,7 @@ mod handle_impl;
mod loaders; mod loaders;
mod message; mod message;
const META_FILE_NAME: &'static str = "metadata.json"; const META_FILE_NAME: &str = "metadata.json";
pub type DumpResult<T> = std::result::Result<T, DumpError>; pub type DumpResult<T> = std::result::Result<T, DumpError>;
@ -138,7 +138,9 @@ pub fn load_dump(
let tmp_dst = tempfile::tempdir_in(dst_dir)?; let tmp_dst = tempfile::tempdir_in(dst_dir)?;
match meta { match meta {
Metadata::V1(meta) => meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size as usize)?, Metadata::V1(meta) => {
meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size as usize)?
}
Metadata::V2(meta) => meta.load_dump( Metadata::V2(meta) => meta.load_dump(
&tmp_src_path, &tmp_src_path,
tmp_dst.path(), tmp_dst.path(),

9
meilisearch-http/src/index_controller/index_actor/actor.rs
View File

@ -6,14 +6,15 @@ use async_stream::stream;
use futures::stream::StreamExt; use futures::stream::StreamExt;
use heed::CompactionOption; use heed::CompactionOption;
use log::debug; use log::debug;
use tokio::{fs, sync::mpsc};
use tokio::task::spawn_blocking; use tokio::task::spawn_blocking;
use tokio::{fs, sync::mpsc};
use uuid::Uuid; use uuid::Uuid;
use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings, update_handler::UpdateHandler}; use crate::index::{
update_handler::UpdateHandler, Checked, Document, SearchQuery, SearchResult, Settings,
};
use crate::index_controller::{ use crate::index_controller::{
get_arc_ownership_blocking, Failed, IndexStats, Processed, get_arc_ownership_blocking, Failed, IndexStats, Processed, Processing,
Processing,
}; };
use crate::option::IndexerOpts; use crate::option::IndexerOpts;

5
meilisearch-http/src/index_controller/index_actor/handle_impl.rs
View File

@ -3,7 +3,10 @@ use std::path::{Path, PathBuf};
use tokio::sync::{mpsc, oneshot}; use tokio::sync::{mpsc, oneshot};
use uuid::Uuid; use uuid::Uuid;
use crate::{index::Checked, index_controller::{IndexSettings, IndexStats, Processing}}; use crate::{
index::Checked,
index_controller::{IndexSettings, IndexStats, Processing},
};
use crate::{ use crate::{
index::{Document, SearchQuery, SearchResult, Settings}, index::{Document, SearchQuery, SearchResult, Settings},
index_controller::{Failed, Processed}, index_controller::{Failed, Processed},

2
meilisearch-http/src/index_controller/index_actor/message.rs
View File

@ -3,7 +3,7 @@ use std::path::PathBuf;
use tokio::sync::oneshot; use tokio::sync::oneshot;
use uuid::Uuid; use uuid::Uuid;
use crate::index::{Document, SearchQuery, SearchResult, Settings, Checked}; use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings};
use crate::index_controller::{Failed, IndexStats, Processed, Processing}; use crate::index_controller::{Failed, IndexStats, Processed, Processing};
use super::{IndexMeta, IndexResult, IndexSettings}; use super::{IndexMeta, IndexResult, IndexSettings};

21
meilisearch-http/src/index_controller/index_actor/mod.rs
View File

@ -15,7 +15,7 @@ use message::IndexMsg;
use store::{IndexStore, MapIndexStore}; use store::{IndexStore, MapIndexStore};
use crate::index::{Checked, Document, Index, SearchQuery, SearchResult, Settings}; use crate::index::{Checked, Document, Index, SearchQuery, SearchResult, Settings};
use crate::index_controller::{Failed, Processed, Processing, IndexStats}; use crate::index_controller::{Failed, IndexStats, Processed, Processing};
use super::IndexSettings; use super::IndexSettings;
@ -44,7 +44,11 @@ impl IndexMeta {
let created_at = index.created_at(&txn)?; let created_at = index.created_at(&txn)?;
let updated_at = index.updated_at(&txn)?; let updated_at = index.updated_at(&txn)?;
let primary_key = index.primary_key(&txn)?.map(String::from); let primary_key = index.primary_key(&txn)?.map(String::from);
Ok(Self { created_at, updated_at, primary_key }) Ok(Self {
created_at,
updated_at,
primary_key,
})
} }
} }
@ -57,7 +61,7 @@ pub enum IndexError {
#[error("Existing primary key")] #[error("Existing primary key")]
ExistingPrimaryKey, ExistingPrimaryKey,
#[error("Internal Index Error: {0}")] #[error("Internal Index Error: {0}")]
Internal(String) Internal(String),
} }
macro_rules! internal_error { macro_rules! internal_error {
@ -72,7 +76,12 @@ macro_rules! internal_error {
} }
} }
internal_error!(anyhow::Error, heed::Error, tokio::task::JoinError, std::io::Error); internal_error!(
anyhow::Error,
heed::Error,
tokio::task::JoinError,
std::io::Error
);
#[async_trait::async_trait] #[async_trait::async_trait]
#[cfg_attr(test, automock)] #[cfg_attr(test, automock)]
@ -190,8 +199,8 @@ mod test {
self.as_ref().snapshot(uuid, path).await self.as_ref().snapshot(uuid, path).await
} }
async fn dump(&self, uid: String, uuid: Uuid, path: PathBuf) -> IndexResult<()> { async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> {
self.as_ref().dump(uid, uuid, path).await self.as_ref().dump(uuid, path).await
} }
async fn get_index_stats(&self, uuid: Uuid) -> IndexResult<IndexStats> { async fn get_index_stats(&self, uuid: Uuid) -> IndexResult<IndexStats> {

6
meilisearch-http/src/index_controller/snapshot.rs
View File

@ -144,7 +144,7 @@ mod test {
use crate::index_controller::update_actor::{ use crate::index_controller::update_actor::{
MockUpdateActorHandle, UpdateActorHandleImpl, UpdateError, MockUpdateActorHandle, UpdateActorHandleImpl, UpdateError,
}; };
use crate::index_controller::uuid_resolver::{MockUuidResolverHandle, UuidError}; use crate::index_controller::uuid_resolver::{MockUuidResolverHandle, UuidResolverError};
#[actix_rt::test] #[actix_rt::test]
async fn test_normal() { async fn test_normal() {
@ -193,7 +193,7 @@ mod test {
.expect_snapshot() .expect_snapshot()
.times(1) .times(1)
// abitrary error // abitrary error
.returning(|_| Box::pin(err(UuidError::NameAlreadyExist))); .returning(|_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
let update_handle = MockUpdateActorHandle::new(); let update_handle = MockUpdateActorHandle::new();
@ -248,7 +248,7 @@ mod test {
// we expect the funtion to be called between 2 and 3 time in the given interval. // we expect the funtion to be called between 2 and 3 time in the given interval.
.times(2..4) .times(2..4)
// abitrary error, to short-circuit the function // abitrary error, to short-circuit the function
.returning(move |_| Box::pin(err(UuidError::NameAlreadyExist))); .returning(move |_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
let update_handle = MockUpdateActorHandle::new(); let update_handle = MockUpdateActorHandle::new();

18
meilisearch-http/src/index_controller/update_actor/store/dump.rs
View File

@ -11,7 +11,10 @@ use uuid::Uuid;
use super::UpdateStore; use super::UpdateStore;
use super::{codec::UpdateKeyCodec, State}; use super::{codec::UpdateKeyCodec, State};
use crate::index_controller::{Enqueued, UpdateStatus, index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path}; use crate::index_controller::{
index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path, Enqueued,
UpdateStatus,
};
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
struct UpdateEntry { struct UpdateEntry {
@ -89,7 +92,7 @@ impl UpdateStore {
}; };
serde_json::to_writer(&mut file, &update_json)?; serde_json::to_writer(&mut file, &update_json)?;
file.write(b"\n")?; file.write_all(b"\n")?;
} }
} }
@ -111,12 +114,12 @@ impl UpdateStore {
for update in updates { for update in updates {
let ((uuid, _), data) = update?; let ((uuid, _), data) = update?;
if uuids.contains(&uuid) { if uuids.contains(&uuid) {
let update = data.decode()?.into(); let update = data.decode()?;
let update_json = UpdateEntry { uuid, update }; let update_json = UpdateEntry { uuid, update };
serde_json::to_writer(&mut file, &update_json)?; serde_json::to_writer(&mut file, &update_json)?;
file.write(b"\n")?; file.write_all(b"\n")?;
} }
} }
@ -131,7 +134,6 @@ impl UpdateStore {
let dst_update_path = dst.as_ref().join("updates/"); let dst_update_path = dst.as_ref().join("updates/");
create_dir_all(&dst_update_path)?; create_dir_all(&dst_update_path)?;
let mut options = EnvOpenOptions::new(); let mut options = EnvOpenOptions::new();
options.map_size(db_size as usize); options.map_size(db_size as usize);
let (store, _) = UpdateStore::new(options, &dst_update_path)?; let (store, _) = UpdateStore::new(options, &dst_update_path)?;
@ -152,7 +154,11 @@ impl UpdateStore {
store.register_raw_updates(&mut wtxn, &update, uuid)?; store.register_raw_updates(&mut wtxn, &update, uuid)?;
// Copy ascociated update path if it exists // Copy ascociated update path if it exists
if let UpdateStatus::Enqueued(Enqueued { content: Some(uuid), .. }) = update { if let UpdateStatus::Enqueued(Enqueued {
content: Some(uuid),
..
}) = update
{
let src = update_uuid_to_file_path(&src_update_path, uuid); let src = update_uuid_to_file_path(&src_update_path, uuid);
let dst = update_uuid_to_file_path(&dst_update_path, uuid); let dst = update_uuid_to_file_path(&dst_update_path, uuid);
std::fs::copy(src, dst)?; std::fs::copy(src, dst)?;

37
meilisearch-http/src/index_controller/update_actor/store/mod.rs
View File

@ -1,10 +1,13 @@
pub mod dump;
mod codec; mod codec;
pub mod dump;
use std::{collections::{BTreeMap, HashSet}, path::PathBuf};
use std::fs::{copy, create_dir_all, remove_file, File}; use std::fs::{copy, create_dir_all, remove_file, File};
use std::path::Path; use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use std::{
collections::{BTreeMap, HashSet},
path::PathBuf,
};
use arc_swap::ArcSwap; use arc_swap::ArcSwap;
use futures::StreamExt; use futures::StreamExt;
@ -20,13 +23,13 @@ use uuid::Uuid;
use codec::*; use codec::*;
use super::UpdateMeta; use super::UpdateMeta;
use crate::{helpers::EnvSizer, index_controller::index_actor::IndexResult};
use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle}; use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle};
use crate::{helpers::EnvSizer, index_controller::index_actor::IndexResult};
#[allow(clippy::upper_case_acronyms)] #[allow(clippy::upper_case_acronyms)]
type BEU64 = U64<heed::byteorder::BE>; type BEU64 = U64<heed::byteorder::BE>;
const UPDATE_DIR: &'static str = "update_files"; const UPDATE_DIR: &str = "update_files";
pub struct UpdateStoreInfo { pub struct UpdateStoreInfo {
/// Size of the update store in bytes. /// Size of the update store in bytes.
@ -441,11 +444,12 @@ impl UpdateStore {
txn.commit()?; txn.commit()?;
uuids_to_remove.iter() uuids_to_remove
.iter()
.map(|uuid| update_uuid_to_file_path(&self.path, *uuid)) .map(|uuid| update_uuid_to_file_path(&self.path, *uuid))
.for_each(|path| { .for_each(|path| {
let _ = remove_file(path); let _ = remove_file(path);
}); });
// We don't care about the currently processing update, since it will be removed by itself // We don't care about the currently processing update, since it will be removed by itself
// once its done processing, and we can't abort a running update. // once its done processing, and we can't abort a running update.
@ -482,7 +486,11 @@ impl UpdateStore {
for entry in pendings { for entry in pendings {
let ((_, uuid, _), pending) = entry?; let ((_, uuid, _), pending) = entry?;
if uuids.contains(&uuid) { if uuids.contains(&uuid) {
if let Enqueued { content: Some(uuid), .. } = pending.decode()? { if let Enqueued {
content: Some(uuid),
..
} = pending.decode()?
{
let path = update_uuid_to_file_path(&self.path, uuid); let path = update_uuid_to_file_path(&self.path, uuid);
copy(path, &update_files_path)?; copy(path, &update_files_path)?;
} }
@ -507,13 +515,16 @@ impl UpdateStore {
Ok(()) Ok(())
} }
pub fn get_info(&self) -> anyhow::Result<UpdateStoreInfo> { pub fn get_info(&self) -> anyhow::Result<UpdateStoreInfo> {
let mut size = self.env.size(); let mut size = self.env.size();
let txn = self.env.read_txn()?; let txn = self.env.read_txn()?;
for entry in self.pending_queue.iter(&txn)? { for entry in self.pending_queue.iter(&txn)? {
let (_, pending) = entry?; let (_, pending) = entry?;
if let Enqueued { content: Some(uuid), .. } = pending { if let Enqueued {
content: Some(uuid),
..
} = pending
{
let path = update_uuid_to_file_path(&self.path, uuid); let path = update_uuid_to_file_path(&self.path, uuid);
size += File::open(path)?.metadata()?.len(); size += File::open(path)?.metadata()?.len();
} }
@ -528,7 +539,9 @@ impl UpdateStore {
} }
fn update_uuid_to_file_path(root: impl AsRef<Path>, uuid: Uuid) -> PathBuf { fn update_uuid_to_file_path(root: impl AsRef<Path>, uuid: Uuid) -> PathBuf {
root.as_ref().join(UPDATE_DIR).join(format!("update_{}", uuid)) root.as_ref()
.join(UPDATE_DIR)
.join(format!("update_{}", uuid))
} }
#[cfg(test)] #[cfg(test)]
@ -577,7 +590,7 @@ mod test {
let store_clone = update_store.clone(); let store_clone = update_store.clone();
tokio::task::spawn_blocking(move || { tokio::task::spawn_blocking(move || {
store_clone store_clone
.register_update(meta, Some("here"), uuid) .register_update(meta, None, uuid)
.unwrap(); .unwrap();
}) })
.await .await

2
meilisearch-http/src/index_controller/uuid_resolver/actor.rs
View File

@ -4,7 +4,7 @@ use log::{info, warn};
use tokio::sync::mpsc; use tokio::sync::mpsc;
use uuid::Uuid; use uuid::Uuid;
use super::{Result, UuidResolverError, UuidResolveMsg, UuidStore}; use super::{Result, UuidResolveMsg, UuidResolverError, UuidStore};
pub struct UuidResolverActor<S> { pub struct UuidResolverActor<S> {
inbox: mpsc::Receiver<UuidResolveMsg>, inbox: mpsc::Receiver<UuidResolveMsg>,

2
meilisearch-http/src/index_controller/uuid_resolver/message.rs
View File

@ -37,5 +37,5 @@ pub enum UuidResolveMsg {
DumpRequest { DumpRequest {
path: PathBuf, path: PathBuf,
ret: oneshot::Sender<Result<HashSet<Uuid>>>, ret: oneshot::Sender<Result<HashSet<Uuid>>>,
} },
} }

4
meilisearch-http/src/index_controller/uuid_resolver/store.rs
View File

@ -164,7 +164,7 @@ impl HeedUuidStore {
let entry = DumpEntry { uuid, uid }; let entry = DumpEntry { uuid, uid };
serde_json::to_writer(&mut dump_file, &entry)?; serde_json::to_writer(&mut dump_file, &entry)?;
dump_file.write(b"\n").unwrap(); dump_file.write_all(b"\n").unwrap();
uuids.insert(uuid); uuids.insert(uuid);
} }
@ -192,7 +192,7 @@ impl HeedUuidStore {
println!("importing {} {}", uid, uuid); println!("importing {} {}", uid, uuid);
db.db.put(&mut txn, &uid, uuid.as_bytes())?; db.db.put(&mut txn, &uid, uuid.as_bytes())?;
} }
Err(e) => Err(e)?, Err(e) => return Err(e.into()),
} }
line.clear(); line.clear();

10
meilisearch-http/src/lib.rs
View File

@ -62,11 +62,11 @@ macro_rules! create_app {
app.wrap( app.wrap(
Cors::default() Cors::default()
.send_wildcard() .send_wildcard()
.allowed_headers(vec!["content-type", "x-meili-api-key"]) .allowed_headers(vec!["content-type", "x-meili-api-key"])
.allow_any_origin() .allow_any_origin()
.allow_any_method() .allow_any_method()
.max_age(86_400) // 24h .max_age(86_400), // 24h
) )
.wrap(middleware::Logger::default()) .wrap(middleware::Logger::default())
.wrap(middleware::Compress::default()) .wrap(middleware::Compress::default())

11
meilisearch-http/src/routes/dump.rs
View File

@ -1,20 +1,17 @@
use actix_web::{post, get, web};
use actix_web::HttpResponse; use actix_web::HttpResponse;
use serde::{Serialize, Deserialize}; use actix_web::{get, post, web};
use serde::{Deserialize, Serialize};
use crate::error::ResponseError; use crate::error::ResponseError;
use crate::helpers::Authentication; use crate::helpers::Authentication;
use crate::Data; use crate::Data;
pub fn services(cfg: &mut web::ServiceConfig) { pub fn services(cfg: &mut web::ServiceConfig) {
cfg.service(create_dump) cfg.service(create_dump).service(get_dump_status);
.service(get_dump_status);
} }
#[post("/dumps", wrap = "Authentication::Private")] #[post("/dumps", wrap = "Authentication::Private")]
async fn create_dump( async fn create_dump(data: web::Data<Data>) -> Result<HttpResponse, ResponseError> {
data: web::Data<Data>,
) -> Result<HttpResponse, ResponseError> {
let res = data.create_dump().await?; let res = data.create_dump().await?;
Ok(HttpResponse::Accepted().json(res)) Ok(HttpResponse::Accepted().json(res))

2
meilisearch-http/src/routes/index.rs
View File

@ -1,7 +1,7 @@
use actix_web::{delete, get, post, put}; use actix_web::{delete, get, post, put};
use actix_web::{web, HttpResponse}; use actix_web::{web, HttpResponse};
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use serde::{Serialize, Deserialize}; use serde::{Deserialize, Serialize};
use crate::error::ResponseError; use crate::error::ResponseError;
use crate::helpers::Authentication; use crate::helpers::Authentication;

2
meilisearch-http/src/routes/mod.rs
View File

@ -2,6 +2,7 @@ use actix_web::{get, HttpResponse};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
pub mod document; pub mod document;
pub mod dump;
pub mod health; pub mod health;
pub mod index; pub mod index;
pub mod key; pub mod key;
@ -9,7 +10,6 @@ pub mod search;
pub mod settings; pub mod settings;
pub mod stats; pub mod stats;
pub mod synonym; pub mod synonym;
pub mod dump;
#[derive(Deserialize)] #[derive(Deserialize)]
pub struct IndexParam { pub struct IndexParam {

2
meilisearch-http/src/routes/settings/mod.rs
View File

@ -1,9 +1,9 @@
use actix_web::{delete, get, post, web, HttpResponse}; use actix_web::{delete, get, post, web, HttpResponse};
use crate::{error::ResponseError, index::Unchecked};
use crate::helpers::Authentication; use crate::helpers::Authentication;
use crate::index::Settings; use crate::index::Settings;
use crate::Data; use crate::Data;
use crate::{error::ResponseError, index::Unchecked};
#[macro_export] #[macro_export]
macro_rules! make_setting_route { macro_rules! make_setting_route {

2
meilisearch-http/tests/common/index.rs
View File

@ -47,7 +47,7 @@ impl Index<'_> {
update_id as u64 update_id as u64
} }
pub async fn create(& self, primary_key: Option<&str>) -> (Value, StatusCode) { pub async fn create(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
let body = json!({ let body = json!({
"uid": self.uid, "uid": self.uid,
"primaryKey": primary_key, "primaryKey": primary_key,

2
meilisearch-http/tests/common/server.rs
View File

@ -44,7 +44,7 @@ impl Server {
} }
/// Returns a view to an index. There is no guarantee that the index exists. /// Returns a view to an index. There is no guarantee that the index exists.
pub fn index(& self, uid: impl AsRef<str>) -> Index<'_> { pub fn index(&self, uid: impl AsRef<str>) -> Index<'_> {
Index { Index {
uid: encode(uid.as_ref()), uid: encode(uid.as_ref()),
service: &self.service, service: &self.service,