searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-03 20:07:09 +02:00
Code Issues Releases Wiki Activity

fix tests

Browse source
This commit is contained in:
mpostma 2021-09-28 18:10:09 +02:00
parent ddfd7def35
commit 692c676625
24 changed files with 325 additions and 481 deletions
Download patch file Download diff file Expand all files Collapse all files

1
meilisearch-http/src/helpers/mod.rs
View file

@ -1,4 +1,3 @@
//pub mod compression;
mod env;
pub use env::EnvSizer;

50
meilisearch-http/src/lib.rs
View file

@ -47,6 +47,9 @@ pub mod analytics;
pub mod helpers;
pub mod option;
pub mod routes;
use std::path::Path;
use std::time::Duration;
use crate::extractors::authentication::AuthConfig;
pub use option::Opt;
@ -81,6 +84,53 @@ impl ApiKeys {
}
}
pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<MeiliSearch> {
let mut meilisearch = MeiliSearch::builder();
meilisearch
.set_max_index_size(opt.max_index_size.get_bytes() as usize)
.set_max_update_store_size(opt.max_udb_size.get_bytes() as usize)
.set_ignore_missing_snapshot(opt.ignore_missing_snapshot)
.set_ignore_snapshot_if_db_exists(opt.ignore_snapshot_if_db_exists)
.set_dump_dst(opt.dumps_dir.clone())
.set_snapshot_interval(Duration::from_secs(opt.snapshot_interval_sec))
.set_snapshot_dir(opt.snapshot_dir.clone());
if let Some(ref path) = opt.import_snapshot {
meilisearch.set_import_snapshot(path.clone());
}
if let Some(ref path) = opt.import_dump {
meilisearch.set_dump_src(path.clone());
}
if opt.schedule_snapshot {
meilisearch.set_schedule_snapshot();
}
meilisearch.build(opt.db_path.clone(), opt.indexer_options.clone())
}
/// Cleans and setup the temporary file folder in the database directory. This must be done after
/// the meilisearch instance has been created, to not interfere with the snapshot and dump loading.
pub fn setup_temp_dir(db_path: impl AsRef<Path>) -> anyhow::Result<()> {
// Set the tempfile directory in the current db path, to avoid cross device references. Also
// remove the previous outstanding files found there
//
// TODO: if two processes open the same db, one might delete the other tmpdir. Need to make
// sure that no one is using it before deleting it.
let temp_path = db_path.as_ref().join("tmp");
// Ignore error if tempdir doesn't exist
let _ = std::fs::remove_dir_all(&temp_path);
std::fs::create_dir_all(&temp_path)?;
if cfg!(windows) {
std::env::set_var("TMP", temp_path);
} else {
std::env::set_var("TMPDIR", temp_path);
}
Ok(())
}
pub fn configure_data(
config: &mut web::ServiceConfig,
data: MeiliSearch,

55
meilisearch-http/src/main.rs
View file

@ -1,7 +1,7 @@
use std::{env, path::Path, time::Duration};
use std::env;
use actix_web::HttpServer;
use meilisearch_http::{create_app, Opt};
use meilisearch_http::{Opt, create_app, setup_meilisearch};
use meilisearch_lib::MeiliSearch;
use structopt::StructOpt;
@ -27,53 +27,6 @@ fn setup(opt: &Opt) -> anyhow::Result<()> {
Ok(())
}
/// Cleans and setup the temporary file folder in the database directory. This must be done after
/// the meilisearch instance has been created, to not interfere with the snapshot and dump loading.
fn setup_temp_dir(db_path: impl AsRef<Path>) -> anyhow::Result<()> {
// Set the tempfile directory in the current db path, to avoid cross device references. Also
// remove the previous outstanding files found there
//
// TODO: if two processes open the same db, one might delete the other tmpdir. Need to make
// sure that no one is using it before deleting it.
let temp_path = db_path.as_ref().join("tmp");
// Ignore error if tempdir doesn't exist
let _ = std::fs::remove_dir_all(&temp_path);
std::fs::create_dir_all(&temp_path)?;
if cfg!(windows) {
std::env::set_var("TMP", temp_path);
} else {
std::env::set_var("TMPDIR", temp_path);
}
Ok(())
}
fn setup_meilisearch(opt: &Opt) -> anyhow::Result<MeiliSearch> {
let mut meilisearch = MeiliSearch::builder();
meilisearch
.set_max_index_size(opt.max_index_size.get_bytes() as usize)
.set_max_update_store_size(opt.max_udb_size.get_bytes() as usize)
.set_ignore_missing_snapshot(opt.ignore_missing_snapshot)
.set_ignore_snapshot_if_db_exists(opt.ignore_snapshot_if_db_exists)
.set_dump_dst(opt.dumps_dir.clone())
.set_snapshot_interval(Duration::from_secs(opt.snapshot_interval_sec))
.set_snapshot_dir(opt.snapshot_dir.clone());
if let Some(ref path) = opt.import_snapshot {
meilisearch.set_import_snapshot(path.clone());
}
if let Some(ref path) = opt.import_dump {
meilisearch.set_dump_src(path.clone());
}
if opt.schedule_snapshot {
meilisearch.set_schedule_snapshot();
}
meilisearch.build(opt.db_path.clone(), opt.indexer_options.clone())
}
#[actix_web::main]
async fn main() -> anyhow::Result<()> {
let opt = Opt::from_args();
@ -92,7 +45,9 @@ async fn main() -> anyhow::Result<()> {
let meilisearch = setup_meilisearch(&opt)?;
setup_temp_dir(&opt.db_path)?;
// Setup the temp directory to be in the db folder. This is important, since temporary file
// don't support to be persisted accross filesystem boundaries.
meilisearch_http::setup_temp_dir(&opt.db_path)?;
#[cfg(all(not(debug_assertions), feature = "analytics"))]
if !opt.no_analytics {

62
meilisearch-http/src/option.rs
View file

@ -1,9 +1,5 @@
use byte_unit::ByteError;
use std::fmt;
use std::io::{BufReader, Read};
use std::ops::Deref;
use std::path::PathBuf;
use std::str::FromStr;
use std::sync::Arc;
use std::fs;
@ -14,7 +10,6 @@ use rustls::{
RootCertStore,
};
use structopt::StructOpt;
use sysinfo::{RefreshKind, System, SystemExt};
use meilisearch_lib::options::IndexerOpts;
const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
@ -177,63 +172,6 @@ impl Opt {
}
}
/// A type used to detect the max memory available and use 2/3 of it.
#[derive(Debug, Clone, Copy)]
pub struct MaxMemory(Option<Byte>);
impl FromStr for MaxMemory {
type Err = ByteError;
fn from_str(s: &str) -> Result<MaxMemory, ByteError> {
Byte::from_str(s).map(Some).map(MaxMemory)
}
}
impl Default for MaxMemory {
fn default() -> MaxMemory {
MaxMemory(
total_memory_bytes()
.map(|bytes| bytes * 2 / 3)
.map(Byte::from_bytes),
)
}
}
impl fmt::Display for MaxMemory {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self.0 {
Some(memory) => write!(f, "{}", memory.get_appropriate_unit(true)),
None => f.write_str("unknown"),
}
}
}
impl Deref for MaxMemory {
type Target = Option<Byte>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl MaxMemory {
pub fn unlimited() -> Self {
Self(None)
}
}
/// Returns the total amount of bytes available or `None` if this system isn't supported.
fn total_memory_bytes() -> Option<u64> {
if System::IS_SUPPORTED {
let memory_kind = RefreshKind::new().with_memory();
let mut system = System::new_with_specifics(memory_kind);
system.refresh_memory();
Some(system.total_memory() * 1024) // KiB into bytes
} else {
None
}
}
fn load_certs(filename: PathBuf) -> anyhow::Result<Vec<rustls::Certificate>> {
let certfile = fs::File::open(filename).map_err(|_| anyhow::anyhow!("cannot open certificate file"))?;
let mut reader = BufReader::new(certfile);

10
meilisearch-http/src/routes/indexes/documents.rs
View file

@ -106,7 +106,7 @@ pub async fn delete_document(
) -> Result<HttpResponse, ResponseError> {
let DocumentParam { document_id, index_uid } = path.into_inner();
let update = Update::DeleteDocuments(vec![document_id]);
let update_status = meilisearch.register_update(index_uid, update).await?;
let update_status = meilisearch.register_update(index_uid, update, false).await?;
debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
@ -170,7 +170,7 @@ pub async fn add_documents(
format: DocumentAdditionFormat::Json,
};
let update_status = meilisearch
.register_update(path.into_inner().index_uid, update)
.register_update(path.into_inner().index_uid, update, true)
.await?;
debug!("returns: {:?}", update_status);
@ -193,7 +193,7 @@ pub async fn update_documents(
format: DocumentAdditionFormat::Json,
};
let update_status = meilisearch
.register_update(path.into_inner().index_uid, update)
.register_update(path.into_inner().index_uid, update, true)
.await?;
debug!("returns: {:?}", update_status);
@ -216,7 +216,7 @@ pub async fn delete_documents(
.collect();
let update = Update::DeleteDocuments(ids);
let update_status = meilisearch.register_update(path.into_inner().index_uid, update).await?;
let update_status = meilisearch.register_update(path.into_inner().index_uid, update, false).await?;
debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
@ -226,7 +226,7 @@ pub async fn clear_all_documents(
path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> {
let update = Update::ClearDocuments;
let update_status = meilisearch.register_update(path.into_inner().index_uid, update).await?;
let update_status = meilisearch.register_update(path.into_inner().index_uid, update, false).await?;
debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}

34
meilisearch-http/src/routes/indexes/mod.rs
View file

@ -18,7 +18,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(
web::resource("")
.route(web::get().to(list_indexes))
//.route(web::post().to(create_index)),
.route(web::post().to(create_index)),
)
.service(
web::scope("/{index_uid}")
@ -26,7 +26,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
web::resource("")
.route(web::get().to(get_index))
.route(web::put().to(update_index))
//.route(web::delete().to(delete_index)),
.route(web::delete().to(delete_index)),
)
.service(web::resource("/stats").route(web::get().to(get_index_stats)))
.service(web::scope("/documents").configure(documents::configure))
@ -49,14 +49,14 @@ pub struct IndexCreateRequest {
primary_key: Option<String>,
}
//pub async fn create_index(
//data: GuardedData<Private, MeiliSearch>,
//body: web::Json<IndexCreateRequest>,
//) -> Result<HttpResponse, ResponseError> {
//let body = body.into_inner();
//let meta = data.create_index(body.uid, body.primary_key).await?;
//Ok(HttpResponse::Created().json(meta))
//}
pub async fn create_index(
meilisearch: GuardedData<Private, MeiliSearch>,
body: web::Json<IndexCreateRequest>,
) -> Result<HttpResponse, ResponseError> {
let body = body.into_inner();
let meta = meilisearch.create_index(body.uid, body.primary_key).await?;
Ok(HttpResponse::Created().json(meta))
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
@ -102,13 +102,13 @@ pub async fn update_index(
Ok(HttpResponse::Ok().json(meta))
}
//pub async fn delete_index(
//data: GuardedData<Private, MeiliSearch>,
//path: web::Path<IndexParam>,
//) -> Result<HttpResponse, ResponseError> {
//data.delete_index(path.index_uid.clone()).await?;
//Ok(HttpResponse::NoContent().finish())
//}
pub async fn delete_index(
meilisearch: GuardedData<Private, MeiliSearch>,
path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> {
meilisearch.delete_index(path.index_uid.clone()).await?;
Ok(HttpResponse::NoContent().finish())
}
pub async fn get_index_stats(
meilisearch: GuardedData<Private, MeiliSearch>,

8
meilisearch-http/src/routes/indexes/settings.rs
View file

@ -30,7 +30,7 @@ macro_rules! make_setting_route {
..Default::default()
};
let update = Update::Settings(settings);
let update_status = meilisearch.register_update(index_uid.into_inner(), update).await?;
let update_status = meilisearch.register_update(index_uid.into_inner(), update, false).await?;
debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
@ -49,7 +49,7 @@ macro_rules! make_setting_route {
};
let update = Update::Settings(settings);
let update_status = meilisearch.register_update(index_uid.into_inner(), update).await?;
let update_status = meilisearch.register_update(index_uid.into_inner(), update, true).await?;
debug!("returns: {:?}", update_status);
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
@ -159,7 +159,7 @@ pub async fn update_all(
let update = Update::Settings(settings);
let update_result = meilisearch
.register_update(index_uid.into_inner(), update)
.register_update(index_uid.into_inner(), update, true)
.await?;
let json = serde_json::json!({ "updateId": update_result.id() });
debug!("returns: {:?}", json);
@ -183,7 +183,7 @@ pub async fn delete_all(
let update = Update::Settings(settings.into_unchecked());
let update_result = data
.register_update(index_uid.into_inner(), update)
.register_update(index_uid.into_inner(), update, false)
.await?;
let json = serde_json::json!({ "updateId": update_result.id() });
debug!("returns: {:?}", json);

9
meilisearch-http/src/routes/mod.rs
View file

@ -280,18 +280,17 @@ pub async fn get_health() -> Result<HttpResponse, ResponseError> {
#[cfg(test)]
mod test {
use super::*;
use crate::data::Data;
use crate::extractors::authentication::GuardedData;
/// A type implemented for a route that uses a authentication policy `Policy`.
///
/// This trait is used for regression testing of route authenticaton policies.
trait Is<Policy, T> {}
trait Is<Policy, Data, T> {}
macro_rules! impl_is_policy {
($($param:ident)*) => {
impl<Policy, Func, $($param,)* Res> Is<Policy, (($($param,)*), Res)> for Func
where Func: Fn(GuardedData<Policy, MeiliSearch>, $($param,)*) -> Res {}
impl<Policy, Func, Data, $($param,)* Res> Is<Policy, Data, (($($param,)*), Res)> for Func
where Func: Fn(GuardedData<Policy, Data>, $($param,)*) -> Res {}
};
}
@ -310,7 +309,7 @@ mod test {
($($policy:ident => { $($route:expr,)*})*) => {
#[test]
fn test_auth() {
$($(let _: &dyn Is<$policy, _> = &$route;)*)*
$($(let _: &dyn Is<$policy, _, _> = &$route;)*)*
}
};
}