2068: chore(http): migrate from structopt to clap3 r=Kerollmops a=MarinPostma

migrate from structopt to clap3

This fix the long lasting issue with flags require a value, such as `--no-analytics` or `--schedule-snapshot`.

All flag arguments now take NO argument, i.e:
`meilisearch --schedule-snapshot true` becomes `meilisearch --schedule-snapshot`

as per https://docs.rs/clap/latest/clap/struct.Arg.html#method.env, the env variable is defines as:
> A false literal is n, no, f, false, off or 0. An absent environment variable will also be considered as false. Anything else will considered as true.

`@gmourier` 
`@curquiza` 
`@meilisearch/docs-team` 

Co-authored-by: mpostma <postma.marin@protonmail.com>
This commit is contained in:
bors[bot] 2022-01-20 10:59:44 +00:00 committed by GitHub
commit f9f075bca2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 100 additions and 175 deletions

101
Cargo.lock generated
View File

@ -278,15 +278,6 @@ dependencies = [
"memchr", "memchr",
] ]
[[package]]
name = "ansi_term"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
dependencies = [
"winapi",
]
[[package]] [[package]]
name = "anyhow" name = "anyhow"
version = "1.0.51" version = "1.0.51"
@ -639,17 +630,32 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "2.34.0" version = "3.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" checksum = "1957aa4a5fb388f0a0a73ce7556c5b42025b874e5cdc2c670775e346e97adec0"
dependencies = [ dependencies = [
"ansi_term",
"atty", "atty",
"bitflags", "bitflags",
"clap_derive",
"indexmap",
"lazy_static",
"os_str_bytes",
"strsim", "strsim",
"termcolor",
"textwrap", "textwrap",
"unicode-width", ]
"vec_map",
[[package]]
name = "clap_derive"
version = "3.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "517358c28fcef6607bf6f76108e02afad7e82297d132a6b846dcc1fc3efcd153"
dependencies = [
"heck",
"proc-macro-error",
"proc-macro2 1.0.32",
"quote 1.0.10",
"syn 1.0.82",
] ]
[[package]] [[package]]
@ -1258,12 +1264,9 @@ dependencies = [
[[package]] [[package]]
name = "heck" name = "heck"
version = "0.3.3" version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
dependencies = [
"unicode-segmentation",
]
[[package]] [[package]]
name = "heed" name = "heed"
@ -1668,6 +1671,7 @@ dependencies = [
"bytes", "bytes",
"cargo_toml", "cargo_toml",
"chrono", "chrono",
"clap",
"crossbeam-channel", "crossbeam-channel",
"either", "either",
"env_logger", "env_logger",
@ -1706,7 +1710,6 @@ dependencies = [
"sha2", "sha2",
"siphasher", "siphasher",
"slice-group-by", "slice-group-by",
"structopt",
"sysinfo", "sysinfo",
"tar", "tar",
"tempfile", "tempfile",
@ -1734,6 +1737,7 @@ dependencies = [
"byte-unit", "byte-unit",
"bytes", "bytes",
"chrono", "chrono",
"clap",
"crossbeam-channel", "crossbeam-channel",
"csv", "csv",
"derivative", "derivative",
@ -1772,7 +1776,6 @@ dependencies = [
"serde_json", "serde_json",
"siphasher", "siphasher",
"slice-group-by", "slice-group-by",
"structopt",
"sysinfo", "sysinfo",
"tar", "tar",
"tempfile", "tempfile",
@ -2055,6 +2058,15 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "os_str_bytes"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "page_size" name = "page_size"
version = "0.4.2" version = "0.4.2"
@ -2890,33 +2902,9 @@ checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.8.0" version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "structopt"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c"
dependencies = [
"clap",
"lazy_static",
"structopt-derive",
]
[[package]]
name = "structopt-derive"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
dependencies = [
"heck",
"proc-macro-error",
"proc-macro2 1.0.32",
"quote 1.0.10",
"syn 1.0.82",
]
[[package]] [[package]]
name = "syn" name = "syn"
@ -3018,12 +3006,9 @@ checksum = "13a4ec180a2de59b57434704ccfad967f789b12737738798fa08798cd5824c16"
[[package]] [[package]]
name = "textwrap" name = "textwrap"
version = "0.11.0" version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"
dependencies = [
"unicode-width",
]
[[package]] [[package]]
name = "thiserror" name = "thiserror"
@ -3274,12 +3259,6 @@ version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
[[package]]
name = "unicode-width"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
[[package]] [[package]]
name = "unicode-xid" name = "unicode-xid"
version = "0.1.0" version = "0.1.0"
@ -3338,12 +3317,6 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vec_map"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]] [[package]]
name = "vergen" name = "vergen"
version = "5.1.18" version = "5.1.18"

View File

@ -67,7 +67,7 @@ serde_json = { version = "1.0.67", features = ["preserve_order"] }
sha2 = "0.9.6" sha2 = "0.9.6"
siphasher = "0.3.7" siphasher = "0.3.7"
slice-group-by = "0.2.6" slice-group-by = "0.2.6"
structopt = "0.3.25" clap = { version = "3.0", features = ["derive", "env"] }
sysinfo = "0.20.2" sysinfo = "0.20.2"
tar = "0.4.37" tar = "0.4.37"
tempfile = "3.2.0" tempfile = "3.2.0"

View File

@ -2,12 +2,12 @@ use std::env;
use std::sync::Arc; use std::sync::Arc;
use actix_web::HttpServer; use actix_web::HttpServer;
use clap::Parser;
use meilisearch_auth::AuthController; use meilisearch_auth::AuthController;
use meilisearch_http::analytics; use meilisearch_http::analytics;
use meilisearch_http::analytics::Analytics; use meilisearch_http::analytics::Analytics;
use meilisearch_http::{create_app, setup_meilisearch, Opt}; use meilisearch_http::{create_app, setup_meilisearch, Opt};
use meilisearch_lib::MeiliSearch; use meilisearch_lib::MeiliSearch;
use structopt::StructOpt;
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
#[global_allocator] #[global_allocator]
@ -29,7 +29,7 @@ fn setup(opt: &Opt) -> anyhow::Result<()> {
#[actix_web::main] #[actix_web::main]
async fn main() -> anyhow::Result<()> { async fn main() -> anyhow::Result<()> {
let opt = Opt::from_args(); let opt = Opt::parse();
setup(&opt)?; setup(&opt)?;
@ -50,7 +50,7 @@ async fn main() -> anyhow::Result<()> {
let auth_controller = AuthController::new(&opt.db_path, &opt.master_key)?; let auth_controller = AuthController::new(&opt.db_path, &opt.master_key)?;
#[cfg(all(not(debug_assertions), feature = "analytics"))] #[cfg(all(not(debug_assertions), feature = "analytics"))]
let (analytics, user) = if opt.analytics() { let (analytics, user) = if !opt.no_analytics {
analytics::SegmentAnalytics::new(&opt, &meilisearch).await analytics::SegmentAnalytics::new(&opt, &meilisearch).await
} else { } else {
analytics::MockAnalytics::new(&opt) analytics::MockAnalytics::new(&opt)
@ -125,7 +125,7 @@ pub fn print_launch_resume(opt: &Opt, user: &str) {
#[cfg(all(not(debug_assertions), feature = "analytics"))] #[cfg(all(not(debug_assertions), feature = "analytics"))]
{ {
if opt.analytics() { if !opt.no_analytics {
eprintln!( eprintln!(
" "
Thank you for using MeiliSearch! Thank you for using MeiliSearch!

View File

@ -4,141 +4,131 @@ use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use byte_unit::Byte; use byte_unit::Byte;
use clap::Parser;
use meilisearch_lib::options::IndexerOpts; use meilisearch_lib::options::IndexerOpts;
use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys}; use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
use rustls::{ use rustls::{
AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth, AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth,
RootCertStore, RootCertStore,
}; };
use structopt::StructOpt;
const POSSIBLE_ENV: [&str; 2] = ["development", "production"]; const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
#[derive(Debug, Clone, StructOpt)] #[derive(Debug, Clone, Parser)]
pub struct Opt { pub struct Opt {
/// The destination where the database must be created. /// The destination where the database must be created.
#[structopt(long, env = "MEILI_DB_PATH", default_value = "./data.ms")] #[clap(long, env = "MEILI_DB_PATH", default_value = "./data.ms")]
pub db_path: PathBuf, pub db_path: PathBuf,
/// The address on which the http server will listen. /// The address on which the http server will listen.
#[structopt(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")] #[clap(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")]
pub http_addr: String, pub http_addr: String,
/// The master key allowing you to do everything on the server. /// The master key allowing you to do everything on the server.
#[structopt(long, env = "MEILI_MASTER_KEY")] #[clap(long, env = "MEILI_MASTER_KEY")]
pub master_key: Option<String>, pub master_key: Option<String>,
/// This environment variable must be set to `production` if you are running in production. /// This environment variable must be set to `production` if you are running in production.
/// If the server is running in development mode more logs will be displayed, /// If the server is running in development mode more logs will be displayed,
/// and the master key can be avoided which implies that there is no security on the updates routes. /// and the master key can be avoided which implies that there is no security on the updates routes.
/// This is useful to debug when integrating the engine with another service. /// This is useful to debug when integrating the engine with another service.
#[structopt(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)] #[clap(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)]
pub env: String, pub env: String,
/// Do not send analytics to Meili. /// Do not send analytics to Meili.
#[cfg(all(not(debug_assertions), feature = "analytics"))] #[cfg(all(not(debug_assertions), feature = "analytics"))]
#[structopt(long, env = "MEILI_NO_ANALYTICS")] #[clap(long, env = "MEILI_NO_ANALYTICS")]
pub no_analytics: Option<Option<bool>>, pub no_analytics: bool,
/// The maximum size, in bytes, of the main lmdb database directory /// The maximum size, in bytes, of the main lmdb database directory
#[structopt(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")] #[clap(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")]
pub max_index_size: Byte, pub max_index_size: Byte,
/// The maximum size, in bytes, of the update lmdb database directory /// The maximum size, in bytes, of the update lmdb database directory
#[structopt(long, env = "MEILI_MAX_TASK_DB_SIZE", default_value = "100 GiB")] #[clap(long, env = "MEILI_MAX_TASK_DB_SIZE", default_value = "100 GiB")]
pub max_task_db_size: Byte, pub max_task_db_size: Byte,
/// The maximum size, in bytes, of accepted JSON payloads /// The maximum size, in bytes, of accepted JSON payloads
#[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")] #[clap(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")]
pub http_payload_size_limit: Byte, pub http_payload_size_limit: Byte,
/// Read server certificates from CERTFILE. /// Read server certificates from CERTFILE.
/// This should contain PEM-format certificates /// This should contain PEM-format certificates
/// in the right order (the first certificate should /// in the right order (the first certificate should
/// certify KEYFILE, the last should be a root CA). /// certify KEYFILE, the last should be a root CA).
#[structopt(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))] #[clap(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))]
pub ssl_cert_path: Option<PathBuf>, pub ssl_cert_path: Option<PathBuf>,
/// Read private key from KEYFILE. This should be a RSA /// Read private key from KEYFILE. This should be a RSA
/// private key or PKCS8-encoded private key, in PEM format. /// private key or PKCS8-encoded private key, in PEM format.
#[structopt(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))] #[clap(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))]
pub ssl_key_path: Option<PathBuf>, pub ssl_key_path: Option<PathBuf>,
/// Enable client authentication, and accept certificates /// Enable client authentication, and accept certificates
/// signed by those roots provided in CERTFILE. /// signed by those roots provided in CERTFILE.
#[structopt(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))] #[clap(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))]
pub ssl_auth_path: Option<PathBuf>, pub ssl_auth_path: Option<PathBuf>,
/// Read DER-encoded OCSP response from OCSPFILE and staple to certificate. /// Read DER-encoded OCSP response from OCSPFILE and staple to certificate.
/// Optional /// Optional
#[structopt(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))] #[clap(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))]
pub ssl_ocsp_path: Option<PathBuf>, pub ssl_ocsp_path: Option<PathBuf>,
/// Send a fatal alert if the client does not complete client authentication. /// Send a fatal alert if the client does not complete client authentication.
#[structopt(long, env = "MEILI_SSL_REQUIRE_AUTH")] #[clap(long, env = "MEILI_SSL_REQUIRE_AUTH")]
pub ssl_require_auth: bool, pub ssl_require_auth: bool,
/// SSL support session resumption /// SSL support session resumption
#[structopt(long, env = "MEILI_SSL_RESUMPTION")] #[clap(long, env = "MEILI_SSL_RESUMPTION")]
pub ssl_resumption: bool, pub ssl_resumption: bool,
/// SSL support tickets. /// SSL support tickets.
#[structopt(long, env = "MEILI_SSL_TICKETS")] #[clap(long, env = "MEILI_SSL_TICKETS")]
pub ssl_tickets: bool, pub ssl_tickets: bool,
/// Defines the path of the snapshot file to import. /// Defines the path of the snapshot file to import.
/// This option will, by default, stop the process if a database already exist or if no snapshot exists at /// This option will, by default, stop the process if a database already exist or if no snapshot exists at
/// the given path. If this option is not specified no snapshot is imported. /// the given path. If this option is not specified no snapshot is imported.
#[structopt(long)] #[clap(long)]
pub import_snapshot: Option<PathBuf>, pub import_snapshot: Option<PathBuf>,
/// The engine will ignore a missing snapshot and not return an error in such case. /// The engine will ignore a missing snapshot and not return an error in such case.
#[structopt(long, requires = "import-snapshot")] #[clap(long, requires = "import-snapshot")]
pub ignore_missing_snapshot: bool, pub ignore_missing_snapshot: bool,
/// The engine will skip snapshot importation and not return an error in such case. /// The engine will skip snapshot importation and not return an error in such case.
#[structopt(long, requires = "import-snapshot")] #[clap(long, requires = "import-snapshot")]
pub ignore_snapshot_if_db_exists: bool, pub ignore_snapshot_if_db_exists: bool,
/// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap. /// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap.
#[structopt(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")] #[clap(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")]
pub snapshot_dir: PathBuf, pub snapshot_dir: PathBuf,
/// Activate snapshot scheduling. /// Activate snapshot scheduling.
#[structopt(long, env = "MEILI_SCHEDULE_SNAPSHOT")] #[clap(long, env = "MEILI_SCHEDULE_SNAPSHOT")]
pub schedule_snapshot: bool, pub schedule_snapshot: bool,
/// Defines time interval, in seconds, between each snapshot creation. /// Defines time interval, in seconds, between each snapshot creation.
#[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h #[clap(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h
pub snapshot_interval_sec: u64, pub snapshot_interval_sec: u64,
/// Folder where dumps are created when the dump route is called. /// Folder where dumps are created when the dump route is called.
#[structopt(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")] #[clap(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")]
pub dumps_dir: PathBuf, pub dumps_dir: PathBuf,
/// Import a dump from the specified path, must be a `.dump` file. /// Import a dump from the specified path, must be a `.dump` file.
#[structopt(long, conflicts_with = "import-snapshot")] #[clap(long, conflicts_with = "import-snapshot")]
pub import_dump: Option<PathBuf>, pub import_dump: Option<PathBuf>,
/// Set the log level /// Set the log level
#[structopt(long, env = "MEILI_LOG_LEVEL", default_value = "info")] #[clap(long, env = "MEILI_LOG_LEVEL", default_value = "info")]
pub log_level: String, pub log_level: String,
#[structopt(skip)] #[clap(skip)]
pub indexer_options: IndexerOpts, pub indexer_options: IndexerOpts,
} }
impl Opt { impl Opt {
/// Wether analytics should be enabled or not.
#[cfg(all(not(debug_assertions), feature = "analytics"))]
pub fn analytics(&self) -> bool {
match self.no_analytics {
None => true,
Some(None) => false,
Some(Some(disabled)) => !disabled,
}
}
pub fn get_ssl_config(&self) -> anyhow::Result<Option<rustls::ServerConfig>> { pub fn get_ssl_config(&self) -> anyhow::Result<Option<rustls::ServerConfig>> {
if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) { if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) {
let client_auth = match &self.ssl_auth_path { let client_auth = match &self.ssl_auth_path {

View File

@ -35,15 +35,12 @@ pub struct Index<'a> {
#[allow(dead_code)] #[allow(dead_code)]
impl Index<'_> { impl Index<'_> {
pub async fn get(&self) -> (Value, StatusCode) { pub async fn get(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string()); let url = format!("/indexes/{}", encode(self.uid.as_ref()));
self.service.get(url).await self.service.get(url).await
} }
pub async fn load_test_set(&self) -> u64 { pub async fn load_test_set(&self) -> u64 {
let url = format!( let url = format!("/indexes/{}/documents", encode(self.uid.as_ref()));
"/indexes/{}/documents",
encode(self.uid.as_ref()).to_string()
);
let (response, code) = self let (response, code) = self
.service .service
.post_str(url, include_str!("../assets/test_set.json")) .post_str(url, include_str!("../assets/test_set.json"))
@ -66,13 +63,13 @@ impl Index<'_> {
let body = json!({ let body = json!({
"primaryKey": primary_key, "primaryKey": primary_key,
}); });
let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string()); let url = format!("/indexes/{}", encode(self.uid.as_ref()));
self.service.put(url, body).await self.service.put(url, body).await
} }
pub async fn delete(&self) -> (Value, StatusCode) { pub async fn delete(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string()); let url = format!("/indexes/{}", encode(self.uid.as_ref()));
self.service.delete(url).await self.service.delete(url).await
} }
@ -84,13 +81,10 @@ impl Index<'_> {
let url = match primary_key { let url = match primary_key {
Some(key) => format!( Some(key) => format!(
"/indexes/{}/documents?primaryKey={}", "/indexes/{}/documents?primaryKey={}",
encode(self.uid.as_ref()).to_string(), encode(self.uid.as_ref()),
key key
), ),
None => format!( None => format!("/indexes/{}/documents", encode(self.uid.as_ref())),
"/indexes/{}/documents",
encode(self.uid.as_ref()).to_string()
),
}; };
self.service.post(url, documents).await self.service.post(url, documents).await
} }
@ -103,13 +97,10 @@ impl Index<'_> {
let url = match primary_key { let url = match primary_key {
Some(key) => format!( Some(key) => format!(
"/indexes/{}/documents?primaryKey={}", "/indexes/{}/documents?primaryKey={}",
encode(self.uid.as_ref()).to_string(), encode(self.uid.as_ref()),
key key
), ),
None => format!( None => format!("/indexes/{}/documents", encode(self.uid.as_ref())),
"/indexes/{}/documents",
encode(self.uid.as_ref()).to_string()
),
}; };
self.service.put(url, documents).await self.service.put(url, documents).await
} }
@ -145,19 +136,12 @@ impl Index<'_> {
id: u64, id: u64,
_options: Option<GetDocumentOptions>, _options: Option<GetDocumentOptions>,
) -> (Value, StatusCode) { ) -> (Value, StatusCode) {
let url = format!( let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
"/indexes/{}/documents/{}",
encode(self.uid.as_ref()).to_string(),
id
);
self.service.get(url).await self.service.get(url).await
} }
pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) { pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
let mut url = format!( let mut url = format!("/indexes/{}/documents?", encode(self.uid.as_ref()));
"/indexes/{}/documents?",
encode(self.uid.as_ref()).to_string()
);
if let Some(limit) = options.limit { if let Some(limit) = options.limit {
url.push_str(&format!("limit={}&", limit)); url.push_str(&format!("limit={}&", limit));
} }
@ -177,26 +161,19 @@ impl Index<'_> {
} }
pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) { pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
let url = format!( let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
"/indexes/{}/documents/{}",
encode(self.uid.as_ref()).to_string(),
id
);
self.service.delete(url).await self.service.delete(url).await
} }
pub async fn clear_all_documents(&self) -> (Value, StatusCode) { pub async fn clear_all_documents(&self) -> (Value, StatusCode) {
let url = format!( let url = format!("/indexes/{}/documents", encode(self.uid.as_ref()));
"/indexes/{}/documents",
encode(self.uid.as_ref()).to_string()
);
self.service.delete(url).await self.service.delete(url).await
} }
pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) { pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
let url = format!( let url = format!(
"/indexes/{}/documents/delete-batch", "/indexes/{}/documents/delete-batch",
encode(self.uid.as_ref()).to_string() encode(self.uid.as_ref())
); );
self.service self.service
.post(url, serde_json::to_value(&ids).unwrap()) .post(url, serde_json::to_value(&ids).unwrap())
@ -204,31 +181,22 @@ impl Index<'_> {
} }
pub async fn settings(&self) -> (Value, StatusCode) { pub async fn settings(&self) -> (Value, StatusCode) {
let url = format!( let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
"/indexes/{}/settings",
encode(self.uid.as_ref()).to_string()
);
self.service.get(url).await self.service.get(url).await
} }
pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) { pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
let url = format!( let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
"/indexes/{}/settings",
encode(self.uid.as_ref()).to_string()
);
self.service.post(url, settings).await self.service.post(url, settings).await
} }
pub async fn delete_settings(&self) -> (Value, StatusCode) { pub async fn delete_settings(&self) -> (Value, StatusCode) {
let url = format!( let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
"/indexes/{}/settings",
encode(self.uid.as_ref()).to_string()
);
self.service.delete(url).await self.service.delete(url).await
} }
pub async fn stats(&self) -> (Value, StatusCode) { pub async fn stats(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}/stats", encode(self.uid.as_ref()).to_string()); let url = format!("/indexes/{}/stats", encode(self.uid.as_ref()));
self.service.get(url).await self.service.get(url).await
} }
@ -253,17 +221,13 @@ impl Index<'_> {
} }
pub async fn search_post(&self, query: Value) -> (Value, StatusCode) { pub async fn search_post(&self, query: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/search", encode(self.uid.as_ref()).to_string()); let url = format!("/indexes/{}/search", encode(self.uid.as_ref()));
self.service.post(url, query).await self.service.post(url, query).await
} }
pub async fn search_get(&self, query: Value) -> (Value, StatusCode) { pub async fn search_get(&self, query: Value) -> (Value, StatusCode) {
let params = serde_url_params::to_string(&query).unwrap(); let params = serde_url_params::to_string(&query).unwrap();
let url = format!( let url = format!("/indexes/{}/search?{}", encode(self.uid.as_ref()), params);
"/indexes/{}/search?{}",
encode(self.uid.as_ref()).to_string(),
params
);
self.service.get(url).await self.service.get(url).await
} }

View File

@ -130,7 +130,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
master_key: None, master_key: None,
env: "development".to_owned(), env: "development".to_owned(),
#[cfg(all(not(debug_assertions), feature = "analytics"))] #[cfg(all(not(debug_assertions), feature = "analytics"))]
no_analytics: Some(Some(true)), no_analytics: true,
max_index_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), max_index_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
max_task_db_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), max_task_db_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(), http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(),

View File

@ -42,7 +42,7 @@ serde = { version = "1.0.130", features = ["derive"] }
serde_json = { version = "1.0.67", features = ["preserve_order"] } serde_json = { version = "1.0.67", features = ["preserve_order"] }
siphasher = "0.3.7" siphasher = "0.3.7"
slice-group-by = "0.2.6" slice-group-by = "0.2.6"
structopt = "0.3.23" clap = { version = "3.0", features = ["derive", "env"] }
tar = "0.4.37" tar = "0.4.37"
tempfile = "3.2.0" tempfile = "3.2.0"
thiserror = "1.0.28" thiserror = "1.0.28"

View File

@ -28,9 +28,7 @@ impl Index {
pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> { pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
// acquire write txn make sure any ongoing write is finished before we start. // acquire write txn make sure any ongoing write is finished before we start.
let txn = self.env.write_txn()?; let txn = self.env.write_txn()?;
let path = path let path = path.as_ref().join(format!("indexes/{}", self.uuid));
.as_ref()
.join(format!("indexes/{}", self.uuid.to_string()));
create_dir_all(&path)?; create_dir_all(&path)?;

View File

@ -2,19 +2,19 @@ use core::fmt;
use std::{ops::Deref, str::FromStr}; use std::{ops::Deref, str::FromStr};
use byte_unit::{Byte, ByteError}; use byte_unit::{Byte, ByteError};
use clap::Parser;
use milli::CompressionType; use milli::CompressionType;
use structopt::StructOpt;
use sysinfo::{RefreshKind, System, SystemExt}; use sysinfo::{RefreshKind, System, SystemExt};
#[derive(Debug, Clone, StructOpt)] #[derive(Debug, Clone, Parser)]
pub struct IndexerOpts { pub struct IndexerOpts {
/// The amount of documents to skip before printing /// The amount of documents to skip before printing
/// a log regarding the indexing advancement. /// a log regarding the indexing advancement.
#[structopt(long, default_value = "100000")] // 100k #[clap(long, default_value = "100000")] // 100k
pub log_every_n: usize, pub log_every_n: usize,
/// Grenad max number of chunks in bytes. /// Grenad max number of chunks in bytes.
#[structopt(long)] #[clap(long)]
pub max_nb_chunks: Option<usize>, pub max_nb_chunks: Option<usize>,
/// The maximum amount of memory the indexer will use. It defaults to 2/3 /// The maximum amount of memory the indexer will use. It defaults to 2/3
@ -24,22 +24,22 @@ pub struct IndexerOpts {
/// In case the engine is unable to retrieve the available memory the engine will /// In case the engine is unable to retrieve the available memory the engine will
/// try to use the memory it needs but without real limit, this can lead to /// try to use the memory it needs but without real limit, this can lead to
/// Out-Of-Memory issues and it is recommended to specify the amount of memory to use. /// Out-Of-Memory issues and it is recommended to specify the amount of memory to use.
#[structopt(long, default_value)] #[clap(long, default_value_t)]
pub max_memory: MaxMemory, pub max_memory: MaxMemory,
/// The name of the compression algorithm to use when compressing intermediate /// The name of the compression algorithm to use when compressing intermediate
/// Grenad chunks while indexing documents. /// Grenad chunks while indexing documents.
/// ///
/// Choosing a fast algorithm will make the indexing faster but may consume more memory. /// Choosing a fast algorithm will make the indexing faster but may consume more memory.
#[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])] #[clap(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])]
pub chunk_compression_type: CompressionType, pub chunk_compression_type: CompressionType,
/// The level of compression of the chosen algorithm. /// The level of compression of the chosen algorithm.
#[structopt(long, requires = "chunk-compression-type")] #[clap(long, requires = "chunk-compression-type")]
pub chunk_compression_level: Option<u32>, pub chunk_compression_level: Option<u32>,
/// Number of parallel jobs for indexing, defaults to # of CPUs. /// Number of parallel jobs for indexing, defaults to # of CPUs.
#[structopt(long)] #[clap(long)]
pub indexing_jobs: Option<usize>, pub indexing_jobs: Option<usize>,
} }