mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-05 04:28:55 +01:00
Merge #2068
2068: chore(http): migrate from structopt to clap3 r=Kerollmops a=MarinPostma migrate from structopt to clap3 This fix the long lasting issue with flags require a value, such as `--no-analytics` or `--schedule-snapshot`. All flag arguments now take NO argument, i.e: `meilisearch --schedule-snapshot true` becomes `meilisearch --schedule-snapshot` as per https://docs.rs/clap/latest/clap/struct.Arg.html#method.env, the env variable is defines as: > A false literal is n, no, f, false, off or 0. An absent environment variable will also be considered as false. Anything else will considered as true. `@gmourier` `@curquiza` `@meilisearch/docs-team` Co-authored-by: mpostma <postma.marin@protonmail.com>
This commit is contained in:
bors[bot]
GitHub
commit
f9f075bca2
101
Cargo.lock
generated
101
Cargo.lock
generated
@ -278,15 +278,6 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ansi_term"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.51"
|
||||
@ -639,17 +630,32 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "2.34.0"
|
||||
version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
|
||||
checksum = "1957aa4a5fb388f0a0a73ce7556c5b42025b874e5cdc2c670775e346e97adec0"
|
||||
dependencies = [
|
||||
"ansi_term",
|
||||
"atty",
|
||||
"bitflags",
|
||||
"clap_derive",
|
||||
"indexmap",
|
||||
"lazy_static",
|
||||
"os_str_bytes",
|
||||
"strsim",
|
||||
"termcolor",
|
||||
"textwrap",
|
||||
"unicode-width",
|
||||
"vec_map",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "517358c28fcef6607bf6f76108e02afad7e82297d132a6b846dcc1fc3efcd153"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro-error",
|
||||
"proc-macro2 1.0.32",
|
||||
"quote 1.0.10",
|
||||
"syn 1.0.82",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1258,12 +1264,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.3.3"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
|
||||
dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
|
||||
|
||||
[[package]]
|
||||
name = "heed"
|
||||
@ -1668,6 +1671,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"cargo_toml",
|
||||
"chrono",
|
||||
"clap",
|
||||
"crossbeam-channel",
|
||||
"either",
|
||||
"env_logger",
|
||||
@ -1706,7 +1710,6 @@ dependencies = [
|
||||
"sha2",
|
||||
"siphasher",
|
||||
"slice-group-by",
|
||||
"structopt",
|
||||
"sysinfo",
|
||||
"tar",
|
||||
"tempfile",
|
||||
@ -1734,6 +1737,7 @@ dependencies = [
|
||||
"byte-unit",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"clap",
|
||||
"crossbeam-channel",
|
||||
"csv",
|
||||
"derivative",
|
||||
@ -1772,7 +1776,6 @@ dependencies = [
|
||||
"serde_json",
|
||||
"siphasher",
|
||||
"slice-group-by",
|
||||
"structopt",
|
||||
"sysinfo",
|
||||
"tar",
|
||||
"tempfile",
|
||||
@ -2055,6 +2058,15 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "os_str_bytes"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "page_size"
|
||||
version = "0.4.2"
|
||||
@ -2890,33 +2902,9 @@ checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.8.0"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
|
||||
|
||||
[[package]]
|
||||
name = "structopt"
|
||||
version = "0.3.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"lazy_static",
|
||||
"structopt-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "structopt-derive"
|
||||
version = "0.4.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro-error",
|
||||
"proc-macro2 1.0.32",
|
||||
"quote 1.0.10",
|
||||
"syn 1.0.82",
|
||||
]
|
||||
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
@ -3018,12 +3006,9 @@ checksum = "13a4ec180a2de59b57434704ccfad967f789b12737738798fa08798cd5824c16"
|
||||
|
||||
[[package]]
|
||||
name = "textwrap"
|
||||
version = "0.11.0"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
|
||||
dependencies = [
|
||||
"unicode-width",
|
||||
]
|
||||
checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
@ -3274,12 +3259,6 @@ version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.1.0"
|
||||
@ -3338,12 +3317,6 @@ version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||
|
||||
[[package]]
|
||||
name = "vec_map"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
|
||||
|
||||
[[package]]
|
||||
name = "vergen"
|
||||
version = "5.1.18"
|
||||
|
||||
@ -67,7 +67,7 @@ serde_json = { version = "1.0.67", features = ["preserve_order"] }
|
||||
sha2 = "0.9.6"
|
||||
siphasher = "0.3.7"
|
||||
slice-group-by = "0.2.6"
|
||||
structopt = "0.3.25"
|
||||
clap = { version = "3.0", features = ["derive", "env"] }
|
||||
sysinfo = "0.20.2"
|
||||
tar = "0.4.37"
|
||||
tempfile = "3.2.0"
|
||||
|
||||
@ -2,12 +2,12 @@ use std::env;
|
||||
use std::sync::Arc;
|
||||
|
||||
use actix_web::HttpServer;
|
||||
use clap::Parser;
|
||||
use meilisearch_auth::AuthController;
|
||||
use meilisearch_http::analytics;
|
||||
use meilisearch_http::analytics::Analytics;
|
||||
use meilisearch_http::{create_app, setup_meilisearch, Opt};
|
||||
use meilisearch_lib::MeiliSearch;
|
||||
use structopt::StructOpt;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[global_allocator]
|
||||
@ -29,7 +29,7 @@ fn setup(opt: &Opt) -> anyhow::Result<()> {
|
||||
|
||||
#[actix_web::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let opt = Opt::from_args();
|
||||
let opt = Opt::parse();
|
||||
|
||||
setup(&opt)?;
|
||||
|
||||
@ -50,7 +50,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
let auth_controller = AuthController::new(&opt.db_path, &opt.master_key)?;
|
||||
|
||||
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
||||
let (analytics, user) = if opt.analytics() {
|
||||
let (analytics, user) = if !opt.no_analytics {
|
||||
analytics::SegmentAnalytics::new(&opt, &meilisearch).await
|
||||
} else {
|
||||
analytics::MockAnalytics::new(&opt)
|
||||
@ -125,7 +125,7 @@ pub fn print_launch_resume(opt: &Opt, user: &str) {
|
||||
|
||||
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
||||
{
|
||||
if opt.analytics() {
|
||||
if !opt.no_analytics {
|
||||
eprintln!(
|
||||
"
|
||||
Thank you for using MeiliSearch!
|
||||
|
||||
@ -4,141 +4,131 @@ use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use byte_unit::Byte;
|
||||
use clap::Parser;
|
||||
use meilisearch_lib::options::IndexerOpts;
|
||||
use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
|
||||
use rustls::{
|
||||
AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth,
|
||||
RootCertStore,
|
||||
};
|
||||
use structopt::StructOpt;
|
||||
|
||||
const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
|
||||
|
||||
#[derive(Debug, Clone, StructOpt)]
|
||||
#[derive(Debug, Clone, Parser)]
|
||||
pub struct Opt {
|
||||
/// The destination where the database must be created.
|
||||
#[structopt(long, env = "MEILI_DB_PATH", default_value = "./data.ms")]
|
||||
#[clap(long, env = "MEILI_DB_PATH", default_value = "./data.ms")]
|
||||
pub db_path: PathBuf,
|
||||
|
||||
/// The address on which the http server will listen.
|
||||
#[structopt(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")]
|
||||
#[clap(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")]
|
||||
pub http_addr: String,
|
||||
|
||||
/// The master key allowing you to do everything on the server.
|
||||
#[structopt(long, env = "MEILI_MASTER_KEY")]
|
||||
#[clap(long, env = "MEILI_MASTER_KEY")]
|
||||
pub master_key: Option<String>,
|
||||
|
||||
/// This environment variable must be set to `production` if you are running in production.
|
||||
/// If the server is running in development mode more logs will be displayed,
|
||||
/// and the master key can be avoided which implies that there is no security on the updates routes.
|
||||
/// This is useful to debug when integrating the engine with another service.
|
||||
#[structopt(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)]
|
||||
#[clap(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)]
|
||||
pub env: String,
|
||||
|
||||
/// Do not send analytics to Meili.
|
||||
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
||||
#[structopt(long, env = "MEILI_NO_ANALYTICS")]
|
||||
pub no_analytics: Option<Option<bool>>,
|
||||
#[clap(long, env = "MEILI_NO_ANALYTICS")]
|
||||
pub no_analytics: bool,
|
||||
|
||||
/// The maximum size, in bytes, of the main lmdb database directory
|
||||
#[structopt(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")]
|
||||
#[clap(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")]
|
||||
pub max_index_size: Byte,
|
||||
|
||||
/// The maximum size, in bytes, of the update lmdb database directory
|
||||
#[structopt(long, env = "MEILI_MAX_TASK_DB_SIZE", default_value = "100 GiB")]
|
||||
#[clap(long, env = "MEILI_MAX_TASK_DB_SIZE", default_value = "100 GiB")]
|
||||
pub max_task_db_size: Byte,
|
||||
|
||||
/// The maximum size, in bytes, of accepted JSON payloads
|
||||
#[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")]
|
||||
#[clap(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")]
|
||||
pub http_payload_size_limit: Byte,
|
||||
|
||||
/// Read server certificates from CERTFILE.
|
||||
/// This should contain PEM-format certificates
|
||||
/// in the right order (the first certificate should
|
||||
/// certify KEYFILE, the last should be a root CA).
|
||||
#[structopt(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))]
|
||||
#[clap(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))]
|
||||
pub ssl_cert_path: Option<PathBuf>,
|
||||
|
||||
/// Read private key from KEYFILE. This should be a RSA
|
||||
/// private key or PKCS8-encoded private key, in PEM format.
|
||||
#[structopt(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))]
|
||||
#[clap(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))]
|
||||
pub ssl_key_path: Option<PathBuf>,
|
||||
|
||||
/// Enable client authentication, and accept certificates
|
||||
/// signed by those roots provided in CERTFILE.
|
||||
#[structopt(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))]
|
||||
#[clap(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))]
|
||||
pub ssl_auth_path: Option<PathBuf>,
|
||||
|
||||
/// Read DER-encoded OCSP response from OCSPFILE and staple to certificate.
|
||||
/// Optional
|
||||
#[structopt(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))]
|
||||
#[clap(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))]
|
||||
pub ssl_ocsp_path: Option<PathBuf>,
|
||||
|
||||
/// Send a fatal alert if the client does not complete client authentication.
|
||||
#[structopt(long, env = "MEILI_SSL_REQUIRE_AUTH")]
|
||||
#[clap(long, env = "MEILI_SSL_REQUIRE_AUTH")]
|
||||
pub ssl_require_auth: bool,
|
||||
|
||||
/// SSL support session resumption
|
||||
#[structopt(long, env = "MEILI_SSL_RESUMPTION")]
|
||||
#[clap(long, env = "MEILI_SSL_RESUMPTION")]
|
||||
pub ssl_resumption: bool,
|
||||
|
||||
/// SSL support tickets.
|
||||
#[structopt(long, env = "MEILI_SSL_TICKETS")]
|
||||
#[clap(long, env = "MEILI_SSL_TICKETS")]
|
||||
pub ssl_tickets: bool,
|
||||
|
||||
/// Defines the path of the snapshot file to import.
|
||||
/// This option will, by default, stop the process if a database already exist or if no snapshot exists at
|
||||
/// the given path. If this option is not specified no snapshot is imported.
|
||||
#[structopt(long)]
|
||||
#[clap(long)]
|
||||
pub import_snapshot: Option<PathBuf>,
|
||||
|
||||
/// The engine will ignore a missing snapshot and not return an error in such case.
|
||||
#[structopt(long, requires = "import-snapshot")]
|
||||
#[clap(long, requires = "import-snapshot")]
|
||||
pub ignore_missing_snapshot: bool,
|
||||
|
||||
/// The engine will skip snapshot importation and not return an error in such case.
|
||||
#[structopt(long, requires = "import-snapshot")]
|
||||
#[clap(long, requires = "import-snapshot")]
|
||||
pub ignore_snapshot_if_db_exists: bool,
|
||||
|
||||
/// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap.
|
||||
#[structopt(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")]
|
||||
#[clap(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")]
|
||||
pub snapshot_dir: PathBuf,
|
||||
|
||||
/// Activate snapshot scheduling.
|
||||
#[structopt(long, env = "MEILI_SCHEDULE_SNAPSHOT")]
|
||||
#[clap(long, env = "MEILI_SCHEDULE_SNAPSHOT")]
|
||||
pub schedule_snapshot: bool,
|
||||
|
||||
/// Defines time interval, in seconds, between each snapshot creation.
|
||||
#[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h
|
||||
#[clap(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h
|
||||
pub snapshot_interval_sec: u64,
|
||||
|
||||
/// Folder where dumps are created when the dump route is called.
|
||||
#[structopt(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")]
|
||||
#[clap(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")]
|
||||
pub dumps_dir: PathBuf,
|
||||
|
||||
/// Import a dump from the specified path, must be a `.dump` file.
|
||||
#[structopt(long, conflicts_with = "import-snapshot")]
|
||||
#[clap(long, conflicts_with = "import-snapshot")]
|
||||
pub import_dump: Option<PathBuf>,
|
||||
|
||||
/// Set the log level
|
||||
#[structopt(long, env = "MEILI_LOG_LEVEL", default_value = "info")]
|
||||
#[clap(long, env = "MEILI_LOG_LEVEL", default_value = "info")]
|
||||
pub log_level: String,
|
||||
|
||||
#[structopt(skip)]
|
||||
#[clap(skip)]
|
||||
pub indexer_options: IndexerOpts,
|
||||
}
|
||||
|
||||
impl Opt {
|
||||
/// Wether analytics should be enabled or not.
|
||||
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
||||
pub fn analytics(&self) -> bool {
|
||||
match self.no_analytics {
|
||||
None => true,
|
||||
Some(None) => false,
|
||||
Some(Some(disabled)) => !disabled,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_ssl_config(&self) -> anyhow::Result<Option<rustls::ServerConfig>> {
|
||||
if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) {
|
||||
let client_auth = match &self.ssl_auth_path {
|
||||
|
||||
@ -35,15 +35,12 @@ pub struct Index<'a> {
|
||||
#[allow(dead_code)]
|
||||
impl Index<'_> {
|
||||
pub async fn get(&self) -> (Value, StatusCode) {
|
||||
let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string());
|
||||
let url = format!("/indexes/{}", encode(self.uid.as_ref()));
|
||||
self.service.get(url).await
|
||||
}
|
||||
|
||||
pub async fn load_test_set(&self) -> u64 {
|
||||
let url = format!(
|
||||
"/indexes/{}/documents",
|
||||
encode(self.uid.as_ref()).to_string()
|
||||
);
|
||||
let url = format!("/indexes/{}/documents", encode(self.uid.as_ref()));
|
||||
let (response, code) = self
|
||||
.service
|
||||
.post_str(url, include_str!("../assets/test_set.json"))
|
||||
@ -66,13 +63,13 @@ impl Index<'_> {
|
||||
let body = json!({
|
||||
"primaryKey": primary_key,
|
||||
});
|
||||
let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string());
|
||||
let url = format!("/indexes/{}", encode(self.uid.as_ref()));
|
||||
|
||||
self.service.put(url, body).await
|
||||
}
|
||||
|
||||
pub async fn delete(&self) -> (Value, StatusCode) {
|
||||
let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string());
|
||||
let url = format!("/indexes/{}", encode(self.uid.as_ref()));
|
||||
self.service.delete(url).await
|
||||
}
|
||||
|
||||
@ -84,13 +81,10 @@ impl Index<'_> {
|
||||
let url = match primary_key {
|
||||
Some(key) => format!(
|
||||
"/indexes/{}/documents?primaryKey={}",
|
||||
encode(self.uid.as_ref()).to_string(),
|
||||
encode(self.uid.as_ref()),
|
||||
key
|
||||
),
|
||||
None => format!(
|
||||
"/indexes/{}/documents",
|
||||
encode(self.uid.as_ref()).to_string()
|
||||
),
|
||||
None => format!("/indexes/{}/documents", encode(self.uid.as_ref())),
|
||||
};
|
||||
self.service.post(url, documents).await
|
||||
}
|
||||
@ -103,13 +97,10 @@ impl Index<'_> {
|
||||
let url = match primary_key {
|
||||
Some(key) => format!(
|
||||
"/indexes/{}/documents?primaryKey={}",
|
||||
encode(self.uid.as_ref()).to_string(),
|
||||
encode(self.uid.as_ref()),
|
||||
key
|
||||
),
|
||||
None => format!(
|
||||
"/indexes/{}/documents",
|
||||
encode(self.uid.as_ref()).to_string()
|
||||
),
|
||||
None => format!("/indexes/{}/documents", encode(self.uid.as_ref())),
|
||||
};
|
||||
self.service.put(url, documents).await
|
||||
}
|
||||
@ -145,19 +136,12 @@ impl Index<'_> {
|
||||
id: u64,
|
||||
_options: Option<GetDocumentOptions>,
|
||||
) -> (Value, StatusCode) {
|
||||
let url = format!(
|
||||
"/indexes/{}/documents/{}",
|
||||
encode(self.uid.as_ref()).to_string(),
|
||||
id
|
||||
);
|
||||
let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
|
||||
self.service.get(url).await
|
||||
}
|
||||
|
||||
pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
|
||||
let mut url = format!(
|
||||
"/indexes/{}/documents?",
|
||||
encode(self.uid.as_ref()).to_string()
|
||||
);
|
||||
let mut url = format!("/indexes/{}/documents?", encode(self.uid.as_ref()));
|
||||
if let Some(limit) = options.limit {
|
||||
url.push_str(&format!("limit={}&", limit));
|
||||
}
|
||||
@ -177,26 +161,19 @@ impl Index<'_> {
|
||||
}
|
||||
|
||||
pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
|
||||
let url = format!(
|
||||
"/indexes/{}/documents/{}",
|
||||
encode(self.uid.as_ref()).to_string(),
|
||||
id
|
||||
);
|
||||
let url = format!("/indexes/{}/documents/{}", encode(self.uid.as_ref()), id);
|
||||
self.service.delete(url).await
|
||||
}
|
||||
|
||||
pub async fn clear_all_documents(&self) -> (Value, StatusCode) {
|
||||
let url = format!(
|
||||
"/indexes/{}/documents",
|
||||
encode(self.uid.as_ref()).to_string()
|
||||
);
|
||||
let url = format!("/indexes/{}/documents", encode(self.uid.as_ref()));
|
||||
self.service.delete(url).await
|
||||
}
|
||||
|
||||
pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
|
||||
let url = format!(
|
||||
"/indexes/{}/documents/delete-batch",
|
||||
encode(self.uid.as_ref()).to_string()
|
||||
encode(self.uid.as_ref())
|
||||
);
|
||||
self.service
|
||||
.post(url, serde_json::to_value(&ids).unwrap())
|
||||
@ -204,31 +181,22 @@ impl Index<'_> {
|
||||
}
|
||||
|
||||
pub async fn settings(&self) -> (Value, StatusCode) {
|
||||
let url = format!(
|
||||
"/indexes/{}/settings",
|
||||
encode(self.uid.as_ref()).to_string()
|
||||
);
|
||||
let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
|
||||
self.service.get(url).await
|
||||
}
|
||||
|
||||
pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
|
||||
let url = format!(
|
||||
"/indexes/{}/settings",
|
||||
encode(self.uid.as_ref()).to_string()
|
||||
);
|
||||
let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
|
||||
self.service.post(url, settings).await
|
||||
}
|
||||
|
||||
pub async fn delete_settings(&self) -> (Value, StatusCode) {
|
||||
let url = format!(
|
||||
"/indexes/{}/settings",
|
||||
encode(self.uid.as_ref()).to_string()
|
||||
);
|
||||
let url = format!("/indexes/{}/settings", encode(self.uid.as_ref()));
|
||||
self.service.delete(url).await
|
||||
}
|
||||
|
||||
pub async fn stats(&self) -> (Value, StatusCode) {
|
||||
let url = format!("/indexes/{}/stats", encode(self.uid.as_ref()).to_string());
|
||||
let url = format!("/indexes/{}/stats", encode(self.uid.as_ref()));
|
||||
self.service.get(url).await
|
||||
}
|
||||
|
||||
@ -253,17 +221,13 @@ impl Index<'_> {
|
||||
}
|
||||
|
||||
pub async fn search_post(&self, query: Value) -> (Value, StatusCode) {
|
||||
let url = format!("/indexes/{}/search", encode(self.uid.as_ref()).to_string());
|
||||
let url = format!("/indexes/{}/search", encode(self.uid.as_ref()));
|
||||
self.service.post(url, query).await
|
||||
}
|
||||
|
||||
pub async fn search_get(&self, query: Value) -> (Value, StatusCode) {
|
||||
let params = serde_url_params::to_string(&query).unwrap();
|
||||
let url = format!(
|
||||
"/indexes/{}/search?{}",
|
||||
encode(self.uid.as_ref()).to_string(),
|
||||
params
|
||||
);
|
||||
let url = format!("/indexes/{}/search?{}", encode(self.uid.as_ref()), params);
|
||||
self.service.get(url).await
|
||||
}
|
||||
|
||||
|
||||
@ -130,7 +130,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
|
||||
master_key: None,
|
||||
env: "development".to_owned(),
|
||||
#[cfg(all(not(debug_assertions), feature = "analytics"))]
|
||||
no_analytics: Some(Some(true)),
|
||||
no_analytics: true,
|
||||
max_index_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
|
||||
max_task_db_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
|
||||
http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(),
|
||||
|
||||
@ -42,7 +42,7 @@ serde = { version = "1.0.130", features = ["derive"] }
|
||||
serde_json = { version = "1.0.67", features = ["preserve_order"] }
|
||||
siphasher = "0.3.7"
|
||||
slice-group-by = "0.2.6"
|
||||
structopt = "0.3.23"
|
||||
clap = { version = "3.0", features = ["derive", "env"] }
|
||||
tar = "0.4.37"
|
||||
tempfile = "3.2.0"
|
||||
thiserror = "1.0.28"
|
||||
|
||||
@ -28,9 +28,7 @@ impl Index {
|
||||
pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
|
||||
// acquire write txn make sure any ongoing write is finished before we start.
|
||||
let txn = self.env.write_txn()?;
|
||||
let path = path
|
||||
.as_ref()
|
||||
.join(format!("indexes/{}", self.uuid.to_string()));
|
||||
let path = path.as_ref().join(format!("indexes/{}", self.uuid));
|
||||
|
||||
create_dir_all(&path)?;
|
||||
|
||||
|
||||
@ -2,19 +2,19 @@ use core::fmt;
|
||||
use std::{ops::Deref, str::FromStr};
|
||||
|
||||
use byte_unit::{Byte, ByteError};
|
||||
use clap::Parser;
|
||||
use milli::CompressionType;
|
||||
use structopt::StructOpt;
|
||||
use sysinfo::{RefreshKind, System, SystemExt};
|
||||
|
||||
#[derive(Debug, Clone, StructOpt)]
|
||||
#[derive(Debug, Clone, Parser)]
|
||||
pub struct IndexerOpts {
|
||||
/// The amount of documents to skip before printing
|
||||
/// a log regarding the indexing advancement.
|
||||
#[structopt(long, default_value = "100000")] // 100k
|
||||
#[clap(long, default_value = "100000")] // 100k
|
||||
pub log_every_n: usize,
|
||||
|
||||
/// Grenad max number of chunks in bytes.
|
||||
#[structopt(long)]
|
||||
#[clap(long)]
|
||||
pub max_nb_chunks: Option<usize>,
|
||||
|
||||
/// The maximum amount of memory the indexer will use. It defaults to 2/3
|
||||
@ -24,22 +24,22 @@ pub struct IndexerOpts {
|
||||
/// In case the engine is unable to retrieve the available memory the engine will
|
||||
/// try to use the memory it needs but without real limit, this can lead to
|
||||
/// Out-Of-Memory issues and it is recommended to specify the amount of memory to use.
|
||||
#[structopt(long, default_value)]
|
||||
#[clap(long, default_value_t)]
|
||||
pub max_memory: MaxMemory,
|
||||
|
||||
/// The name of the compression algorithm to use when compressing intermediate
|
||||
/// Grenad chunks while indexing documents.
|
||||
///
|
||||
/// Choosing a fast algorithm will make the indexing faster but may consume more memory.
|
||||
#[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])]
|
||||
#[clap(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])]
|
||||
pub chunk_compression_type: CompressionType,
|
||||
|
||||
/// The level of compression of the chosen algorithm.
|
||||
#[structopt(long, requires = "chunk-compression-type")]
|
||||
#[clap(long, requires = "chunk-compression-type")]
|
||||
pub chunk_compression_level: Option<u32>,
|
||||
|
||||
/// Number of parallel jobs for indexing, defaults to # of CPUs.
|
||||
#[structopt(long)]
|
||||
#[clap(long)]
|
||||
pub indexing_jobs: Option<usize>,
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user