mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-05 04:28:55 +01:00
Merge #349
349: Enable the grenad tempfile feature back r=ManyTheFish a=Kerollmops
This PR enables the grenad `tempfile` feature back, [when this is feature is disabled the sorter writes the entries in memory](7c082d05bf/src/sorter.rs (L470-L476)
) instead of on disk and therefore, consumes more memory. By enabling this feature grenad merges on disk by using the `tempfile` dependency.
This PR also bumps milli to v0.3.1 where `@ManyTheFish` added an assert for when the allocator can't allocate and disable the default snappy compression in the `http-ui` crate.
Co-authored-by: Kerollmops <clement@meilisearch.com>
This commit is contained in:
commit
63bc231243
@ -103,8 +103,8 @@ pub struct IndexerOpt {
|
||||
/// chunks during indexing documents.
|
||||
///
|
||||
/// Choosing a fast algorithm will make the indexing faster but may consume more memory.
|
||||
#[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])]
|
||||
pub chunk_compression_type: CompressionType,
|
||||
#[structopt(long, possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])]
|
||||
pub chunk_compression_type: Option<CompressionType>,
|
||||
|
||||
/// The level of compression of the chosen algorithm.
|
||||
#[structopt(long, requires = "chunk-compression-type")]
|
||||
@ -343,7 +343,9 @@ async fn main() -> anyhow::Result<()> {
|
||||
update_builder.thread_pool(GLOBAL_THREAD_POOL.get().unwrap());
|
||||
update_builder.log_every_n(indexer_opt_cloned.log_every_n);
|
||||
update_builder.max_memory(indexer_opt_cloned.max_memory.get_bytes() as usize);
|
||||
update_builder.chunk_compression_type(indexer_opt_cloned.chunk_compression_type);
|
||||
update_builder.chunk_compression_type(
|
||||
indexer_opt_cloned.chunk_compression_type.unwrap_or(CompressionType::None),
|
||||
);
|
||||
|
||||
let before_update = Instant::now();
|
||||
// we extract the update type and execute the update itself.
|
||||
|
@ -15,7 +15,7 @@ either = "1.6.1"
|
||||
flate2 = "1.0.20"
|
||||
fst = "0.4.5"
|
||||
fxhash = "0.2.1"
|
||||
grenad = { version = "0.3.0", default-features = false }
|
||||
grenad = { version = "0.3.1", default-features = false, features = ["tempfile"] }
|
||||
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1", default-features = false, features = ["lmdb", "sync-read-txn"] }
|
||||
human_format = "1.0.3"
|
||||
levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
|
||||
|
Loading…
Reference in New Issue
Block a user