searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-23 13:24:27 +01:00
Code Issues Releases Wiki Activity

Merge #349

Browse Source 
349: Enable the grenad tempfile feature back r=ManyTheFish a=Kerollmops

This PR enables the grenad `tempfile` feature back, [when this is feature is disabled the sorter writes the entries in memory](7c082d05bf/src/sorter.rs (L470-L476)) instead of on disk and therefore, consumes more memory. By enabling this feature grenad merges on disk by using the `tempfile` dependency.

This PR also bumps milli to v0.3.1 where `@ManyTheFish` added an assert for when the allocator can't allocate and disable the default snappy compression in the `http-ui` crate.

Co-authored-by: Kerollmops <clement@meilisearch.com>
This commit is contained in:
bors[bot] 2021-09-08 13:23:57 +00:00 committed by GitHub
commit 63bc231243
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
http-ui/src/main.rs
View File

@ -103,8 +103,8 @@ pub struct IndexerOpt {
/// chunks during indexing documents. /// chunks during indexing documents.
/// ///
/// Choosing a fast algorithm will make the indexing faster but may consume more memory. /// Choosing a fast algorithm will make the indexing faster but may consume more memory.
#[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])] #[structopt(long, possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])]
pub chunk_compression_type: CompressionType, pub chunk_compression_type: Option<CompressionType>,
/// The level of compression of the chosen algorithm. /// The level of compression of the chosen algorithm.
#[structopt(long, requires = "chunk-compression-type")] #[structopt(long, requires = "chunk-compression-type")]
@ -343,7 +343,9 @@ async fn main() -> anyhow::Result<()> {
update_builder.thread_pool(GLOBAL_THREAD_POOL.get().unwrap()); update_builder.thread_pool(GLOBAL_THREAD_POOL.get().unwrap());
update_builder.log_every_n(indexer_opt_cloned.log_every_n); update_builder.log_every_n(indexer_opt_cloned.log_every_n);
update_builder.max_memory(indexer_opt_cloned.max_memory.get_bytes() as usize); update_builder.max_memory(indexer_opt_cloned.max_memory.get_bytes() as usize);
update_builder.chunk_compression_type(indexer_opt_cloned.chunk_compression_type); update_builder.chunk_compression_type(
indexer_opt_cloned.chunk_compression_type.unwrap_or(CompressionType::None),
);
let before_update = Instant::now(); let before_update = Instant::now();
// we extract the update type and execute the update itself. // we extract the update type and execute the update itself.

2
milli/Cargo.toml
View File

@ -15,7 +15,7 @@ either = "1.6.1"
flate2 = "1.0.20" flate2 = "1.0.20"
fst = "0.4.5" fst = "0.4.5"
fxhash = "0.2.1" fxhash = "0.2.1"
grenad = { version = "0.3.0", default-features = false } grenad = { version = "0.3.1", default-features = false, features = ["tempfile"] }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1", default-features = false, features = ["lmdb", "sync-read-txn"] } heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1", default-features = false, features = ["lmdb", "sync-read-txn"] }
human_format = "1.0.3" human_format = "1.0.3"
levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] } levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }