searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-01-23 19:57:30 +01:00
Code Issues Releases Wiki Activity

Remove memory usage test that fails when many tests are run in parallel

Browse Source
This commit is contained in:
Loïc Lecrenier 2022-11-28 12:39:43 +01:00
parent 80588daae5
commit f70856bab1
1 changed files with 61 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
milli/src/search/query_tree.rs
View File

@ -866,9 +866,7 @@ pub fn maximum_proximity(operation: &Operation) -> usize {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use std::alloc::{GlobalAlloc, System};
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::atomic::{self, AtomicI64};
use charabia::Tokenize; use charabia::Tokenize;
use maplit::hashmap; use maplit::hashmap;
@ -1395,59 +1393,66 @@ mod test {
)); ));
} }
#[global_allocator] // The memory usage test below is disabled because `cargo test` runs multiple tests in parallel,
static ALLOC: CountingAlloc = // which invalidates the measurements of memory usage. Nevertheless, it is a useful test to run
CountingAlloc { resident: AtomicI64::new(0), allocated: AtomicI64::new(0) }; // manually from time to time, so I kept it here, commented-out.
pub struct CountingAlloc { // use std::alloc::{GlobalAlloc, System};
pub resident: AtomicI64, // use std::sync::atomic::{self, AtomicI64};
pub allocated: AtomicI64, //
} // #[global_allocator]
unsafe impl GlobalAlloc for CountingAlloc { // static ALLOC: CountingAlloc =
unsafe fn alloc(&self, layout: std::alloc::Layout) -> *mut u8 { // CountingAlloc { resident: AtomicI64::new(0), allocated: AtomicI64::new(0) };
self.allocated.fetch_add(layout.size() as i64, atomic::Ordering::Relaxed); //
self.resident.fetch_add(layout.size() as i64, atomic::Ordering::Relaxed); // pub struct CountingAlloc {
// pub resident: AtomicI64,
System.alloc(layout) // pub allocated: AtomicI64,
} // }
// unsafe impl GlobalAlloc for CountingAlloc {
unsafe fn dealloc(&self, ptr: *mut u8, layout: std::alloc::Layout) { // unsafe fn alloc(&self, layout: std::alloc::Layout) -> *mut u8 {
self.resident.fetch_sub(layout.size() as i64, atomic::Ordering::Relaxed); // self.allocated.fetch_add(layout.size() as i64, atomic::Ordering::Relaxed);
System.dealloc(ptr, layout) // self.resident.fetch_add(layout.size() as i64, atomic::Ordering::Relaxed);
} //
} // System.alloc(layout)
// }
#[test] //
fn memory_usage_of_ten_word_query() { // unsafe fn dealloc(&self, ptr: *mut u8, layout: std::alloc::Layout) {
let resident_before = ALLOC.resident.load(atomic::Ordering::SeqCst); // self.resident.fetch_sub(layout.size() as i64, atomic::Ordering::Relaxed);
let allocated_before = ALLOC.allocated.load(atomic::Ordering::SeqCst); // System.dealloc(ptr, layout)
// }
let index = TempIndex::new(); // }
let rtxn = index.read_txn().unwrap(); //
let query = "a beautiful summer house by the beach overlooking what seems"; // #[test]
let mut builder = QueryTreeBuilder::new(&rtxn, &index).unwrap(); // fn memory_usage_of_ten_word_query() {
builder.words_limit(10); // let resident_before = ALLOC.resident.load(atomic::Ordering::SeqCst);
let x = builder.build(query.tokenize()).unwrap().unwrap(); // let allocated_before = ALLOC.allocated.load(atomic::Ordering::SeqCst);
let resident_after = ALLOC.resident.load(atomic::Ordering::SeqCst); //
let allocated_after = ALLOC.allocated.load(atomic::Ordering::SeqCst); // let index = TempIndex::new();
// let rtxn = index.read_txn().unwrap();
// Weak check on the memory usage // let query = "a beautiful summer house by the beach overlooking what seems";
// Don't keep more than 5MB. (Arguably 5MB is already too high) // let mut builder = QueryTreeBuilder::new(&rtxn, &index).unwrap();
assert!(resident_after - resident_before < 5_000_000); // builder.words_limit(10);
// Don't allocate more than 10MB. // let x = builder.build(query.tokenize()).unwrap().unwrap();
assert!(allocated_after - allocated_before < 10_000_000); // let resident_after = ALLOC.resident.load(atomic::Ordering::SeqCst);
// let allocated_after = ALLOC.allocated.load(atomic::Ordering::SeqCst);
// Use these snapshots to measure the exact memory usage. //
// The values below were correct at the time I wrote them. // // Weak check on the memory usage
// insta::assert_snapshot!(format!("{}", resident_after - resident_before), @"4486950"); // // Don't keep more than 5MB. (Arguably 5MB is already too high)
// insta::assert_snapshot!(format!("{}", allocated_after - allocated_before), @"7107502"); // assert!(resident_after - resident_before < 5_000_000);
// // Don't allocate more than 10MB.
// Note, with the matching word cache deactivated, the memory usage was: // assert!(allocated_after - allocated_before < 10_000_000);
// insta::assert_snapshot!(format!("{}", resident_after - resident_before), @"91248697"); //
// insta::assert_snapshot!(format!("{}", allocated_after - allocated_before), @"125697588"); // // Use these snapshots to measure the exact memory usage.
// or about 20x more resident memory (90MB vs 4.5MB) // // The values below were correct at the time I wrote them.
// // insta::assert_snapshot!(format!("{}", resident_after - resident_before), @"4486950");
// Use x // // insta::assert_snapshot!(format!("{}", allocated_after - allocated_before), @"7107502");
let _x = x; //
} // // Note, with the matching word cache deactivated, the memory usage was:
// // insta::assert_snapshot!(format!("{}", resident_after - resident_before), @"91248697");
// // insta::assert_snapshot!(format!("{}", allocated_after - allocated_before), @"125697588");
// // or about 20x more resident memory (90MB vs 4.5MB)
//
// // Use x
// let _x = x;
// }
} }