2022-09-01 11:40:29 +02:00
|
|
|
use std::fs::File;
|
|
|
|
use std::io;
|
|
|
|
|
|
|
|
use heed::BytesEncode;
|
|
|
|
|
2022-08-29 16:01:54 +02:00
|
|
|
use super::helpers::{create_sorter, sorter_into_reader, try_split_array_at, GrenadParameters};
|
2022-10-12 09:42:55 +02:00
|
|
|
use crate::heed_codec::facet::{FacetGroupKey, FacetGroupKeyCodec};
|
|
|
|
use crate::heed_codec::StrRefCodec;
|
2022-11-16 14:03:27 +01:00
|
|
|
use crate::update::index_documents::merge_cbo_roaring_bitmaps;
|
2022-11-24 09:00:53 +01:00
|
|
|
use crate::{FieldId, Result, MAX_FACET_VALUE_LENGTH};
|
2021-08-16 13:36:30 +02:00
|
|
|
|
|
|
|
/// Extracts the facet string and the documents ids where this facet string appear.
|
|
|
|
///
|
|
|
|
/// Returns a grenad reader with the list of extracted facet strings and
|
|
|
|
/// documents ids from the given chunk of docid facet string positions.
|
2021-08-24 13:55:53 +02:00
|
|
|
#[logging_timer::time]
|
2022-02-16 15:28:48 +01:00
|
|
|
pub fn extract_facet_string_docids<R: io::Read + io::Seek>(
|
|
|
|
docid_fid_facet_string: grenad::Reader<R>,
|
2021-08-16 13:36:30 +02:00
|
|
|
indexer: GrenadParameters,
|
|
|
|
) -> Result<grenad::Reader<File>> {
|
|
|
|
let max_memory = indexer.max_memory_by_thread();
|
|
|
|
|
|
|
|
let mut facet_string_docids_sorter = create_sorter(
|
2022-09-13 10:40:37 +02:00
|
|
|
grenad::SortAlgorithm::Stable,
|
2022-09-07 18:03:52 +02:00
|
|
|
merge_cbo_roaring_bitmaps,
|
2021-08-16 13:36:30 +02:00
|
|
|
indexer.chunk_compression_type,
|
|
|
|
indexer.chunk_compression_level,
|
|
|
|
indexer.max_nb_chunks,
|
|
|
|
max_memory,
|
|
|
|
);
|
|
|
|
|
2022-02-16 15:28:48 +01:00
|
|
|
let mut cursor = docid_fid_facet_string.into_cursor()?;
|
2022-08-31 08:27:16 +02:00
|
|
|
while let Some((key, _original_value_bytes)) = cursor.move_on_next()? {
|
2021-08-16 13:36:30 +02:00
|
|
|
let (field_id_bytes, bytes) = try_split_array_at(key).unwrap();
|
|
|
|
let field_id = FieldId::from_be_bytes(field_id_bytes);
|
|
|
|
|
2022-08-31 08:27:16 +02:00
|
|
|
let (document_id_bytes, normalized_value_bytes) =
|
|
|
|
try_split_array_at::<_, 4>(bytes).unwrap();
|
2022-09-01 09:51:43 +02:00
|
|
|
let document_id = u32::from_be_bytes(document_id_bytes);
|
2021-08-16 13:36:30 +02:00
|
|
|
|
2022-11-16 14:03:27 +01:00
|
|
|
let mut normalised_value = std::str::from_utf8(normalized_value_bytes)?;
|
|
|
|
|
|
|
|
let normalised_truncated_value: String;
|
|
|
|
if normalised_value.len() > MAX_FACET_VALUE_LENGTH {
|
|
|
|
normalised_truncated_value = normalised_value
|
|
|
|
.char_indices()
|
2023-08-08 12:00:51 +02:00
|
|
|
.take_while(|(idx, _)| *idx < MAX_FACET_VALUE_LENGTH)
|
2022-11-16 14:03:27 +01:00
|
|
|
.map(|(_, c)| c)
|
|
|
|
.collect();
|
|
|
|
normalised_value = normalised_truncated_value.as_str();
|
|
|
|
}
|
2022-09-05 13:01:36 +02:00
|
|
|
let key = FacetGroupKey { field_id, level: 0, left_bound: normalised_value };
|
|
|
|
let key_bytes = FacetGroupKeyCodec::<StrRefCodec>::bytes_encode(&key).unwrap();
|
2022-11-16 14:03:27 +01:00
|
|
|
// document id is encoded in native-endian because of the CBO roaring bitmap codec
|
|
|
|
facet_string_docids_sorter.insert(&key_bytes, document_id.to_ne_bytes())?;
|
2021-08-16 13:36:30 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
sorter_into_reader(facet_string_docids_sorter, indexer)
|
|
|
|
}
|