2022-09-08 11:53:01 +02:00
|
|
|
#![cfg_attr(all(test, fuzzing), feature(no_coverage))]
|
2023-02-28 11:49:24 +01:00
|
|
|
#![allow(clippy::type_complexity)]
|
2023-03-20 09:41:37 +01:00
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
#[global_allocator]
|
2023-03-02 21:27:42 +01:00
|
|
|
pub static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
|
|
|
|
2021-08-31 11:44:15 +02:00
|
|
|
#[macro_use]
|
|
|
|
pub mod documents;
|
|
|
|
|
2021-09-22 15:18:39 +02:00
|
|
|
mod asc_desc;
|
2020-08-12 10:43:02 +02:00
|
|
|
mod criterion;
|
2023-06-14 14:58:51 +02:00
|
|
|
pub mod distance;
|
2021-06-09 17:05:46 +02:00
|
|
|
mod error;
|
2020-11-22 14:48:42 +01:00
|
|
|
mod external_documents_ids;
|
2020-11-11 15:48:24 +01:00
|
|
|
pub mod facet;
|
2021-06-16 18:33:33 +02:00
|
|
|
mod fields_ids_map;
|
2020-08-28 14:16:37 +02:00
|
|
|
pub mod heed_codec;
|
2021-02-14 18:55:15 +01:00
|
|
|
pub mod index;
|
2020-09-22 10:53:20 +02:00
|
|
|
pub mod proximity;
|
2023-06-21 17:07:02 +02:00
|
|
|
mod readable_slices;
|
2023-06-06 18:06:02 +02:00
|
|
|
pub mod score_details;
|
2021-06-16 18:33:33 +02:00
|
|
|
mod search;
|
2020-10-25 18:32:01 +01:00
|
|
|
pub mod update;
|
2020-06-04 20:25:51 +02:00
|
|
|
|
2022-08-03 08:45:26 +02:00
|
|
|
#[cfg(test)]
|
|
|
|
#[macro_use]
|
|
|
|
pub mod snapshot_tests;
|
|
|
|
|
2021-06-17 17:05:34 +02:00
|
|
|
use std::collections::{BTreeMap, HashMap};
|
2021-07-06 11:31:24 +02:00
|
|
|
use std::convert::{TryFrom, TryInto};
|
2020-05-31 16:09:34 +02:00
|
|
|
use std::hash::BuildHasherDefault;
|
2020-10-31 16:10:15 +01:00
|
|
|
|
2023-03-29 10:57:02 +02:00
|
|
|
use charabia::normalizer::{CharNormalizer, CompatibilityDecompositionNormalizer};
|
2023-06-27 12:29:40 +02:00
|
|
|
pub use distance::dot_product_similarity;
|
2022-12-03 19:37:33 +01:00
|
|
|
pub use filter_parser::{Condition, FilterCondition, Span, Token};
|
2020-06-29 22:25:59 +02:00
|
|
|
use fxhash::{FxHasher32, FxHasher64};
|
2021-08-24 18:15:31 +02:00
|
|
|
pub use grenad::CompressionType;
|
2023-03-27 17:49:18 +02:00
|
|
|
pub use search::new::{
|
2023-04-13 13:45:34 +02:00
|
|
|
execute_search, DefaultSearchLogger, GeoSortStrategy, SearchContext, SearchLogger,
|
|
|
|
VisualSearchLogger,
|
2023-03-27 17:49:18 +02:00
|
|
|
};
|
2022-06-15 15:36:27 +02:00
|
|
|
use serde_json::Value;
|
2022-06-02 15:47:28 +02:00
|
|
|
pub use {charabia as tokenizer, heed};
|
2020-06-04 20:25:51 +02:00
|
|
|
|
2021-09-27 19:07:22 +02:00
|
|
|
pub use self::asc_desc::{AscDesc, AscDescError, Member, SortError};
|
2021-09-22 16:02:07 +02:00
|
|
|
pub use self::criterion::{default_criteria, Criterion, CriterionError};
|
2021-06-17 15:16:20 +02:00
|
|
|
pub use self::error::{
|
|
|
|
Error, FieldIdMapMissingEntry, InternalError, SerializationError, UserError,
|
|
|
|
};
|
2020-11-22 17:53:33 +01:00
|
|
|
pub use self::external_documents_ids::ExternalDocumentsIds;
|
2020-10-23 14:11:00 +02:00
|
|
|
pub use self::fields_ids_map::FieldsIdsMap;
|
2021-06-16 18:33:33 +02:00
|
|
|
pub use self::heed_codec::{
|
2023-07-20 17:57:07 +02:00
|
|
|
BEU16StrCodec, BEU32StrCodec, BoRoaringBitmapCodec, BoRoaringBitmapLenCodec,
|
|
|
|
CboRoaringBitmapCodec, CboRoaringBitmapLenCodec, FieldIdWordCountCodec, ObkvCodec,
|
|
|
|
RoaringBitmapCodec, RoaringBitmapLenCodec, StrBEU32Codec, U8StrStrCodec,
|
|
|
|
UncheckedU8StrStrCodec,
|
2021-06-16 18:33:33 +02:00
|
|
|
};
|
2020-10-21 15:55:48 +02:00
|
|
|
pub use self::index::Index;
|
2022-03-30 10:50:23 +02:00
|
|
|
pub use self::search::{
|
2023-04-26 17:09:59 +02:00
|
|
|
FacetDistribution, FacetValueHit, Filter, FormatOptions, MatchBounds, MatcherBuilder,
|
2023-05-29 15:32:09 +02:00
|
|
|
MatchingWords, OrderBy, Search, SearchForFacetValues, SearchResult, TermsMatchingStrategy,
|
2023-04-13 17:38:20 +02:00
|
|
|
DEFAULT_VALUES_PER_FACET,
|
2022-03-30 10:50:23 +02:00
|
|
|
};
|
2020-05-31 16:09:34 +02:00
|
|
|
|
2021-06-14 16:46:19 +02:00
|
|
|
pub type Result<T> = std::result::Result<T, error::Error>;
|
|
|
|
|
2022-06-15 15:36:27 +02:00
|
|
|
pub type Attribute = u32;
|
2022-06-16 08:24:16 +02:00
|
|
|
pub type BEU16 = heed::zerocopy::U16<heed::byteorder::BE>;
|
2020-05-31 16:09:34 +02:00
|
|
|
pub type BEU32 = heed::zerocopy::U32<heed::byteorder::BE>;
|
2020-10-18 15:16:57 +02:00
|
|
|
pub type BEU64 = heed::zerocopy::U64<heed::byteorder::BE>;
|
2020-11-26 17:38:08 +01:00
|
|
|
pub type DocumentId = u32;
|
2022-06-15 15:36:27 +02:00
|
|
|
pub type FastMap4<K, V> = HashMap<K, V, BuildHasherDefault<FxHasher32>>;
|
|
|
|
pub type FastMap8<K, V> = HashMap<K, V, BuildHasherDefault<FxHasher64>>;
|
|
|
|
pub type FieldDistribution = BTreeMap<String, u64>;
|
2021-07-06 11:31:24 +02:00
|
|
|
pub type FieldId = u16;
|
2022-06-15 15:36:27 +02:00
|
|
|
pub type Object = serde_json::Map<String, serde_json::Value>;
|
2020-07-07 12:21:22 +02:00
|
|
|
pub type Position = u32;
|
2021-09-22 17:48:24 +02:00
|
|
|
pub type RelativePosition = u16;
|
2022-06-15 15:36:27 +02:00
|
|
|
pub type SmallString32 = smallstr::SmallString<[u8; 32]>;
|
|
|
|
pub type SmallVec16<T> = smallvec::SmallVec<[T; 16]>;
|
|
|
|
pub type SmallVec32<T> = smallvec::SmallVec<[T; 32]>;
|
|
|
|
pub type SmallVec8<T> = smallvec::SmallVec<[T; 8]>;
|
2021-12-14 12:21:24 +01:00
|
|
|
|
|
|
|
/// A GeoPoint is a point in cartesian plan, called xyz_point in the code. Its metadata
|
|
|
|
/// is a tuple composed of 1. the DocumentId of the associated document and 2. the original point
|
|
|
|
/// expressed in term of latitude and longitude.
|
|
|
|
pub type GeoPoint = rstar::primitives::GeomWithData<[f64; 3], (DocumentId, [f64; 2])>;
|
2020-10-31 16:10:15 +01:00
|
|
|
|
2022-11-24 09:00:53 +01:00
|
|
|
/// The maximum length a LMDB key can be.
|
|
|
|
///
|
|
|
|
/// Note that the actual allowed length is a little bit higher, but
|
|
|
|
/// we keep a margin of safety.
|
|
|
|
const MAX_LMDB_KEY_LENGTH: usize = 500;
|
|
|
|
|
|
|
|
/// The maximum length a field value can be when inserted in an LMDB key.
|
|
|
|
///
|
|
|
|
/// This number is determined by the keys of the different facet databases
|
|
|
|
/// and adding a margin of safety.
|
2023-08-08 12:00:51 +02:00
|
|
|
pub const MAX_FACET_VALUE_LENGTH: usize = MAX_LMDB_KEY_LENGTH - 32;
|
2022-11-24 09:00:53 +01:00
|
|
|
|
|
|
|
/// The maximum length a word can be
|
|
|
|
pub const MAX_WORD_LENGTH: usize = MAX_LMDB_KEY_LENGTH / 2;
|
|
|
|
|
2021-09-22 17:48:24 +02:00
|
|
|
pub const MAX_POSITION_PER_ATTRIBUTE: u32 = u16::MAX as u32 + 1;
|
|
|
|
|
|
|
|
// Convert an absolute word position into a relative position.
|
|
|
|
// Return the field id of the attribute related to the absolute position
|
|
|
|
// and the relative position in the attribute.
|
|
|
|
pub fn relative_from_absolute_position(absolute: Position) -> (FieldId, RelativePosition) {
|
|
|
|
((absolute >> 16) as u16, (absolute & 0xFFFF) as u16)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Compute the absolute word position with the field id of the attribute and relative position in the attribute.
|
|
|
|
pub fn absolute_from_relative_position(field_id: FieldId, relative: RelativePosition) -> Position {
|
|
|
|
(field_id as u32) << 16 | (relative as u32)
|
|
|
|
}
|
2023-03-23 09:22:01 +01:00
|
|
|
// TODO: this is wrong, but will do for now
|
|
|
|
/// Compute the "bucketed" absolute position from the field id and relative position in the field.
|
|
|
|
///
|
|
|
|
/// In a bucketed position, the accuracy of the relative position is reduced exponentially as it gets larger.
|
|
|
|
pub fn bucketed_position(relative: u16) -> u16 {
|
|
|
|
// The first few relative positions are kept intact.
|
|
|
|
if relative < 16 {
|
|
|
|
relative
|
|
|
|
} else if relative < 24 {
|
|
|
|
// Relative positions between 16 and 24 all become equal to 24
|
|
|
|
24
|
|
|
|
} else {
|
|
|
|
// Then, groups of positions that have the same base-2 logarithm are reduced to
|
|
|
|
// the same relative position: the smallest power of 2 that is greater than them
|
|
|
|
(relative as f64).log2().ceil().exp2() as u16
|
|
|
|
}
|
|
|
|
}
|
2021-09-22 17:48:24 +02:00
|
|
|
|
2020-11-05 13:34:15 +01:00
|
|
|
/// Transform a raw obkv store into a JSON Object.
|
|
|
|
pub fn obkv_to_json(
|
2020-11-26 17:38:08 +01:00
|
|
|
displayed_fields: &[FieldId],
|
2020-11-05 13:34:15 +01:00
|
|
|
fields_ids_map: &FieldsIdsMap,
|
2022-11-24 16:08:23 +01:00
|
|
|
obkv: obkv::KvReaderU16,
|
2022-06-15 15:36:27 +02:00
|
|
|
) -> Result<Object> {
|
2021-06-16 18:33:33 +02:00
|
|
|
displayed_fields
|
|
|
|
.iter()
|
2020-11-05 13:34:15 +01:00
|
|
|
.copied()
|
|
|
|
.flat_map(|id| obkv.get(id).map(|value| (id, value)))
|
|
|
|
.map(|(id, value)| {
|
2021-06-14 16:46:19 +02:00
|
|
|
let name = fields_ids_map.name(id).ok_or(error::FieldIdMapMissingEntry::FieldId {
|
|
|
|
field_id: id,
|
2021-06-15 11:10:50 +02:00
|
|
|
process: "obkv_to_json",
|
2021-06-14 16:46:19 +02:00
|
|
|
})?;
|
|
|
|
let value = serde_json::from_slice(value).map_err(error::InternalError::SerdeJson)?;
|
2020-11-05 13:34:15 +01:00
|
|
|
Ok((name.to_owned(), value))
|
|
|
|
})
|
|
|
|
.collect()
|
|
|
|
}
|
2020-11-06 16:15:07 +01:00
|
|
|
|
2022-11-24 03:18:58 +01:00
|
|
|
/// Transform every field of a raw obkv store into a JSON Object.
|
2022-11-24 16:08:23 +01:00
|
|
|
pub fn all_obkv_to_json(obkv: obkv::KvReaderU16, fields_ids_map: &FieldsIdsMap) -> Result<Object> {
|
2022-11-24 03:18:58 +01:00
|
|
|
let all_keys = obkv.iter().map(|(k, _v)| k).collect::<Vec<_>>();
|
|
|
|
obkv_to_json(all_keys.as_slice(), fields_ids_map, obkv)
|
|
|
|
}
|
|
|
|
|
2020-11-06 16:15:07 +01:00
|
|
|
/// Transform a JSON value into a string that can be indexed.
|
2020-11-11 17:33:05 +01:00
|
|
|
pub fn json_to_string(value: &Value) -> Option<String> {
|
|
|
|
fn inner(value: &Value, output: &mut String) -> bool {
|
2020-11-06 16:15:07 +01:00
|
|
|
use std::fmt::Write;
|
|
|
|
match value {
|
|
|
|
Value::Null => false,
|
|
|
|
Value::Bool(boolean) => write!(output, "{}", boolean).is_ok(),
|
|
|
|
Value::Number(number) => write!(output, "{}", number).is_ok(),
|
|
|
|
Value::String(string) => write!(output, "{}", string).is_ok(),
|
|
|
|
Value::Array(array) => {
|
|
|
|
let mut count = 0;
|
|
|
|
for value in array {
|
|
|
|
if inner(value, output) {
|
|
|
|
output.push_str(". ");
|
|
|
|
count += 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// check that at least one value was written
|
|
|
|
count != 0
|
2021-06-16 18:33:33 +02:00
|
|
|
}
|
2020-11-06 16:15:07 +01:00
|
|
|
Value::Object(object) => {
|
|
|
|
let mut buffer = String::new();
|
|
|
|
let mut count = 0;
|
|
|
|
for (key, value) in object {
|
|
|
|
buffer.clear();
|
|
|
|
let _ = write!(&mut buffer, "{}: ", key);
|
|
|
|
if inner(value, &mut buffer) {
|
|
|
|
buffer.push_str(". ");
|
|
|
|
// We write the "key: value. " pair only when
|
|
|
|
// we are sure that the value can be written.
|
|
|
|
output.push_str(&buffer);
|
|
|
|
count += 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// check that at least one value was written
|
|
|
|
count != 0
|
2021-06-16 18:33:33 +02:00
|
|
|
}
|
2020-11-06 16:15:07 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut string = String::new();
|
|
|
|
if inner(value, &mut string) {
|
|
|
|
Some(string)
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-07-06 11:31:24 +02:00
|
|
|
/// Divides one slice into two at an index, returns `None` if mid is out of bounds.
|
|
|
|
fn try_split_at<T>(slice: &[T], mid: usize) -> Option<(&[T], &[T])> {
|
|
|
|
if mid <= slice.len() {
|
|
|
|
Some(slice.split_at(mid))
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Divides one slice into an array and the tail at an index,
|
|
|
|
/// returns `None` if `N` is out of bounds.
|
|
|
|
fn try_split_array_at<T, const N: usize>(slice: &[T]) -> Option<([T; N], &[T])>
|
|
|
|
where
|
|
|
|
[T; N]: for<'a> TryFrom<&'a [T]>,
|
|
|
|
{
|
|
|
|
let (head, tail) = try_split_at(slice, N)?;
|
|
|
|
let head = head.try_into().ok()?;
|
|
|
|
Some((head, tail))
|
|
|
|
}
|
|
|
|
|
2021-09-09 15:19:47 +02:00
|
|
|
/// Return the distance between two points in meters. Each points are composed of two f64,
|
|
|
|
/// one latitude and one longitude.
|
|
|
|
pub fn distance_between_two_points(a: &[f64; 2], b: &[f64; 2]) -> f64 {
|
2021-09-09 18:11:38 +02:00
|
|
|
let a = geoutils::Location::new(a[0], a[1]);
|
|
|
|
let b = geoutils::Location::new(b[0], b[1]);
|
2021-09-07 12:11:03 +02:00
|
|
|
|
2021-09-09 18:11:38 +02:00
|
|
|
a.haversine_distance_to(&b).meters()
|
2021-09-07 12:11:03 +02:00
|
|
|
}
|
|
|
|
|
2021-12-14 12:21:24 +01:00
|
|
|
/// Convert a point expressed in terms of latitude and longitude to a point in the
|
|
|
|
/// cartesian coordinate expressed in terms of x, y and z.
|
|
|
|
pub fn lat_lng_to_xyz(coord: &[f64; 2]) -> [f64; 3] {
|
|
|
|
let [lat, lng] = coord.map(|f| f.to_radians());
|
|
|
|
let x = lat.cos() * lng.cos();
|
|
|
|
let y = lat.cos() * lng.sin();
|
|
|
|
let z = lat.sin();
|
|
|
|
|
|
|
|
[x, y, z]
|
|
|
|
}
|
|
|
|
|
2022-03-23 17:28:41 +01:00
|
|
|
/// Returns `true` if the field match one of the faceted fields.
|
|
|
|
/// See the function [`is_faceted_by`] below to see what “matching” means.
|
|
|
|
pub fn is_faceted(field: &str, faceted_fields: impl IntoIterator<Item = impl AsRef<str>>) -> bool {
|
2022-10-13 22:02:54 +02:00
|
|
|
faceted_fields.into_iter().any(|facet| is_faceted_by(field, facet.as_ref()))
|
2022-03-23 17:28:41 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Returns `true` if the field match the facet.
|
|
|
|
/// ```
|
|
|
|
/// use milli::is_faceted_by;
|
|
|
|
/// // -- the valid basics
|
|
|
|
/// assert!(is_faceted_by("animaux", "animaux"));
|
|
|
|
/// assert!(is_faceted_by("animaux.chien", "animaux"));
|
|
|
|
/// assert!(is_faceted_by("animaux.chien.race.bouvier bernois.fourrure.couleur", "animaux"));
|
|
|
|
/// assert!(is_faceted_by("animaux.chien.race.bouvier bernois.fourrure.couleur", "animaux.chien"));
|
|
|
|
/// assert!(is_faceted_by("animaux.chien.race.bouvier bernois.fourrure.couleur", "animaux.chien.race.bouvier bernois"));
|
|
|
|
/// assert!(is_faceted_by("animaux.chien.race.bouvier bernois.fourrure.couleur", "animaux.chien.race.bouvier bernois.fourrure"));
|
|
|
|
/// assert!(is_faceted_by("animaux.chien.race.bouvier bernois.fourrure.couleur", "animaux.chien.race.bouvier bernois.fourrure.couleur"));
|
|
|
|
///
|
|
|
|
/// // -- the wrongs
|
|
|
|
/// assert!(!is_faceted_by("chien", "chat"));
|
|
|
|
/// assert!(!is_faceted_by("animaux", "animaux.chien"));
|
|
|
|
/// assert!(!is_faceted_by("animaux.chien", "animaux.chat"));
|
|
|
|
///
|
|
|
|
/// // -- the strange edge cases
|
|
|
|
/// assert!(!is_faceted_by("animaux.chien", "anima"));
|
|
|
|
/// assert!(!is_faceted_by("animaux.chien", "animau"));
|
|
|
|
/// assert!(!is_faceted_by("animaux.chien", "animaux."));
|
|
|
|
/// assert!(!is_faceted_by("animaux.chien", "animaux.c"));
|
|
|
|
/// assert!(!is_faceted_by("animaux.chien", "animaux.ch"));
|
|
|
|
/// assert!(!is_faceted_by("animaux.chien", "animaux.chi"));
|
|
|
|
/// assert!(!is_faceted_by("animaux.chien", "animaux.chie"));
|
|
|
|
/// ```
|
|
|
|
pub fn is_faceted_by(field: &str, facet: &str) -> bool {
|
|
|
|
field.starts_with(facet)
|
|
|
|
&& field[facet.len()..].chars().next().map(|c| c == '.').unwrap_or(true)
|
|
|
|
}
|
|
|
|
|
2023-03-29 10:57:02 +02:00
|
|
|
pub fn normalize_facet(original: &str) -> String {
|
|
|
|
CompatibilityDecompositionNormalizer.normalize_str(original.trim()).to_lowercase()
|
|
|
|
}
|
|
|
|
|
2023-06-20 15:54:28 +02:00
|
|
|
/// Represents either a vector or an array of multiple vectors.
|
|
|
|
#[derive(serde::Serialize, serde::Deserialize, Debug)]
|
|
|
|
#[serde(transparent)]
|
|
|
|
pub struct VectorOrArrayOfVectors {
|
2023-08-14 16:03:55 +02:00
|
|
|
#[serde(with = "either::serde_untagged_optional")]
|
|
|
|
inner: Option<either::Either<Vec<f32>, Vec<Vec<f32>>>>,
|
2023-06-20 15:54:28 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
impl VectorOrArrayOfVectors {
|
2023-08-14 16:03:55 +02:00
|
|
|
pub fn into_array_of_vectors(self) -> Option<Vec<Vec<f32>>> {
|
|
|
|
match self.inner? {
|
|
|
|
either::Either::Left(vector) => Some(vec![vector]),
|
|
|
|
either::Either::Right(vectors) => Some(vectors),
|
2023-06-20 15:54:28 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-06-27 12:29:40 +02:00
|
|
|
/// Normalize a vector by dividing the dimensions by the length of it.
|
2023-06-20 11:45:29 +02:00
|
|
|
pub fn normalize_vector(mut vector: Vec<f32>) -> Vec<f32> {
|
|
|
|
let squared: f32 = vector.iter().map(|x| x * x).sum();
|
|
|
|
let length = squared.sqrt();
|
|
|
|
if length <= f32::EPSILON {
|
|
|
|
vector
|
|
|
|
} else {
|
2023-06-20 13:48:39 +02:00
|
|
|
vector.iter_mut().for_each(|x| *x /= length);
|
2023-06-20 11:45:29 +02:00
|
|
|
vector
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-11-06 16:15:07 +01:00
|
|
|
#[cfg(test)]
|
|
|
|
mod tests {
|
|
|
|
use serde_json::json;
|
|
|
|
|
2021-06-16 18:33:33 +02:00
|
|
|
use super::*;
|
|
|
|
|
2020-11-06 16:15:07 +01:00
|
|
|
#[test]
|
|
|
|
fn json_to_string_object() {
|
|
|
|
let value = json!({
|
|
|
|
"name": "John Doe",
|
|
|
|
"age": 43,
|
|
|
|
"not_there": null,
|
|
|
|
});
|
|
|
|
|
2020-11-11 17:33:05 +01:00
|
|
|
let string = json_to_string(&value).unwrap();
|
2020-11-06 16:15:07 +01:00
|
|
|
assert_eq!(string, "name: John Doe. age: 43. ");
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn json_to_string_array() {
|
|
|
|
let value = json!([
|
|
|
|
{ "name": "John Doe" },
|
|
|
|
43,
|
|
|
|
"hello",
|
|
|
|
[ "I", "am", "fine" ],
|
|
|
|
null,
|
|
|
|
]);
|
|
|
|
|
2020-11-11 17:33:05 +01:00
|
|
|
let string = json_to_string(&value).unwrap();
|
2020-11-06 16:15:07 +01:00
|
|
|
// We don't care about having two point (.) after the other as
|
|
|
|
// the distance of hard separators is clamped to 8 anyway.
|
|
|
|
assert_eq!(string, "name: John Doe. . 43. hello. I. am. fine. . ");
|
|
|
|
}
|
2021-09-22 17:48:24 +02:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_relative_position_conversion() {
|
|
|
|
assert_eq!((0x0000, 0x0000), relative_from_absolute_position(0x00000000));
|
|
|
|
assert_eq!((0x0000, 0xFFFF), relative_from_absolute_position(0x0000FFFF));
|
|
|
|
assert_eq!((0xFFFF, 0x0000), relative_from_absolute_position(0xFFFF0000));
|
|
|
|
assert_eq!((0xFF00, 0xFF00), relative_from_absolute_position(0xFF00FF00));
|
|
|
|
assert_eq!((0xFF00, 0x00FF), relative_from_absolute_position(0xFF0000FF));
|
|
|
|
assert_eq!((0x1234, 0x5678), relative_from_absolute_position(0x12345678));
|
|
|
|
assert_eq!((0xFFFF, 0xFFFF), relative_from_absolute_position(0xFFFFFFFF));
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_absolute_position_conversion() {
|
|
|
|
assert_eq!(0x00000000, absolute_from_relative_position(0x0000, 0x0000));
|
|
|
|
assert_eq!(0x0000FFFF, absolute_from_relative_position(0x0000, 0xFFFF));
|
|
|
|
assert_eq!(0xFFFF0000, absolute_from_relative_position(0xFFFF, 0x0000));
|
|
|
|
assert_eq!(0xFF00FF00, absolute_from_relative_position(0xFF00, 0xFF00));
|
|
|
|
assert_eq!(0xFF0000FF, absolute_from_relative_position(0xFF00, 0x00FF));
|
|
|
|
assert_eq!(0x12345678, absolute_from_relative_position(0x1234, 0x5678));
|
|
|
|
assert_eq!(0xFFFFFFFF, absolute_from_relative_position(0xFFFF, 0xFFFF));
|
|
|
|
}
|
2022-11-24 03:18:58 +01:00
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_all_obkv_to_json() {
|
|
|
|
let mut fields_ids_map = FieldsIdsMap::new();
|
|
|
|
let id1 = fields_ids_map.insert("field1").unwrap();
|
|
|
|
let id2 = fields_ids_map.insert("field2").unwrap();
|
|
|
|
|
|
|
|
let mut writer = obkv::KvWriterU16::memory();
|
|
|
|
writer.insert(id1, b"1234").unwrap();
|
|
|
|
writer.insert(id2, b"4321").unwrap();
|
|
|
|
let contents = writer.into_inner().unwrap();
|
|
|
|
let obkv = obkv::KvReaderU16::new(&contents);
|
|
|
|
|
|
|
|
let expected = json!({
|
|
|
|
"field1": 1234,
|
|
|
|
"field2": 4321,
|
|
|
|
});
|
|
|
|
let expected = expected.as_object().unwrap();
|
2022-11-24 16:08:23 +01:00
|
|
|
let actual = all_obkv_to_json(obkv, &fields_ids_map).unwrap();
|
2022-11-24 03:18:58 +01:00
|
|
|
|
|
|
|
assert_eq!(&actual, expected);
|
|
|
|
}
|
2020-11-06 16:15:07 +01:00
|
|
|
}
|