Merge branch 'main' into tmp-release-v1.11.0

This commit is contained in:
Tamo 2024-11-04 16:14:44 +01:00
commit cf6ad1ae5e
1071 changed files with 263 additions and 106 deletions

View file

@ -0,0 +1,78 @@
use std::io::{Read, Write};
use actix_http::header::TryIntoHeaderPair;
use bytes::Bytes;
use flate2::read::{GzDecoder, ZlibDecoder};
use flate2::write::{GzEncoder, ZlibEncoder};
use flate2::Compression;
#[derive(Clone, Copy)]
pub enum Encoder {
Plain,
Gzip,
Deflate,
Brotli,
}
impl Encoder {
pub fn encode(self: &Encoder, body: impl Into<Bytes>) -> impl Into<Bytes> {
match self {
Self::Gzip => {
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(&body.into()).expect("Failed to encode request body");
encoder.finish().expect("Failed to encode request body")
}
Self::Deflate => {
let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
encoder.write_all(&body.into()).expect("Failed to encode request body");
encoder.finish().unwrap()
}
Self::Plain => Vec::from(body.into()),
Self::Brotli => {
let mut encoder = brotli::CompressorWriter::new(Vec::new(), 32 * 1024, 3, 22);
encoder.write_all(&body.into()).expect("Failed to encode request body");
encoder.flush().expect("Failed to encode request body");
encoder.into_inner()
}
}
}
pub fn decode(self: &Encoder, bytes: impl Into<Bytes>) -> impl Into<Bytes> {
let mut buffer = Vec::new();
let input = bytes.into();
match self {
Self::Gzip => {
GzDecoder::new(input.as_ref())
.read_to_end(&mut buffer)
.expect("Invalid gzip stream");
}
Self::Deflate => {
ZlibDecoder::new(input.as_ref())
.read_to_end(&mut buffer)
.expect("Invalid zlib stream");
}
Self::Plain => {
buffer.write_all(input.as_ref()).expect("Unexpected memory copying issue");
}
Self::Brotli => {
brotli::Decompressor::new(input.as_ref(), 4096)
.read_to_end(&mut buffer)
.expect("Invalid brotli stream");
}
};
buffer
}
pub fn header(self: &Encoder) -> Option<impl TryIntoHeaderPair> {
match self {
Self::Plain => None,
Self::Gzip => Some(("Content-Encoding", "gzip")),
Self::Deflate => Some(("Content-Encoding", "deflate")),
Self::Brotli => Some(("Content-Encoding", "br")),
}
}
pub fn iterator() -> impl Iterator<Item = Self> {
[Self::Plain, Self::Gzip, Self::Deflate, Self::Brotli].iter().copied()
}
}

View file

@ -0,0 +1,512 @@
use std::fmt::Write;
use std::marker::PhantomData;
use std::panic::{catch_unwind, resume_unwind, UnwindSafe};
use std::time::Duration;
use actix_web::http::StatusCode;
use tokio::time::sleep;
use urlencoding::encode as urlencode;
use super::encoder::Encoder;
use super::service::Service;
use super::Value;
use super::{Owned, Shared};
use crate::json;
pub struct Index<'a, State = Owned> {
pub uid: String,
pub service: &'a Service,
pub(super) encoder: Encoder,
pub(super) marker: PhantomData<State>,
}
impl<'a> Index<'a, Owned> {
pub fn to_shared(&self) -> Index<'a, Shared> {
Index {
uid: self.uid.clone(),
service: self.service,
encoder: self.encoder,
marker: PhantomData,
}
}
pub async fn load_test_set(&self) -> u64 {
let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref()));
let (response, code) = self
.service
.post_str(
url,
include_str!("../assets/test_set.json"),
vec![("content-type", "application/json")],
)
.await;
assert_eq!(code, 202);
let update_id = response["taskUid"].as_i64().unwrap();
self.wait_task(update_id as u64).await;
update_id as u64
}
pub async fn load_test_set_ndjson(&self) -> u64 {
let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref()));
let (response, code) = self
.service
.post_str(
url,
include_str!("../assets/test_set.ndjson"),
vec![("content-type", "application/x-ndjson")],
)
.await;
assert_eq!(code, 202);
let update_id = response["taskUid"].as_i64().unwrap();
self.wait_task(update_id as u64).await;
update_id as u64
}
pub async fn create(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
self._create(primary_key).await
}
pub async fn update_raw(&self, body: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
self.service.patch_encoded(url, body, self.encoder).await
}
pub async fn update(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
let body = json!({
"primaryKey": primary_key,
});
let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
self.service.patch_encoded(url, body, self.encoder).await
}
pub async fn delete(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
self.service.delete(url).await
}
pub async fn add_documents(
&self,
documents: Value,
primary_key: Option<&str>,
) -> (Value, StatusCode) {
self._add_documents(documents, primary_key).await
}
pub async fn raw_add_documents(
&self,
payload: &str,
headers: Vec<(&str, &str)>,
query_parameter: &str,
) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents{}", urlencode(self.uid.as_ref()), query_parameter);
self.service.post_str(url, payload, headers).await
}
pub async fn update_documents(
&self,
documents: Value,
primary_key: Option<&str>,
) -> (Value, StatusCode) {
let url = match primary_key {
Some(key) => {
format!("/indexes/{}/documents?primaryKey={}", urlencode(self.uid.as_ref()), key)
}
None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
};
self.service.put_encoded(url, documents, self.encoder).await
}
pub async fn raw_update_documents(
&self,
payload: &str,
content_type: Option<&str>,
query_parameter: &str,
) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents{}", urlencode(self.uid.as_ref()), query_parameter);
if let Some(content_type) = content_type {
self.service.put_str(url, payload, vec![("Content-Type", content_type)]).await
} else {
self.service.put_str(url, payload, Vec::new()).await
}
}
pub async fn list_tasks(&self) -> (Value, StatusCode) {
let url = format!("/tasks?indexUids={}", self.uid);
self.service.get(url).await
}
pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
self.service.delete(url).await
}
pub async fn delete_document_by_filter(&self, body: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents/delete", urlencode(self.uid.as_ref()));
self.service.post_encoded(url, body, self.encoder).await
}
pub async fn clear_all_documents(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents", urlencode(self.uid.as_ref()));
self.service.delete(url).await
}
pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents/delete-batch", urlencode(self.uid.as_ref()));
self.service
.post_encoded(url, serde_json::to_value(&ids).unwrap().into(), self.encoder)
.await
}
pub async fn delete_batch_raw(&self, body: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents/delete-batch", urlencode(self.uid.as_ref()));
self.service.post_encoded(url, body, self.encoder).await
}
pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
self._update_settings(settings).await
}
pub async fn update_settings_displayed_attributes(
&self,
settings: Value,
) -> (Value, StatusCode) {
let url =
format!("/indexes/{}/settings/displayed-attributes", urlencode(self.uid.as_ref()));
self.service.put_encoded(url, settings, self.encoder).await
}
pub async fn update_settings_searchable_attributes(
&self,
settings: Value,
) -> (Value, StatusCode) {
let url =
format!("/indexes/{}/settings/searchable-attributes", urlencode(self.uid.as_ref()));
self.service.put_encoded(url, settings, self.encoder).await
}
pub async fn update_settings_filterable_attributes(
&self,
settings: Value,
) -> (Value, StatusCode) {
let url =
format!("/indexes/{}/settings/filterable-attributes", urlencode(self.uid.as_ref()));
self.service.put_encoded(url, settings, self.encoder).await
}
pub async fn update_settings_sortable_attributes(
&self,
settings: Value,
) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings/sortable-attributes", urlencode(self.uid.as_ref()));
self.service.put_encoded(url, settings, self.encoder).await
}
pub async fn update_settings_ranking_rules(&self, settings: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings/ranking-rules", urlencode(self.uid.as_ref()));
self.service.put_encoded(url, settings, self.encoder).await
}
pub async fn update_settings_stop_words(&self, settings: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings/stop-words", urlencode(self.uid.as_ref()));
self.service.put_encoded(url, settings, self.encoder).await
}
pub async fn update_settings_synonyms(&self, settings: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings/synonyms", urlencode(self.uid.as_ref()));
self.service.put_encoded(url, settings, self.encoder).await
}
pub async fn update_settings_distinct_attribute(&self, settings: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings/distinct-attribute", urlencode(self.uid.as_ref()));
self.service.put_encoded(url, settings, self.encoder).await
}
pub async fn update_settings_typo_tolerance(&self, settings: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings/typo-tolerance", urlencode(self.uid.as_ref()));
self.service.patch_encoded(url, settings, self.encoder).await
}
pub async fn update_settings_faceting(&self, settings: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings/faceting", urlencode(self.uid.as_ref()));
self.service.patch_encoded(url, settings, self.encoder).await
}
pub async fn update_settings_pagination(&self, settings: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings/pagination", urlencode(self.uid.as_ref()));
self.service.patch_encoded(url, settings, self.encoder).await
}
pub async fn update_settings_search_cutoff_ms(&self, settings: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings/search-cutoff-ms", urlencode(self.uid.as_ref()));
self.service.put_encoded(url, settings, self.encoder).await
}
pub async fn delete_settings(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
self.service.delete(url).await
}
pub async fn update_distinct_attribute(&self, value: Value) -> (Value, StatusCode) {
let url =
format!("/indexes/{}/settings/{}", urlencode(self.uid.as_ref()), "distinct-attribute");
self.service.put_encoded(url, value, self.encoder).await
}
}
impl<'a> Index<'a, Shared> {
/// You cannot modify the content of a shared index, thus the delete_document_by_filter call
/// must fail. If the task successfully enqueue itself, we'll wait for the task to finishes,
/// and if it succeed the function will panic.
pub async fn delete_document_by_filter_fail(&self, body: Value) -> (Value, StatusCode) {
let (mut task, code) = self._delete_document_by_filter(body).await;
if code.is_success() {
task = self.wait_task(task.uid()).await;
if task.is_success() {
panic!(
"`delete_document_by_filter_fail` succeeded: {}",
serde_json::to_string_pretty(&task).unwrap()
);
}
}
(task, code)
}
pub async fn delete_index_fail(&self) -> (Value, StatusCode) {
let (mut task, code) = self._delete().await;
if code.is_success() {
task = self.wait_task(task.uid()).await;
if task.is_success() {
panic!(
"`delete_index_fail` succeeded: {}",
serde_json::to_string_pretty(&task).unwrap()
);
}
}
(task, code)
}
}
#[allow(dead_code)]
impl<State> Index<'_, State> {
pub async fn get(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
self.service.get(url).await
}
/// add_documents is not allowed on shared index but we need to use it to initialize
/// a bunch of very common indexes in `common/mod.rs`.
pub(super) async fn _add_documents(
&self,
documents: Value,
primary_key: Option<&str>,
) -> (Value, StatusCode) {
let url = match primary_key {
Some(key) => {
format!("/indexes/{}/documents?primaryKey={}", urlencode(self.uid.as_ref()), key)
}
None => format!("/indexes/{}/documents", urlencode(self.uid.as_ref())),
};
self.service.post_encoded(url, documents, self.encoder).await
}
pub(super) async fn _update_settings(&self, settings: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
self.service.patch_encoded(url, settings, self.encoder).await
}
pub(super) async fn _delete_document_by_filter(&self, body: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents/delete", urlencode(self.uid.as_ref()));
self.service.post_encoded(url, body, self.encoder).await
}
pub(super) async fn _create(&self, primary_key: Option<&str>) -> (Value, StatusCode) {
let body = json!({
"uid": self.uid,
"primaryKey": primary_key,
});
self.service.post_encoded("/indexes", body, self.encoder).await
}
pub(super) async fn _delete(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}", urlencode(self.uid.as_ref()));
self.service.delete(url).await
}
pub async fn wait_task(&self, update_id: u64) -> Value {
// try several times to get status, or panic to not wait forever
let url = format!("/tasks/{}", update_id);
for _ in 0..100 {
let (response, status_code) = self.service.get(&url).await;
assert_eq!(200, status_code, "response: {}", response);
if response["status"] == "succeeded" || response["status"] == "failed" {
return response;
}
// wait 0.5 second.
sleep(Duration::from_millis(500)).await;
}
panic!("Timeout waiting for update id");
}
pub async fn get_task(&self, update_id: u64) -> (Value, StatusCode) {
let url = format!("/tasks/{}", update_id);
self.service.get(url).await
}
pub async fn filtered_tasks(
&self,
types: &[&str],
statuses: &[&str],
canceled_by: &[&str],
) -> (Value, StatusCode) {
let mut url = format!("/tasks?indexUids={}", self.uid);
if !types.is_empty() {
let _ = write!(url, "&types={}", types.join(","));
}
if !statuses.is_empty() {
let _ = write!(url, "&statuses={}", statuses.join(","));
}
if !canceled_by.is_empty() {
let _ = write!(url, "&canceledBy={}", canceled_by.join(","));
}
self.service.get(url).await
}
pub async fn get_document(&self, id: u64, options: Option<Value>) -> (Value, StatusCode) {
let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
if let Some(options) = options {
write!(url, "{}", yaup::to_string(&options).unwrap()).unwrap();
}
self.service.get(url).await
}
pub async fn get_document_by_filter(&self, payload: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents/fetch", urlencode(self.uid.as_ref()));
self.service.post(url, payload).await
}
pub async fn get_all_documents_raw(&self, options: &str) -> (Value, StatusCode) {
let url = format!("/indexes/{}/documents{}", urlencode(self.uid.as_ref()), options);
self.service.get(url).await
}
pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
let url = format!(
"/indexes/{}/documents{}",
urlencode(self.uid.as_ref()),
yaup::to_string(&options).unwrap()
);
self.service.get(url).await
}
pub async fn settings(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref()));
self.service.get(url).await
}
pub async fn stats(&self) -> (Value, StatusCode) {
let url = format!("/indexes/{}/stats", urlencode(self.uid.as_ref()));
self.service.get(url).await
}
/// Performs both GET and POST search queries
pub async fn search(
&self,
query: Value,
test: impl Fn(Value, StatusCode) + UnwindSafe + Clone,
) {
let post = self.search_post(query.clone()).await;
let query = yaup::to_string(&query).unwrap();
let get = self.search_get(&query).await;
insta::allow_duplicates! {
let (response, code) = post;
let t = test.clone();
if let Err(e) = catch_unwind(move || t(response, code)) {
eprintln!("Error with post search");
resume_unwind(e);
}
let (response, code) = get;
if let Err(e) = catch_unwind(move || test(response, code)) {
eprintln!("Error with get search");
resume_unwind(e);
}
}
}
pub async fn search_post(&self, query: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/search", urlencode(self.uid.as_ref()));
self.service.post_encoded(url, query, self.encoder).await
}
pub async fn search_get(&self, query: &str) -> (Value, StatusCode) {
let url = format!("/indexes/{}/search{}", urlencode(self.uid.as_ref()), query);
self.service.get(url).await
}
/// Performs both GET and POST similar queries
pub async fn similar(
&self,
query: Value,
test: impl Fn(Value, StatusCode) + UnwindSafe + Clone,
) {
let post = self.similar_post(query.clone()).await;
let query = yaup::to_string(&query).unwrap();
let get = self.similar_get(&query).await;
insta::allow_duplicates! {
let (response, code) = post;
let t = test.clone();
if let Err(e) = catch_unwind(move || t(response, code)) {
eprintln!("Error with post search");
resume_unwind(e);
}
let (response, code) = get;
if let Err(e) = catch_unwind(move || test(response, code)) {
eprintln!("Error with get search");
resume_unwind(e);
}
}
}
pub async fn similar_post(&self, query: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/similar", urlencode(self.uid.as_ref()));
self.service.post_encoded(url, query, self.encoder).await
}
pub async fn similar_get(&self, query: &str) -> (Value, StatusCode) {
let url = format!("/indexes/{}/similar{}", urlencode(self.uid.as_ref()), query);
self.service.get(url).await
}
pub async fn facet_search(&self, query: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/facet-search", urlencode(self.uid.as_ref()));
self.service.post_encoded(url, query, self.encoder).await
}
pub async fn get_distinct_attribute(&self) -> (Value, StatusCode) {
let url =
format!("/indexes/{}/settings/{}", urlencode(self.uid.as_ref()), "distinct-attribute");
self.service.get(url).await
}
}
#[derive(Debug, Default, serde::Serialize)]
#[serde(rename_all = "camelCase")]
pub struct GetAllDocumentsOptions {
#[serde(skip_serializing_if = "Option::is_none")]
pub limit: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub offset: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub fields: Option<Vec<&'static str>>,
pub retrieve_vectors: bool,
}

View file

@ -0,0 +1,391 @@
pub mod encoder;
pub mod index;
pub mod server;
pub mod service;
use std::fmt::{self, Display};
#[allow(unused)]
pub use index::GetAllDocumentsOptions;
use meili_snap::json_string;
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
#[allow(unused)]
pub use server::{default_settings, Server};
use tokio::sync::OnceCell;
use crate::common::index::Index;
pub enum Shared {}
pub enum Owned {}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct Value(pub serde_json::Value);
impl Value {
#[track_caller]
pub fn uid(&self) -> u64 {
if let Some(uid) = self["uid"].as_u64() {
uid
} else if let Some(uid) = self["taskUid"].as_u64() {
uid
} else {
panic!("Didn't find any task id in: {self}");
}
}
/// Return `true` if the `status` field is set to `succeeded`.
/// Panic if the `status` field doesn't exists.
#[track_caller]
pub fn is_success(&self) -> bool {
if !self["status"].is_string() {
panic!("Called `is_success` on {}", serde_json::to_string_pretty(&self.0).unwrap());
}
self["status"] == serde_json::Value::String(String::from("succeeded"))
}
// Panic if the json doesn't contain the `status` field set to "succeeded"
#[track_caller]
pub fn succeeded(&self) -> &Self {
if !self.is_success() {
panic!("Called succeeded on {}", serde_json::to_string_pretty(&self.0).unwrap());
}
self
}
}
impl From<serde_json::Value> for Value {
fn from(value: serde_json::Value) -> Self {
Value(value)
}
}
impl std::ops::Deref for Value {
type Target = serde_json::Value;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for Value {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl PartialEq<serde_json::Value> for Value {
fn eq(&self, other: &serde_json::Value) -> bool {
&self.0 == other
}
}
impl PartialEq<Value> for serde_json::Value {
fn eq(&self, other: &Value) -> bool {
self == &other.0
}
}
impl PartialEq<&str> for Value {
fn eq(&self, other: &&str) -> bool {
self.0.eq(other)
}
}
impl Display for Value {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{}",
json_string!(self, {
".uid" => "[uid]",
".enqueuedAt" => "[date]",
".startedAt" => "[date]",
".finishedAt" => "[date]",
".duration" => "[duration]",
".processingTimeMs" => "[duration]",
".details.embedders.*.url" => "[url]"
})
)
}
}
impl From<Vec<Value>> for Value {
fn from(value: Vec<Value>) -> Self {
Self(value.into_iter().map(|value| value.0).collect::<serde_json::Value>())
}
}
#[macro_export]
macro_rules! json {
($($json:tt)+) => {
$crate::common::Value(serde_json::json!($($json)+))
};
}
/// Performs a search test on both post and get routes
#[macro_export]
macro_rules! test_post_get_search {
($server:expr, $query:expr, |$response:ident, $status_code:ident | $block:expr) => {
let post_query: meilisearch::routes::search::SearchQueryPost =
serde_json::from_str(&$query.clone().to_string()).unwrap();
let get_query: meilisearch::routes::search::SearchQuery = post_query.into();
let get_query = ::serde_url_params::to_string(&get_query).unwrap();
let ($response, $status_code) = $server.search_get(&get_query).await;
let _ = ::std::panic::catch_unwind(|| $block)
.map_err(|e| panic!("panic in get route: {:?}", e.downcast_ref::<&str>().unwrap()));
let ($response, $status_code) = $server.search_post($query).await;
let _ = ::std::panic::catch_unwind(|| $block)
.map_err(|e| panic!("panic in post route: {:?}", e.downcast_ref::<&str>().unwrap()));
};
}
pub async fn shared_does_not_exists_index() -> &'static Index<'static, Shared> {
static INDEX: Lazy<Index<'static, Shared>> = Lazy::new(|| {
let server = Server::new_shared();
server._index("DOES_NOT_EXISTS").to_shared()
});
&INDEX
}
pub async fn shared_empty_index() -> &'static Index<'static, Shared> {
static INDEX: OnceCell<Index<'static, Shared>> = OnceCell::const_new();
INDEX
.get_or_init(|| async {
let server = Server::new_shared();
let index = server._index("EMPTY_INDEX").to_shared();
let (response, _code) = index._create(None).await;
index.wait_task(response.uid()).await.succeeded();
index
})
.await
}
pub static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
json!([
{
"title": "Shazam!",
"id": "287947",
"color": ["green", "blue"],
"_vectors": { "manual": [1, 2, 3]},
},
{
"title": "Captain Marvel",
"id": "299537",
"color": ["yellow", "blue"],
"_vectors": { "manual": [1, 2, 54] },
},
{
"title": "Escape Room",
"id": "522681",
"color": ["yellow", "red"],
"_vectors": { "manual": [10, -23, 32] },
},
{
"title": "How to Train Your Dragon: The Hidden World",
"id": "166428",
"color": ["green", "red"],
"_vectors": { "manual": [-100, 231, 32] },
},
{
"title": "Gläss",
"id": "450465",
"color": ["blue", "red"],
"_vectors": { "manual": [-100, 340, 90] },
}
])
});
pub async fn shared_index_with_documents() -> &'static Index<'static, Shared> {
static INDEX: OnceCell<Index<'static, Shared>> = OnceCell::const_new();
INDEX.get_or_init(|| async {
let server = Server::new_shared();
let index = server._index("SHARED_DOCUMENTS").to_shared();
let documents = DOCUMENTS.clone();
let (response, _code) = index._add_documents(documents, None).await;
index.wait_task(response.uid()).await.succeeded();
let (response, _code) = index
._update_settings(
json!({"filterableAttributes": ["id", "title"], "sortableAttributes": ["id", "title"]}),
)
.await;
index.wait_task(response.uid()).await.succeeded();
index
}).await
}
pub static SCORE_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
json!([
{
"title": "Batman the dark knight returns: Part 1",
"id": "A",
},
{
"title": "Batman the dark knight returns: Part 2",
"id": "B",
},
{
"title": "Batman Returns",
"id": "C",
},
{
"title": "Batman",
"id": "D",
},
{
"title": "Badman",
"id": "E",
}
])
});
pub static NESTED_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
json!([
{
"id": 852,
"father": "jean",
"mother": "michelle",
"doggos": [
{
"name": "bobby",
"age": 2,
},
{
"name": "buddy",
"age": 4,
},
],
"cattos": "pésti",
"_vectors": { "manual": [1, 2, 3]},
},
{
"id": 654,
"father": "pierre",
"mother": "sabine",
"doggos": [
{
"name": "gros bill",
"age": 8,
},
],
"cattos": ["simba", "pestiféré"],
"_vectors": { "manual": [1, 2, 54] },
},
{
"id": 750,
"father": "romain",
"mother": "michelle",
"cattos": ["enigma"],
"_vectors": { "manual": [10, 23, 32] },
},
{
"id": 951,
"father": "jean-baptiste",
"mother": "sophie",
"doggos": [
{
"name": "turbo",
"age": 5,
},
{
"name": "fast",
"age": 6,
},
],
"cattos": ["moumoute", "gomez"],
"_vectors": { "manual": [10, 23, 32] },
},
])
});
pub async fn shared_index_with_nested_documents() -> &'static Index<'static, Shared> {
static INDEX: OnceCell<Index<'static, Shared>> = OnceCell::const_new();
INDEX.get_or_init(|| async {
let server = Server::new_shared();
let index = server._index("SHARED_NESTED_DOCUMENTS").to_shared();
let documents = NESTED_DOCUMENTS.clone();
let (response, _code) = index._add_documents(documents, None).await;
index.wait_task(response.uid()).await.succeeded();
let (response, _code) = index
._update_settings(
json!({"filterableAttributes": ["father", "doggos"], "sortableAttributes": ["doggos"]}),
)
.await;
index.wait_task(response.uid()).await.succeeded();
index
}).await
}
pub static FRUITS_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
json!([
{
"name": "Exclusive sale: green apple",
"id": "green-apple-boosted",
"BOOST": true
},
{
"name": "Pear",
"id": "pear",
},
{
"name": "Red apple gala",
"id": "red-apple-gala",
},
{
"name": "Exclusive sale: Red Tomato",
"id": "red-tomatoes-boosted",
"BOOST": true
},
{
"name": "Exclusive sale: Red delicious apple",
"id": "red-delicious-boosted",
"BOOST": true,
}
])
});
pub static VECTOR_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
json!([
{
"id": "A",
"description": "the dog barks at the cat",
"_vectors": {
// dimensions [canine, feline, young]
"animal": [0.9, 0.8, 0.05],
// dimensions [negative/positive, energy]
"sentiment": [-0.1, 0.55]
}
},
{
"id": "B",
"description": "the kitten scratched the beagle",
"_vectors": {
// dimensions [canine, feline, young]
"animal": [0.8, 0.9, 0.5],
// dimensions [negative/positive, energy]
"sentiment": [-0.2, 0.65]
}
},
{
"id": "C",
"description": "the dog had to stay alone today",
"_vectors": {
// dimensions [canine, feline, young]
"animal": [0.85, 0.02, 0.1],
// dimensions [negative/positive, energy]
"sentiment": [-1.0, 0.1]
}
},
{
"id": "D",
"description": "the little boy pets the puppy",
"_vectors": {
// dimensions [canine, feline, young]
"animal": [0.8, 0.09, 0.8],
// dimensions [negative/positive, energy]
"sentiment": [0.8, 0.3]
}
},
])
});

View file

@ -0,0 +1,404 @@
#![allow(dead_code)]
use std::marker::PhantomData;
use std::path::Path;
use std::str::FromStr;
use std::time::Duration;
use actix_http::body::MessageBody;
use actix_web::dev::ServiceResponse;
use actix_web::http::StatusCode;
use byte_unit::{Byte, Unit};
use clap::Parser;
use meilisearch::option::{IndexerOpts, MaxMemory, MaxThreads, Opt};
use meilisearch::setup_meilisearch;
use once_cell::sync::Lazy;
use tempfile::TempDir;
use tokio::sync::OnceCell;
use tokio::time::sleep;
use uuid::Uuid;
use super::index::Index;
use super::service::Service;
use super::{Owned, Shared};
use crate::common::encoder::Encoder;
use crate::common::Value;
use crate::json;
pub struct Server<State = Owned> {
pub service: Service,
// hold ownership to the tempdir while we use the server instance.
_dir: Option<TempDir>,
_marker: PhantomData<State>,
}
pub static TEST_TEMP_DIR: Lazy<TempDir> = Lazy::new(|| TempDir::new().unwrap());
impl Server<Owned> {
fn into_shared(self) -> Server<Shared> {
Server { service: self.service, _dir: self._dir, _marker: PhantomData }
}
pub async fn new() -> Self {
let dir = TempDir::new().unwrap();
if cfg!(windows) {
std::env::set_var("TMP", TEST_TEMP_DIR.path());
} else {
std::env::set_var("TMPDIR", TEST_TEMP_DIR.path());
}
let options = default_settings(dir.path());
let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
let service = Service { index_scheduler, auth, options, api_key: None };
Server { service, _dir: Some(dir), _marker: PhantomData }
}
pub async fn new_auth_with_options(mut options: Opt, dir: TempDir) -> Self {
if cfg!(windows) {
std::env::set_var("TMP", TEST_TEMP_DIR.path());
} else {
std::env::set_var("TMPDIR", TEST_TEMP_DIR.path());
}
options.master_key = Some("MASTER_KEY".to_string());
let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
let service = Service { index_scheduler, auth, options, api_key: None };
Server { service, _dir: Some(dir), _marker: PhantomData }
}
pub async fn new_auth() -> Self {
let dir = TempDir::new().unwrap();
let options = default_settings(dir.path());
Self::new_auth_with_options(options, dir).await
}
pub async fn new_with_options(options: Opt) -> Result<Self, anyhow::Error> {
let (index_scheduler, auth) = setup_meilisearch(&options)?;
let service = Service { index_scheduler, auth, options, api_key: None };
Ok(Server { service, _dir: None, _marker: PhantomData })
}
pub fn use_api_key(&mut self, api_key: impl AsRef<str>) {
self.service.api_key = Some(api_key.as_ref().to_string());
}
/// Fetch and use the default admin key for nexts http requests.
pub async fn use_admin_key(&mut self, master_key: impl AsRef<str>) {
self.use_api_key(master_key);
let (response, code) = self.list_api_keys("").await;
assert_eq!(200, code, "{:?}", response);
let admin_key = &response["results"][1]["key"];
self.use_api_key(admin_key.as_str().unwrap());
}
pub async fn add_api_key(&self, content: Value) -> (Value, StatusCode) {
let url = "/keys";
self.service.post(url, content).await
}
pub async fn patch_api_key(&self, key: impl AsRef<str>, content: Value) -> (Value, StatusCode) {
let url = format!("/keys/{}", key.as_ref());
self.service.patch(url, content).await
}
pub async fn delete_api_key(&self, key: impl AsRef<str>) -> (Value, StatusCode) {
let url = format!("/keys/{}", key.as_ref());
self.service.delete(url).await
}
/// Returns a view to an index. There is no guarantee that the index exists.
pub fn index(&self, uid: impl AsRef<str>) -> Index<'_> {
self.index_with_encoder(uid, Encoder::Plain)
}
pub async fn create_index(&self, body: Value) -> (Value, StatusCode) {
self.service.post("/indexes", body).await
}
pub fn index_with_encoder(&self, uid: impl AsRef<str>, encoder: Encoder) -> Index<'_> {
Index {
uid: uid.as_ref().to_string(),
service: &self.service,
encoder,
marker: PhantomData,
}
}
pub async fn list_indexes(
&self,
offset: Option<usize>,
limit: Option<usize>,
) -> (Value, StatusCode) {
let (offset, limit) = (
offset.map(|offset| format!("offset={offset}")),
limit.map(|limit| format!("limit={limit}")),
);
let query_parameter = offset
.as_ref()
.zip(limit.as_ref())
.map(|(offset, limit)| format!("{offset}&{limit}"))
.or_else(|| offset.xor(limit));
if let Some(query_parameter) = query_parameter {
self.service.get(format!("/indexes?{query_parameter}")).await
} else {
self.service.get("/indexes").await
}
}
pub async fn stats(&self) -> (Value, StatusCode) {
self.service.get("/stats").await
}
pub async fn tasks(&self) -> (Value, StatusCode) {
self.service.get("/tasks").await
}
pub async fn set_features(&self, value: Value) -> (Value, StatusCode) {
self.service.patch("/experimental-features", value).await
}
pub async fn get_metrics(&self) -> (Value, StatusCode) {
self.service.get("/metrics").await
}
}
impl Server<Shared> {
fn init_new_shared_instance() -> Server<Shared> {
let dir = TempDir::new().unwrap();
if cfg!(windows) {
std::env::set_var("TMP", TEST_TEMP_DIR.path());
} else {
std::env::set_var("TMPDIR", TEST_TEMP_DIR.path());
}
let options = default_settings(dir.path());
let (index_scheduler, auth) = setup_meilisearch(&options).unwrap();
let service = Service { index_scheduler, auth, api_key: None, options };
Server { service, _dir: Some(dir), _marker: PhantomData }
}
pub fn new_shared() -> &'static Server<Shared> {
static SERVER: Lazy<Server<Shared>> = Lazy::new(Server::init_new_shared_instance);
&SERVER
}
pub async fn new_shared_with_admin_key() -> &'static Server<Shared> {
static SERVER: OnceCell<Server<Shared>> = OnceCell::const_new();
SERVER
.get_or_init(|| async {
let mut server = Server::new_auth().await;
server.use_admin_key("MASTER_KEY").await;
server.into_shared()
})
.await
}
/// You shouldn't access random indexes on a shared instance thus this method
/// must fail.
pub async fn get_index_fail(&self, uid: impl AsRef<str>) -> (Value, StatusCode) {
let url = format!("/indexes/{}", urlencoding::encode(uid.as_ref()));
let (value, code) = self.service.get(url).await;
if code.is_success() {
panic!("`get_index_fail` succeeded with uid: {}", uid.as_ref());
}
(value, code)
}
pub async fn delete_index_fail(&self, uid: impl AsRef<str>) -> (Value, StatusCode) {
let url = format!("/indexes/{}", urlencoding::encode(uid.as_ref()));
let (value, code) = self.service.delete(url).await;
if code.is_success() {
panic!("`delete_index_fail` succeeded with uid: {}", uid.as_ref());
}
(value, code)
}
pub async fn update_raw_index_fail(
&self,
uid: impl AsRef<str>,
body: Value,
) -> (Value, StatusCode) {
let url = format!("/indexes/{}", urlencoding::encode(uid.as_ref()));
let (value, code) = self.service.patch_encoded(url, body, Encoder::Plain).await;
if code.is_success() {
panic!("`update_raw_index_fail` succeeded with uid: {}", uid.as_ref());
}
(value, code)
}
/// Since this call updates the state of the instance, it must fail.
/// If it doesn't fail, the test will panic to help you debug what
/// is going on.
pub async fn create_index_fail(&self, body: Value) -> (Value, StatusCode) {
let (mut task, code) = self._create_index(body).await;
if code.is_success() {
task = self.wait_task(task.uid()).await;
if task.is_success() {
panic!(
"`create_index_fail` succeeded: {}",
serde_json::to_string_pretty(&task).unwrap()
);
}
}
(task, code)
}
}
impl<State> Server<State> {
pub async fn init_web_app(
&self,
) -> impl actix_web::dev::Service<
actix_http::Request,
Response = ServiceResponse<impl MessageBody>,
Error = actix_web::Error,
> {
self.service.init_web_app().await
}
pub async fn list_api_keys(&self, params: &str) -> (Value, StatusCode) {
let url = format!("/keys{params}");
self.service.get(url).await
}
pub async fn dummy_request(
&self,
method: impl AsRef<str>,
url: impl AsRef<str>,
) -> (Value, StatusCode) {
match method.as_ref() {
"POST" => self.service.post(url, json!({})).await,
"PUT" => self.service.put(url, json!({})).await,
"PATCH" => self.service.patch(url, json!({})).await,
"GET" => self.service.get(url).await,
"DELETE" => self.service.delete(url).await,
_ => unreachable!(),
}
}
pub async fn get_api_key(&self, key: impl AsRef<str>) -> (Value, StatusCode) {
let url = format!("/keys/{}", key.as_ref());
self.service.get(url).await
}
pub(super) fn _index(&self, uid: impl AsRef<str>) -> Index<'_> {
Index {
uid: uid.as_ref().to_string(),
service: &self.service,
encoder: Encoder::Plain,
marker: PhantomData,
}
}
/// Returns a view to an index. There is no guarantee that the index exists.
pub fn unique_index(&self) -> Index<'_> {
let uuid = Uuid::new_v4();
Index {
uid: uuid.to_string(),
service: &self.service,
encoder: Encoder::Plain,
marker: PhantomData,
}
}
pub fn unique_index_with_encoder(&self, encoder: Encoder) -> Index<'_> {
let uuid = Uuid::new_v4();
Index { uid: uuid.to_string(), service: &self.service, encoder, marker: PhantomData }
}
pub(super) async fn _create_index(&self, body: Value) -> (Value, StatusCode) {
self.service.post("/indexes", body).await
}
pub async fn multi_search(&self, queries: Value) -> (Value, StatusCode) {
self.service.post("/multi-search", queries).await
}
pub async fn list_indexes_raw(&self, parameters: &str) -> (Value, StatusCode) {
self.service.get(format!("/indexes{parameters}")).await
}
pub async fn tasks_filter(&self, filter: &str) -> (Value, StatusCode) {
self.service.get(format!("/tasks?{}", filter)).await
}
pub async fn version(&self) -> (Value, StatusCode) {
self.service.get("/version").await
}
pub async fn create_dump(&self) -> (Value, StatusCode) {
self.service.post("/dumps", json!(null)).await
}
pub async fn create_snapshot(&self) -> (Value, StatusCode) {
self.service.post("/snapshots", json!(null)).await
}
pub async fn index_swap(&self, value: Value) -> (Value, StatusCode) {
self.service.post("/swap-indexes", value).await
}
pub async fn cancel_tasks(&self, value: &str) -> (Value, StatusCode) {
self.service.post(format!("/tasks/cancel?{}", value), json!(null)).await
}
pub async fn delete_tasks(&self, value: &str) -> (Value, StatusCode) {
self.service.delete(format!("/tasks?{}", value)).await
}
pub async fn wait_task(&self, update_id: u64) -> Value {
// try several times to get status, or panic to not wait forever
let url = format!("/tasks/{}", update_id);
for _ in 0..100 {
let (response, status_code) = self.service.get(&url).await;
assert_eq!(200, status_code, "response: {}", response);
if response["status"] == "succeeded" || response["status"] == "failed" {
return response;
}
// wait 0.5 second.
sleep(Duration::from_millis(500)).await;
}
panic!("Timeout waiting for update id");
}
pub async fn get_task(&self, update_id: u64) -> (Value, StatusCode) {
let url = format!("/tasks/{}", update_id);
self.service.get(url).await
}
pub async fn get_features(&self) -> (Value, StatusCode) {
self.service.get("/experimental-features").await
}
}
pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
Opt {
db_path: dir.as_ref().join("db"),
dump_dir: dir.as_ref().join("dumps"),
env: "development".to_owned(),
no_analytics: true,
max_index_size: Byte::from_u64_with_unit(100, Unit::MiB).unwrap(),
max_task_db_size: Byte::from_u64_with_unit(1, Unit::GiB).unwrap(),
http_payload_size_limit: Byte::from_u64_with_unit(10, Unit::MiB).unwrap(),
snapshot_dir: ".".into(),
indexer_options: IndexerOpts {
// memory has to be unlimited because several meilisearch are running in test context.
max_indexing_memory: MaxMemory::unlimited(),
skip_index_budget: true,
// Having 2 threads makes the tests way faster
max_indexing_threads: MaxThreads::from_str("2").unwrap(),
},
experimental_enable_metrics: false,
..Parser::parse_from(None as Option<&str>)
}
}

View file

@ -0,0 +1,177 @@
use std::num::NonZeroUsize;
use std::sync::Arc;
use actix_web::body::MessageBody;
use actix_web::dev::ServiceResponse;
use actix_web::http::header::ContentType;
use actix_web::http::StatusCode;
use actix_web::test;
use actix_web::test::TestRequest;
use actix_web::web::Data;
use index_scheduler::IndexScheduler;
use meilisearch::analytics::Analytics;
use meilisearch::search_queue::SearchQueue;
use meilisearch::{create_app, Opt, SubscriberForSecondLayer};
use meilisearch_auth::AuthController;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::Layer;
use crate::common::encoder::Encoder;
use crate::common::Value;
pub struct Service {
pub index_scheduler: Arc<IndexScheduler>,
pub auth: Arc<AuthController>,
pub options: Opt,
pub api_key: Option<String>,
}
impl Service {
pub async fn post(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
self.post_encoded(url, body, Encoder::Plain).await
}
pub async fn post_encoded(
&self,
url: impl AsRef<str>,
body: Value,
encoder: Encoder,
) -> (Value, StatusCode) {
let mut req = test::TestRequest::post().uri(url.as_ref());
req = self.encode(req, body, encoder);
self.request(req).await
}
/// Send a test post request from a text body.
pub async fn post_str(
&self,
url: impl AsRef<str>,
body: impl AsRef<str>,
headers: Vec<(&str, &str)>,
) -> (Value, StatusCode) {
let mut req =
test::TestRequest::post().uri(url.as_ref()).set_payload(body.as_ref().to_string());
for header in headers {
req = req.insert_header(header);
}
self.request(req).await
}
pub async fn get(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
let req = test::TestRequest::get().uri(url.as_ref());
self.request(req).await
}
pub async fn put(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
self.put_encoded(url, body, Encoder::Plain).await
}
/// Send a test put request from a text body.
pub async fn put_str(
&self,
url: impl AsRef<str>,
body: impl AsRef<str>,
headers: Vec<(&str, &str)>,
) -> (Value, StatusCode) {
let mut req =
test::TestRequest::put().uri(url.as_ref()).set_payload(body.as_ref().to_string());
for header in headers {
req = req.insert_header(header);
}
self.request(req).await
}
pub async fn put_encoded(
&self,
url: impl AsRef<str>,
body: Value,
encoder: Encoder,
) -> (Value, StatusCode) {
let mut req = test::TestRequest::put().uri(url.as_ref());
req = self.encode(req, body, encoder);
self.request(req).await
}
pub async fn patch(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
self.patch_encoded(url, body, Encoder::Plain).await
}
pub async fn patch_encoded(
&self,
url: impl AsRef<str>,
body: Value,
encoder: Encoder,
) -> (Value, StatusCode) {
let mut req = test::TestRequest::patch().uri(url.as_ref());
req = self.encode(req, body, encoder);
self.request(req).await
}
pub async fn delete(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
let req = test::TestRequest::delete().uri(url.as_ref());
self.request(req).await
}
pub async fn init_web_app(
&self,
) -> impl actix_web::dev::Service<
actix_http::Request,
Response = ServiceResponse<impl MessageBody>,
Error = actix_web::Error,
> {
let (_route_layer, route_layer_handle) =
tracing_subscriber::reload::Layer::new(None.with_filter(
tracing_subscriber::filter::Targets::new().with_target("", LevelFilter::OFF),
));
let (_stderr_layer, stderr_layer_handle) = tracing_subscriber::reload::Layer::new(
(Box::new(
tracing_subscriber::fmt::layer()
.with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE),
)
as Box<dyn tracing_subscriber::Layer<SubscriberForSecondLayer> + Send + Sync>)
.with_filter(tracing_subscriber::filter::Targets::new()),
);
let search_queue = SearchQueue::new(
self.options.experimental_search_queue_size,
NonZeroUsize::new(1).unwrap(),
);
actix_web::test::init_service(create_app(
self.index_scheduler.clone().into(),
self.auth.clone().into(),
Data::new(search_queue),
self.options.clone(),
(route_layer_handle, stderr_layer_handle),
Data::new(Analytics::no_analytics()),
true,
))
.await
}
pub async fn request(&self, mut req: test::TestRequest) -> (Value, StatusCode) {
let app = self.init_web_app().await;
if let Some(api_key) = &self.api_key {
req = req.insert_header(("Authorization", ["Bearer ", api_key].concat()));
}
let req = req.to_request();
let res = test::call_service(&app, req).await;
let status_code = res.status();
let body = test::read_body(res).await;
let response = serde_json::from_slice(&body).unwrap_or_default();
(response, status_code)
}
fn encode(&self, req: TestRequest, body: Value, encoder: Encoder) -> TestRequest {
let bytes = serde_json::to_string(&body).expect("Failed to serialize test data to json");
let encoded_body = encoder.encode(bytes);
let header = encoder.header();
match header {
Some(header) => req.insert_header(header),
None => req,
}
.set_payload(encoded_body)
.insert_header(ContentType::json())
}
}