searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-03 11:57:07 +02:00
Code Issues Releases Wiki Activity

Store content files into the S3

Browse source
This commit is contained in:
Kerollmops 2023-09-11 18:17:22 +02:00
parent 719fdd701b
commit a53a0fdb77
No known key found for this signature in database
GPG key ID: F250A4C4E3AE5F5F
5 changed files with 75 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

14
meilisearch/src/lib.rs
View file

@ -245,13 +245,15 @@ fn open_or_create_database_unchecked(
instance_features,
zookeeper: zookeeper.clone(),
s3: opt.s3_url.as_ref().map(|url| {
Bucket::new(
"test-rust-s3",
Region::Custom { region: "eu-central-1".to_owned(), endpoint: url.clone() },
Credentials::default().unwrap(),
Arc::new(
Bucket::new(
"test-rust-s3",
Region::Custom { region: "eu-central-1".to_owned(), endpoint: url.clone() },
Credentials::default().unwrap(),
)
.unwrap()
.with_path_style(),
)
.unwrap()
.with_path_style()
}),
}))
.map_err(anyhow::Error::from);

10
meilisearch/src/routes/indexes/documents.rs
View file

@ -1,4 +1,4 @@
use std::io::ErrorKind;
use std::io::{BufReader, ErrorKind};
use actix_web::http::header::CONTENT_TYPE;
use actix_web::web::Data;
@ -400,6 +400,7 @@ async fn document_addition(
}
let read_file = buffer.into_inner().into_std().await;
let s3 = index_scheduler.s3.clone();
let documents_count = tokio::task::spawn_blocking(move || {
let documents_count = match format {
PayloadType::Json => read_json(&read_file, update_file.as_file_mut())?,
@ -408,8 +409,15 @@ async fn document_addition(
}
PayloadType::Ndjson => read_ndjson(&read_file, update_file.as_file_mut())?,
};
if let Some(s3) = s3 {
let mut reader = BufReader::new(&*update_file);
s3.put_object_stream(&mut reader, format!("/update-files/{}", uuid)).unwrap();
}
// we NEED to persist the file here because we moved the `udpate_file` in another task.
update_file.persist()?;
Ok(documents_count)
})
.await;