searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-27 07:14:26 +01:00
Code Issues Releases Wiki Activity

Compress updates content using gzip

Browse Source
This commit is contained in:
Clément Renault 2020-12-17 10:00:46 +01:00
parent cd5605bb86
commit 5a23417499
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
3 changed files with 23 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
http-ui/Cargo.lock generated
View File

@ -80,6 +80,19 @@ dependencies = [
"warp", "warp",
] ]
[[package]]
name = "async-compression"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb1ff21a63d3262af46b9f33a826a8d134e2d0d9b2179c86034948b732ea8b2a"
dependencies = [
"flate2",
"futures-core",
"memchr",
"pin-project-lite",
"tokio",
]
[[package]] [[package]]
name = "atty" name = "atty"
version = "0.2.14" version = "0.2.14"
@ -745,6 +758,7 @@ dependencies = [
"anyhow", "anyhow",
"askama", "askama",
"askama_warp", "askama_warp",
"async-compression",
"bytes", "bytes",
"flate2", "flate2",
"futures", "futures",

1
http-ui/Cargo.toml
View File

@ -7,6 +7,7 @@ edition = "2018"
[dependencies] [dependencies]
anyhow = "1.0.28" anyhow = "1.0.28"
async-compression = { version = "0.3.6", features = ["gzip", "tokio-02"] }
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
heed = "0.10.5" heed = "0.10.5"
memmap = "0.7.0" memmap = "0.7.0"

11
http-ui/src/main.rs
View File

@ -10,6 +10,7 @@ use std::time::Instant;
use std::{mem, io}; use std::{mem, io};
use askama_warp::Template; use askama_warp::Template;
use async_compression::tokio_02::write::GzipEncoder;
use flate2::read::GzDecoder; use flate2::read::GzDecoder;
use futures::stream; use futures::stream;
use futures::{FutureExt, StreamExt}; use futures::{FutureExt, StreamExt};
@ -340,7 +341,7 @@ async fn main() -> anyhow::Result<()> {
otherwise => panic!("invalid indexing method {:?}", otherwise), otherwise => panic!("invalid indexing method {:?}", otherwise),
}; };
let gzipped = false; let gzipped = true;
let reader = if gzipped { let reader = if gzipped {
Box::new(GzDecoder::new(content)) Box::new(GzDecoder::new(content))
} else { } else {
@ -704,13 +705,17 @@ async fn main() -> anyhow::Result<()> {
) -> Result<impl warp::Reply, warp::Rejection> ) -> Result<impl warp::Reply, warp::Rejection>
{ {
let file = tokio::task::block_in_place(tempfile::tempfile).unwrap(); let file = tokio::task::block_in_place(tempfile::tempfile).unwrap();
let mut file = TFile::from_std(file); let file = TFile::from_std(file);
let mut encoder = GzipEncoder::new(file);
while let Some(result) = stream.next().await { while let Some(result) = stream.next().await {
let bytes = result.unwrap().to_bytes(); let bytes = result.unwrap().to_bytes();
file.write_all(&bytes[..]).await.unwrap(); encoder.write_all(&bytes[..]).await.unwrap();
} }
encoder.shutdown().await.unwrap();
let mut file = encoder.into_inner();
file.sync_all().await.unwrap();
let file = file.into_std().await; let file = file.into_std().await;
let mmap = unsafe { memmap::Mmap::map(&file).unwrap() }; let mmap = unsafe { memmap::Mmap::map(&file).unwrap() };