diff --git a/benchmarks/README.md b/benchmarks/README.md index 16838e488..81447c4f2 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -54,7 +54,7 @@ MILLI_BENCH_DATASETS_PATH=~/datasets cargo bench --bench songs # the code is com ## Comparison between benchmarks -The benchmark reports we push are generated with `critcmp`. Thus, we use `critcmp` to generate comparison results between 2 benchmarks. +The benchmark reports we push are generated with `critcmp`. Thus, we use `critcmp` to show the result of a benchmark, or compare results between multiple benchmarks. We provide a script to download and display the comparison report. @@ -71,12 +71,18 @@ List the available file in the DO Space: ```bash songs_main_09a4321.json songs_geosearch_24ec456.json +search_songs_main_cb45a10b.json ``` Run the comparison script: ```bash +# we get the result of ONE benchmark, this give you an idea of how much time an operation took +./benchmarks/scripts/compare.sh son songs_geosearch_24ec456.json +# we compare two benchmarks ./benchmarks/scripts/compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json +# we compare three benchmarks +./benchmarks/scripts/compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json search_songs_main_cb45a10b.json ``` ## Datasets diff --git a/benchmarks/scripts/compare.sh b/benchmarks/scripts/compare.sh index 506c94015..84d1dc0e6 100755 --- a/benchmarks/scripts/compare.sh +++ b/benchmarks/scripts/compare.sh @@ -17,21 +17,9 @@ if [[ "$?" -ne 0 ]]; then exit 1 fi -if [[ $# -ne 2 ]] - then - echo 'Need 2 arguments.' - echo 'Usage: ' - echo ' $ ./compare.sh old new' - echo 'Ex:' - echo ' $ ./compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json' - exit 1 -fi - -old_file="$1" -new_file="$2" s3_url='https://milli-benchmarks.fra1.digitaloceanspaces.com/critcmp_results' -for file in $old_file $new_file +for file in $@ do file_s3_url="$s3_url/$file" file_local_path="/tmp/$file" @@ -45,6 +33,12 @@ do fi done -# Print the diff changes between the old and new benchmarks -# by only displaying the lines that have a diff of more than 5%. -critcmp --threshold 5 "/tmp/$old_file" "/tmp/$new_file" +path_list=$(echo " $@" | sed 's/ / \/tmp\//g') + +if [[ ${#@} -gt 1 ]]; then + # Print the diff changes between the old and new benchmarks + # by only displaying the lines that have a diff of more than 5%. + critcmp --threshold 5 $path_list +else + critcmp $path_list +fi