From cd780b122de227f5e634415355d64b33d139c1dc Mon Sep 17 00:00:00 2001 From: H1ghBre4k3r Date: Wed, 8 Dec 2021 18:19:27 +0100 Subject: [PATCH] First refactorings for contacts and bloom api --- src/contacts.c | 272 +++++++++++++--------------------------------- src/contacts.h | 5 +- src/ens/records.c | 2 +- src/ens/records.h | 14 ++- src/ens/storage.c | 12 +- src/ens/storage.h | 8 +- src/main.c | 7 +- zephyr/Kconfig | 25 ++++- 8 files changed, 133 insertions(+), 212 deletions(-) diff --git a/src/contacts.c b/src/contacts.c index 33b42b0..72ccdbc 100644 --- a/src/contacts.c +++ b/src/contacts.c @@ -27,6 +27,8 @@ #include "ens/storage.h" #include "exposure-notification.h" +// #define CONFIG_INTERVAL_SPREAD 100 + void print_key(_ENBaseKey* key) { for (int i = 0; i < sizeof(key->b); i++) { printk("%02x", key->b[i]); @@ -54,31 +56,6 @@ int register_record(record_t* record) { return rc; } -// set size to test -#define TEST_DATA_SIZE 100000 - -/** - * Dumb implementation, where a single iterator is created for iterating over the entire flash. - */ -int get_number_of_infected_for_multiple_intervals_dumb(infected_for_interval_ident_ctx_t* ctx, int count) { - record_iterator_t iterator; - int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL); - if (rc) { - // there was a general error, so just do nothing - return rc; - } - while (!iterator.finished) { - for (int i = 0; i < count; i++) { - if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier, - sizeof(rolling_proximity_identifier_t)) == 0) { - ctx[i].infected++; - } - } - ens_records_iterator_next(&iterator); - } - return 0; -} - /** * Simple implementation, where an iterator is created for every element in the passed arrray. */ @@ -90,50 +67,17 @@ int get_number_of_infected_for_multiple_intervals_simple(infected_for_interval_i // on error, skip this rpi continue; } - while (!iterator.finished) { - if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier, + record_t* current; + while (current = ens_records_iterator_next(&iterator)) { + if (memcmp(&(current->rolling_proximity_identifier), &ctx[i].interval_identifier, sizeof(rolling_proximity_identifier_t)) == 0) { ctx[i].infected++; } - ens_records_iterator_next(&iterator); } } return 0; } -/** - * Optimized implementation, where overlapping search intervals for consecutive RPI are merged. - */ -int get_number_of_infected_for_multiple_intervals_optimized(infected_for_interval_ident_ctx_t* ctx, int count) { - record_iterator_t iterator; - int i = 0; - while (i < count) { - // determine start and end of iterator - int start = i; - int end = i; - while (end + 1 < count && ctx[end + 1].search_start <= ctx[end].search_end) { - end++; - } - // init iterator with start and end - int rc = ens_records_iterator_init_timerange(&iterator, &ctx[start].search_start, &ctx[end].search_end); - if (rc) { - goto end; - } - while (!iterator.finished) { - for (int j = start; j <= end; j++) { - if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[j].interval_identifier, - sizeof(iterator.current.rolling_proximity_identifier)) == 0) { - ctx[j].infected++; - } - } - ens_records_iterator_next(&iterator); - } - end: - i = end + 1; - } - return 0; -} - /** * Fill the bloom filter with all stored records. */ @@ -147,9 +91,9 @@ void fill_bloom_with_stored_records(bloom_filter_t* bloom) { } // fill bloom filter with records - while (!iterator.finished) { - bloom_add_record(bloom, &iterator.current.rolling_proximity_identifier); - ens_records_iterator_next(&iterator); + record_t* current; + while (current = ens_records_iterator_next(&iterator)) { + bloom_add_record(bloom, &(current->rolling_proximity_identifier)); } } @@ -163,19 +107,8 @@ int64_t test_bloom_performance(infected_for_interval_ident_ctx_t* ctx, int count return -1; } - // Measure bloom creation time - timing_t start_time, end_time; - uint64_t total_cycles; - uint64_t total_ns; - start_time = timing_counter_get(); - fill_bloom_with_stored_records(bloom); - end_time = timing_counter_get(); - total_cycles = timing_cycles_get(&start_time, &end_time); - total_ns = timing_cycles_to_ns(total_cycles); - printk("\nbloom init took %lld ms\n\n", total_ns / 1000000); - // test bloom performance for (int i = 0; i < count; i++) { if (bloom_probably_has_record(bloom, &ctx[i].interval_identifier)) { @@ -191,72 +124,13 @@ int64_t test_bloom_performance(infected_for_interval_ident_ctx_t* ctx, int count amount++; } } - printk("amount of infected records: %d\n", amount); - // int ret = get_number_of_infected_for_multiple_intervals_dumb(ctx, amount); int ret = get_number_of_infected_for_multiple_intervals_simple(ctx, amount); - // int ret = get_number_of_infected_for_multiple_intervals_optimized(ctx, amount); bloom_destroy(bloom); return ret; } -/** - * Fill bloom with passed RPIs and test flash records against it. - */ -int64_t test_bloom_reverse_performance(infected_for_interval_ident_ctx_t* ctx, int count) { - bloom_filter_t* bloom = bloom_init(count * 2); - if (!bloom) { - printk("bloom init failed\n"); - return -1; - } - - // Measure bloom creation time - timing_t start_time, end_time; - uint64_t total_cycles; - uint64_t total_ns; - start_time = timing_counter_get(); - for (int i = 0; i < count; i++) { - bloom_add_record(bloom, &ctx[i].interval_identifier); - } - end_time = timing_counter_get(); - total_cycles = timing_cycles_get(&start_time, &end_time); - total_ns = timing_cycles_to_ns(total_cycles); - printk("\nbloom init took %lld ms\n\n", total_ns / 1000000); - - int64_t amount = 0; - for (int round = 0; round < TEST_DATA_SIZE / count; round++) { - record_iterator_t iterator; - int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL); - if (rc) { - printk("init iterator failed (err %d)\n", rc); - amount = rc; - goto cleanup; - } - - while (!iterator.finished) { - if (bloom_probably_has_record(bloom, &iterator.current.rolling_proximity_identifier)) { - for (int i = 0; i < count; i++) { - if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier, - sizeof(iterator.current.rolling_proximity_identifier)) == 0) { - amount++; - break; - } - if (iterator.current.timestamp > ctx[i].search_end) { - break; - } - } - } - ens_records_iterator_next(&iterator); - } - printk("round %d\n", round); - } - -cleanup: - bloom_destroy(bloom); - return amount; -} - //////////////////// // FILL TEST DATA // //////////////////// @@ -313,63 +187,6 @@ void measure_perf(test_func_t func, printk("\n'%s' took %lld ms\n---------------------------\n", label, total_ns / 1000000); } -//////////////////// -// SETUP DATA // -//////////////////// - -void setup_test_data() { - en_derive_period_identifier_key(&infectedPik, &infectedPeriodKey); - en_derive_period_identifier_key(&dummyPik, &dummyPeriodKey); - -// every 100th record is infected -#define INTERVAL_SPREAD 100 - - for (int i = 0; i < EN_TEK_ROLLING_PERIOD; i++) { - // create infected record - record_t infectedRecord; - infectedRecord.timestamp = i * EN_INTERVAL_LENGTH; - en_derive_interval_identifier((ENIntervalIdentifier*)&infectedRecord.rolling_proximity_identifier, &infectedPik, - i); - int rc; - if (i % INTERVAL_SPREAD == 0 && (rc = add_record(&infectedRecord))) { - printk("err %d\n", rc); - } - - record_t dummyRecord; - en_derive_interval_identifier((ENIntervalIdentifier*)&dummyRecord.rolling_proximity_identifier, &dummyPik, i); - int spread = 1; - - for (int j = 0; j < EN_INTERVAL_LENGTH / spread; j++) { - dummyRecord.timestamp = i * EN_INTERVAL_LENGTH + j * spread + 1; - if ((rc = add_record(&dummyRecord))) { - printk("err %d\n", rc); - } - } - printk("period %d\n", i); - } - -#define INFECTED_INTERVALS_COUNT 2000 - // setup our ordered array with infected RPIs - static infected_for_interval_ident_ctx_t infectedIntervals[INFECTED_INTERVALS_COUNT]; - - printk("Starting measurements with %d RPIs to seach and an infection rate of every %d. interval\n", - INFECTED_INTERVALS_COUNT, INTERVAL_SPREAD); - - // measure_perf(get_number_of_infected_for_multiple_intervals_dumb, "dumb", infectedIntervals, - // INFECTED_INTERVALS_COUNT); - // measure_perf(get_number_of_infected_for_multiple_intervals_simple, "simple", infectedIntervals, - // INFECTED_INTERVALS_COUNT); - // measure_perf(get_number_of_infected_for_multiple_intervals_optimized, "optimized", infectedIntervals, - // INFECTED_INTERVALS_COUNT); - - // TODO lome: prepare massive tests for bloom simple & reverse - // TODO lome: document everything - - // measure_perf(test_bloom_performance, "bloom", infectedIntervals, INFECTED_INTERVALS_COUNT); - - measure_perf(test_bloom_reverse_performance, "bloom reverse", infectedIntervals, INFECTED_INTERVALS_COUNT); -} - int reverse_bloom_filter(infected_for_interval_ident_ctx_t* ctx, int count) { bloom_filter_t* bloom = bloom_init(count * 2); if (!bloom) { @@ -393,21 +210,22 @@ int reverse_bloom_filter(infected_for_interval_ident_ctx_t* ctx, int count) { goto cleanup; } - while (!iterator.finished) { - if (bloom_probably_has_record(bloom, &iterator.current.rolling_proximity_identifier)) { + record_t* current; + while (current = ens_records_iterator_next(&iterator)) { + if (bloom_probably_has_record(bloom, &(current->rolling_proximity_identifier))) { for (int i = 0; i < count; i++) { - if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier, - sizeof(iterator.current.rolling_proximity_identifier)) == 0) { + if (memcmp(&(current->rolling_proximity_identifier), &ctx[i].interval_identifier, + sizeof(current->rolling_proximity_identifier)) == 0) { ctx[i].infected = 1; amount++; break; } - if (iterator.current.timestamp > ctx[i].search_end) { + // TODO lome: maybe dont require orderred array, so this is not useful + if (current->timestamp > ctx[i].search_end) { break; } } } - ens_records_iterator_next(&iterator); } cleanup: @@ -416,8 +234,64 @@ cleanup: return amount; } -// risc_assessment -// TODo lome: liste von Tagesschlüsseln rein und liste von tagesschlüsseln raus +//////////////////// +// SETUP DATA // +//////////////////// + +void setup_test_data() { + en_derive_period_identifier_key(&infectedPik, &infectedPeriodKey); + en_derive_period_identifier_key(&dummyPik, &dummyPeriodKey); + + int counter = 0; + for (int i = 0; i < EN_TEK_ROLLING_PERIOD; i++) { + record_t infectedRecord; + record_t dummyRecord; + en_derive_interval_identifier((ENIntervalIdentifier*)&infectedRecord.rolling_proximity_identifier, &infectedPik, + i); + en_derive_interval_identifier((ENIntervalIdentifier*)&dummyRecord.rolling_proximity_identifier, &dummyPik, i); + + for (int j = 0; j < CONFIG_TEST_RECORDS_PER_INTERVAL; j++) { + counter += CONFIG_TEST_INFECTED_RATE; + record_t* curRecord; + if (counter >= 100) { + counter -= 100; + curRecord = &infectedRecord; + } else { + curRecord = &dummyRecord; + } + curRecord->timestamp = i * EN_INTERVAL_LENGTH + j * (EN_INTERVAL_LENGTH / CONFIG_TEST_RECORDS_PER_INTERVAL); + int rc = add_record(curRecord); + if (rc) { + printk("err %d\n", rc); + } + } + } + +#define INFECTED_INTERVALS_COUNT 2000 + // setup our ordered array with infected RPIs + static infected_for_interval_ident_ctx_t infectedIntervals[INFECTED_INTERVALS_COUNT]; + + printk("Starting measurements with %d RPIs to seach and an infection rate of %d\n", INFECTED_INTERVALS_COUNT, + CONFIG_TEST_INFECTED_RATE); + + // measure_perf(get_number_of_infected_for_multiple_intervals_dumb, "dumb", infectedIntervals, + // INFECTED_INTERVALS_COUNT); + // measure_perf(get_number_of_infected_for_multiple_intervals_simple, "simple", infectedIntervals, + // INFECTED_INTERVALS_COUNT); + // measure_perf(get_number_of_infected_for_multiple_intervals_optimized, "optimized", infectedIntervals, + // INFECTED_INTERVALS_COUNT); + + // measure_perf(test_bloom_performance, "bloom", infectedIntervals, INFECTED_INTERVALS_COUNT); + + measure_perf(reverse_bloom_filter, "bloom reverse", infectedIntervals, INFECTED_INTERVALS_COUNT); +} + +int init_contacts() { +#if CONFIG_CONTACTS_PERFORM_RISC_CHECK_TEST + setup_test_data(); +#endif + return 0; +} /** * Check for a list of specified interval identifiers, whether they were probably met or not. diff --git a/src/contacts.h b/src/contacts.h index 7a92fdf..e9e99a9 100644 --- a/src/contacts.h +++ b/src/contacts.h @@ -56,9 +56,6 @@ int get_number_of_infected_for_period(ENPeriodKey* key, time_t timestamp); */ // int get_number_of_infected_for_multiple_intervals(infected_for_period_key_ctx_t* ctx, int count); -/** - * Setup fixed test data for storage. - */ -void setup_test_data(); +int init_contacts(); #endif diff --git a/src/ens/records.c b/src/ens/records.c index 4f9ddca..ad10efa 100644 --- a/src/ens/records.c +++ b/src/ens/records.c @@ -205,4 +205,4 @@ uint8_t ens_records_iterate_with_callback(record_iterator_t* iter, ens_record_it cb(NULL, userdata); // we call the callback one last time but with null data } return 0; -} \ No newline at end of file +} diff --git a/src/ens/records.h b/src/ens/records.h index a46a895..e8ec035 100644 --- a/src/ens/records.h +++ b/src/ens/records.h @@ -11,9 +11,21 @@ #include "storage.h" typedef struct record_iterator { + /** + * @internal + */ record_t current; + /** + * @internal + */ record_sequence_number_t sn_next; + /** + * @internal + */ record_sequence_number_t sn_end; // the last sn to include + /** + * @internal + */ uint8_t finished; } record_iterator_t; @@ -46,4 +58,4 @@ enum { // load_record function is thread safe?!) uint8_t ens_records_iterate_with_callback(record_iterator_t* iter, ens_record_iterator_cb_t cb, void* userdata); -#endif \ No newline at end of file +#endif diff --git a/src/ens/storage.c b/src/ens/storage.c index 7a2a0c2..0c61d35 100644 --- a/src/ens/storage.c +++ b/src/ens/storage.c @@ -22,8 +22,6 @@ static ens_fs_t ens_fs; // Information about currently stored contacts static stored_records_information_t record_information = {.oldest_contact = 0, .count = 0}; - - inline storage_id_t convert_sn_to_storage_id(record_sequence_number_t sn) { return (storage_id_t)(sn % CONFIG_ENS_MAX_CONTACTS); } @@ -110,6 +108,14 @@ int init_record_storage(bool clean) { return rc; } +int reset_record_storage() { + k_mutex_lock(&info_fs_lock, K_FOREVER); + record_information.count = 0; + record_information.oldest_contact = 0; + save_storage_information(); + k_mutex_unlock(&info_fs_lock); +} + int load_record(record_t* dest, record_sequence_number_t sn) { storage_id_t id = convert_sn_to_storage_id(sn); int rc = ens_fs_read(&ens_fs, id, dest); @@ -203,7 +209,7 @@ record_sequence_number_t get_oldest_sequence_number() { return record_information.oldest_contact; } -int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t *latest) { +int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t* latest) { int ret = -1; // we lock so that the interval is always valid (e.g. not overlapping) k_mutex_lock(&info_fs_lock, K_FOREVER); diff --git a/src/ens/storage.h b/src/ens/storage.h index fadaf03..8f6741b 100644 --- a/src/ens/storage.h +++ b/src/ens/storage.h @@ -29,6 +29,11 @@ typedef struct stored_records_information { */ int init_record_storage(bool clean); +/** + * Reset state of record storage. + */ +int reset_record_storage(); + /** * Loads the record with number sn into the destination struct * @param dest @@ -70,7 +75,6 @@ record_sequence_number_t get_oldest_sequence_number(); */ uint32_t get_num_records(); - -int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t *latest); +int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t* latest); #endif diff --git a/src/main.c b/src/main.c index 2c89b3b..75f193b 100644 --- a/src/main.c +++ b/src/main.c @@ -47,7 +47,12 @@ void main(void) { printk("init storage failed (err %d)\n", err); return; } - // setup_test_data(); + + err = init_contacts(); + if (err) { + printk("init contacts failed (err %d)\n", err); + return; + } #endif err = init_io(); diff --git a/zephyr/Kconfig b/zephyr/Kconfig index 2a39a31..307de82 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -5,7 +5,7 @@ config ENS_MAX_CONTACTS int "Max contacts in storage" default 2048 help - The maximum amount of contacts, that can be stored on the devies. Needs to be a power of 2! + The maximum amount of contacts, that can be stored on the devices. Needs to be a power of 2! endmenu @@ -21,3 +21,26 @@ config TEST_UNPACK_KEYS select TIMING_FUNCTIONS endmenu + +menu "Contact Tests" + +config TEST_RECORDS_PER_INTERVAL + int "Records per interval" + default 120 + help + Number of records per interval. + +config TEST_INFECTED_RATE + int "" + range 0 100 + default 1 + help + The rate for infected intervals in test data. + +config CONTACTS_PERFORM_RISC_CHECK_TEST + bool "" + default n + help + Flag for performing tests. + +endmenu