covid19/CovidBracelet
covid19/CovidBracelet
1
0
Fork 0
mirror of https://github.com/CovidBraceletPrj/CovidBracelet.git synced 2024-12-05 00:55:43 +01:00
Code Releases Activity

First refactorings for contacts and bloom api

Browse Source
This commit is contained in:
H1ghBre4k3r 2021-12-08 18:19:27 +01:00 committed by Patrick Rathje
parent 26d55cc91f
commit cd780b122d
8 changed files with 133 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

272
src/contacts.c
View File

@ -27,6 +27,8 @@
#include "ens/storage.h" #include "ens/storage.h"
#include "exposure-notification.h" #include "exposure-notification.h"
// #define CONFIG_INTERVAL_SPREAD 100
void print_key(_ENBaseKey* key) { void print_key(_ENBaseKey* key) {
for (int i = 0; i < sizeof(key->b); i++) { for (int i = 0; i < sizeof(key->b); i++) {
printk("%02x", key->b[i]); printk("%02x", key->b[i]);
@ -54,31 +56,6 @@ int register_record(record_t* record) {
return rc; return rc;
} }
// set size to test
#define TEST_DATA_SIZE 100000
/**
* Dumb implementation, where a single iterator is created for iterating over the entire flash.
*/
int get_number_of_infected_for_multiple_intervals_dumb(infected_for_interval_ident_ctx_t* ctx, int count) {
record_iterator_t iterator;
int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL);
if (rc) {
// there was a general error, so just do nothing
return rc;
}
while (!iterator.finished) {
for (int i = 0; i < count; i++) {
if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier,
sizeof(rolling_proximity_identifier_t)) == 0) {
ctx[i].infected++;
}
}
ens_records_iterator_next(&iterator);
}
return 0;
}
/** /**
* Simple implementation, where an iterator is created for every element in the passed arrray. * Simple implementation, where an iterator is created for every element in the passed arrray.
*/ */
@ -90,50 +67,17 @@ int get_number_of_infected_for_multiple_intervals_simple(infected_for_interval_i
// on error, skip this rpi // on error, skip this rpi
continue; continue;
} }
while (!iterator.finished) { record_t* current;
if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier, while (current = ens_records_iterator_next(&iterator)) {
if (memcmp(&(current->rolling_proximity_identifier), &ctx[i].interval_identifier,
sizeof(rolling_proximity_identifier_t)) == 0) { sizeof(rolling_proximity_identifier_t)) == 0) {
ctx[i].infected++; ctx[i].infected++;
} }
ens_records_iterator_next(&iterator);
} }
} }
return 0; return 0;
} }
/**
* Optimized implementation, where overlapping search intervals for consecutive RPI are merged.
*/
int get_number_of_infected_for_multiple_intervals_optimized(infected_for_interval_ident_ctx_t* ctx, int count) {
record_iterator_t iterator;
int i = 0;
while (i < count) {
// determine start and end of iterator
int start = i;
int end = i;
while (end + 1 < count && ctx[end + 1].search_start <= ctx[end].search_end) {
end++;
}
// init iterator with start and end
int rc = ens_records_iterator_init_timerange(&iterator, &ctx[start].search_start, &ctx[end].search_end);
if (rc) {
goto end;
}
while (!iterator.finished) {
for (int j = start; j <= end; j++) {
if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[j].interval_identifier,
sizeof(iterator.current.rolling_proximity_identifier)) == 0) {
ctx[j].infected++;
}
}
ens_records_iterator_next(&iterator);
}
end:
i = end + 1;
}
return 0;
}
/** /**
* Fill the bloom filter with all stored records. * Fill the bloom filter with all stored records.
*/ */
@ -147,9 +91,9 @@ void fill_bloom_with_stored_records(bloom_filter_t* bloom) {
} }
// fill bloom filter with records // fill bloom filter with records
while (!iterator.finished) { record_t* current;
bloom_add_record(bloom, &iterator.current.rolling_proximity_identifier); while (current = ens_records_iterator_next(&iterator)) {
ens_records_iterator_next(&iterator); bloom_add_record(bloom, &(current->rolling_proximity_identifier));
} }
} }
@ -163,19 +107,8 @@ int64_t test_bloom_performance(infected_for_interval_ident_ctx_t* ctx, int count
return -1; return -1;
} }
// Measure bloom creation time
timing_t start_time, end_time;
uint64_t total_cycles;
uint64_t total_ns;
start_time = timing_counter_get();
fill_bloom_with_stored_records(bloom); fill_bloom_with_stored_records(bloom);
end_time = timing_counter_get();
total_cycles = timing_cycles_get(&start_time, &end_time);
total_ns = timing_cycles_to_ns(total_cycles);
printk("\nbloom init took %lld ms\n\n", total_ns / 1000000);
// test bloom performance // test bloom performance
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
if (bloom_probably_has_record(bloom, &ctx[i].interval_identifier)) { if (bloom_probably_has_record(bloom, &ctx[i].interval_identifier)) {
@ -191,72 +124,13 @@ int64_t test_bloom_performance(infected_for_interval_ident_ctx_t* ctx, int count
amount++; amount++;
} }
} }
printk("amount of infected records: %d\n", amount);
// int ret = get_number_of_infected_for_multiple_intervals_dumb(ctx, amount);
int ret = get_number_of_infected_for_multiple_intervals_simple(ctx, amount); int ret = get_number_of_infected_for_multiple_intervals_simple(ctx, amount);
// int ret = get_number_of_infected_for_multiple_intervals_optimized(ctx, amount);
bloom_destroy(bloom); bloom_destroy(bloom);
return ret; return ret;
} }
/**
* Fill bloom with passed RPIs and test flash records against it.
*/
int64_t test_bloom_reverse_performance(infected_for_interval_ident_ctx_t* ctx, int count) {
bloom_filter_t* bloom = bloom_init(count * 2);
if (!bloom) {
printk("bloom init failed\n");
return -1;
}
// Measure bloom creation time
timing_t start_time, end_time;
uint64_t total_cycles;
uint64_t total_ns;
start_time = timing_counter_get();
for (int i = 0; i < count; i++) {
bloom_add_record(bloom, &ctx[i].interval_identifier);
}
end_time = timing_counter_get();
total_cycles = timing_cycles_get(&start_time, &end_time);
total_ns = timing_cycles_to_ns(total_cycles);
printk("\nbloom init took %lld ms\n\n", total_ns / 1000000);
int64_t amount = 0;
for (int round = 0; round < TEST_DATA_SIZE / count; round++) {
record_iterator_t iterator;
int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL);
if (rc) {
printk("init iterator failed (err %d)\n", rc);
amount = rc;
goto cleanup;
}
while (!iterator.finished) {
if (bloom_probably_has_record(bloom, &iterator.current.rolling_proximity_identifier)) {
for (int i = 0; i < count; i++) {
if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier,
sizeof(iterator.current.rolling_proximity_identifier)) == 0) {
amount++;
break;
}
if (iterator.current.timestamp > ctx[i].search_end) {
break;
}
}
}
ens_records_iterator_next(&iterator);
}
printk("round %d\n", round);
}
cleanup:
bloom_destroy(bloom);
return amount;
}
//////////////////// ////////////////////
// FILL TEST DATA // // FILL TEST DATA //
//////////////////// ////////////////////
@ -313,63 +187,6 @@ void measure_perf(test_func_t func,
printk("\n'%s' took %lld ms\n---------------------------\n", label, total_ns / 1000000); printk("\n'%s' took %lld ms\n---------------------------\n", label, total_ns / 1000000);
} }
////////////////////
// SETUP DATA //
////////////////////
void setup_test_data() {
en_derive_period_identifier_key(&infectedPik, &infectedPeriodKey);
en_derive_period_identifier_key(&dummyPik, &dummyPeriodKey);
// every 100th record is infected
#define INTERVAL_SPREAD 100
for (int i = 0; i < EN_TEK_ROLLING_PERIOD; i++) {
// create infected record
record_t infectedRecord;
infectedRecord.timestamp = i * EN_INTERVAL_LENGTH;
en_derive_interval_identifier((ENIntervalIdentifier*)&infectedRecord.rolling_proximity_identifier, &infectedPik,
i);
int rc;
if (i % INTERVAL_SPREAD == 0 && (rc = add_record(&infectedRecord))) {
printk("err %d\n", rc);
}
record_t dummyRecord;
en_derive_interval_identifier((ENIntervalIdentifier*)&dummyRecord.rolling_proximity_identifier, &dummyPik, i);
int spread = 1;
for (int j = 0; j < EN_INTERVAL_LENGTH / spread; j++) {
dummyRecord.timestamp = i * EN_INTERVAL_LENGTH + j * spread + 1;
if ((rc = add_record(&dummyRecord))) {
printk("err %d\n", rc);
}
}
printk("period %d\n", i);
}
#define INFECTED_INTERVALS_COUNT 2000
// setup our ordered array with infected RPIs
static infected_for_interval_ident_ctx_t infectedIntervals[INFECTED_INTERVALS_COUNT];
printk("Starting measurements with %d RPIs to seach and an infection rate of every %d. interval\n",
INFECTED_INTERVALS_COUNT, INTERVAL_SPREAD);
// measure_perf(get_number_of_infected_for_multiple_intervals_dumb, "dumb", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(get_number_of_infected_for_multiple_intervals_simple, "simple", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(get_number_of_infected_for_multiple_intervals_optimized, "optimized", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// TODO lome: prepare massive tests for bloom simple & reverse
// TODO lome: document everything
// measure_perf(test_bloom_performance, "bloom", infectedIntervals, INFECTED_INTERVALS_COUNT);
measure_perf(test_bloom_reverse_performance, "bloom reverse", infectedIntervals, INFECTED_INTERVALS_COUNT);
}
int reverse_bloom_filter(infected_for_interval_ident_ctx_t* ctx, int count) { int reverse_bloom_filter(infected_for_interval_ident_ctx_t* ctx, int count) {
bloom_filter_t* bloom = bloom_init(count * 2); bloom_filter_t* bloom = bloom_init(count * 2);
if (!bloom) { if (!bloom) {
@ -393,21 +210,22 @@ int reverse_bloom_filter(infected_for_interval_ident_ctx_t* ctx, int count) {
goto cleanup; goto cleanup;
} }
while (!iterator.finished) { record_t* current;
if (bloom_probably_has_record(bloom, &iterator.current.rolling_proximity_identifier)) { while (current = ens_records_iterator_next(&iterator)) {
if (bloom_probably_has_record(bloom, &(current->rolling_proximity_identifier))) {
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier, if (memcmp(&(current->rolling_proximity_identifier), &ctx[i].interval_identifier,
sizeof(iterator.current.rolling_proximity_identifier)) == 0) { sizeof(current->rolling_proximity_identifier)) == 0) {
ctx[i].infected = 1; ctx[i].infected = 1;
amount++; amount++;
break; break;
} }
if (iterator.current.timestamp > ctx[i].search_end) { // TODO lome: maybe dont require orderred array, so this is not useful
if (current->timestamp > ctx[i].search_end) {
break; break;
} }
} }
} }
ens_records_iterator_next(&iterator);
} }
cleanup: cleanup:
@ -416,8 +234,64 @@ cleanup:
return amount; return amount;
} }
// risc_assessment ////////////////////
// TODo lome: liste von Tagesschlüsseln rein und liste von tagesschlüsseln raus // SETUP DATA //
////////////////////
void setup_test_data() {
en_derive_period_identifier_key(&infectedPik, &infectedPeriodKey);
en_derive_period_identifier_key(&dummyPik, &dummyPeriodKey);
int counter = 0;
for (int i = 0; i < EN_TEK_ROLLING_PERIOD; i++) {
record_t infectedRecord;
record_t dummyRecord;
en_derive_interval_identifier((ENIntervalIdentifier*)&infectedRecord.rolling_proximity_identifier, &infectedPik,
i);
en_derive_interval_identifier((ENIntervalIdentifier*)&dummyRecord.rolling_proximity_identifier, &dummyPik, i);
for (int j = 0; j < CONFIG_TEST_RECORDS_PER_INTERVAL; j++) {
counter += CONFIG_TEST_INFECTED_RATE;
record_t* curRecord;
if (counter >= 100) {
counter -= 100;
curRecord = &infectedRecord;
} else {
curRecord = &dummyRecord;
}
curRecord->timestamp = i * EN_INTERVAL_LENGTH + j * (EN_INTERVAL_LENGTH / CONFIG_TEST_RECORDS_PER_INTERVAL);
int rc = add_record(curRecord);
if (rc) {
printk("err %d\n", rc);
}
}
}
#define INFECTED_INTERVALS_COUNT 2000
// setup our ordered array with infected RPIs
static infected_for_interval_ident_ctx_t infectedIntervals[INFECTED_INTERVALS_COUNT];
printk("Starting measurements with %d RPIs to seach and an infection rate of %d\n", INFECTED_INTERVALS_COUNT,
CONFIG_TEST_INFECTED_RATE);
// measure_perf(get_number_of_infected_for_multiple_intervals_dumb, "dumb", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(get_number_of_infected_for_multiple_intervals_simple, "simple", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(get_number_of_infected_for_multiple_intervals_optimized, "optimized", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(test_bloom_performance, "bloom", infectedIntervals, INFECTED_INTERVALS_COUNT);
measure_perf(reverse_bloom_filter, "bloom reverse", infectedIntervals, INFECTED_INTERVALS_COUNT);
}
int init_contacts() {
#if CONFIG_CONTACTS_PERFORM_RISC_CHECK_TEST
setup_test_data();
#endif
return 0;
}
/** /**
* Check for a list of specified interval identifiers, whether they were probably met or not. * Check for a list of specified interval identifiers, whether they were probably met or not.

5
src/contacts.h
View File

@ -56,9 +56,6 @@ int get_number_of_infected_for_period(ENPeriodKey* key, time_t timestamp);
*/ */
// int get_number_of_infected_for_multiple_intervals(infected_for_period_key_ctx_t* ctx, int count); // int get_number_of_infected_for_multiple_intervals(infected_for_period_key_ctx_t* ctx, int count);
/** int init_contacts();
* Setup fixed test data for storage.
*/
void setup_test_data();
#endif #endif

2
src/ens/records.c
View File

@ -205,4 +205,4 @@ uint8_t ens_records_iterate_with_callback(record_iterator_t* iter, ens_record_it
cb(NULL, userdata); // we call the callback one last time but with null data cb(NULL, userdata); // we call the callback one last time but with null data
} }
return 0; return 0;
} }

14
src/ens/records.h
View File

@ -11,9 +11,21 @@
#include "storage.h" #include "storage.h"
typedef struct record_iterator { typedef struct record_iterator {
/**
* @internal
*/
record_t current; record_t current;
/**
* @internal
*/
record_sequence_number_t sn_next; record_sequence_number_t sn_next;
/**
* @internal
*/
record_sequence_number_t sn_end; // the last sn to include record_sequence_number_t sn_end; // the last sn to include
/**
* @internal
*/
uint8_t finished; uint8_t finished;
} record_iterator_t; } record_iterator_t;
@ -46,4 +58,4 @@ enum {
// load_record function is thread safe?!) // load_record function is thread safe?!)
uint8_t ens_records_iterate_with_callback(record_iterator_t* iter, ens_record_iterator_cb_t cb, void* userdata); uint8_t ens_records_iterate_with_callback(record_iterator_t* iter, ens_record_iterator_cb_t cb, void* userdata);
#endif #endif

12
src/ens/storage.c
View File

@ -22,8 +22,6 @@ static ens_fs_t ens_fs;
// Information about currently stored contacts // Information about currently stored contacts
static stored_records_information_t record_information = {.oldest_contact = 0, .count = 0}; static stored_records_information_t record_information = {.oldest_contact = 0, .count = 0};
inline storage_id_t convert_sn_to_storage_id(record_sequence_number_t sn) { inline storage_id_t convert_sn_to_storage_id(record_sequence_number_t sn) {
return (storage_id_t)(sn % CONFIG_ENS_MAX_CONTACTS); return (storage_id_t)(sn % CONFIG_ENS_MAX_CONTACTS);
} }
@ -110,6 +108,14 @@ int init_record_storage(bool clean) {
return rc; return rc;
} }
int reset_record_storage() {
k_mutex_lock(&info_fs_lock, K_FOREVER);
record_information.count = 0;
record_information.oldest_contact = 0;
save_storage_information();
k_mutex_unlock(&info_fs_lock);
}
int load_record(record_t* dest, record_sequence_number_t sn) { int load_record(record_t* dest, record_sequence_number_t sn) {
storage_id_t id = convert_sn_to_storage_id(sn); storage_id_t id = convert_sn_to_storage_id(sn);
int rc = ens_fs_read(&ens_fs, id, dest); int rc = ens_fs_read(&ens_fs, id, dest);
@ -203,7 +209,7 @@ record_sequence_number_t get_oldest_sequence_number() {
return record_information.oldest_contact; return record_information.oldest_contact;
} }
int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t *latest) { int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t* latest) {
int ret = -1; int ret = -1;
// we lock so that the interval is always valid (e.g. not overlapping) // we lock so that the interval is always valid (e.g. not overlapping)
k_mutex_lock(&info_fs_lock, K_FOREVER); k_mutex_lock(&info_fs_lock, K_FOREVER);

8
src/ens/storage.h
View File

@ -29,6 +29,11 @@ typedef struct stored_records_information {
*/ */
int init_record_storage(bool clean); int init_record_storage(bool clean);
/**
* Reset state of record storage.
*/
int reset_record_storage();
/** /**
* Loads the record with number sn into the destination struct * Loads the record with number sn into the destination struct
* @param dest * @param dest
@ -70,7 +75,6 @@ record_sequence_number_t get_oldest_sequence_number();
*/ */
uint32_t get_num_records(); uint32_t get_num_records();
int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t* latest);
int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t *latest);
#endif #endif

7
src/main.c
View File

@ -47,7 +47,12 @@ void main(void) {
printk("init storage failed (err %d)\n", err); printk("init storage failed (err %d)\n", err);
return; return;
} }
// setup_test_data();
err = init_contacts();
if (err) {
printk("init contacts failed (err %d)\n", err);
return;
}
#endif #endif
err = init_io(); err = init_io();

25
zephyr/Kconfig
View File

@ -5,7 +5,7 @@ config ENS_MAX_CONTACTS
int "Max contacts in storage" int "Max contacts in storage"
default 2048 default 2048
help help
The maximum amount of contacts, that can be stored on the devies. Needs to be a power of 2! The maximum amount of contacts, that can be stored on the devices. Needs to be a power of 2!
endmenu endmenu
@ -21,3 +21,26 @@ config TEST_UNPACK_KEYS
select TIMING_FUNCTIONS select TIMING_FUNCTIONS
endmenu endmenu
menu "Contact Tests"
config TEST_RECORDS_PER_INTERVAL
int "Records per interval"
default 120
help
Number of records per interval.
config TEST_INFECTED_RATE
int ""
range 0 100
default 1
help
The rate for infected intervals in test data.
config CONTACTS_PERFORM_RISC_CHECK_TEST
bool ""
default n
help
Flag for performing tests.
endmenu