covid19/CovidBracelet
covid19
/
CovidBracelet
mirror of https://github.com/CovidBraceletPrj/CovidBracelet.git
1
0
Fork 0
Code Releases Activity

First refactorings for contacts and bloom api

This commit is contained in:
H1ghBre4k3r 2021-12-08 18:19:27 +01:00 committed by Patrick Rathje
parent 26d55cc91f
commit cd780b122d
8 changed files with 133 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

272
src/contacts.c
View File

@ -27,6 +27,8 @@
#include "ens/storage.h"
#include "exposure-notification.h"
// #define CONFIG_INTERVAL_SPREAD 100
void print_key(_ENBaseKey* key) {
for (int i = 0; i < sizeof(key->b); i++) {
printk("%02x", key->b[i]);
@ -54,31 +56,6 @@ int register_record(record_t* record) {
return rc;
}
// set size to test
#define TEST_DATA_SIZE 100000
/**
* Dumb implementation, where a single iterator is created for iterating over the entire flash.
*/
int get_number_of_infected_for_multiple_intervals_dumb(infected_for_interval_ident_ctx_t* ctx, int count) {
record_iterator_t iterator;
int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL);
if (rc) {
// there was a general error, so just do nothing
return rc;
}
while (!iterator.finished) {
for (int i = 0; i < count; i++) {
if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier,
sizeof(rolling_proximity_identifier_t)) == 0) {
ctx[i].infected++;
}
}
ens_records_iterator_next(&iterator);
}
return 0;
}
/**
* Simple implementation, where an iterator is created for every element in the passed arrray.
*/
@ -90,50 +67,17 @@ int get_number_of_infected_for_multiple_intervals_simple(infected_for_interval_i
// on error, skip this rpi
continue;
}
while (!iterator.finished) {
if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier,
record_t* current;
while (current = ens_records_iterator_next(&iterator)) {
if (memcmp(&(current->rolling_proximity_identifier), &ctx[i].interval_identifier,
sizeof(rolling_proximity_identifier_t)) == 0) {
ctx[i].infected++;
}
ens_records_iterator_next(&iterator);
}
}
return 0;
}
/**
* Optimized implementation, where overlapping search intervals for consecutive RPI are merged.
*/
int get_number_of_infected_for_multiple_intervals_optimized(infected_for_interval_ident_ctx_t* ctx, int count) {
record_iterator_t iterator;
int i = 0;
while (i < count) {
// determine start and end of iterator
int start = i;
int end = i;
while (end + 1 < count && ctx[end + 1].search_start <= ctx[end].search_end) {
end++;
}
// init iterator with start and end
int rc = ens_records_iterator_init_timerange(&iterator, &ctx[start].search_start, &ctx[end].search_end);
if (rc) {
goto end;
}
while (!iterator.finished) {
for (int j = start; j <= end; j++) {
if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[j].interval_identifier,
sizeof(iterator.current.rolling_proximity_identifier)) == 0) {
ctx[j].infected++;
}
}
ens_records_iterator_next(&iterator);
}
end:
i = end + 1;
}
return 0;
}
/**
* Fill the bloom filter with all stored records.
*/
@ -147,9 +91,9 @@ void fill_bloom_with_stored_records(bloom_filter_t* bloom) {
}
// fill bloom filter with records
while (!iterator.finished) {
bloom_add_record(bloom, &iterator.current.rolling_proximity_identifier);
ens_records_iterator_next(&iterator);
record_t* current;
while (current = ens_records_iterator_next(&iterator)) {
bloom_add_record(bloom, &(current->rolling_proximity_identifier));
}
}
@ -163,19 +107,8 @@ int64_t test_bloom_performance(infected_for_interval_ident_ctx_t* ctx, int count
return -1;
}
// Measure bloom creation time
timing_t start_time, end_time;
uint64_t total_cycles;
uint64_t total_ns;
start_time = timing_counter_get();
fill_bloom_with_stored_records(bloom);
end_time = timing_counter_get();
total_cycles = timing_cycles_get(&start_time, &end_time);
total_ns = timing_cycles_to_ns(total_cycles);
printk("\nbloom init took %lld ms\n\n", total_ns / 1000000);
// test bloom performance
for (int i = 0; i < count; i++) {
if (bloom_probably_has_record(bloom, &ctx[i].interval_identifier)) {
@ -191,72 +124,13 @@ int64_t test_bloom_performance(infected_for_interval_ident_ctx_t* ctx, int count
amount++;
}
}
printk("amount of infected records: %d\n", amount);
// int ret = get_number_of_infected_for_multiple_intervals_dumb(ctx, amount);
int ret = get_number_of_infected_for_multiple_intervals_simple(ctx, amount);
// int ret = get_number_of_infected_for_multiple_intervals_optimized(ctx, amount);
bloom_destroy(bloom);
return ret;
}
/**
* Fill bloom with passed RPIs and test flash records against it.
*/
int64_t test_bloom_reverse_performance(infected_for_interval_ident_ctx_t* ctx, int count) {
bloom_filter_t* bloom = bloom_init(count * 2);
if (!bloom) {
printk("bloom init failed\n");
return -1;
}
// Measure bloom creation time
timing_t start_time, end_time;
uint64_t total_cycles;
uint64_t total_ns;
start_time = timing_counter_get();
for (int i = 0; i < count; i++) {
bloom_add_record(bloom, &ctx[i].interval_identifier);
}
end_time = timing_counter_get();
total_cycles = timing_cycles_get(&start_time, &end_time);
total_ns = timing_cycles_to_ns(total_cycles);
printk("\nbloom init took %lld ms\n\n", total_ns / 1000000);
int64_t amount = 0;
for (int round = 0; round < TEST_DATA_SIZE / count; round++) {
record_iterator_t iterator;
int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL);
if (rc) {
printk("init iterator failed (err %d)\n", rc);
amount = rc;
goto cleanup;
}
while (!iterator.finished) {
if (bloom_probably_has_record(bloom, &iterator.current.rolling_proximity_identifier)) {
for (int i = 0; i < count; i++) {
if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier,
sizeof(iterator.current.rolling_proximity_identifier)) == 0) {
amount++;
break;
}
if (iterator.current.timestamp > ctx[i].search_end) {
break;
}
}
}
ens_records_iterator_next(&iterator);
}
printk("round %d\n", round);
}
cleanup:
bloom_destroy(bloom);
return amount;
}
////////////////////
// FILL TEST DATA //
////////////////////
@ -313,63 +187,6 @@ void measure_perf(test_func_t func,
printk("\n'%s' took %lld ms\n---------------------------\n", label, total_ns / 1000000);
}
////////////////////
// SETUP DATA //
////////////////////
void setup_test_data() {
en_derive_period_identifier_key(&infectedPik, &infectedPeriodKey);
en_derive_period_identifier_key(&dummyPik, &dummyPeriodKey);
// every 100th record is infected
#define INTERVAL_SPREAD 100
for (int i = 0; i < EN_TEK_ROLLING_PERIOD; i++) {
// create infected record
record_t infectedRecord;
infectedRecord.timestamp = i * EN_INTERVAL_LENGTH;
en_derive_interval_identifier((ENIntervalIdentifier*)&infectedRecord.rolling_proximity_identifier, &infectedPik,
i);
int rc;
if (i % INTERVAL_SPREAD == 0 && (rc = add_record(&infectedRecord))) {
printk("err %d\n", rc);
}
record_t dummyRecord;
en_derive_interval_identifier((ENIntervalIdentifier*)&dummyRecord.rolling_proximity_identifier, &dummyPik, i);
int spread = 1;
for (int j = 0; j < EN_INTERVAL_LENGTH / spread; j++) {
dummyRecord.timestamp = i * EN_INTERVAL_LENGTH + j * spread + 1;
if ((rc = add_record(&dummyRecord))) {
printk("err %d\n", rc);
}
}
printk("period %d\n", i);
}
#define INFECTED_INTERVALS_COUNT 2000
// setup our ordered array with infected RPIs
static infected_for_interval_ident_ctx_t infectedIntervals[INFECTED_INTERVALS_COUNT];
printk("Starting measurements with %d RPIs to seach and an infection rate of every %d. interval\n",
INFECTED_INTERVALS_COUNT, INTERVAL_SPREAD);
// measure_perf(get_number_of_infected_for_multiple_intervals_dumb, "dumb", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(get_number_of_infected_for_multiple_intervals_simple, "simple", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(get_number_of_infected_for_multiple_intervals_optimized, "optimized", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// TODO lome: prepare massive tests for bloom simple & reverse
// TODO lome: document everything
// measure_perf(test_bloom_performance, "bloom", infectedIntervals, INFECTED_INTERVALS_COUNT);
measure_perf(test_bloom_reverse_performance, "bloom reverse", infectedIntervals, INFECTED_INTERVALS_COUNT);
}
int reverse_bloom_filter(infected_for_interval_ident_ctx_t* ctx, int count) {
bloom_filter_t* bloom = bloom_init(count * 2);
if (!bloom) {
@ -393,21 +210,22 @@ int reverse_bloom_filter(infected_for_interval_ident_ctx_t* ctx, int count) {
goto cleanup;
}
while (!iterator.finished) {
if (bloom_probably_has_record(bloom, &iterator.current.rolling_proximity_identifier)) {
record_t* current;
while (current = ens_records_iterator_next(&iterator)) {
if (bloom_probably_has_record(bloom, &(current->rolling_proximity_identifier))) {
for (int i = 0; i < count; i++) {
if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier,
sizeof(iterator.current.rolling_proximity_identifier)) == 0) {
if (memcmp(&(current->rolling_proximity_identifier), &ctx[i].interval_identifier,
sizeof(current->rolling_proximity_identifier)) == 0) {
ctx[i].infected = 1;
amount++;
break;
}
if (iterator.current.timestamp > ctx[i].search_end) {
// TODO lome: maybe dont require orderred array, so this is not useful
if (current->timestamp > ctx[i].search_end) {
break;
}
}
}
ens_records_iterator_next(&iterator);
}
cleanup:
@ -416,8 +234,64 @@ cleanup:
return amount;
}
// risc_assessment
// TODo lome: liste von Tagesschlüsseln rein und liste von tagesschlüsseln raus
////////////////////
// SETUP DATA //
////////////////////
void setup_test_data() {
en_derive_period_identifier_key(&infectedPik, &infectedPeriodKey);
en_derive_period_identifier_key(&dummyPik, &dummyPeriodKey);
int counter = 0;
for (int i = 0; i < EN_TEK_ROLLING_PERIOD; i++) {
record_t infectedRecord;
record_t dummyRecord;
en_derive_interval_identifier((ENIntervalIdentifier*)&infectedRecord.rolling_proximity_identifier, &infectedPik,
i);
en_derive_interval_identifier((ENIntervalIdentifier*)&dummyRecord.rolling_proximity_identifier, &dummyPik, i);
for (int j = 0; j < CONFIG_TEST_RECORDS_PER_INTERVAL; j++) {
counter += CONFIG_TEST_INFECTED_RATE;
record_t* curRecord;
if (counter >= 100) {
counter -= 100;
curRecord = &infectedRecord;
} else {
curRecord = &dummyRecord;
}
curRecord->timestamp = i * EN_INTERVAL_LENGTH + j * (EN_INTERVAL_LENGTH / CONFIG_TEST_RECORDS_PER_INTERVAL);
int rc = add_record(curRecord);
if (rc) {
printk("err %d\n", rc);
}
}
}
#define INFECTED_INTERVALS_COUNT 2000
// setup our ordered array with infected RPIs
static infected_for_interval_ident_ctx_t infectedIntervals[INFECTED_INTERVALS_COUNT];
printk("Starting measurements with %d RPIs to seach and an infection rate of %d\n", INFECTED_INTERVALS_COUNT,
CONFIG_TEST_INFECTED_RATE);
// measure_perf(get_number_of_infected_for_multiple_intervals_dumb, "dumb", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(get_number_of_infected_for_multiple_intervals_simple, "simple", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(get_number_of_infected_for_multiple_intervals_optimized, "optimized", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(test_bloom_performance, "bloom", infectedIntervals, INFECTED_INTERVALS_COUNT);
measure_perf(reverse_bloom_filter, "bloom reverse", infectedIntervals, INFECTED_INTERVALS_COUNT);
}
int init_contacts() {
#if CONFIG_CONTACTS_PERFORM_RISC_CHECK_TEST
setup_test_data();
#endif
return 0;
}
/**
* Check for a list of specified interval identifiers, whether they were probably met or not.

5
src/contacts.h
View File

@ -56,9 +56,6 @@ int get_number_of_infected_for_period(ENPeriodKey* key, time_t timestamp);
*/
// int get_number_of_infected_for_multiple_intervals(infected_for_period_key_ctx_t* ctx, int count);
/**
* Setup fixed test data for storage.
*/
void setup_test_data();
int init_contacts();
#endif

2
src/ens/records.c
View File

@ -205,4 +205,4 @@ uint8_t ens_records_iterate_with_callback(record_iterator_t* iter, ens_record_it
cb(NULL, userdata); // we call the callback one last time but with null data
}
return 0;
}
}

14
src/ens/records.h
View File

@ -11,9 +11,21 @@
#include "storage.h"
typedef struct record_iterator {
/**
* @internal
*/
record_t current;
/**
* @internal
*/
record_sequence_number_t sn_next;
/**
* @internal
*/
record_sequence_number_t sn_end; // the last sn to include
/**
* @internal
*/
uint8_t finished;
} record_iterator_t;
@ -46,4 +58,4 @@ enum {
// load_record function is thread safe?!)
uint8_t ens_records_iterate_with_callback(record_iterator_t* iter, ens_record_iterator_cb_t cb, void* userdata);
#endif
#endif

12
src/ens/storage.c
View File

@ -22,8 +22,6 @@ static ens_fs_t ens_fs;
// Information about currently stored contacts
static stored_records_information_t record_information = {.oldest_contact = 0, .count = 0};
inline storage_id_t convert_sn_to_storage_id(record_sequence_number_t sn) {
return (storage_id_t)(sn % CONFIG_ENS_MAX_CONTACTS);
}
@ -110,6 +108,14 @@ int init_record_storage(bool clean) {
return rc;
}
int reset_record_storage() {
k_mutex_lock(&info_fs_lock, K_FOREVER);
record_information.count = 0;
record_information.oldest_contact = 0;
save_storage_information();
k_mutex_unlock(&info_fs_lock);
}
int load_record(record_t* dest, record_sequence_number_t sn) {
storage_id_t id = convert_sn_to_storage_id(sn);
int rc = ens_fs_read(&ens_fs, id, dest);
@ -203,7 +209,7 @@ record_sequence_number_t get_oldest_sequence_number() {
return record_information.oldest_contact;
}
int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t *latest) {
int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t* latest) {
int ret = -1;
// we lock so that the interval is always valid (e.g. not overlapping)
k_mutex_lock(&info_fs_lock, K_FOREVER);

8
src/ens/storage.h
View File

@ -29,6 +29,11 @@ typedef struct stored_records_information {
*/
int init_record_storage(bool clean);
/**
* Reset state of record storage.
*/
int reset_record_storage();
/**
* Loads the record with number sn into the destination struct
* @param dest
@ -70,7 +75,6 @@ record_sequence_number_t get_oldest_sequence_number();
*/
uint32_t get_num_records();
int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t *latest);
int get_sequence_number_interval(record_sequence_number_t* oldest, record_sequence_number_t* latest);
#endif

7
src/main.c
View File

@ -47,7 +47,12 @@ void main(void) {
printk("init storage failed (err %d)\n", err);
return;
}
// setup_test_data();
err = init_contacts();
if (err) {
printk("init contacts failed (err %d)\n", err);
return;
}
#endif
err = init_io();

25
zephyr/Kconfig
View File

@ -5,7 +5,7 @@ config ENS_MAX_CONTACTS
int "Max contacts in storage"
default 2048
help
The maximum amount of contacts, that can be stored on the devies. Needs to be a power of 2!
The maximum amount of contacts, that can be stored on the devices. Needs to be a power of 2!
endmenu
@ -21,3 +21,26 @@ config TEST_UNPACK_KEYS
select TIMING_FUNCTIONS
endmenu
menu "Contact Tests"
config TEST_RECORDS_PER_INTERVAL
int "Records per interval"
default 120
help
Number of records per interval.
config TEST_INFECTED_RATE
int ""
range 0 100
default 1
help
The rate for infected intervals in test data.
config CONTACTS_PERFORM_RISC_CHECK_TEST
bool ""
default n
help
Flag for performing tests.
endmenu