Browse Source

Merge branch 'master' into feature/num_channels

pull/138/head
Teque5 2 months ago
parent
commit
96c205ac0b
  1. 6
      README.md
  2. 27
      sigmf/sigmffile.py
  3. 97
      sigmf/validate.py
  4. 124
      tests/test_validation.py
  5. 29
      tests/testdata.py

6
README.md

@ -62,6 +62,12 @@ handle.get_captures() # returns list of 'captures' dictionaries
handle.get_annotations() # returns list of all annotations
```
### Verify SigMF dataset integrity & compliance
```bash
sigmf_validate example.sigmf
```
### Load a SigMF dataset; read its annotation, metadata, and samples
```python

27
sigmf/sigmffile.py

@ -290,8 +290,13 @@ class SigMFFile():
Calculates the hash of the data file and adds it to the global section.
Also returns a string representation of the hash.
"""
the_hash = sigmf_hash.calculate_sha512(self.data_file)
return self.set_global_field(self.HASH_KEY, the_hash)
old_hash = self.get_global_field(self.HASH_KEY)
new_hash = sigmf_hash.calculate_sha512(self.data_file)
if old_hash:
if old_hash != new_hash:
raise SigMFFileError('Calculated file hash does not match associated metadata.')
return self.set_global_field(self.HASH_KEY, new_hash)
def set_data_file(self, data_file):
"""
@ -427,7 +432,6 @@ class SigMFFile():
raise IOError('Number of samples must be greater than zero, or -1 for all samples.')
elif start_index + count > self.sample_count:
raise IOError("Cannot read beyond EOF.")
if self.data_file is None:
raise SigMFFileError("No signal data file has been associated with the metadata.")
@ -560,13 +564,22 @@ def fromarchive(archive_path, dir=None):
def fromfile(filename, skip_checksum=False):
"""
'''
Creates and returns a returns a SigMFFile instance with metadata loaded from the specified file.
The filename may be that of either a sigmf-meta file, a sigmf-data file, or a sigmf archive.
Keyword arguments:
filename -- the SigMF filename
"""
Parameters
----------
filename: str
Path for SigMF dataset with or without extension.
skip_checksum: bool, default False
When True will not read entire dataset to caculate hash.
Returns
-------
object
SigMFFile object with dataset & metadata.
'''
fns = get_sigmf_filenames(filename)
meta_fn = fns['meta_fn']
data_fn = fns['data_fn']

97
sigmf/validate.py

@ -20,11 +20,13 @@
'''SigMF Validator'''
from . import schema
import json
class ValidationResult(object):
" Amends a validation result (True, False) with an error string. "
'''Amends a validation result (True, False) with an error string.'''
def __init__(self, value=False, error=None):
self.error = error
self.value = value
@ -40,46 +42,49 @@ class ValidationResult(object):
def match_type(value, our_type):
" Checks if value matches our_type "
'''Checks if value matches our_type'''
return value is None or {
'string': lambda x: isinstance(x, str) or isinstance(x, unicode), # FIXME make py3k compatible
'string': lambda x: isinstance(x, str),
'uint': lambda x: isinstance(x, int) and x >= 0,
'double': lambda x: isinstance(x, float) or isinstance(x, int),
}[our_type](value)
def validate_key(data_value, ref_dict, section, key):
"""
'''
Validates a key/value pair entry in a chunk.
Parameters:
data_value -- The value. May be None.
ref_dict -- A dictionary containing reference information.
section -- The section in which this key/value pair is stored ("global",
etc.). This is for better error reporting only.
key -- The key of this key/value pair ("core:Version", etc.). This is for
better error reporting only.
"""
Parameters
----------
data_value
Valid or invaid entry in metadata for validation.
ref_dict: dict
A dictionary containing reference information.
section: str
The section in which this key/value pair is stored ("global", etc.).
This is for better error reporting only.
key: str
The key of this key/value pair ("core:Version", etc.). This is for better error reporting only.
Returns
-------
True or ValidationResult
'''
if ref_dict.get('required') and data_value is None:
return ValidationResult(
False,
"In Section `{sec}', an entry is missing required key `{key}'.".format(
sec=section,
key=key
))
f'In Section `{section}`, an entry is missing required key `{key}`'
)
if 'type' in ref_dict and not match_type(data_value, ref_dict["type"]):
return ValidationResult(
False,
"In Section `{sec}', entry `{key}={value}' is not of type `{type}'.".format(
sec=section,
value=data_value,
key=key,
type=ref_dict["type"]
))
f'In Section `{section}`, entry `{key}={data_value}` is not of type `{ref_dict["type"]}`'
)
# if "py_re" in ref_dict and not re.match(ref_dict["py_re"], data_value):
# return ValidationResult(False, "regex fail")
return True
def validate_key_throw(*args):
"""
Like validate_key, but throws a ValueError when invalid.
@ -89,12 +94,10 @@ def validate_key_throw(*args):
raise ValueError(str(validation_result))
return validation_result
def validate_section_dict(data_section, ref_section, section):
if not isinstance(data_section, dict):
return ValidationResult(
False,
"Section `{sec}' exists, but is not a dict.".format(sec=section)
)
return ValidationResult(False, f'Section `{section}` exists, but is not a dict.')
key_validation_results = (
validate_key(
data_section.get(key),
@ -107,22 +110,20 @@ def validate_section_dict(data_section, ref_section, section):
return result
return True
def validate_section_dict_list(data_section, ref_section, section):
if not isinstance(data_section, list) or \
not all((isinstance(x, dict) for x in data_section)):
return ValidationResult(
False,
"Section `{sec}' exists, but is not a list of dicts.".format(sec=section)
)
sort_key = ref_section.get("sort")
return ValidationResult(False, f'Section `{section}` exists, but is not a list of dicts.')
sort_key = ref_section.get('sort')
last_index = (data_section[0].get(sort_key, 0) if len(data_section) else 0) - 1
for chunk in data_section:
key_validation_results = (
validate_key(
chunk.get(key),
ref_section["keys"].get(key),
ref_section['keys'].get(key),
section, key
) for key in ref_section["keys"]
) for key in ref_section['keys']
)
for result in key_validation_results:
if not bool(result):
@ -131,46 +132,41 @@ def validate_section_dict_list(data_section, ref_section, section):
if this_index <= last_index:
return ValidationResult(
False,
"In Section `{sec}', chunk starting at index {idx} "\
"is ahead of previous section.".format(
sec=section, idx=this_index
f'In Section `{section}`, chunk starting at index {this_index} is ahead of previous section.'
)
)
last_index = this_index
return True
def validate_section(data_section, ref_section, section):
"""
Validates a section (e.g. global, capture, etc.).
"""
'''Validates a section (e.g. global, capture, etc.).'''
if ref_section["required"] and data_section is None:
return ValidationResult(
False,
"Required section `{sec}' not found.".format(sec=section)
)
return ValidationResult(False, f'Required section `{section}` not found.')
return {
'dict': validate_section_dict,
'dict_list': validate_section_dict_list,
}[ref_section["type"]](data_section, ref_section, section)
}[ref_section['type']](data_section, ref_section, section)
def validate(data, ref=None):
if ref is None:
from . import schema
ref = schema.get_schema()
for result in (validate_section(data.get(key), ref.get(key), key) for key in ref):
if not result:
return result
return True
def main():
import argparse
import logging
import warnings
from . import sigmffile
from . import error
parser = argparse.ArgumentParser(description='Validate SigMF Archive or file pair against JSON schema.')
parser.add_argument('filename', help='SigMF path (extension optional).')
parser.add_argument('--skip-checksum', action='store_true', help='Skip reading dataset to validate checksum.')
parser.add_argument('-v', '--verbose', action='count', default=0)
args = parser.parse_args()
@ -183,7 +179,11 @@ def main():
logging.basicConfig(level=level_lut[min(args.verbose, 2)])
try:
signal = sigmffile.fromfile(args.filename)
signal = sigmffile.fromfile(args.filename, skip_checksum=args.skip_checksum)
except error.SigMFFileError as err:
# this happens if checksum fails
log.error(err)
exit(1)
except IOError as err:
log.error(err)
log.error('Unable to read SigMF, bad path?')
@ -199,5 +199,6 @@ def main():
log.error(result)
exit(1)
if __name__ == '__main__':
main()

124
tests/test_validation.py

@ -18,99 +18,61 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from sigmf.sigmffile import SigMFFile
from sigmf import __version__
import tempfile
import sigmf
from sigmf import SigMFFile
from .testdata import TEST_FLOAT32_DATA, TEST_METADATA
def test_valid_data():
assert SigMFFile(TEST_METADATA).validate()
MD_VALID = """
{
"global": {
"core:datatype": "cf32",
"core:offset": 0,
"core:version": "X.X.X",
"core:license": "CC0",
"core:datetime": "foo",
"core:url": "foo",
"core:sha512": "69a014f8855058d25b30b1caf4f9d15bb7b38afa26e28b24a63545734e534a861d658eddae1dbc666b33ca1d18c1ca85722f1f2f010703a7dbbef08189a1d0e5"
},
"captures": [
{
"core:sample_start": 0,
"core:sample_rate": 10000000,
"core:frequency": 950000000,
"core:datetime": "2017-02-01T11:33:17,053240428+01:00",
"uhd:gain": 34
},
{
"core:sample_start": 100000,
"core:frequency": 950000000,
"uhd:gain": 54
}
],
"annotations": [
{
"core:sample_start": 100000,
"core:sample_count": 120000,
"core:comment": "Some textual comment about stuff happenning",
"gsm:xxx": 111
}
]
}
"""
MD_VALID = MD_VALID.replace("X.X.X", __version__)
MD_INVALID_SEQUENCE_CAP = """
{
"global": {
"core:datatype": "cf32"
},
"captures": [
def test_invalid_capture_sequence():
'''metadata must have captures in order'''
invalid_metadata = dict(TEST_METADATA)
invalid_metadata[SigMFFile.CAPTURE_KEY] = [
{
"core:sample_start": 10
SigMFFile.START_INDEX_KEY: 10
},
{
"core:sample_start": 9
}
],
"annotations": [
{
"core:sample_start": 100000,
"core:sample_count": 120000,
"core:comment": "stuff"
SigMFFile.START_INDEX_KEY: 9
}
]
}
"""
assert not SigMFFile(invalid_metadata).validate()
MD_INVALID_SEQUENCE_ANN = """
{
"global": {
"core:datatype": "cf32"
},
"captures": [
{
"core:sample_start": 0
}
],
"annotations": [
def test_invalid_annotation_sequence():
'''metadata must have annotations in order'''
invalid_metadata = dict(TEST_METADATA)
invalid_metadata[SigMFFile.ANNOTATION_KEY] = [
{
"core:sample_start": 2,
"core:sample_count": 120000,
"core:comment": "stuff"
SigMFFile.START_INDEX_KEY: 2,
SigMFFile.LENGTH_INDEX_KEY: 120000,
SigMFFile.COMMENT_KEY: "stuff"
},
{
"core:sample_start": 1,
"core:sample_count": 120000,
"core:comment": "stuff"
SigMFFile.START_INDEX_KEY: 1,
SigMFFile.LENGTH_INDEX_KEY: 120000,
SigMFFile.COMMENT_KEY: "stuff"
}
]
}
"""
def test_valid_data():
assert SigMFFile(MD_VALID).validate()
assert not SigMFFile(invalid_metadata).validate()
def test_invalid_capture_seq():
assert not SigMFFile(MD_INVALID_SEQUENCE_CAP).validate()
assert not SigMFFile(MD_INVALID_SEQUENCE_ANN).validate()
def test_invalid_hash():
'''metadata must have captures in order'''
invalid_metadata = dict(TEST_METADATA)
invalid_metadata[SigMFFile.GLOBAL_KEY][SigMFFile.HASH_KEY] = 'derp'
temp_path = tempfile.mkstemp()[1]
TEST_FLOAT32_DATA.tofile(temp_path)
try:
SigMFFile(metadata=invalid_metadata, data_file=temp_path)
except sigmf.error.SigMFFileError:
# this should occur since the hash is wrong
pass
else:
# this only happens if no error occurs
assert False

29
tests/testdata.py

@ -23,28 +23,19 @@
import numpy as np
from sigmf import __version__
from sigmf import SigMFFile
TEST_FLOAT32_DATA = np.arange(16, dtype=np.float32)
TEST_METADATA = {
"global": {
"core:datatype": "rf32_le",
"core:num_channels": 1,
"core:offset": 0,
"core:sha512": "f4984219b318894fa7144519185d1ae81ea721c6113243a52b51e444512a39d74cf41a4cec3c5d000bd7277cc71232c04d7a946717497e18619bdbe94bfeadd6",
"core:version": __version__
},
"captures": [
{
"core:sample_start": 0
}
],
"annotations": [
{
"core:sample_count": 16,
"core:sample_start": 0
}
]
SigMFFile.ANNOTATION_KEY: [{SigMFFile.LENGTH_INDEX_KEY: 16, SigMFFile.START_INDEX_KEY: 0}],
SigMFFile.CAPTURE_KEY: [{SigMFFile.START_INDEX_KEY: 0}],
SigMFFile.GLOBAL_KEY: {
SigMFFile.DATATYPE_KEY: 'rf32_le',
SigMFFile.HASH_KEY: 'f4984219b318894fa7144519185d1ae81ea721c6113243a52b51e444512a39d74cf41a4cec3c5d000bd7277cc71232c04d7a946717497e18619bdbe94bfeadd6',
SigMFFile.NUM_CHANNELS_KEY: 1,
SigMFFile.START_OFFSET_KEY: 0,
SigMFFile.VERSION_KEY: __version__
}
}

Loading…
Cancel
Save