diff --git a/src/pynxtools/data/NXtest.nxdl.xml b/src/pynxtools/data/NXtest.nxdl.xml index 8695a20c9..2d6547698 100644 --- a/src/pynxtools/data/NXtest.nxdl.xml +++ b/src/pynxtools/data/NXtest.nxdl.xml @@ -28,9 +28,13 @@ + A dummy entry for a float value. + + A dummy entry for a number value. + A dummy entry for a bool value. @@ -53,6 +57,12 @@ + + + + + + diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index 71d4a4b9f..3cc5ce652 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -24,7 +24,7 @@ from datetime import datetime, timezone from enum import Enum from functools import lru_cache -from typing import Any, Callable, List, Optional, Tuple, Union +from typing import Any, Callable, List, Optional, Tuple, Union, Sequence import h5py import lxml.etree as ET @@ -81,11 +81,11 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar if log_type == ValidationProblem.UnitWithoutDocumentation: logger.warning( - f"The unit, {path} = {value}, is being written but has no documentation" + f"The unit, {path} = {value}, is being written but has no documentation." ) elif log_type == ValidationProblem.InvalidEnum: logger.warning( - f"The value at {path} should be on of the following strings: {value}" + f"The value at {path} should be one of the following: {value}" ) elif log_type == ValidationProblem.MissingRequiredGroup: logger.warning(f"The required group, {path}, hasn't been supplied.") @@ -96,7 +96,7 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar ) elif log_type == ValidationProblem.InvalidType: logger.warning( - f"The value at {path} should be one of: {value}" + f"The value at {path} should be one of the following Python types: {value}" f", as defined in the NXDL as {args[0] if args else ''}." ) elif log_type == ValidationProblem.InvalidDatetime: @@ -114,7 +114,10 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar f"Expected a group at {path} but found a field or attribute." ) elif log_type == ValidationProblem.MissingDocumentation: - logger.warning(f"Field {path} written without documentation.") + if "@" in path.rsplit("/")[-1]: + logger.warning(f"Attribute {path} written without documentation.") + else: + logger.warning(f"Field {path} written without documentation.") elif log_type == ValidationProblem.MissingUnit: logger.warning( f"Field {path} requires a unit in the unit category {value}." @@ -122,7 +125,7 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar elif log_type == ValidationProblem.MissingRequiredAttribute: logger.warning(f'Missing attribute: "{path}"') elif log_type == ValidationProblem.UnitWithoutField: - logger.warning(f"Unit {path} in dataset without its field {value}") + logger.warning(f"Unit {path} in dataset without its field {value}.") elif log_type == ValidationProblem.AttributeForNonExistingField: logger.warning( f"There were attributes set for the field {path}, " @@ -158,9 +161,9 @@ def collect_and_log( "NX_ANY", ): return - if self.logging: + if self.logging and path + str(log_type) + str(value) not in self.data: self._log(path, log_type, value, *args, **kwargs) - self.data.add(path) + self.data.add(path + str(log_type) + str(value)) def has_validation_problems(self): """Returns True if there were any validation problems.""" @@ -215,7 +218,6 @@ def get_nxdl_name_for(xml_elem: ET._Element) -> Optional[str]: The name of the element. None if the xml element has no name or type attribute. """ - """""" if "name" in xml_elem.attrib: return xml_elem.attrib["name"] if "type" in xml_elem.attrib: @@ -575,86 +577,47 @@ def is_value_valid_element_of_enum(value, elist) -> Tuple[bool, list]: return True, [] -NUMPY_FLOAT_TYPES = (np.half, np.float16, np.single, np.double, np.longdouble) -NUMPY_INT_TYPES = (np.short, np.intc, np.int_) -NUMPY_UINT_TYPES = (np.ushort, np.uintc, np.uint) -# np int for np version 1.26.0 -np_int = ( - np.intc, - np.int_, - np.intp, - np.int8, - np.int16, - np.int32, - np.int64, - np.uint8, - np.uint16, - np.uint32, - np.uint64, - np.unsignedinteger, - np.signedinteger, -) -np_float = (np.float16, np.float32, np.float64, np.floating) -np_bytes = (np.bytes_, np.byte, np.ubyte) -np_char = (np.str_, np.char.chararray, *np_bytes) -np_bool = (np.bool_,) -np_complex = (np.complex64, np.complex128, np.cdouble, np.csingle) +nx_char = (str, np.character) +nx_int = (int, np.integer) +nx_float = (float, np.floating) +nx_number = nx_int + nx_float + NEXUS_TO_PYTHON_DATA_TYPES = { "ISO8601": (str,), - "NX_BINARY": ( - bytes, - bytearray, - np.ndarray, - *np_bytes, - ), - "NX_BOOLEAN": (bool, np.ndarray, *np_bool), - "NX_CHAR": (str, np.ndarray, *np_char), + "NX_BINARY": (bytes, bytearray, np.bytes_), + "NX_BOOLEAN": (bool, np.bool_), + "NX_CHAR": nx_char, "NX_DATE_TIME": (str,), - "NX_FLOAT": (float, np.ndarray, *np_float), - "NX_INT": (int, np.ndarray, *np_int), - "NX_UINT": (np.ndarray, np.unsignedinteger), - "NX_NUMBER": ( - int, - float, - np.ndarray, - *np_int, - *np_float, - dict, - ), - "NX_POSINT": ( - int, - np.ndarray, - np.signedinteger, - ), # > 0 is checked in is_valid_data_field() - "NX_COMPLEX": (complex, np.ndarray, *np_complex), - "NXDL_TYPE_UNAVAILABLE": (str,), # Defaults to a string if a type is not provided. - "NX_CHAR_OR_NUMBER": ( - str, - int, - float, - np.ndarray, - *np_char, - *np_int, - *np_float, - dict, + "NX_FLOAT": nx_float, + "NX_INT": nx_int, + "NX_UINT": (np.unsignedinteger,), + "NX_NUMBER": nx_number, + "NX_POSINT": nx_int, # > 0 is checked in is_valid_data_field() + "NX_COMPLEX": ( + complex, + np.complexfloating, ), + "NX_CHAR_OR_NUMBER": nx_char + nx_number, + "NXDL_TYPE_UNAVAILABLE": ( + nx_char, + ), # Defaults to a string if a type is not provided. } -def check_all_children_for_callable(objects: list, check: Callable, *args) -> bool: - """Checks whether all objects in list are validated by given callable.""" - for obj in objects: - if not check(obj, *args): - return False +def check_all_children_for_callable( + objects: Union[list, np.ndarray], check_function: Optional[Callable] = None, *args +) -> bool: + """Checks whether all objects in list or numpy array are validated + by given callable and types. + """ + if not isinstance(objects, np.ndarray): + objects = np.array(objects) - return True + return all([check_function(o, *args) for o in objects.flat]) def is_valid_data_type(value, accepted_types): """Checks whether the given value or its children are of an accepted type.""" - if not isinstance(value, list): - return isinstance(value, accepted_types) - return check_all_children_for_callable(value, isinstance, accepted_types) @@ -662,59 +625,55 @@ def is_positive_int(value): """Checks whether the given value or its children are positive.""" def is_greater_than(num): - return num.flat[0] > 0 if isinstance(num, np.ndarray) else num > 0 - - if isinstance(value, list): - return check_all_children_for_callable(value, is_greater_than) + return num > 0 - return value.flat[0] > 0 if isinstance(value, np.ndarray) else value > 0 + return check_all_children_for_callable( + objects=value, check_function=is_greater_than + ) -def convert_str_to_bool_safe(value): +def convert_str_to_bool_safe(value: str) -> Optional[bool]: """Only returns True or False if someone mistakenly adds quotation marks but mean a bool. - For everything else it returns None. + For everything else it raises a ValueError. """ if value.lower() == "true": return True if value.lower() == "false": return False - return None + raise ValueError(f"Could not interpret string '{value}' as boolean.") -def is_valid_data_field(value, nxdl_type, path): - # todo: Check this funciton and wtire test for it. It seems the funciton is not +def is_valid_data_field(value: Any, nxdl_type: str, nxdl_enum: list, path: str) -> Any: + # todo: Check this function and write test for it. It seems the function is not # working as expected. - """Checks whether a given value is valid according to what is defined in the NXDL. + """Checks whether a given value is valid according to the type defined in the NXDL. - This function will also try to convert typical types, for example int to float, - and return the successful conversion. + This function only tries to convert boolean value in str format (e.g. "true" ) to + python Boolean (True). In case, it fails to convert, it raises an Exception. - If it fails to convert, it raises an Exception. - - Returns two values: first, boolean (True if the the value corresponds to nxdl_type, - False otherwise) and second, result of attempted conversion or the original value - (if conversion is not needed or impossible) + Return: + value: the possibly converted data value """ - accepted_types = NEXUS_TO_PYTHON_DATA_TYPES[nxdl_type] - output_value = value + accepted_types = NEXUS_TO_PYTHON_DATA_TYPES[nxdl_type] + # Do not count the dict as it represents a link value if not isinstance(value, dict) and not is_valid_data_type(value, accepted_types): - try: - if accepted_types[0] is bool and isinstance(value, str): + # try to convert string to bool + if accepted_types[0] is bool and isinstance(value, str): + try: value = convert_str_to_bool_safe(value) - if value is None: - raise ValueError - output_value = accepted_types[0](value) - except ValueError: + except (ValueError, TypeError): + collector.collect_and_log( + path, ValidationProblem.InvalidType, accepted_types, nxdl_type + ) + else: collector.collect_and_log( path, ValidationProblem.InvalidType, accepted_types, nxdl_type ) - return False, value if nxdl_type == "NX_POSINT" and not is_positive_int(value): collector.collect_and_log(path, ValidationProblem.IsNotPosInt, value) - return False, value if nxdl_type in ("ISO8601", "NX_DATE_TIME"): iso8601 = re.compile( @@ -724,9 +683,16 @@ def is_valid_data_field(value, nxdl_type, path): results = iso8601.search(value) if results is None: collector.collect_and_log(path, ValidationProblem.InvalidDatetime, value) - return False, value - return True, output_value + # Check enumeration + if nxdl_enum is not None and value not in nxdl_enum: + collector.collect_and_log( + path, + ValidationProblem.InvalidEnum, + nxdl_enum, + ) + + return value @lru_cache(maxsize=None) diff --git a/src/pynxtools/dataconverter/nexus_tree.py b/src/pynxtools/dataconverter/nexus_tree.py index bbba22c09..77349df49 100644 --- a/src/pynxtools/dataconverter/nexus_tree.py +++ b/src/pynxtools/dataconverter/nexus_tree.py @@ -761,7 +761,7 @@ class NexusEntity(NexusNode): type: Literal["field", "attribute"] unit: Optional[NexusUnitCategory] = None dtype: NexusType = "NX_CHAR" - items: Optional[List[str]] = None + items: Optional[List[Any]] = None shape: Optional[Tuple[Optional[int], ...]] = None def _set_type(self): @@ -790,14 +790,23 @@ def _set_items(self): based on the values in the inheritance chain. The first vale found is used. """ - if not self.dtype == "NX_CHAR": - return for elem in self.inheritance: enum = elem.find(f"nx:enumeration", namespaces=namespaces) if enum is not None: self.items = [] for items in enum.findall(f"nx:item", namespaces=namespaces): - self.items.append(items.attrib["value"]) + value = items.attrib["value"] + if value[0] == "[" and value[-1] == "]": + import ast + + try: + self.items.append(ast.literal_eval(value)) + except (ValueError, SyntaxError): + raise Exception( + f"Error parsing enumeration item in the provided NXDL: {value}" + ) + else: + self.items.append(value) return def _set_shape(self): diff --git a/src/pynxtools/dataconverter/readers/example/reader.py b/src/pynxtools/dataconverter/readers/example/reader.py index fefe37f5c..7e368a264 100644 --- a/src/pynxtools/dataconverter/readers/example/reader.py +++ b/src/pynxtools/dataconverter/readers/example/reader.py @@ -58,7 +58,11 @@ def read( # outputs with --generate-template for a provided NXDL file if ( k.startswith("/ENTRY[entry]/required_group") - or k == "/ENTRY[entry]/optional_parent/req_group_in_opt_group" + or k + in ( + "/ENTRY[entry]/optional_parent/req_group_in_opt_group", + "/ENTRY[entry]/NXODD_name[nxodd_name]/anamethatRENAMES[anamethatrenames]", + ) or k.startswith("/ENTRY[entry]/OPTIONAL_group") ): continue diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 4b599b43a..4c4d35cc8 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -20,7 +20,7 @@ from collections import defaultdict from functools import reduce from operator import getitem -from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import h5py import lxml.etree as ET @@ -165,14 +165,14 @@ def best_namefit_of(name: str, keys: Iterable[str]) -> Optional[str]: def validate_dict_against( - appdef: str, mapping: Mapping[str, Any], ignore_undocumented: bool = False + appdef: str, mapping: MutableMapping[str, Any], ignore_undocumented: bool = False ) -> Tuple[bool, List]: """ - Validates a mapping against the NeXus tree for applicationd definition `appdef`. + Validates a mapping against the NeXus tree for application definition `appdef`. Args: appdef (str): The appdef name to validate against. - mapping (Mapping[str, Any]): + mapping (MutableMapping[str, Any]): The mapping containing the data to validate. This should be a dict of `/` separated paths. Attributes are denoted with `@` in front of the last element. @@ -248,6 +248,14 @@ def check_nxdata(): prev_path=prev_path, ) + # check NXdata attributes + for attr in ("signal", "auxiliary_signals", "axes"): + handle_attribute( + node.search_add_child_for(attr), + keys, + prev_path=prev_path, + ) + for i, axis in enumerate(axes): if axis == ".": continue @@ -392,17 +400,17 @@ def _follow_link( def handle_field(node: NexusNode, keys: Mapping[str, Any], prev_path: str): full_path = remove_from_not_visited(f"{prev_path}/{node.name}") variants = get_variations_of(node, keys) - if not variants: - if node.optionality == "required" and node.type in missing_type_err: - collector.collect_and_log( - full_path, missing_type_err.get(node.type), None - ) - + if ( + not variants + and node.optionality == "required" + and node.type in missing_type_err + ): + collector.collect_and_log(full_path, missing_type_err.get(node.type), None) return for variant in variants: if node.optionality == "required" and isinstance(keys[variant], Mapping): - # Check if all fields in the dict are actual attributes (startwith @) + # Check if all fields in the dict are actual attributes (startswith @) all_attrs = True for entry in keys[variant]: if not entry.startswith("@"): @@ -422,21 +430,13 @@ def handle_field(node: NexusNode, keys: Mapping[str, Any], prev_path: str): continue # Check general validity - is_valid_data_field( - mapping[f"{prev_path}/{variant}"], node.dtype, f"{prev_path}/{variant}" + mapping[f"{prev_path}/{variant}"] = is_valid_data_field( + mapping[f"{prev_path}/{variant}"], + node.dtype, + node.items, + f"{prev_path}/{variant}", ) - # Check enumeration - if ( - node.items is not None - and mapping[f"{prev_path}/{variant}"] not in node.items - ): - collector.collect_and_log( - f"{prev_path}/{variant}", - ValidationProblem.InvalidEnum, - node.items, - ) - # Check unit category if node.unit is not None: remove_from_not_visited(f"{prev_path}/{variant}/@units") @@ -460,19 +460,23 @@ def handle_field(node: NexusNode, keys: Mapping[str, Any], prev_path: str): def handle_attribute(node: NexusNode, keys: Mapping[str, Any], prev_path: str): full_path = remove_from_not_visited(f"{prev_path}/@{node.name}") variants = get_variations_of(node, keys) - if not variants: - if node.optionality == "required" and node.type in missing_type_err: - collector.collect_and_log( - full_path, missing_type_err.get(node.type), None - ) + if ( + not variants + and node.optionality == "required" + and node.type in missing_type_err + ): + collector.collect_and_log(full_path, missing_type_err.get(node.type), None) return for variant in variants: - is_valid_data_field( + mapping[ + f"{prev_path}/{variant if variant.startswith('@') else f'@{variant}'}" + ] = is_valid_data_field( mapping[ f"{prev_path}/{variant if variant.startswith('@') else f'@{variant}'}" ], node.dtype, + node.items, f"{prev_path}/{variant if variant.startswith('@') else f'@{variant}'}", ) @@ -504,19 +508,26 @@ def handle_unknown_type(node: NexusNode, keys: Mapping[str, Any], prev_path: str # TODO: Raise error or log the issue? pass - def is_documented(key: str, node: NexusNode) -> bool: - if mapping.get(key) is None: - # This value is not really set. Skip checking it's documentation. - return True - + def add_best_matches_for(key: str, node: NexusNode) -> Optional[NexusNode]: for name in key[1:].replace("@", "").split("/"): children = node.get_all_direct_children_names() best_name = best_namefit_of(name, children) if best_name is None: - return False + return None node = node.search_add_child_for(best_name) + return node + + def is_documented(key: str, node: NexusNode) -> bool: + if mapping.get(key) is None: + # This value is not really set. Skip checking it's documentation. + return True + + node = add_best_matches_for(key, node) + if node is None: + return False + if isinstance(mapping[key], dict) and "link" in mapping[key]: # TODO: Follow link and check consistency with current field return True @@ -526,6 +537,13 @@ def is_documented(key: str, node: NexusNode) -> bool: if "@" in key and node.type != "attribute": return False + # if we arrive here, the key is supposed to be documented. + # We still do some further checks before returning. + + # Check general validity + mapping[key] = is_valid_data_field(mapping[key], node.dtype, node.items, key) + + # Check main field exists for units if ( isinstance(node, NexusEntity) and node.unit is not None @@ -535,7 +553,7 @@ def is_documented(key: str, node: NexusNode) -> bool: f"{key}", ValidationProblem.MissingUnit, node.unit ) - return is_valid_data_field(mapping[key], node.dtype, key)[0] + return True def recurse_tree( node: NexusNode, @@ -556,7 +574,7 @@ def check_attributes_of_nonexisting_field( ) -> list: """ This method runs through the mapping dictionary and checks if there are any - attributes assigned to the fields (not groups!) which are not expicitly + attributes assigned to the fields (not groups!) which are not explicitly present in the mapping. If there are any found, a warning is logged and the corresponding items are added to the list returned by the method. @@ -573,9 +591,10 @@ def check_attributes_of_nonexisting_field( for key in mapping: last_index = key.rfind("/") - if key[last_index + 1] == "@": + if key[last_index + 1] == "@" and key[last_index + 1 :] != "@units": # key is an attribute. Find a corresponding parent, check all the other # children of this parent + # ignore units here, they are checked separately attribute_parent_checked = False for key_iterating in mapping: # check if key_iterating starts with parent of the key OR any @@ -657,7 +676,7 @@ def check_type_with_tree( if (next_child_class is not None) or (next_child_name is not None): output = None for child in node.children: - # regexs to separarte the class and the name from full name of the child + # regexs to separate the class and the name from full name of the child child_class_from_node = re.sub( r"(\@.*)*(\[.*?\])*(\(.*?\))*([a-z]\_)*(\_[a-z])*[a-z]*\s*", "", @@ -745,22 +764,55 @@ def startswith_with_variations( not_visited = list(mapping) recurse_tree(tree, nested_keys) + keys_to_remove = check_attributes_of_nonexisting_field(tree) + for not_visited_key in not_visited: if not_visited_key.endswith("/@units"): - if is_documented(not_visited_key.rsplit("/", 1)[0], tree): - continue - if not_visited_key.rsplit("/", 1)[0] not in not_visited: + # check that parent exists + if not_visited_key.rsplit("/", 1)[0] not in mapping.keys(): collector.collect_and_log( not_visited_key, ValidationProblem.UnitWithoutField, not_visited_key.rsplit("/", 1)[0], ) - if not ignore_undocumented: collector.collect_and_log( not_visited_key, - ValidationProblem.UnitWithoutDocumentation, - mapping[not_visited_key], + ValidationProblem.KeyToBeRemoved, + None, ) + keys_to_remove.append(not_visited_key) + else: + # check that parent has units + node = add_best_matches_for(not_visited_key.rsplit("/", 1)[0], tree) + if node.unit is None: + collector.collect_and_log( + not_visited_key, + ValidationProblem.UnitWithoutDocumentation, + mapping[not_visited_key], + ) + + # parent key will be checked on its own if it exists, because it is in the list + continue + + if "@" in not_visited_key.rsplit("/")[-1]: + # check that parent exists + if not_visited_key.rsplit("/", 1)[0] not in mapping.keys(): + # check that parent is not a group + node = add_best_matches_for(not_visited_key.rsplit("/", 1)[0], tree) + if node.type != "group": + collector.collect_and_log( + not_visited_key.rsplit("/", 1)[0], + ValidationProblem.AttributeForNonExistingField, + None, + ) + collector.collect_and_log( + not_visited_key, + ValidationProblem.KeyToBeRemoved, + None, + ) + keys_to_remove.append(not_visited_key) + continue + if is_documented(not_visited_key, tree): continue @@ -769,7 +821,6 @@ def startswith_with_variations( not_visited_key, ValidationProblem.MissingDocumentation, None ) - keys_to_remove = check_attributes_of_nonexisting_field(tree) return (not collector.has_validation_problems(), keys_to_remove) @@ -802,6 +853,6 @@ def populate_full_tree(node: NexusNode, max_depth: Optional[int] = 5, depth: int # Backwards compatibility def validate_data_dict( - _: Mapping[str, Any], read_data: Mapping[str, Any], root: ET._Element + _: MutableMapping[str, Any], read_data: MutableMapping[str, Any], root: ET._Element ) -> bool: return validate_dict_against(root.attrib["name"], read_data)[0] diff --git a/tests/data/dataconverter/readers/example/testdata.json b/tests/data/dataconverter/readers/example/testdata.json index 21deb40c3..e66af9962 100644 --- a/tests/data/dataconverter/readers/example/testdata.json +++ b/tests/data/dataconverter/readers/example/testdata.json @@ -7,6 +7,8 @@ "float_value_units": "nm", "int_value": -3, "int_value_units": "eV", + "number_value": 3, + "number_value_units": "eV", "posint_value": 7, "posint_value_units": "kg", "definition": "NXtest", @@ -17,5 +19,6 @@ "date_value_units": "", "required_child": 1, "optional_child": 1, - "@version": "1.0" + "@version": "1.0", + "@array": [0, 1, 2] } \ No newline at end of file diff --git a/tests/dataconverter/test_helpers.py b/tests/dataconverter/test_helpers.py index 0ef64ea0d..b01ba9e3c 100644 --- a/tests/dataconverter/test_helpers.py +++ b/tests/dataconverter/test_helpers.py @@ -30,18 +30,6 @@ from pynxtools.dataconverter.validation import validate_dict_against -def remove_optional_parent(data_dict: Template): - """Completely removes the optional group from the test Template.""" - internal_dict = Template(data_dict) - del internal_dict["/ENTRY[my_entry]/optional_parent/required_child"] - del internal_dict["/ENTRY[my_entry]/optional_parent/optional_child"] - del internal_dict[ - "/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/DATA[data]" - ] - - return internal_dict - - def alter_dict(data_dict: Template, key: str, value: object): """Helper function to alter a single entry in dict for parametrize.""" if data_dict is not None: @@ -52,83 +40,6 @@ def alter_dict(data_dict: Template, key: str, value: object): return None -def set_to_none_in_dict(data_dict: Optional[Template], key: str, optionality: str): - """Helper function to forcefully set path to 'None'""" - if data_dict is None: - return None - - internal_dict = Template(data_dict) - internal_dict[optionality][key] = None - return internal_dict - - -def set_whole_group_to_none( - data_dict: Optional[Template], key: str, optionality: str -) -> Optional[Template]: - """Set a whole path to None in the dict""" - if data_dict is None: - return None - - internal_dict = Template(data_dict) - for path in data_dict[optionality]: - if path.startswith(key): - internal_dict[optionality][path] = None - return internal_dict - - -def remove_from_dict(data_dict: Template, key: str, optionality: str = "optional"): - """Helper function to remove a key from dict""" - if data_dict is not None and key in data_dict[optionality]: - internal_dict = Template(data_dict) - del internal_dict[optionality][key] - return internal_dict - - return None - - -def listify_template(data_dict: Template): - """Helper function to turn most values in the Template into lists""" - listified_template = Template() - for optionality in ("optional", "recommended", "required", "undocumented"): - for path in data_dict[optionality]: - if path[path.rindex("/") + 1 :] in ( - "@units", - "type", - "definition", - "date_value", - ): - listified_template[optionality][path] = data_dict[optionality][path] - else: - listified_template[optionality][path] = [data_dict[optionality][path]] - return listified_template - - -@pytest.mark.parametrize( - "input_data, expected_output", - [ - ("2.4E-23", 2.4e-23), - ("28", 28), - ("45.98", 45.98), - ("test", "test"), - (["59", "3.00005", "498E-36"], np.array([59.0, 3.00005, 4.98e-34])), - ("23 34 444 5000", np.array([23.0, 34.0, 444.0, 5000.0])), - ("xrd experiment", "xrd experiment"), - (None, None), - ], -) -def test_transform_to_intended_dt(input_data, expected_output): - """Transform to possible numerical method.""" - result = helpers.transform_to_intended_dt(input_data) - - # Use pytest.approx for comparing floating-point numbers - if isinstance(expected_output, np.ndarray): - np.testing.assert_allclose(result, expected_output, rtol=1e-3) - elif isinstance(expected_output, float): - assert result == pytest.approx(expected_output, rel=1e-5) - else: - assert result == expected_output - - @pytest.fixture(name="template") def fixture_template(): """pytest fixture to use the same template in all tests""" @@ -155,6 +66,9 @@ def fixture_filled_test_data(template, tmp_path): ) template.clear() + template[ + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/anamethatRENAMES[anamethatichangetothis]" + ] = 2 template["/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value"] = 2.0 template["/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value/@units"] = "nm" template["/ENTRY[my_entry]/optional_parent/required_child"] = 1 @@ -162,6 +76,8 @@ def fixture_filled_test_data(template, tmp_path): template["/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value"] = True template["/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value"] = 2 template["/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value/@units"] = "eV" + template["/ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value"] = 2 + template["/ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value/@units"] = "eV" template["/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value"] = np.array( [1, 2, 3], dtype=np.int8 ) @@ -183,378 +99,30 @@ def fixture_filled_test_data(template, tmp_path): return template -TEMPLATE = Template() -TEMPLATE["optional"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value"] = 2.0 # pylint: disable=E1126 -TEMPLATE["optional"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value/@units"] = ( - "nm" # pylint: disable=E1126 -) -TEMPLATE["optional"]["/ENTRY[my_entry]/optional_parent/required_child"] = 1 # pylint: disable=E1126 -TEMPLATE["optional"]["/ENTRY[my_entry]/optional_parent/optional_child"] = 1 # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value"] = True # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value/@units"] = "" -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value"] = 2 # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value/@units"] = "eV" # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value"] = np.array( - [1, 2, 3], # pylint: disable=E1126 - dtype=np.int8, -) # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value/@units"] = ( - "kg" # pylint: disable=E1126 -) -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value"] = ( - "just chars" # pylint: disable=E1126 -) -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value/@units"] = "" -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value"] = True # pylint: disable=E1126 -TEMPLATE["required"][ - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value/@units" -] = "" -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/int_value"] = 2 # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/int_value/@units"] = ( - "eV" # pylint: disable=E1126 -) -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/posint_value"] = ( - np.array( - [1, 2, 3], # pylint: disable=E1126 - dtype=np.int8, - ) -) # pylint: disable=E1126 -TEMPLATE["required"][ - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/posint_value/@units" -] = "kg" # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/char_value"] = ( - "just chars" # pylint: disable=E1126 -) -TEMPLATE["required"][ - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/char_value/@units" -] = "" -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/type"] = "2nd type" # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/date_value"] = ( - "2022-01-22T12:14:12.05018+00:00" # pylint: disable=E1126 -) -TEMPLATE["required"][ - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/date_value/@units" -] = "" -TEMPLATE["required"]["/ENTRY[my_entry]/OPTIONAL_group[my_group]/required_field"] = 1 # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/definition"] = "NXtest" # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/definition/@version"] = "2.4.6" # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/program_name"] = "Testing program" # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/type"] = "2nd type" # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value"] = ( - "2022-01-22T12:14:12.05018+00:00" # pylint: disable=E1126 -) -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value/@units"] = "" -TEMPLATE["optional"]["/ENTRY[my_entry]/OPTIONAL_group[my_group]/optional_field"] = 1 -TEMPLATE["optional"]["/ENTRY[my_entry]/required_group/description"] = ( - "An example description" -) -TEMPLATE["optional"]["/ENTRY[my_entry]/required_group2/description"] = ( - "An example description" -) -TEMPLATE["required"][ - "/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/DATA[data]" -] = 1 -TEMPLATE["lone_groups"] = [ - "/ENTRY[entry]/required_group", - "/ENTRY[entry]/required_group2", - "/ENTRY[entry]/optional_parent/req_group_in_opt_group", -] -TEMPLATE["optional"]["/@default"] = "Some NXroot attribute" - -# "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/in" -# "t_value should be one of: (, , )," -# " as defined in the NXDL as NX_INT." - - -# pylint: disable=too-many-arguments @pytest.mark.parametrize( - "data_dict,error_message", + "input_data, expected_output", [ - pytest.param( - alter_dict( - TEMPLATE, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value", - "not_a_num", - ), - ( - "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/in" - "t_value should be one of: (, , , ," - " , , , , , " - ", , , , , ), as defined in " - "the NXDL as NX_INT." - ), - id="string-instead-of-int", - ), - pytest.param( - alter_dict( - TEMPLATE, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value", - "NOT_TRUE_OR_FALSE", - ), - ( - "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value sh" - "ould be one of: (, , , , )," - " as defined in the NXDL as NX_CHAR." - ), - id="int-instead-of-chars", - ), - pytest.param( - alter_dict( - TEMPLATE, "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", None - ), - "", - id="empty-optional-field", - ), - pytest.param( - set_to_none_in_dict( - TEMPLATE, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value", - "required", - ), - ( - "The data entry corresponding to /ENTRY[my_entry]/NXODD_name[nxodd_name]" - "/bool_value is" - " required and hasn't been supplied by the reader." - ), - id="empty-required-field", - ), - pytest.param( - set_to_none_in_dict( - TEMPLATE, - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value", - "required", - ), - ( - "The data entry corresponding to /ENTRY[my_entry]/" - "NXODD_name[nxodd_two_name]/bool_value is" - " required and hasn't been supplied by the reader." - ), - id="empty-required-field", - ), - pytest.param( - remove_from_dict( - remove_from_dict( - TEMPLATE, - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value", - "required", - ), - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value", - "required", - ), - ( - "The data entry corresponding to /ENTRY[my_entry]/NXODD_name[nxodd_name]" - "/bool_value is" - " required and hasn't been supplied by the reader." - ), - id="empty-required-field", - ), - pytest.param( - set_whole_group_to_none( - set_whole_group_to_none( - TEMPLATE, - "/ENTRY[my_entry]/NXODD_name", - "required", - ), - "/ENTRY[my_entry]/NXODD_name", - "optional", - ), - ("The required group, /ENTRY[my_entry]/NXODD_name, hasn't been supplied."), - id="all-required-fields-set-to-none", - ), - pytest.param( - alter_dict( - TEMPLATE, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value", - "2022-01-22T12:14:12.05018+00:00", - ), - "", - id="UTC-with-+00:00", - ), - pytest.param( - alter_dict( - TEMPLATE, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value", - "2022-01-22T12:14:12.05018Z", - ), - "", - id="UTC-with-Z", - ), - pytest.param( - alter_dict( - TEMPLATE, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value", - "2022-01-22T12:14:12.05018-00:00", - ), - "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value" - " = 2022-01-22T12:14:12.05018-00:00 should be a timezone aware" - " ISO8601 formatted str. For example, 2022-01-22T12:14:12.05018Z or 2022-01-22" - "T12:14:12.05018+00:00.", - id="UTC-with--00:00", - ), - pytest.param(listify_template(TEMPLATE), "", id="lists"), - pytest.param( - alter_dict( - TEMPLATE, "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type", "Wrong option" - ), - ( - "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type should " - "be on of the following" - " strings: ['1st type', '2nd type', '3rd type', '4th type']" - ), - id="wrong-enum-choice", - ), - pytest.param( - set_to_none_in_dict( - TEMPLATE, "/ENTRY[my_entry]/optional_parent/required_child", "optional" - ), - ( - "The data entry corresponding to /ENTRY[my_entry]/optional_parent/" - "required_child is required and hasn't been supplied by the reader." - ), - id="atleast-one-required-child-not-provided-optional-parent", - ), - pytest.param( - set_to_none_in_dict( - TEMPLATE, - "/ENTRY[my_entry]/OPTIONAL_group[my_group]/required_field", - "required", - ), - ( - "The data entry corresponding to /ENTRY[my_entry]/" - "OPTIONAL_group[my_group]/required_field " - "is required and hasn't been supplied by the reader." - ), - id="required-field-not-provided-in-variadic-optional-group", - ), - pytest.param( - set_to_none_in_dict( - TEMPLATE, - "/ENTRY[my_entry]/OPTIONAL_group[my_group]/optional_field", - "required", - ), - (""), - id="required-field-provided-in-variadic-optional-group", - ), - pytest.param( - alter_dict( - alter_dict( - TEMPLATE, "/ENTRY[my_entry]/optional_parent/required_child", None - ), - "/ENTRY[my_entry]/optional_parent/optional_child", - None, - ), - (""), - id="no-child-provided-optional-parent", - ), - pytest.param(TEMPLATE, "", id="valid-data-dict"), - pytest.param( - remove_from_dict(TEMPLATE, "/ENTRY[my_entry]/required_group/description"), - "The required group, /ENTRY[my_entry]/required_group, hasn't been supplied.", - id="missing-empty-yet-required-group", - ), - pytest.param( - remove_from_dict(TEMPLATE, "/ENTRY[my_entry]/required_group2/description"), - "The required group, /ENTRY[my_entry]/required_group2, hasn't been supplied.", - id="missing-empty-yet-required-group2", - ), - pytest.param( - alter_dict( - remove_from_dict( - TEMPLATE, "/ENTRY[my_entry]/required_group/description" - ), - "/ENTRY[entry]/required_group", - None, - ), - "The required group, /ENTRY[my_entry]/required_group, hasn't been supplied.", - id="allow-required-and-empty-group", - ), - pytest.param( - remove_from_dict( - TEMPLATE, - "/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/DATA[data]", - "required", - ), - ( - "The required group, /ENTRY[my_entry]/" - "optional_parent/req_group_in_opt_group, " - "hasn't been supplied." - ), - id="req-group-in-opt-parent-removed", - ), - pytest.param( - remove_optional_parent(TEMPLATE), (""), id="opt-group-completely-removed" - ), + ("2.4E-23", 2.4e-23), + ("28", 28), + ("45.98", 45.98), + ("test", "test"), + (["59", "3.00005", "498E-36"], np.array([59.0, 3.00005, 4.98e-34])), + ("23 34 444 5000", np.array([23.0, 34.0, 444.0, 5000.0])), + ("xrd experiment", "xrd experiment"), + (None, None), ], ) -def test_validate_data_dict(caplog, data_dict, error_message, request): - """Unit test for the data validation routine.""" - if request.node.callspec.id in ( - "valid-data-dict", - "lists", - "empty-optional-field", - "UTC-with-+00:00", - "UTC-with-Z", - "no-child-provided-optional-parent", - "int-instead-of-chars", - "link-dict-instead-of-bool", - "opt-group-completely-removed", - "required-field-provided-in-variadic-optional-group", - ): - with caplog.at_level(logging.WARNING): - assert validate_dict_against("NXtest", data_dict)[0] - assert caplog.text == "" - # Missing required fields caught by logger with warning - elif request.node.callspec.id in ( - "empty-required-field", - "allow-required-and-empty-group", - "req-group-in-opt-parent-removed", - "missing-empty-yet-required-group", - "missing-empty-yet-required-group2", - ): - assert "" == caplog.text - captured_logs = caplog.records - assert not validate_dict_against("NXtest", data_dict)[0] - assert any(error_message in rec.message for rec in captured_logs) - else: - with caplog.at_level(logging.WARNING): - assert not validate_dict_against("NXtest", data_dict)[0] +def test_transform_to_intended_dt(input_data, expected_output): + """Transform to possible numerical method.""" + result = helpers.transform_to_intended_dt(input_data) - assert error_message in caplog.text + # Use pytest.approx for comparing floating-point numbers + if isinstance(expected_output, np.ndarray): + np.testing.assert_allclose(result, expected_output, rtol=1e-3) + elif isinstance(expected_output, float): + assert result == pytest.approx(expected_output, rel=1e-5) + else: + assert result == expected_output @pytest.mark.parametrize( diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index 2c946a3a1..b5a309131 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -17,129 +17,939 @@ # limitations under the License. # import logging -from typing import Any, Dict, List, Tuple, Union +from typing import Optional import numpy as np import pytest +from pynxtools.dataconverter.template import Template from pynxtools.dataconverter.validation import validate_dict_against +from .test_helpers import ( # pylint: disable=unused-import + alter_dict, + fixture_filled_test_data, + fixture_template, +) -def get_data_dict(): - return { - "/ENTRY[my_entry]/optional_parent/required_child": 1, - "/ENTRY[my_entry]/optional_parent/optional_child": 1, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value_no_attr": 2.0, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value": 2.0, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value/@units": "nm", - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value": True, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value/@units": "", - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value": 2, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value/@units": "eV", - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value": np.array( - [1, 2, 3], dtype=np.int8 - ), - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value/@units": "kg", - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value": "just chars", - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value/@units": "", - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type": "2nd type", - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value": "2022-01-22T12:14:12.05018+00:00", - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value/@units": "", - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value": True, - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value/@units": "", - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/int_value": 2, - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/int_value/@units": "eV", - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/posint_value": np.array( - [1, 2, 3], dtype=np.int8 - ), - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/posint_value/@units": "kg", - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/char_value": "just chars", - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/char_value/@units": "", - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/type": "2nd type", - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/date_value": "2022-01-22T12:14:12.05018+00:00", - "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/date_value/@units": "", - "/ENTRY[my_entry]/OPTIONAL_group[my_group]/required_field": 1, - "/ENTRY[my_entry]/definition": "NXtest", - "/ENTRY[my_entry]/definition/@version": "2.4.6", - "/ENTRY[my_entry]/program_name": "Testing program", - "/ENTRY[my_entry]/OPTIONAL_group[my_group]/optional_field": 1, - "/ENTRY[my_entry]/required_group/description": "An example description", - "/ENTRY[my_entry]/required_group2/description": "An example description", - "/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/data": 1, - "/@default": "Some NXroot attribute", - } - - -def remove_from_dict(keys: Union[Union[List[str], Tuple[str, ...]], str], data_dict): - if isinstance(keys, (list, tuple)): - for key in keys: - data_dict.pop(key, None) - else: - data_dict.pop(keys) - return data_dict +def set_to_none_in_dict(data_dict: Optional[Template], key: str, optionality: str): + """Helper function to forcefully set path to 'None'""" + if data_dict is None: + return None + + internal_dict = Template(data_dict) + internal_dict[optionality][key] = None + return internal_dict + + +def set_whole_group_to_none( + data_dict: Optional[Template], key: str, optionality: str +) -> Optional[Template]: + """Set a whole path to None in the dict""" + if data_dict is None: + return None + + internal_dict = Template(data_dict) + for path in data_dict[optionality]: + if path.startswith(key): + internal_dict[optionality][path] = None + return internal_dict + + +def remove_from_dict(data_dict: Template, key: str, optionality: str = "optional"): + """Helper function to remove a key from dict""" + if data_dict is not None and key in data_dict[optionality]: + internal_dict = Template(data_dict) + del internal_dict[optionality][key] + return internal_dict + + return None -def alter_dict(new_values: Dict[str, Any], data_dict: Dict[str, Any]) -> Dict[str, Any]: - for key, value in new_values.items(): - data_dict[key] = value - return data_dict +def listify_template(data_dict: Template): + """Helper function to turn most values in the Template into lists""" + listified_template = Template() + for optionality in ("optional", "recommended", "required", "undocumented"): + for path in data_dict[optionality]: + if path[path.rindex("/") + 1 :] in ( + "@units", + "type", + "definition", + "date_value", + ) or isinstance(data_dict[optionality][path], list): + listified_template[optionality][path] = data_dict[optionality][path] + else: + listified_template[optionality][path] = [data_dict[optionality][path]] + return listified_template + + +TEMPLATE = Template() +TEMPLATE["optional"][ + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/anamethatRENAMES[anamethatichangetothis]" +] = 2 +TEMPLATE["optional"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value"] = 2.0 # pylint: disable=E1126 +TEMPLATE["optional"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value/@units"] = ( + "nm" # pylint: disable=E1126 +) +TEMPLATE["optional"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value_no_attr"] = ( + 2.0, +) +TEMPLATE["optional"]["/ENTRY[my_entry]/optional_parent/required_child"] = 1 # pylint: disable=E1126 +TEMPLATE["optional"]["/ENTRY[my_entry]/optional_parent/optional_child"] = 1 # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value"] = True # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value/@units"] = "" +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value"] = 2 # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value/@units"] = "eV" # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value"] = 2 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value/@units"] = ( + "eV" +) +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value"] = np.array( + [1, 2, 3], # pylint: disable=E1126 + dtype=np.int8, +) # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value/@units"] = ( + "kg" # pylint: disable=E1126 +) +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value"] = ( + "just chars" # pylint: disable=E1126 +) +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value/@units"] = "" +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value"] = True # pylint: disable=E1126 +TEMPLATE["required"][ + "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value/@units" +] = "" +TEMPLATE["required"][ + "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/anamethatRENAMES[anamethatichangetothis]" +] = 2 # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/int_value"] = 2 # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/int_value/@units"] = ( + "eV" # pylint: disable=E1126 +) +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/posint_value"] = ( + np.array( + [1, 2, 3], # pylint: disable=E1126 + dtype=np.int8, + ) +) # pylint: disable=E1126 +TEMPLATE["required"][ + "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/posint_value/@units" +] = "kg" # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/char_value"] = ( + "just chars" # pylint: disable=E1126 +) +TEMPLATE["required"][ + "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/char_value/@units" +] = "" +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/type"] = "2nd type" # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/type/@array"] = [ + 0, + 1, + 2, +] +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/date_value"] = ( + "2022-01-22T12:14:12.05018+00:00" # pylint: disable=E1126 +) +TEMPLATE["required"][ + "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/date_value/@units" +] = "" +TEMPLATE["required"]["/ENTRY[my_entry]/OPTIONAL_group[my_group]/required_field"] = 1 # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/definition"] = "NXtest" # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/definition/@version"] = "2.4.6" # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/program_name"] = "Testing program" # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/type"] = "2nd type" # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/type/@array"] = [0, 1, 2] +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value"] = ( + "2022-01-22T12:14:12.05018+00:00" # pylint: disable=E1126 +) +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value/@units"] = "" +TEMPLATE["optional"]["/ENTRY[my_entry]/OPTIONAL_group[my_group]/optional_field"] = 1 +TEMPLATE["optional"]["/ENTRY[my_entry]/required_group/description"] = ( + "An example description" +) +TEMPLATE["optional"]["/ENTRY[my_entry]/required_group2/description"] = ( + "An example description" +) +TEMPLATE["required"][ + "/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/DATA[data]" +] = 1 +TEMPLATE["lone_groups"] = [ + "/ENTRY[entry]/required_group", + "/ENTRY[entry]/required_group2", + "/ENTRY[entry]/optional_parent/req_group_in_opt_group", +] +TEMPLATE["optional"]["/@default"] = "Some NXroot attribute" +# keys not registered in appdef +TEMPLATE["required"]["/ENTRY[my_entry]/duration"] = 1 # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/duration/@units"] = "s" # pylint: disable=E1126 +TEMPLATE["required"][ + "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/type" +] = "Ion Source" # pylint: disable=E1126 +# pylint: disable=too-many-arguments @pytest.mark.parametrize( - "data_dict", + "data_dict,error_message", [ - pytest.param(get_data_dict(), id="valid-unaltered-data-dict"), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/anamethatRENAMES[anamethatichangetothis]", + "not_a_num", + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/anamethatRENAMES[anamethatichangetothis]" + " should be one of the following Python types: (, ), as defined in " + "the NXDL as NX_INT." + ), + id="variadic-field-str-instead-of-int", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value", + "not_a_num", + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/in" + "t_value should be one of the following Python types: (, ), as defined in " + "the NXDL as NX_INT." + ), + id="string-instead-of-int", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value", + "NOT_TRUE_OR_FALSE", + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value should be one of the following Python types: (, ), as defined in the NXDL as NX_BOOLEAN." + ), + id="string-instead-of-bool", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value", + ["1", "2", "3"], + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value should" + " be one of the following Python types: (, ), as defined in the NXDL as NX_INT." + ), + id="list-of-int-str-instead-of-int", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value", + np.array([2.0, 3.0, 4.0], dtype=np.float32), + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value should be" + " one of the following Python types: (, ), as defined in the NXDL as NX_INT." + ), + id="array-of-float-instead-of-int", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value", + [2, 3, 4], + ), + (""), + id="list-of-int-instead-of-int", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value", + np.array([2, 3, 4], dtype=np.int32), + ), + (""), + id="array-of-int32-instead-of-int", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value", + "2022-01-22T12:14:12.05018-00:00", + ), + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value" + " = 2022-01-22T12:14:12.05018-00:00 should be a timezone aware" + " ISO8601 formatted str. For example, 2022-01-22T12:14:12.05018Z or 2022-01-22" + "T12:14:12.05018+00:00.", + id="int-instead-of-date", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", + 0, + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value should be one of the following Python types: (, ), as defined in the NXDL as NX_FLOAT." + ), + id="int-instead-of-float", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value", + "0", + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value should be one of the following Python types: (, , , ), as defined in the NXDL as NX_NUMBER." + ), + id="str-instead-of-number", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value", + np.array([0.0, 2]), + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value should be one" + " of the following Python types: (, ), as" + " defined in the NXDL as NX_CHAR." + ), + id="wrong-type-ndarray-instead-of-char", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value", + np.array(["x", "2"]), + ), + (""), + id="valid-ndarray-instead-of-char", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value", + {"link": "/a-link"}, + ), + (""), + id="link-dict-instead-of-int", + ), + pytest.param( + alter_dict( + TEMPLATE, "/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value", -1 + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value " + "should be a positive int, but is -1." + ), + id="negative-posint", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value", + [-1, 2], + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value " + "should be a positive int, but is [-1, 2]." + ), + id="negative-posint-list", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value", + np.array([-1, 2], dtype=np.int8), + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value should" + " be a positive int, but is [-1 2]." + ), + id="negative-posint-array", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value", + [1, 2], + ), + (""), + id="positive-posint-list", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value", + np.array([1, 2], dtype=np.int8), + ), + (""), + id="positive-posint-array", + ), + pytest.param( + alter_dict( + TEMPLATE, "/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value", 3 + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value should be one of the following Python types:" + " (, )," + " as defined in the NXDL as NX_CHAR." + ), + id="int-instead-of-chars", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value", + np.array(["1", "2", "3"], dtype=np.str_), + ), + (""), + id="array-of-chars", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value", + np.array(["1", "2", "3"], dtype=np.bytes_), + ), + (""), + id="array-of-bytes-chars", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value", + ["list", "of", "chars"], + ), + "", + id="list-of-string-instead-of-chars", + ), + pytest.param( + alter_dict( + TEMPLATE, "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", None + ), + "", + id="empty-optional-field", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", + np.array([2.0, 3.0, 4.0], dtype=np.float32), + ), + "", + id="array-of-float-instead-of-float", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", + np.array(["2.0", "3.0"], dtype=np.str_), + ), + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value should be " + "one of the following Python types: (, ), as defined in the NXDL " + "as NX_FLOAT.", + id="array-of-str-instead-of-float", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", + [2], # pylint: disable=E1126 + ), + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value should be " + "one of the following Python types: (, ), as defined in the NXDL " + "as NX_FLOAT.", + id="list-of-int-instead-of-float", + ), + pytest.param( + set_to_none_in_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value", + "required", + ), + ( + "The data entry corresponding to /ENTRY[my_entry]/NXODD_name[nxodd_name]" + "/bool_value is" + " required and hasn't been supplied by the reader." + ), + id="empty-required-field", + ), + pytest.param( + set_to_none_in_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value", + "required", + ), + ( + "The data entry corresponding to /ENTRY[my_entry]/" + "NXODD_name[nxodd_two_name]/bool_value is" + " required and hasn't been supplied by the reader." + ), + id="empty-required-field", + ), pytest.param( remove_from_dict( + remove_from_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value", + "required", + ), + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value", + "required", + ), + ( + "The data entry corresponding to /ENTRY[my_entry]/NXODD_name[nxodd_name]" + "/bool_value is" + " required and hasn't been supplied by the reader." + ), + id="empty-required-field", + ), + pytest.param( + remove_from_dict( + TEMPLATE, "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value_no_attr", - get_data_dict(), + "optional", ), + "", id="removed-optional-value", ), - ], -) -def test_valid_data_dict(caplog, data_dict): - with caplog.at_level(logging.WARNING): - assert validate_dict_against("NXtest", data_dict)[0] - assert caplog.text == "" - - -@pytest.mark.parametrize( - "data_dict, error_message_1, error_message_2", - [ pytest.param( remove_from_dict( - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", get_data_dict() + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", + "optional", + ), + "Unit /ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value/@units in dataset without its field /ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value.", + id="removed-optional-value-with-attribute-remaining", + ), + pytest.param( + remove_from_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", + "optional", ), "The attribute /ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value/@units will not be written.", - "There were attributes set for the field /ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value, but the field does not exist.", id="removed-optional-value-with-attribute-remaining", ), - ], -) -def test_data_dict_attr_with_no_field( - caplog, data_dict, error_message_1, error_message_2 -): - with caplog.at_level(logging.WARNING): - assert not validate_dict_against("NXtest", data_dict)[0] - assert error_message_1 in caplog.text - assert error_message_2 in caplog.text - - -@pytest.mark.parametrize( - "data_dict, error_message", - [ pytest.param( remove_from_dict( - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value", get_data_dict() + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value", + "required", ), "The data entry corresponding to /ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value is required and hasn't been supplied by the reader.", id="missing-required-value", - ) + ), + pytest.param( + set_whole_group_to_none( + set_whole_group_to_none( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name", + "required", + ), + "/ENTRY[my_entry]/NXODD_name", + "optional", + ), + ("The required group, /ENTRY[my_entry]/NXODD_name, hasn't been supplied."), + id="all-required-fields-set-to-none", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value", + "2022-01-22T12:14:12.05018+00:00", + ), + "", + id="UTC-with-+00:00", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value", + "2022-01-22T12:14:12.05018Z", + ), + "", + id="UTC-with-Z", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value", + "2022-01-22T12:14:12.05018-00:00", + ), + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value" + " = 2022-01-22T12:14:12.05018-00:00 should be a timezone aware" + " ISO8601 formatted str. For example, 2022-01-22T12:14:12.05018Z or 2022-01-22" + "T12:14:12.05018+00:00.", + id="UTC-with--00:00", + ), + pytest.param(listify_template(TEMPLATE), "", id="lists"), + pytest.param( + alter_dict( + TEMPLATE, "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type", "Wrong option" + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type should " + "be one of the following" + ": ['1st type', '2nd type', '3rd type', '4th type']" + ), + id="wrong-enum-choice", + ), + pytest.param( + set_to_none_in_dict( + TEMPLATE, "/ENTRY[my_entry]/optional_parent/required_child", "optional" + ), + ( + "The data entry corresponding to /ENTRY[my_entry]/optional_parent/" + "required_child is required and hasn't been supplied by the reader." + ), + id="atleast-one-required-child-not-provided-optional-parent", + ), + pytest.param( + set_to_none_in_dict( + TEMPLATE, + "/ENTRY[my_entry]/OPTIONAL_group[my_group]/required_field", + "required", + ), + ( + "The data entry corresponding to /ENTRY[my_entry]/" + "OPTIONAL_group[my_group]/required_field " + "is required and hasn't been supplied by the reader." + ), + id="required-field-not-provided-in-variadic-optional-group", + ), + pytest.param( + set_to_none_in_dict( + TEMPLATE, + "/ENTRY[my_entry]/OPTIONAL_group[my_group]/optional_field", + "required", + ), + (""), + id="required-field-provided-in-variadic-optional-group", + ), + pytest.param( + alter_dict( + alter_dict( + TEMPLATE, "/ENTRY[my_entry]/optional_parent/required_child", None + ), + "/ENTRY[my_entry]/optional_parent/optional_child", + None, + ), + (""), + id="no-child-provided-optional-parent", + ), + pytest.param(TEMPLATE, "", id="valid-data-dict"), + pytest.param( + remove_from_dict(TEMPLATE, "/ENTRY[my_entry]/required_group/description"), + "The required group, /ENTRY[my_entry]/required_group, hasn't been supplied.", + id="missing-empty-yet-required-group", + ), + pytest.param( + remove_from_dict(TEMPLATE, "/ENTRY[my_entry]/required_group2/description"), + "The required group, /ENTRY[my_entry]/required_group2, hasn't been supplied.", + id="missing-empty-yet-required-group2", + ), + pytest.param( + alter_dict( + remove_from_dict( + TEMPLATE, "/ENTRY[my_entry]/required_group/description" + ), + "/ENTRY[entry]/required_group", + None, + ), + "The required group, /ENTRY[my_entry]/required_group, hasn't been supplied.", + id="allow-required-and-empty-group", + ), + pytest.param( + remove_from_dict( + TEMPLATE, + "/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/DATA[data]", + "required", + ), + ( + "The required group, /ENTRY[my_entry]/" + "optional_parent/req_group_in_opt_group, " + "hasn't been supplied." + ), + id="req-group-in-opt-parent-removed", + ), + pytest.param((TEMPLATE), (""), id="opt-group-completely-removed"), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type/@array", + ["0", 1, 2], + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type/@array should be one of the following: [[0, 1, 2], [2, 3, 4]]" + ), + id="wrong-type-array-in-attribute", + ), + pytest.param( + alter_dict( + TEMPLATE, "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type/@array", [1, 2] + ), + ( + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type/@array should be one of the following: [[0, 1, 2], [2, 3, 4]]" + ), + id="wrong-value-array-in-attribute", + ), + pytest.param( + remove_from_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value/@units", + "required", + ), + "Field /ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value requires a unit in the unit category NX_ENERGY.", + id="missing-unit", + ), + pytest.param( + remove_from_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value", + "required", + ), + "Unit /ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value/@units in dataset without its field /ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value.", + id="unit-missing-field", + ), + pytest.param( + remove_from_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value", + "required", + ), + "The attribute /ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value/@units will not be written.", + id="unit-missing-field", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/required_group/illegal_name", + 1, + ), + ( + "Field /ENTRY[my_entry]/required_group/illegal_name written without documentation." + ), + id="add-undocumented-field", + ), + pytest.param( + alter_dict( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/required_group/author", + "author", + ), + "/ENTRY[my_entry]/required_group/author/@illegal", + "illegal_attribute", + ), + ( + "Attribute /ENTRY[my_entry]/required_group/author/@illegal written without documentation." + ), + id="add-undocumented-attribute", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/BEAM[my_beam]/@default", + "unknown", + ), + "", + id="group-with-only-attributes", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/BEAM[my_beam]/@illegal", + "unknown", + ), + ( + "Attribute /ENTRY[my_entry]/INSTRUMENT[my_instrument]/BEAM[my_beam]/@illegal written without documentation." + ), + id="group-with-illegal-attributes", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/optional_parent/required_child/@units", + "s", + ), + ( + "The unit, /ENTRY[my_entry]/optional_parent/required_child/@units = s, is being written but has no documentation." + ), + id="field-with-illegal-unit", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/duration", + np.array([2.0, 3.0, 4.0], dtype=np.float32), + ), + ( + "The value at /ENTRY[my_entry]/duration should be" + " one of the following Python types: (, ), as defined in the NXDL as NX_INT." + ), + id="baseclass-wrong-dtype", + ), + pytest.param( + remove_from_dict( + TEMPLATE, + "/ENTRY[my_entry]/duration/@units", + "required", + ), + "Field /ENTRY[my_entry]/duration requires a unit in the unit category NX_TIME.", + id="baseclass-missing-unit", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/collection_time/@illegal", + "s", + ), + ( + "There were attributes set for the field /ENTRY[my_entry]/collection_time, but the field does not exist." + ), + id="baseclass-attribute-missing-field", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/collection_time/@illegal", + "s", + ), + ( + "The attribute /ENTRY[my_entry]/collection_time/@illegal will not be written." + ), + id="baseclass-attribute-missing-field", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/type", + "Wrong source type", + ), + ( + "The value at /ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/type " + "should be one of the following: ['Spallation Neutron Source', 'Pulsed Reactor Neutron Source', " + "'Reactor Neutron Source', 'Synchrotron X-ray Source', 'Pulsed Muon Source', 'Rotating Anode X-ray', " + "'Fixed Tube X-ray', 'UV Laser', 'Free-Electron Laser', 'Optical Laser', 'Ion Source', 'UV Plasma Source', " + "'Metal Jet X-ray', 'Laser', 'Dye-Laser', 'Broadband Tunable Light Source', 'Halogen lamp', 'LED', " + "'Mercury Cadmium Telluride', 'Deuterium Lamp', 'Xenon Lamp', 'Globar', 'other']" + ), + id="baseclass-wrong-enum", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/illegal_name", + 1, + ), + ( + "Field /ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/illegal_name written without documentation." + ), + id="baseclass-add-undocumented-field", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/type/@illegal", + "illegal_attribute", + ), + ( + "Attribute /ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/type/@illegal written without documentation." + ), + id="baseclass-add-undocumented-attribute", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/illegal/@units", + "illegal_attribute", + ), + ( + "Unit /ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/illegal/@units " + "in dataset without its field /ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/illegal." + ), + id="baseclass-add-unit-of-missing-undocumented-field", + ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/illegal/@units", + "illegal_attribute", + ), + ( + "The attribute /ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/illegal/@units will not be written." + ), + id="baseclass-add-unit-of-missing-undocumented-field", + ), + pytest.param( + alter_dict( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/required_group/author", + "author", + ), + "/ENTRY[my_entry]/required_group/author/@units", + "s", + ), + ( + "The unit, /ENTRY[my_entry]/required_group/author/@units = s, is being written but has no documentation." + ), + id="baseclass-field-with-illegal-unit", + ), ], ) -def test_validation_shows_warning(caplog, data_dict, error_message): - with caplog.at_level(logging.WARNING): - assert not validate_dict_against("NXtest", data_dict)[0] +def test_validate_data_dict(caplog, data_dict, error_message, request): + """Unit test for the data validation routine.""" + + def format_error_message(msg: str) -> str: + return msg[msg.rfind("G: ") + 3 :].rstrip("\n") - assert error_message in caplog.text + if request.node.callspec.id in ( + "valid-data-dict", + "lists", + "empty-optional-field", + "UTC-with-+00:00", + "UTC-with-Z", + "no-child-provided-optional-parent", + "link-dict-instead-of-int", + "opt-group-completely-removed", + "required-field-provided-in-variadic-optional-group", + "valid-ndarray-instead-of-char", + "list-of-int-instead-of-int", + "list-of-string-instead-of-chars", + "array-of-int32-instead-of-int", + "List-of-int-instead-of-int", + "positive-posint-list", + "positive-posint-array", + "array-of-chars", + "array-of-bytes-chars", + "array-of-float-instead-of-float", + "numpy-chararray", + "removed-optional-value", + "group-with-only-attributes", + ): + with caplog.at_level(logging.WARNING): + assert validate_dict_against("NXtest", data_dict)[0] + assert caplog.text == "" + # Missing required fields caught by logger with warning + elif request.node.callspec.id in ( + "empty-required-field", + "allow-required-and-empty-group", + "req-group-in-opt-parent-removed", + "missing-empty-yet-required-group", + "missing-empty-yet-required-group2", + ): + assert "" == caplog.text + captured_logs = caplog.records + assert not validate_dict_against("NXtest", data_dict)[0] + assert any( + error_message == format_error_message(rec.message) for rec in captured_logs + ) + else: + with caplog.at_level(logging.WARNING): + assert not validate_dict_against("NXtest", data_dict)[0] + assert any( + error_message == format_error_message(rec.message) for rec in caplog.records + )