diff --git a/.github/workflows/test-schemas.yml b/.github/workflows/test-schemas.yml new file mode 100644 index 0000000..33708aa --- /dev/null +++ b/.github/workflows/test-schemas.yml @@ -0,0 +1,36 @@ +name: Test schemas + +on: + push: + branches: [main] + + pull_request: + branches: [main] + + workflow_dispatch: + +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Verify schemas up-to-date + run: | + python3 utils/gen_schema.py + + if [ -n "$(git status --porcelain)" ]; then + echo "::error:: Uncommited schema changes. Please run 'python3 utils/gen_schema.py' locally and commit the changes." + exit 1 + fi diff --git a/docs_src/examples.md b/docs_src/examples.md index 5636643..c07661e 100644 --- a/docs_src/examples.md +++ b/docs_src/examples.md @@ -2,7 +2,7 @@ The OpenPrintTag specification comes with [a set of utilities written in Python]({{repo}}/tree/main/utils) that serve as a baseline/reference implementation. ## Reading a tag -The `rec_info` utility can be used to parse data on the NFC tag into a YAML file that is readable by both humans and computers. +The `rec_info` utility can be used to parse data on the NFC tag into a YAML file that is readable by both humans and computers. This format is intended to be standardized across all OpenPrintTag libraries and is defined in [opt_json.schema.json]({{repo}}/tree/main/utils/schema/opt_json.schema.json). {{ show_example("cat sample_data/sample_tag.bin | >rec_info.py --show-data --opt-check") }} ## Updating a tag diff --git a/docs_src/sample_data/compact_data_to_fill.yaml b/docs_src/sample_data/compact_data_to_fill.yaml index 2f3951a..70310d2 100644 --- a/docs_src/sample_data/compact_data_to_fill.yaml +++ b/docs_src/sample_data/compact_data_to_fill.yaml @@ -5,7 +5,7 @@ data: material_type: PLA brand_specific_material_id: 1 tags: [glitter] - primary_color: [0x3D, 0x3E, 0x3D] + primary_color: "#3d3e3d" nominal_netto_full_weight: 1000 transmission_distance: 0.2 min_print_temperature: 205 diff --git a/docs_src/sample_data/data_to_fill.yaml b/docs_src/sample_data/data_to_fill.yaml index ba0f219..e6fff8f 100644 --- a/docs_src/sample_data/data_to_fill.yaml +++ b/docs_src/sample_data/data_to_fill.yaml @@ -6,7 +6,7 @@ data: material_name: PLA Galaxy Black brand_specific_material_id: 1 tags: [glitter] - primary_color: [0x3D, 0x3E, 0x3D] + primary_color: "#3d3e3d" manufactured_date: 1739371290 nominal_netto_full_weight: 1000 actual_netto_full_weight: 1012 diff --git a/requirements.txt b/requirements.txt index c48162b..d102d04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ cbor2~=5.7.1 jinja2~=3.1.5 numpy~=2.2.3 simple_parsing~=0.1.7 +referencing~=0.37.0 +jsonschema~=4.25.1 diff --git a/tests/encode_decode/01_data.bin b/tests/encode_decode/01_data.bin index de48f23..1442939 100644 Binary files a/tests/encode_decode/01_data.bin and b/tests/encode_decode/01_data.bin differ diff --git a/tests/encode_decode/01_info.yaml b/tests/encode_decode/01_info.yaml index 1bdbcae..3512c2b 100644 --- a/tests/encode_decode/01_info.yaml +++ b/tests/encode_decode/01_info.yaml @@ -8,7 +8,7 @@ regions: payload_offset: 4 absolute_offset: 70 size: 206 - used_size: 148 + used_size: 149 aux: payload_offset: 210 absolute_offset: 276 @@ -18,8 +18,8 @@ root: data_size: 312 payload_size: 245 overhead: 67 - payload_used_size: 153 - total_used_size: 220 + payload_used_size: 154 + total_used_size: 221 data: meta: aux_region_offset: 210 @@ -34,8 +34,7 @@ data: nominal_netto_full_weight: 1000 actual_netto_full_weight: 1012 empty_container_weight: 280 - primary_color: - hex: 3d3e3d + primary_color: '#3d3e3dff' tags: - glitter density: 1.24 @@ -57,9 +56,12 @@ data: aux: {} raw_data: meta: a10218d2 - main: bf041b000007d0fcab45f9056a33333463353466303838080009000a76504c412050727573612047616c61787920426c61636b0b6950727573616d656e740e1a68d3c7d7101903e8111903f41219011813433d3e3d181c9f17ff181df93cf6182218cd182318e1182418aa182518281826183c18271218281828182914182a1840182b18c8182c1864182d183418389f0001ffff00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + main: bf041b000007d0fcab45f9056a33333463353466303838080009000a76504c412050727573612047616c61787920426c61636b0b6950727573616d656e740e1a68d3c7d7101903e8111903f41219011813443d3e3dff181c9f17ff181df93cf6182218cd182318e1182418aa182518281826183c18271218281828182914182a1840182b18c8182c1864182d183418389f0001ffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 aux: a000000000000000000000000000000000000000000000000000000000000000000000 uri: https://3dtag.org/s/334c54f088 +validate: + warnings: [] + errors: [] opt_check: warnings: [] errors: [] diff --git a/tests/encode_decode/01_input.yaml b/tests/encode_decode/01_input.yaml index 049ecc4..6e5c38f 100644 --- a/tests/encode_decode/01_input.yaml +++ b/tests/encode_decode/01_input.yaml @@ -8,8 +8,7 @@ data: material_type: PLA brand_name: Prusament material_name: PLA Prusa Galaxy Black - primary_color: - hex: 3D3E3D + primary_color: "#3d3e3dff" tags: [glitter] certifications: [ul_2818, ul_94_v0] density: 1.24 diff --git a/tests/encode_decode/02_info.yaml b/tests/encode_decode/02_info.yaml index e0184a8..a894c8a 100644 --- a/tests/encode_decode/02_info.yaml +++ b/tests/encode_decode/02_info.yaml @@ -34,8 +34,7 @@ data: nominal_netto_full_weight: 1000 actual_netto_full_weight: 1050 empty_container_weight: 280 - primary_color: - hex: 24292a + primary_color: '#24292a' tags: [] density: 1.27 min_print_temperature: 240 @@ -56,6 +55,9 @@ raw_data: main: bf041b000007d0fcab465c056a37616232616362353039080009010a6e50455447204a657420426c61636b0b6950727573616d656e740e1a68c01ae7101903e81119041a12190118134324292a181c9fff181df93d14182218f01823190104182418aa182518461826185a1827121828183c18291823182a1840182b18c8182c1864182d1834ff0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 aux: a000000000000000000000000000000000000000000000000000000000000000000000 uri: https://3dtag.org/s/7ab2acb509 +validate: + warnings: [] + errors: [] opt_check: warnings: [] errors: [] diff --git a/tests/encode_decode/02_input.yaml b/tests/encode_decode/02_input.yaml index 37ea07b..692d3ba 100644 --- a/tests/encode_decode/02_input.yaml +++ b/tests/encode_decode/02_input.yaml @@ -7,8 +7,7 @@ data: material_type: PETG brand_name: Prusament material_name: PETG Jet Black - primary_color: - hex: 24292A + primary_color: "#24292a" tags: [] density: 1.27 diff --git a/tests/encode_decode/03_info.yaml b/tests/encode_decode/03_info.yaml index 340f2c4..6acd2ea 100644 --- a/tests/encode_decode/03_info.yaml +++ b/tests/encode_decode/03_info.yaml @@ -33,8 +33,7 @@ data: nominal_netto_full_weight: 1000 actual_netto_full_weight: 1050 empty_container_weight: 280 - primary_color: - hex: 24292a + primary_color: '#24292a' tags: [] density: 1.27 min_print_temperature: 240 @@ -58,6 +57,9 @@ uri: null opt_check: warnings: - Missing recommended field 'brand_name' + - Failed to deduce brand_uuid + - Failed to deduce material_uuid + - Failed to deduce package_uuid errors: - 'Fields preheat_temperature (300), min_print_temperature (240): a <= b' - 'Fields preheat_temperature (300), max_print_temperature (210): a <= b' @@ -66,9 +68,6 @@ opt_check: - 'Fields min_chamber_temperature (90), max_chamber_temperature (60): a <= b' - 'Fields container_hole_diameter (210), container_inner_diameter (100): a <= b' - 'Fields container_hole_diameter (210), container_outer_diameter (200): a <= b' - - Failed to deduce brand_uuid - - Failed to deduce material_uuid - - Failed to deduce package_uuid notes: [] uuids: brand_uuid: null diff --git a/tests/encode_decode/03_input.yaml b/tests/encode_decode/03_input.yaml index 9478193..7a32d8f 100644 --- a/tests/encode_decode/03_input.yaml +++ b/tests/encode_decode/03_input.yaml @@ -1,13 +1,13 @@ test_config: extra_required_fields: null + expect_success: false data: main: # Material information material_class: FFF material_type: PETG material_name: PETG Jet Black - primary_color: - hex: 24292A + primary_color: "#24292a" tags: [] density: 1.27 diff --git a/tests/encode_decode/04_info.yaml b/tests/encode_decode/04_info.yaml index 2ed0e20..715824b 100644 --- a/tests/encode_decode/04_info.yaml +++ b/tests/encode_decode/04_info.yaml @@ -32,8 +32,7 @@ data: nominal_netto_full_weight: 750 actual_netto_full_weight: 756 empty_container_weight: 310 - primary_color: - hex: '303030' + primary_color: '#303030' tags: - abrasive - contains_carbon_fiber @@ -59,12 +58,15 @@ raw_data: main: bf080009150a7819436172626f6e582050454b4b2d412b4346313520426c61636b0b67334458546563680e1a690c5d9b101902ee111902f4121901361343303030181c9f04181f181eff181df93d8f1821f9366618221901681823190186182518781826188c1827183c1828188c1829185a182a183d182b18c6182c1864182d183518346750454b4b2d434618361a000372d0ff0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 aux: a000000000000000000000000000000000000000000000000000000000000000000000 uri: https://www.3dxtech.com/ -opt_check: +validate: warnings: - Missing recommended field 'gtin' - Missing recommended field 'preheat_temperature' - errors: + errors: [] +opt_check: + warnings: - Failed to deduce package_uuid + errors: [] notes: [] uuids: brand_uuid: 0d616a90-9d18-567b-92f9-ce471171f898 diff --git a/tests/encode_decode/04_input.yaml b/tests/encode_decode/04_input.yaml index 1a8ba28..7c200a6 100644 --- a/tests/encode_decode/04_input.yaml +++ b/tests/encode_decode/04_input.yaml @@ -10,8 +10,7 @@ data: brand_name: 3DXTech material_name: CarbonX PEKK-A+CF15 Black material_abbreviation: PEKK-CF - primary_color: - hex: "303030" + primary_color: "#303030" tags: [abrasive, contains_carbon_fiber, contains_carbon] density: 1.39 diff --git a/tests/encode_decode/05_data.bin b/tests/encode_decode/05_data.bin new file mode 100644 index 0000000..7135abd Binary files /dev/null and b/tests/encode_decode/05_data.bin differ diff --git a/tests/encode_decode/05_info.yaml b/tests/encode_decode/05_info.yaml new file mode 100644 index 0000000..896af03 --- /dev/null +++ b/tests/encode_decode/05_info.yaml @@ -0,0 +1,69 @@ +regions: + meta: + payload_offset: 0 + absolute_offset: 42 + size: 4 + used_size: 4 + main: + payload_offset: 4 + absolute_offset: 46 + size: 230 + used_size: 23 + aux: + payload_offset: 234 + absolute_offset: 276 + size: 35 + used_size: 1 +root: + data_size: 312 + payload_size: 269 + overhead: 43 + payload_used_size: 28 + total_used_size: 71 +data: + meta: + aux_region_offset: 234 + main: + material_class: FFF + transmission_distance: 5.1 + aux: {} +unknown_fields: + main: + 19ffff: '6474657374' +raw_data: + meta: a10218ea + main: bf0800181bfb401466666666666619ffff6474657374ff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + aux: a000000000000000000000000000000000000000000000000000000000000000000000 +uri: null +validate: + warnings: + - Region 'main' contains unknown fields + - Missing recommended field 'gtin' + - Missing recommended field 'material_type' + - Missing recommended field 'material_name' + - Missing recommended field 'brand_name' + - Missing recommended field 'manufactured_date' + - Missing recommended field 'nominal_netto_full_weight' + - Missing recommended field 'actual_netto_full_weight' + - Missing recommended field 'empty_container_weight' + - Missing recommended field 'primary_color' + - Missing recommended field 'tags' + - Missing recommended field 'density' + - Missing recommended field 'min_print_temperature' + - Missing recommended field 'max_print_temperature' + - Missing recommended field 'preheat_temperature' + - Missing recommended field 'min_bed_temperature' + - Missing recommended field 'max_bed_temperature' + errors: [] +opt_check: + warnings: + - Failed to deduce brand_uuid + - Failed to deduce material_uuid + - Failed to deduce package_uuid + errors: [] + notes: [] + uuids: + brand_uuid: null + material_uuid: null + package_uuid: null + instance_uuid: null diff --git a/tests/encode_decode/05_input.yaml b/tests/encode_decode/05_input.yaml new file mode 100644 index 0000000..8105794 --- /dev/null +++ b/tests/encode_decode/05_input.yaml @@ -0,0 +1,12 @@ +test_config: + extra_required_fields: null +data: + main: + material_class: FFF +unknown_fields: + main: + # 27 (= transmission distance): 5.1 + "181b": "fb4014666666666666" + + # 65535 (invalid value): "test" + "19ffff": "6474657374" diff --git a/tests/run_tests.py b/tests/run_tests.py index 81305f0..59a183e 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -118,12 +118,15 @@ def check_util_output(name, data, compare_fn): if val := yml.get("tag_uid"): info_args += ["--tag-uid", val] + expect_success = yml.get("expect_success", True) + utils_test( init_args=init_args, update_args=[str(file)], info_args=info_args, expected_info_fn=fn_info, expected_data_fn=f"{fn_base}_data.bin", + expect_success=expect_success, ) diff --git a/tests/specific/unknown_info_1.yaml b/tests/specific/unknown_info_1.yaml index f6b8035..b8c0d01 100644 --- a/tests/specific/unknown_info_1.yaml +++ b/tests/specific/unknown_info_1.yaml @@ -25,6 +25,9 @@ data: aux_region_offset: 234 main: {} aux: {} +unknown_fields: + main: + 1926fd: 6d48656c6c6f2c20776f726c6421 raw_data: meta: a10218ea main: bf1926fd6d48656c6c6f2c20776f726c6421ff00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 diff --git a/tests/specific/unknown_info_2.yaml b/tests/specific/unknown_info_2.yaml index 2a0cda5..80fbb57 100644 --- a/tests/specific/unknown_info_2.yaml +++ b/tests/specific/unknown_info_2.yaml @@ -26,6 +26,9 @@ data: main: material_class: FFF aux: {} +unknown_fields: + main: + 1926fd: 6d48656c6c6f2c20776f726c6421 raw_data: meta: a10218ea main: bf08001926fd6d48656c6c6f2c20776f726c6421ff0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 diff --git a/tests/validate/01.yaml b/tests/validate/01.yaml index 9e6978b..7f6347c 100644 --- a/tests/validate/01.yaml +++ b/tests/validate/01.yaml @@ -5,8 +5,7 @@ data: material_type: PLA brand_name: Prusament material_name: PLA Prusa Galaxy Black - primary_color: - hex: 3D3E3D + primary_color: "#3d3e3d" tags: [glitter] density: 1.24 diff --git a/utils/fields.py b/utils/fields.py index 8736c9d..1d5ab0d 100644 --- a/utils/fields.py +++ b/utils/fields.py @@ -7,6 +7,7 @@ import cbor2 import io import dataclasses +import re @dataclasses.dataclass @@ -41,10 +42,19 @@ def __init__(self, num: float): self.value = num +# Represent a raw CBOR data that are to be encoded verbatim +class RawCBORData: + data: bytes + + def __init__(self, data: bytes): + self.data = data + + class Field: key: int name: str required: bool + type_name: str def __init__(self, config, config_dir): self.type_name = config["type"] @@ -158,48 +168,16 @@ def encode(self, data): return result -class BytesField(Field): - max_len: int | None - - def __init__(self, config, config_dir): - super().__init__(config, config_dir) - assert "max_length" in config, f"max_length not specified for '{config['name']}'" - self.max_len = config["max_length"] - +class ColorRGBAField(Field): def decode(self, data): assert isinstance(data, bytes) - return {"hex": data.hex()} + return f"#{data.hex()}" def encode(self, data): - if isinstance(data, bytes): - result = data - - elif isinstance(data, str): - result = data.encode("utf-8") - - elif isinstance(data, int): - return data.to_bytes(64, "little").rstrip(b"\x00") - - elif isinstance(data, list): - result = bytearray(data) - - elif isinstance(data, dict): - result = bytearray.fromhex(data["hex"]) - - else: - assert False, f"Cannot encode type {type(data)} to bytes" - - assert self.max_len is None or len(result) <= self.max_len - return result - - -class ColorRGBAField(BytesField): - def __init__(self, config, config_dir): - if "max_length" not in config: - # default to RGBA, but - # leave the door open for RGB or other formats in the future - config["max_length"] = 4 - super().__init__(config, config_dir) + assert isinstance(data, str) + m = re.match(r"^#([0-9a-f]{6}([0-9a-f]{2})?)$", data) + assert m + return bytes.fromhex(m.group(1)) class UUIDField(Field): @@ -218,7 +196,6 @@ def encode(self, data): "enum": EnumField, "enum_array": EnumArrayField, "timestamp": IntField, - "bytes": BytesField, "color_rgba": ColorRGBAField, "uuid": UUIDField, } @@ -259,14 +236,15 @@ def from_file(file: str): # Decodes the fields and values from the CBOR binary data # If out_unknown_fields is provided, unknown fields are written into it instead of asserting - def decode(self, binary_data: typing.IO[bytes], out_unknown_fields: dict[any, any] = None): + def decode(self, binary_data: typing.IO[bytes], out_unknown_fields: dict[str, str] = None): data = cbor2.load(binary_data) result = dict() for key, value in data.items(): field = self.fields_by_key.get(key) if field is None and out_unknown_fields is not None: - out_unknown_fields[key] = value + # TODO: These would ideally be passed verbatim, avoiding the deserialize-serialize loop + out_unknown_fields[cbor2.dumps(key).hex()] = cbor2.dumps(value).hex() continue assert field, f"Unknown CBOR key '{key}'" @@ -283,7 +261,14 @@ def decode(self, binary_data: typing.IO[bytes], out_unknown_fields: dict[any, an def encode(self, data: dict[str, any], config: EncodeConfig = EncodeConfig()) -> bytes: return self.update(update_fields=data, config=config) - def update(self, original_data: typing.IO[bytes] = None, update_fields: dict[str, any] = {}, remove_fields: list[str] = [], config: EncodeConfig = EncodeConfig()) -> bytes: + def update( + self, + original_data: typing.IO[bytes] = None, + update_fields: dict[str, any] = {}, + update_unknown_fields: dict[str, str] = {}, + remove_fields: list[str] = [], + config: EncodeConfig = EncodeConfig(), + ) -> bytes: if original_data: result = cbor2.load(original_data) else: @@ -310,11 +295,19 @@ def update(self, original_data: typing.IO[bytes] = None, update_fields: dict[str if isinstance(value, float): result[field_name] = CompactFloat(value) + # Unknown fields pass verbatim + for key, value in update_unknown_fields.items(): + result[RawCBORData(bytes.fromhex(key))] = RawCBORData(bytes.fromhex(value)) + def default_enc(enc: cbor2.CBOREncoder, data: typing.Any): if isinstance(data, CompactFloat): # Always encode floats canonically # Noncanonically, floats would always be encoded in 8 B, which is a lot of wasted space cbor2.CBOREncoder(enc.fp, canonical=True).encode(data.value) + + elif isinstance(data, RawCBORData): + enc.fp.write(data.data) + else: raise RuntimeError(f"Unsupported type {type(data)} to encode") @@ -328,21 +321,3 @@ def default_enc(enc: cbor2.CBOREncoder, data: typing.Any): encoder.encode(result) return data_io.getvalue() - - def validate(self, decoded_data): - for field_name, field in self.fields_by_name.items(): - if field_name in decoded_data: - continue - - match field.required: - case False: - pass - - case True: - assert False, f"Missing required field '{field.name}'" - - case "recommended": - print(f"Missing recommended field '{field.name}'", file=sys.stderr) - - case _: - assert False, f"Invalid field '{field.name}' 'required' value '{field.required}'" diff --git a/utils/gen_schema.py b/utils/gen_schema.py new file mode 100644 index 0000000..eff333a --- /dev/null +++ b/utils/gen_schema.py @@ -0,0 +1,34 @@ +from fields import Fields, Field +import json +from pathlib import Path + +current_dir = Path(__file__).parent + +properties = {} + +for key in ["meta", "main", "aux"]: + fields = Fields.from_file(current_dir / ".." / "data" / f"{key}_fields.yaml") + props = {} + + field: Field + for field in fields.fields_by_key.values(): + props[field.name] = {"$ref": f"field_types.schema.json#/definitions/{field.type_name}"} + + properties[key] = { + "type": "object", + "properties": props, + "unevaluatedProperties": False, + } + +result = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Auto-generated by gen_schema.py", + "type": "object", + "properties": properties, + "unevaluatedProperties": False, +} +with open(current_dir / "schema" / "fields.schema.json", "w", encoding="utf-8") as f: + json.dump(result, f, indent=4) + + # To stop precommit from complaining + f.write("\n") diff --git a/utils/opt_check.py b/utils/opt_check.py index f34e085..0b616df 100644 --- a/utils/opt_check.py +++ b/utils/opt_check.py @@ -16,7 +16,9 @@ def opt_check(rec: Record, tag_uid: bytes = None): notes = list() uuids = dict() - main_data = rec.main_region.read() + # Pass empty dict to out_unknown_fields so that the script does not crash if the tag has unkown fields + # Unknown fields should by reported by rec_info --validate + main_data = rec.main_region.read(out_unknown_fields={}) # Aux region checks if rec.aux_region is None: @@ -25,15 +27,6 @@ def opt_check(rec: Record, tag_uid: bytes = None): if len(rec.aux_region.memory) < 16: warnings.append("Aux region is smaller than 16 bytes") - # Check we have all required & recommended fields - for field in rec.main_region.fields.fields_by_name.values(): - if field.name in main_data: - pass # Has the field, no problem - elif field.required == "recommended": - warnings.append(f"Missing recommended field '{field.name}'") - elif field.required: - errors.append(f"Missing required field '{field.name}'") - # Check tag transitivities data_tags = main_data.get("tags", []) for tag_data in rec.main_region.fields.fields_by_name["tags"].items_yaml: @@ -47,7 +40,8 @@ def opt_check(rec: Record, tag_uid: bytes = None): for implication in tag_data.get("implies", []): if implication not in data_tags: - errors.append(f"Tag '{tag_name}' present but implied tag '{implication}' not") + # Not an error, just a warning - if the data is older, the implied tag might not have existed + warnings.append(f"Tag '{tag_name}' present but implied tag '{implication}' not") for hint in tag_data.get("hints", []): if hint not in data_tags: @@ -99,7 +93,7 @@ def deduce_uuid(field, generated_uuid, report_deduce_fail: bool = True): else: if report_deduce_fail: - errors.append(f"Failed to deduce {field}") + warnings.append(f"Failed to deduce {field}") result = None diff --git a/utils/rec_info.py b/utils/rec_info.py index d2aa5ef..f6fd3c2 100644 --- a/utils/rec_info.py +++ b/utils/rec_info.py @@ -5,6 +5,12 @@ from record import Record from common import default_config_file from opt_check import opt_check +from pathlib import Path +import referencing +import urllib.parse +import jsonschema +import jsonschema.validators +import json parser = argparse.ArgumentParser(prog="rec_info", description="Reads a record from the STDIN and prints various information about it in the YAML format") parser.add_argument("-c", "--config-file", type=str, default=default_config_file, help="Record configuration YAML file") @@ -41,6 +47,7 @@ record = Record(args.config_file, memoryview(data)) output = {} +return_fail = False if args.show_region_info or args.show_root_info: regions_info = dict() @@ -72,11 +79,11 @@ if name == "meta" and not args.show_meta: continue - unknown_fields = dict() - data[name] = region.read(out_unknown_fields=unknown_fields) + region_unknown_fields = dict() + data[name] = region.read(out_unknown_fields=region_unknown_fields) - if len(unknown_fields) > 0: - unknown_fields[name] = unknown_fields + if len(region_unknown_fields) > 0: + unknown_fields[name] = region_unknown_fields output["data"] = data @@ -95,9 +102,12 @@ if args.show_uri: output["uri"] = record.uri -if args.validate: - for name, region in record.regions.items(): - region.fields.validate(region.read()) +if args.validate or args.opt_check: + validate_result = record.validate() + output["validate"] = validate_result + + if len(validate_result["errors"]) > 0: + return_fail = True if args.extra_required_fields: with open(args.extra_required_fields, "r", encoding="utf-8") as f: @@ -118,7 +128,29 @@ else: tag_uid = None - output["opt_check"] = opt_check(record, tag_uid) + opt_check_result = opt_check(record, tag_uid) + output["opt_check"] = opt_check_result + + if len(opt_check_result["errors"]) > 0: + return_fail = True + + +# Check that the output of this utility is up to the spec +def validate_output_with_json_schema(): + def file_retrieve(uri): + path = Path(__file__).parent / "schema" / urllib.parse.urlparse(uri).path + result = json.loads(path.read_text(encoding="utf-8")) + return referencing.Resource.from_contents(result) + + registry = referencing.Registry(retrieve=file_retrieve) + entry = "opt_json.schema.json" + + schema = registry.get_or_retrieve(entry).value.contents + validator = jsonschema.validators.validator_for(schema)(schema, registry=registry) + validator.validate(output) + + +validate_output_with_json_schema() def yaml_hex_bytes_representer(dumper: yaml.SafeDumper, data: bytes): @@ -131,3 +163,5 @@ class InfoDumper(yaml.SafeDumper): InfoDumper.add_representer(bytes, yaml_hex_bytes_representer) yaml.dump(output, stream=sys.stdout, Dumper=InfoDumper, sort_keys=False) + +sys.exit(1 if return_fail else 0) diff --git a/utils/rec_update.py b/utils/rec_update.py index c2dbffb..835931f 100644 --- a/utils/rec_update.py +++ b/utils/rec_update.py @@ -23,6 +23,7 @@ region.update( update_fields=update_data.get("data", dict()).get(region_name, dict()), remove_fields=update_data.get("remove", dict()).get(region_name, dict()), + update_unknown_fields=update_data.get("unknown_fields", dict()).get(region_name, dict()), clear=args.clear, ) diff --git a/utils/record.py b/utils/record.py index f1b18f5..6c05bc9 100644 --- a/utils/record.py +++ b/utils/record.py @@ -63,12 +63,18 @@ def read(self, out_unknown_fields: dict[any, any] = None) -> dict[str, any]: def write(self, data: dict[str, any]): return self.update(data, clear=True) - def update(self, update_fields: dict[str, any], remove_fields: list[str] = [], clear: bool = False): + def update(self, update_fields: dict[str, any], update_unknown_fields: dict[str, str] = {}, remove_fields: list[str] = [], clear: bool = False): if len(update_fields) == 0 and len(remove_fields) == 0 and not clear: # Nothing to do return - encoded = self.fields.update(original_data=io.BytesIO(self.memory) if not clear else None, update_fields=update_fields, remove_fields=remove_fields, config=self.record.encode_config) + encoded = self.fields.update( + original_data=io.BytesIO(self.memory) if not clear else None, + update_fields=update_fields, + remove_fields=remove_fields, + update_unknown_fields=update_unknown_fields, + config=self.record.encode_config, + ) encoded_len = len(encoded) assert encoded_len <= len(self.memory), f"Data of size {encoded_len} does not fit into region of size {len(self.memory)}" @@ -164,6 +170,34 @@ def __init__(self, config_file: str, data: memoryview): assert type(self.payload) is memoryview self._setup_regions() + # Validates the region and reports possible errors + def validate(self): + warnings = list() + errors = list() + + # Check we have all required & recommended fields + for region_name, region in self.regions.items(): + unknown_fields = {} + region_data = region.read(out_unknown_fields=unknown_fields) + + if len(unknown_fields) > 0: + warnings.append(f"Region '{region_name}' contains unknown fields") + + for field in region.fields.fields_by_name.values(): + if field.name in region_data: + pass # Has the field, no problem + + elif field.required == "recommended": + warnings.append(f"Missing recommended field '{field.name}'") + + elif field.required: + errors.append(f"Missing required field '{field.name}'") + + return { + "warnings": warnings, + "errors": errors, + } + def _setup_regions(self): if "meta_fields" not in self.config.__dict__: # If meta region is not present, we only have the main region which spans the entire payload diff --git a/utils/schema/field_types.schema.json b/utils/schema/field_types.schema.json new file mode 100644 index 0000000..f6912b1 --- /dev/null +++ b/utils/schema/field_types.schema.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "JSON representations of the OpenPrintTag field types", + "definitions": { + "int": { + "type": "integer" + }, + "number": { + "type": "number" + }, + "string": { + "type": "string" + }, + "enum": { + "anyOf": [ + { + "description": "Known enumeration value", + "type": "string" + }, + { + "description": "Unknown enumeration value", + "type": "integer", + "minimum": 0 + } + ] + }, + "enum_array": { + "type": "array", + "items": { + "$ref": "#/definitions/enum" + } + }, + "uuid": { + "type": "string", + "format": "uuid" + }, + "timestamp": { + "description": "UNIX timestamp", + "type": "integer" + }, + "color_rgba": { + "type": "string", + "description": "RGB(A) color in a standard hex notation '#RRGGBB(AA)'", + "pattern": "^#[0-9a-f]{6}([0-9a-f]{2})?$" + } + } +} diff --git a/utils/schema/fields.schema.json b/utils/schema/fields.schema.json new file mode 100644 index 0000000..abccba7 --- /dev/null +++ b/utils/schema/fields.schema.json @@ -0,0 +1,212 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Auto-generated by gen_schema.py", + "type": "object", + "properties": { + "meta": { + "type": "object", + "properties": { + "main_region_offset": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "main_region_size": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "aux_region_offset": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "aux_region_size": { + "$ref": "field_types.schema.json#/definitions/int" + } + }, + "unevaluatedProperties": false + }, + "main": { + "type": "object", + "properties": { + "instance_uuid": { + "$ref": "field_types.schema.json#/definitions/uuid" + }, + "package_uuid": { + "$ref": "field_types.schema.json#/definitions/uuid" + }, + "material_uuid": { + "$ref": "field_types.schema.json#/definitions/uuid" + }, + "brand_uuid": { + "$ref": "field_types.schema.json#/definitions/uuid" + }, + "gtin": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "brand_specific_instance_id": { + "$ref": "field_types.schema.json#/definitions/string" + }, + "brand_specific_package_id": { + "$ref": "field_types.schema.json#/definitions/string" + }, + "brand_specific_material_id": { + "$ref": "field_types.schema.json#/definitions/string" + }, + "material_class": { + "$ref": "field_types.schema.json#/definitions/enum" + }, + "material_type": { + "$ref": "field_types.schema.json#/definitions/enum" + }, + "material_name": { + "$ref": "field_types.schema.json#/definitions/string" + }, + "material_abbreviation": { + "$ref": "field_types.schema.json#/definitions/string" + }, + "brand_name": { + "$ref": "field_types.schema.json#/definitions/string" + }, + "write_protection": { + "$ref": "field_types.schema.json#/definitions/enum" + }, + "manufactured_date": { + "$ref": "field_types.schema.json#/definitions/timestamp" + }, + "country_of_origin": { + "$ref": "field_types.schema.json#/definitions/string" + }, + "expiration_date": { + "$ref": "field_types.schema.json#/definitions/timestamp" + }, + "nominal_netto_full_weight": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "actual_netto_full_weight": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "nominal_full_length": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "actual_full_length": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "empty_container_weight": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "primary_color": { + "$ref": "field_types.schema.json#/definitions/color_rgba" + }, + "secondary_color_0": { + "$ref": "field_types.schema.json#/definitions/color_rgba" + }, + "secondary_color_1": { + "$ref": "field_types.schema.json#/definitions/color_rgba" + }, + "secondary_color_2": { + "$ref": "field_types.schema.json#/definitions/color_rgba" + }, + "secondary_color_3": { + "$ref": "field_types.schema.json#/definitions/color_rgba" + }, + "secondary_color_4": { + "$ref": "field_types.schema.json#/definitions/color_rgba" + }, + "transmission_distance": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "tags": { + "$ref": "field_types.schema.json#/definitions/enum_array" + }, + "certifications": { + "$ref": "field_types.schema.json#/definitions/enum_array" + }, + "density": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "filament_diameter": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "shore_hardness_a": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "shore_hardness_d": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "min_nozzle_diameter": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "min_print_temperature": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "max_print_temperature": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "preheat_temperature": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "min_bed_temperature": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "max_bed_temperature": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "min_chamber_temperature": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "max_chamber_temperature": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "chamber_temperature": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "container_width": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "container_outer_diameter": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "container_inner_diameter": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "container_hole_diameter": { + "$ref": "field_types.schema.json#/definitions/int" + }, + "viscosity_18c": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "viscosity_25c": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "viscosity_40c": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "viscosity_60c": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "container_volumetric_capacity": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "cure_wavelength": { + "$ref": "field_types.schema.json#/definitions/int" + } + }, + "unevaluatedProperties": false + }, + "aux": { + "type": "object", + "properties": { + "consumed_weight": { + "$ref": "field_types.schema.json#/definitions/number" + }, + "workgroup": { + "$ref": "field_types.schema.json#/definitions/string" + }, + "general_purpose_range_user": { + "$ref": "field_types.schema.json#/definitions/string" + }, + "last_stir_time": { + "$ref": "field_types.schema.json#/definitions/timestamp" + } + }, + "unevaluatedProperties": false + } + }, + "unevaluatedProperties": false +} diff --git a/utils/schema/opt_json.schema.json b/utils/schema/opt_json.schema.json new file mode 100644 index 0000000..e3e1171 --- /dev/null +++ b/utils/schema/opt_json.schema.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OpenPrintTag JSON representation", + "description": "Decoded JSON/YAML representation of the binary data stored on an OpenPrintTag", + "type": "object", + "properties": { + "data": { + "description": "Decoded data on regions of the tag", + "$ref": "fields.schema.json", + "unevaluatedProperties": false + }, + "unknown_fields": { + "description": "Fields that are not in the specification - specification mandates to preserve them. Kept as a key-value binary hex strings.", + "properties": { + "meta": { + "$ref": "#/definitions/unknown_fields_region" + }, + "main": { + "$ref": "#/definitions/unknown_fields_region" + }, + "aux": { + "$ref": "#/definitions/unknown_fields_region" + } + }, + "additionalProperties": false + } + }, + "definitions": { + "unknown_fields_region": { + "type": "object", + "patternProperties": { + "^([0-9a-f]{2})+$": { + "description": "Both key and value are raw CBOR data (of the respective CBOR map key and value) encoded as a hex string.", + "type": "string", + "pattern": "^([0-9a-f]{2})+$" + } + }, + "unevaluatedProperties": false + } + } +}