diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 715104f1..582dfc83 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -91,7 +91,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] name: Merge tests 2 - Python (${{ matrix.python-version }}) functional tests (Ubuntu) steps: - name: Checkout the code @@ -153,7 +153,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - name: Install zcbor uses: ./.github/actions/install_zcbor @@ -248,7 +248,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] asserts: ["", "-x VERBOSE=ON -x ASSERTS=ON"] name: Release tests 3 (Python ${{ matrix.python-version }}${{ matrix.asserts != '' && ' with asserts' || '' }}) needs: @@ -284,7 +284,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] asserts: ["", "-x VERBOSE=ON -x ASSERTS=ON"] name: Release tests 4 (Python ${{ matrix.python-version }}${{ matrix.asserts != '' && ' with asserts' || '' }}) needs: diff --git a/README.md b/README.md index 713ad357..0e7e3834 100644 --- a/README.md +++ b/README.md @@ -438,8 +438,8 @@ zcbor code --help usage: zcbor code [-h] -c CDDL [--no-prelude] [-v] [--default-max-qty DEFAULT_MAX_QTY] [--output-c OUTPUT_C] [--output-h OUTPUT_H] [--output-h-types OUTPUT_H_TYPES] - [--copy-sources] [--output-cmake OUTPUT_CMAKE] -t - ENTRY_TYPES [ENTRY_TYPES ...] [-d] [-e] [--time-header] + [--copy-sources] [--output-cmake OUTPUT_CMAKE] + -t ENTRY_TYPES [ENTRY_TYPES ...] [-d] [-e] [--time-header] [--git-sha-header] [-b {32,64}] [--include-prefix INCLUDE_PREFIX] [-s] [--file-header FILE_HEADER] @@ -461,14 +461,14 @@ This script requires 'regex' for lookaround functionality not present in 're'. options: -h, --help show this help message and exit - -c CDDL, --cddl CDDL Path to one or more input CDDL file(s). Passing + -c, --cddl CDDL Path to one or more input CDDL file(s). Passing multiple files is equivalent to concatenating them. --no-prelude Exclude the standard CDDL prelude from the build. The prelude can be viewed at zcbor/prelude.cddl in the repo, or together with the script. -v, --verbose Print more information while parsing CDDL and generating code. - --default-max-qty DEFAULT_MAX_QTY, --dq DEFAULT_MAX_QTY + --default-max-qty, --dq DEFAULT_MAX_QTY Default maximum number of repetitions when no maximum is specified. This is needed to construct complete C types. The default_max_qty can usually be set to a @@ -477,7 +477,7 @@ options: as sometimes the value is needed for internal computations. If so, the script will raise an exception. - --output-c OUTPUT_C, --oc OUTPUT_C + --output-c, --oc OUTPUT_C Path to output C file. If both --decode and --encode are specified, _decode and _encode will be appended to the filename when creating the two files. If not @@ -486,7 +486,7 @@ options: next to the cmake file, and the C file will be placed there with the same name (except the file extension) as the cmake file. - --output-h OUTPUT_H, --oh OUTPUT_H + --output-h, --oh OUTPUT_H Path to output header file. If both --decode and --encode are specified, _decode and _encode will be appended to the filename when creating the two files. @@ -495,7 +495,7 @@ options: be created next to the cmake file, and the C file will be placed there with the same name (except the file extension) as the cmake file. - --output-h-types OUTPUT_H_TYPES, --oht OUTPUT_H_TYPES + --output-h-types, --oht OUTPUT_H_TYPES Path to output header file with typedefs (shared between decode and encode). If not specified, the path and name will be taken from the output header file @@ -513,7 +513,7 @@ options: include() in your CMakeLists.txt file, and link the target to your program. This option works with or without the --copy-sources option. - -t ENTRY_TYPES [ENTRY_TYPES ...], --entry-types ENTRY_TYPES [ENTRY_TYPES ...] + -t, --entry-types ENTRY_TYPES [ENTRY_TYPES ...] Names of the types which should have their xcode functions exposed. -d, --decode Generate decoding code. Either --decode or --encode or @@ -524,7 +524,7 @@ options: files. --git-sha-header Put the current git sha of zcbor in a comment in the generated files. - -b {32,64}, --default-bit-size {32,64} + -b, --default-bit-size {32,64} Default bit size of integers in code. When integers have no explicit bounds, assume they have this bit width. Should follow the bit width of the architecture @@ -560,25 +560,24 @@ CDDL schema file. options: -h, --help show this help message and exit - -c CDDL, --cddl CDDL Path to one or more input CDDL file(s). Passing + -c, --cddl CDDL Path to one or more input CDDL file(s). Passing multiple files is equivalent to concatenating them. --no-prelude Exclude the standard CDDL prelude from the build. The prelude can be viewed at zcbor/prelude.cddl in the repo, or together with the script. -v, --verbose Print more information while parsing CDDL and generating code. - -i INPUT, --input INPUT - Input data file. The option --input-as specifies how + -i, --input INPUT Input data file. The option --input-as specifies how to interpret the contents. Use "-" to indicate stdin. --input-as {yaml,json,cbor,cborhex} Which format to interpret the input file as. If omitted, the format is inferred from the file name. .yaml, .yml => YAML, .json => JSON, .cborhex => CBOR as hex string, everything else => CBOR - -t ENTRY_TYPE, --entry-type ENTRY_TYPE + -t, --entry-type ENTRY_TYPE Name of the type (from the CDDL) to interpret the data as. - --default-max-qty DEFAULT_MAX_QTY, --dq DEFAULT_MAX_QTY + --default-max-qty, --dq DEFAULT_MAX_QTY Default maximum number of repetitions when no maximum is specified. It is only relevant when handling data that will be decoded by generated code. If omitted, a @@ -613,25 +612,24 @@ conform. 'zcbor validate' can be used if only validate is needed. options: -h, --help show this help message and exit - -c CDDL, --cddl CDDL Path to one or more input CDDL file(s). Passing + -c, --cddl CDDL Path to one or more input CDDL file(s). Passing multiple files is equivalent to concatenating them. --no-prelude Exclude the standard CDDL prelude from the build. The prelude can be viewed at zcbor/prelude.cddl in the repo, or together with the script. -v, --verbose Print more information while parsing CDDL and generating code. - -i INPUT, --input INPUT - Input data file. The option --input-as specifies how + -i, --input INPUT Input data file. The option --input-as specifies how to interpret the contents. Use "-" to indicate stdin. --input-as {yaml,json,cbor,cborhex} Which format to interpret the input file as. If omitted, the format is inferred from the file name. .yaml, .yml => YAML, .json => JSON, .cborhex => CBOR as hex string, everything else => CBOR - -t ENTRY_TYPE, --entry-type ENTRY_TYPE + -t, --entry-type ENTRY_TYPE Name of the type (from the CDDL) to interpret the data as. - --default-max-qty DEFAULT_MAX_QTY, --dq DEFAULT_MAX_QTY + --default-max-qty, --dq DEFAULT_MAX_QTY Default maximum number of repetitions when no maximum is specified. It is only relevant when handling data that will be decoded by generated code. If omitted, a @@ -644,8 +642,7 @@ options: supports. bytestrings (BSTR), tags, undefined, and maps with non-text keys need special handling. See the zcbor README for more information. - -o OUTPUT, --output OUTPUT - Output data file. The option --output-as specifies how + -o, --output OUTPUT Output data file. The option --output-as specifies how to interpret the contents. Use "-" to indicate stdout. --output-as {yaml,json,cbor,cborhex,c_code} Which format to interpret the output file as. If diff --git a/samples/hello_world/src/main.c b/samples/hello_world/src/main.c index e7ac53cc..a4a34e56 100644 --- a/samples/hello_world/src/main.c +++ b/samples/hello_world/src/main.c @@ -9,7 +9,7 @@ #include #include -void main(void) +int main(void) { uint8_t cbor_payload[15]; bool success; @@ -23,7 +23,7 @@ void main(void) if (!success) { printf("Encoding failed: %d\r\n", zcbor_peek_error(encoding_state)); - return; + return 1; } /* Create zcbor state variable for decoding. */ @@ -34,8 +34,9 @@ void main(void) if (!success) { printf("Decoding failed: %d\r\n", zcbor_peek_error(decoding_state)); - return; + return 1; } printf("Decoded string: '%.*s'\r\n", (int)decoded_string.len, decoded_string.value); + return 0; } diff --git a/samples/pet/src/main.c b/samples/pet/src/main.c index 22aa9852..eedabf8f 100644 --- a/samples/pet/src/main.c +++ b/samples/pet/src/main.c @@ -120,9 +120,10 @@ static void get_pet3(void) print_pet(&decoded_pet); } -void main(void) +int main(void) { get_pet1(); get_pet2(); get_pet3(); + return 0; } diff --git a/scripts/requirements-test.txt b/scripts/requirements-test.txt index 3fa33988..9c8c0bdb 100644 --- a/scripts/requirements-test.txt +++ b/scripts/requirements-test.txt @@ -2,3 +2,4 @@ pyelftools pycodestyle west ecdsa +setuptools diff --git a/tests/decode/test1_suit_old_formats/CMakeLists.txt b/tests/decode/test1_suit_old_formats/CMakeLists.txt index c8ecad03..a3b427ca 100644 --- a/tests/decode/test1_suit_old_formats/CMakeLists.txt +++ b/tests/decode/test1_suit_old_formats/CMakeLists.txt @@ -37,6 +37,10 @@ set(py_command4 execute_process( COMMAND ${py_command3} + COMMAND_ERROR_IS_FATAL ANY +) + +execute_process( COMMAND ${py_command4} COMMAND_ERROR_IS_FATAL ANY diff --git a/tests/scripts/test_repo_files.py b/tests/scripts/test_repo_files.py index ae62a867..cdec2e23 100644 --- a/tests/scripts/test_repo_files.py +++ b/tests/scripts/test_repo_files.py @@ -8,7 +8,7 @@ from pathlib import Path from re import search, S, compile from urllib import request -from urllib.error import HTTPError +from urllib.error import HTTPError, URLError from argparse import ArgumentParser from subprocess import Popen, check_output, PIPE, run from pycodestyle import StyleGuide @@ -127,6 +127,39 @@ def test_pet_file_header(self): class TestDocs(TestCase): + @staticmethod + def _repo_url_to_github_https(repo_url: str) -> str | None: + """Convert common git remote URL formats to https://github.com/...""" + repo_url = repo_url.strip() + + # git@github.com:org/repo(.git) + if repo_url.startswith("git@github.com:"): + repo_url = "https://github.com/" + repo_url.removeprefix("git@github.com:") + + # git@github.com/org/repo(.git) + elif repo_url.startswith("git@github.com/"): + repo_url = "https://github.com/" + repo_url.removeprefix("git@github.com/") + + # ssh://git@github.com/org/repo(.git) + elif repo_url.startswith("ssh://git@github.com/"): + repo_url = "https://github.com/" + repo_url.removeprefix("ssh://git@github.com/") + + # https://github.com/org/repo(.git) + elif repo_url.startswith("https://github.com/"): + pass + + # http://github.com/org/repo(.git) + elif repo_url.startswith("http://github.com/"): + repo_url = "https://github.com/" + repo_url.removeprefix("http://github.com/") + + else: + return None + + if repo_url.endswith(".git"): + repo_url = repo_url.removesuffix(".git") + + return repo_url + def __init__(self, *args, **kwargs): """Overridden to get base URL for relative links from remote tracking branch.""" super(TestDocs, self).__init__(*args, **kwargs) @@ -136,12 +169,11 @@ def __init__(self, *args, **kwargs): if remote_tracking: remote, remote_branch = remote_tracking.split('/', 1) # '1' to only split one time. repo_url_args = ['git', 'remote', 'get-url', remote] - repo_url = check_output(repo_url_args).decode('utf-8').strip().strip('.git') - if 'github.com' in repo_url: - self.base_url = (repo_url + '/tree/' + remote_branch + '/') - else: - # The URL is not in github.com, so we are not sure it is constructed correctly. - self.base_url = None + repo_url_raw = check_output(repo_url_args).decode('utf-8').strip() + repo_url = self._repo_url_to_github_https(repo_url_raw) + + # Use /blob// so relative file links resolve correctly. + self.base_url = (repo_url + '/blob/' + remote_branch + '/') if repo_url else None elif "GITHUB_SHA" in os.environ and "GITHUB_REPOSITORY" in os.environ: repo = os.environ["GITHUB_REPOSITORY"] sha = os.environ["GITHUB_SHA"] @@ -155,10 +187,20 @@ def __init__(self, *args, **kwargs): def check_code(self, link, codes): """Check the status code of a URL link. Assert if not 200 (OK).""" try: - call = request.urlopen(link) + req = request.Request( + link, + headers={ + # Some sites (e.g. Wikipedia) may reject urllib's default user-agent. + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) zcbor-tests", + }, + ) + call = request.urlopen(req, timeout=10) code = call.getcode() except HTTPError as e: code = e.code + except Exception as e: + # Avoid noisy thread stack traces; surface the root cause in the assertion. + code = f"{type(e).__name__}: {e}" codes.append((link, code)) def do_test_links(self, path, allow_local=True): @@ -175,7 +217,6 @@ def do_test_links(self, path, allow_local=True): matches = self.link_regex.findall(text) codes = list() - threads = list() for m in matches: link = m if allow_local: @@ -186,12 +227,16 @@ def do_test_links(self, path, allow_local=True): link = self.base_url + relative_path + link else: self.assertTrue(link.startswith("https://"), "Link is not a URL") - threads.append(t := Thread(target=self.check_code, args=(link, codes), daemon=True)) - t.start() - for t in threads: - t.join() + + # Run sequentially to avoid noisy threaded exception dumps when + # the environment has SSL/HTTP issues. + self.check_code(link, codes) + for link, code in codes: - self.assertEqual(code, 200, f"'{link}' gives code {code}") + if isinstance(code, int): + self.assertEqual(code, 200, f"'{link}' gives code {code}") + else: + self.fail(f"'{link}' failed: {code}") def test_readme_links(self): self.do_test_links(p_readme) diff --git a/tests/test.sh b/tests/test.sh index 2b2d2d7a..864df824 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -5,13 +5,20 @@ # SPDX-License-Identifier: Apache-2.0 # -pushd "scripts" +set -eu + +pushd "tests/scripts" python3 -m unittest test_zcbor test_repo_files [[ $? -ne 0 ]] && popd && exit 1 popd -if [[ -z "$ZEHPYR_BASE" ]]; then - ZEPHYR_BASE=$(west topdir)/zephyr +if ! command -v west >/dev/null 2>&1; then + echo "west not found; cannot run Zephyr twister tests." >&2 + exit 1 fi -$ZEPHYR_BASE/scripts/twister -M -v -T . -W --exclude-tag release --platform native_sim --platform native_sim/native/64 --platform mps2/an521/cpu0 $* +west twister -M -v -T . -W --exclude-tag release \ + --platform native_sim \ + --platform native_sim/native/64 \ + --platform mps2/an521/cpu0 \ + "$@" diff --git a/tests/unit/test2_cpp/prj.conf b/tests/unit/test2_cpp/prj.conf index 6c71f2a8..8165e844 100644 --- a/tests/unit/test2_cpp/prj.conf +++ b/tests/unit/test2_cpp/prj.conf @@ -4,4 +4,4 @@ # SPDX-License-Identifier: Apache-2.0 # -CONFIG_CPLUSPLUS=y +CONFIG_CPP=y diff --git a/tests/unit/test3_float16/CMakeLists.txt b/tests/unit/test3_float16/CMakeLists.txt index c84206dd..7b5b2fe3 100644 --- a/tests/unit/test3_float16/CMakeLists.txt +++ b/tests/unit/test3_float16/CMakeLists.txt @@ -21,24 +21,44 @@ if (VERBOSE) return() # Because the output volume is too large endif() -if (CONFIG_NATIVE_APPLICATION AND NOT CONFIG_64BIT) +if (CONFIG_NATIVE_BUILD AND NOT CONFIG_64BIT) set (LD_SIMULATION -m elf_i386) endif() set(LD ld) -if(NOT CONFIG_NATIVE_APPLICATION) +if(NOT CONFIG_NATIVE_BUILD) set(LD ${CROSS_COMPILE_TARGET}-ld) if(DEFINED ENV{ZEPHYR_SDK_INSTALL_DIR}) set(LD $ENV{ZEPHYR_SDK_INSTALL_DIR}/${CROSS_COMPILE_TARGET}/bin/${LD}) endif() endif() -execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/include RESULT_VARIABLE err1) -execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/fp_bytes_decode.bin ${PROJECT_BINARY_DIR}/fp_bytes_decode.bin RESULT_VARIABLE err2) # copy before ld so symbol names are shorter. -execute_process(COMMAND ${LD} ${LD_SIMULATION} -r -b binary -o fp_bytes_decode.o fp_bytes_decode.bin WORKING_DIRECTORY ${PROJECT_BINARY_DIR} RESULT_VARIABLE err3) -target_link_libraries(app PRIVATE ${PROJECT_BINARY_DIR}/fp_bytes_decode.o) +if(DEFINED CMAKE_LINKER) + set(LD ${CMAKE_LINKER}) +endif() + +execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/include) + +function(zcbor_add_binary_obj OUT_VAR BIN_NAME) + set(bin_src ${CMAKE_CURRENT_LIST_DIR}/${BIN_NAME}.bin) + set(bin_dst ${PROJECT_BINARY_DIR}/${BIN_NAME}.bin) + set(obj_out ${PROJECT_BINARY_DIR}/${BIN_NAME}.o) + + add_custom_command( + OUTPUT ${obj_out} + COMMAND ${CMAKE_COMMAND} -E copy ${bin_src} ${bin_dst} + COMMAND ${LD} ${LD_SIMULATION} -r -b binary -o ${obj_out} ${BIN_NAME}.bin + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + DEPENDS ${bin_src} + VERBATIM + ) + + set(${OUT_VAR} ${obj_out} PARENT_SCOPE) +endfunction() -execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/fp_bytes_encode.bin ${PROJECT_BINARY_DIR}/fp_bytes_encode.bin RESULT_VARIABLE err4) # copy before ld so symbol names are shorter. -execute_process(COMMAND ${LD} ${LD_SIMULATION} -r -b binary -o fp_bytes_encode.o fp_bytes_encode.bin WORKING_DIRECTORY ${PROJECT_BINARY_DIR} RESULT_VARIABLE err5) +zcbor_add_binary_obj(FP_DECODE_OBJ fp_bytes_decode) +zcbor_add_binary_obj(FP_ENCODE_OBJ fp_bytes_encode) -target_link_libraries(app PRIVATE ${PROJECT_BINARY_DIR}/fp_bytes_encode.o) +add_custom_target(fp_bytes_objs DEPENDS ${FP_DECODE_OBJ} ${FP_ENCODE_OBJ}) +add_dependencies(app fp_bytes_objs) +target_link_libraries(app PRIVATE ${FP_DECODE_OBJ} ${FP_ENCODE_OBJ}) diff --git a/zcbor/zcbor.py b/zcbor/zcbor.py index ea649a01..625ae2d5 100755 --- a/zcbor/zcbor.py +++ b/zcbor/zcbor.py @@ -1789,16 +1789,17 @@ def _to_yaml_obj(self, obj): retval[key] = self._to_yaml_obj(val) return retval elif isinstance(obj, bytes): - f = BytesIO(obj) try: - bstr_obj = self._to_yaml_obj(load(f)) + decoded = loads(obj) + # Only treat it as nested CBOR if it round-trips exactly. + # This avoids mis-classifying arbitrary byte sequences like: + # 0x01 0x23 0x45 ... (which "loads" will happily parse as 1). + if dumps(decoded) == obj and not isinstance(decoded, bytes): + bstr_obj = self._to_yaml_obj(decoded) + else: + bstr_obj = obj.hex() except (CBORDecodeValueError, CBORDecodeEOF): - # failed decoding bstr_obj = obj.hex() - else: - if f.read(1) != b'': - # not fully decoded - bstr_obj = obj.hex() return {"zcbor_bstr": bstr_obj} elif isinstance(obj, CBORTag): return {"zcbor_tag": obj.tag, "zcbor_tag_val": self._to_yaml_obj(obj.value)}