diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 56fd40e7..b72ec6db 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -175,6 +175,8 @@ There are up to 4 files constructed: - The H file with all the struct definitions (the type file). If both decoding and encoding files are generated for the same CDDL, they can share the same type file. - An optional cmake file for building the generated code together with the zcbor C libraries. +When `zcbor code` is invoked with `--stream-encode`, CodeRenderer also emits streaming encode entrypoints (`cbor_stream_encode_`) and the associated provider structs used to supply repeated elements and chunked `tstr`/`bstr` values. + CodeRenderer conducts some pruning and deduplication of the list of types and functions received from CodeGenerator. diff --git a/README.md b/README.md index 0e7e3834..5d82cd46 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ There are samples in the [samples](samples) directory that demonstrate different 1. The [hello_world sample](samples/hello_world/README.md) is a minimum examples of encoding and decoding using the C library. 2. The [pet sample](samples/pet/README.md) shows a how to use the C library together with generated code, and how to use the script tool to do code generation and data conversion. +3. The [streaming_chunks sample](samples/streaming_chunks/README.md) shows how to use streaming encode and decode entrypoints with chunk callbacks. The [tests](tests) also demonstrate how to use zcbor in different ways. The [encoding](tests/encode), [decoding](tests/decode), and [unit](tests/unit) tests run using [Zephyr](https://github.com/zephyrproject-rtos/zephyr) (the samples do not use Zephyr). @@ -196,6 +197,34 @@ The tests require [Zephyr](https://github.com/zephyrproject-rtos/zephyr) (if you The generated C code is C++ compatible. +Streaming encode/decode entrypoints +----------------------------------- + +For low-RAM paths, `zcbor code` can optionally generate streaming encode and decode entrypoints: + +- Encode: `cbor_stream_encode_(zcbor_stream_write_fn stream_write, void *stream_user_data, const *input, const struct cbor_stream_io_ *prov, size_t *bytes_written_out)` +- Decode: `cbor_stream_decode_(const uint8_t *payload, size_t payload_len, *result, const struct cbor_stream_io_ *prov, size_t *payload_len_out)` + +Enable them with `--stream-encode` and `--stream-decode`. + +Streaming encode entrypoints write CBOR via the supplied callback (no output buffer), and can stream: + +- the `zcbor_stream_write_fn` callback signature is: + `size_t (*)(void *user_data, const uint8_t *data, size_t len)` and should return + the number of bytes written (must equal `len` for success) +- repeated fields via iterator callbacks in `struct cbor_stream_io_` +- `bstr`/`tstr` values via chunk callbacks (use `chunks_out_*` fields) that emit chunks into the encoder + +Streaming decode entrypoints let you consume `bstr`/`tstr` chunks without reassembly: + +- the `zcbor_stream_chunk_in` callback signature is: + `bool (*)(void *ctx, const uint8_t *data, size_t len)` +- chunk callbacks live in `struct cbor_stream_io_` (use `chunks_in_*` fields) +- when chunk io is present, the generated decode logic calls + `zcbor_tstr_chunk_in()` / `zcbor_bstr_chunk_in()` instead of + decoding into a single contiguous buffer, so the corresponding `zcbor_string` + field is not populated and the callback is responsible for capturing data + Build system ------------ @@ -436,7 +465,8 @@ zcbor code --help ``` usage: zcbor code [-h] -c CDDL [--no-prelude] [-v] - [--default-max-qty DEFAULT_MAX_QTY] [--output-c OUTPUT_C] + [--default-max-qty DEFAULT_MAX_QTY] [--repeated-as-pointers] + [--stream-encode] [--stream-decode] [--output-c OUTPUT_C] [--output-h OUTPUT_H] [--output-h-types OUTPUT_H_TYPES] [--copy-sources] [--output-cmake OUTPUT_CMAKE] -t ENTRY_TYPES [ENTRY_TYPES ...] [-d] [-e] [--time-header] @@ -477,6 +507,25 @@ options: as sometimes the value is needed for internal computations. If so, the script will raise an exception. + --repeated-as-pointers + Represent repeated fields (max_qty > 1) as pointer + + count instead of embedding a fixed-size array in the + generated types. This can significantly reduce the + size of top-level unions/structs at the cost of + requiring the caller to provide storage for decode, + and a readable array for encode. + --stream-encode Also generate streaming encode entrypoints + (cbor_stream_encode_) for each entry type. These + use zcbor_new_encode_state_streaming() and are + intended for low-RAM UART write paths. Streaming + encode entrypoints support: - repeated fields via + zcbor_multi_encode_iter_minmax() (iterator callback) - + tstr/bstr fields via chunk callbacks + --stream-decode Also generate streaming decode entrypoints + (cbor_stream_decode_) for each entry type. These + use zcbor_tstr_chunk_in()/zcbor_bstr_chunk_in() + callbacks and allow indefinite-length tstr/bstr values + to be processed without reassembly. --output-c, --oc OUTPUT_C Path to output C file. If both --decode and --encode are specified, _decode and _encode will be appended to diff --git a/include/zcbor_common.h b/include/zcbor_common.h index 6d0f8362..9df755ac 100644 --- a/include/zcbor_common.h +++ b/include/zcbor_common.h @@ -152,8 +152,61 @@ struct zcbor_state_constant { #ifdef ZCBOR_MAP_SMART_SEARCH uint8_t *map_search_elem_state_end; /**< The end of the @ref map_search_elem_state buffer. */ #endif + size_t (*stream_write)(void *user_data, const uint8_t *data, size_t len); + void *stream_user_data; + size_t stream_bytes_written; /**< Total bytes written in streaming mode. */ + + const void *stream_io; + }; +#ifndef ZCBOR_STREAM_IO_HELPERS +#define ZCBOR_STREAM_IO_HELPERS +static inline void zcbor_set_stream_io(zcbor_state_t *state, const void *io) +{ + if (state && state->constant_state) { + state->constant_state->stream_io = io; + } +} + +static inline const void *zcbor_get_stream_io(const zcbor_state_t *state) +{ + return (state && state->constant_state) ? state->constant_state->stream_io : NULL; +} +#endif + +/* Alignment helper for state storage. */ +#if defined(__cplusplus) && (__cplusplus >= 201103L) +#define ZCBOR_ALIGNAS(type) alignas(type) +#define ZCBOR_ALIGNOF(type) alignof(type) +#define ZCBOR_STATIC_ASSERT(cond, msg) static_assert((cond), msg) +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +#define ZCBOR_ALIGNAS(type) _Alignas(type) +#define ZCBOR_ALIGNOF(type) _Alignof(type) +#define ZCBOR_STATIC_ASSERT(cond, msg) _Static_assert((cond), msg) +#else +/* Require GCC extensions when C11/C++11 alignment is unavailable. */ +#if defined(__GNUC__) +#define ZCBOR_ALIGNAS(type) __attribute__((aligned(__alignof__(type)))) +#define ZCBOR_ALIGNOF(type) __alignof__(type) +#define ZCBOR_STATIC_ASSERT(cond, msg) _Static_assert((cond), msg) +#else +#error "Unsupported compiler: zcbor requires alignment and static assertion support." +#endif +#endif + +/* We store struct zcbor_state_constant in state storage that is aligned as + * zcbor_state_t. Ensure that is sufficient for the constant state object. + */ +ZCBOR_STATIC_ASSERT((ZCBOR_ALIGNOF(zcbor_state_t) >= ZCBOR_ALIGNOF(struct zcbor_state_constant)), + "zcbor_state_t alignment must be >= zcbor_state_constant alignment."); + +/** Number of zcbor_state_t slots required to store a struct zcbor_state_constant + * object at the end of the state array. + */ +#define ZCBOR_CONST_STATE_SLOTS \ + ((sizeof(struct zcbor_state_constant) + sizeof(zcbor_state_t) - 1) / sizeof(zcbor_state_t)) + #ifdef ZCBOR_CANONICAL #define ZCBOR_ENFORCE_CANONICAL_DEFAULT true #else @@ -285,6 +338,8 @@ do { \ #define ZCBOR_ERR_MAP_FLAGS_NOT_AVAILABLE 20 #define ZCBOR_ERR_INVALID_VALUE_ENCODING 21 ///! When ZCBOR_CANONICAL is defined, and the incoming data is not encoded with minimal length, or uses indefinite length array. #define ZCBOR_ERR_CONSTANT_STATE_MISSING 22 +#define ZCBOR_ERR_STREAM_WRITE_FAILED 23 ///! Streaming callback returned error or wrote fewer bytes than requested +#define ZCBOR_ERR_STREAM_READ_FAILED 24 ///! Streaming provider callback returned error or invalid data #define ZCBOR_ERR_UNKNOWN 31 /** The largest possible elem_count. */ diff --git a/include/zcbor_decode.h b/include/zcbor_decode.h index 90319d21..712b9388 100644 --- a/include/zcbor_decode.h +++ b/include/zcbor_decode.h @@ -44,10 +44,15 @@ void zcbor_new_decode_state(zcbor_state_t *state_array, size_t n_states, * including elements in nested unordered maps. */ #define ZCBOR_STATE_D(name, num_backups, payload, payload_size, elem_count, n_flags) \ -zcbor_state_t name[((num_backups) + 2 + ZCBOR_FLAG_STATES(n_flags))]; \ +ZCBOR_ALIGNAS(zcbor_state_t) uint8_t name##_storage[ \ + (((num_backups) + 1 + ZCBOR_FLAG_STATES(n_flags) + ZCBOR_CONST_STATE_SLOTS) * sizeof(zcbor_state_t)) \ +]; \ +zcbor_state_t *name = (zcbor_state_t *)name##_storage; \ do { \ - zcbor_new_decode_state(name, ZCBOR_ARRAY_SIZE(name), payload, payload_size, elem_count, \ - (uint8_t *)&name[(num_backups) + 1], ZCBOR_FLAG_STATES(n_flags) * sizeof(zcbor_state_t)); \ + zcbor_new_decode_state(name, ((num_backups) + 1 + ZCBOR_FLAG_STATES(n_flags) + ZCBOR_CONST_STATE_SLOTS), \ + payload, payload_size, elem_count, \ + (uint8_t *)&name[(num_backups) + 1], \ + ZCBOR_FLAG_STATES(n_flags) * sizeof(zcbor_state_t)); \ } while(0) @@ -73,6 +78,10 @@ bool zcbor_uint_decode(zcbor_state_t *state, void *result, size_t result_size); bool zcbor_bstr_decode(zcbor_state_t *state, struct zcbor_string *result); /* bstr */ bool zcbor_tstr_decode(zcbor_state_t *state, struct zcbor_string *result); /* tstr */ bool zcbor_tag_decode(zcbor_state_t *state, uint32_t *result); /* CBOR tag */ + +typedef bool (*zcbor_stream_chunk_in)(void *ctx, const uint8_t *data, size_t len); +bool zcbor_tstr_chunk_in(zcbor_state_t *state, zcbor_stream_chunk_in call, void *ctx); +bool zcbor_bstr_chunk_in(zcbor_state_t *state, zcbor_stream_chunk_in call, void *ctx); bool zcbor_bool_decode(zcbor_state_t *state, bool *result); /* boolean CBOR simple value */ bool zcbor_float16_decode(zcbor_state_t *state, float *result); /* IEEE754 float16 */ bool zcbor_float16_bytes_decode(zcbor_state_t *state, uint16_t *result); /* IEEE754 float16 raw bytes */ diff --git a/include/zcbor_encode.h b/include/zcbor_encode.h index 9bd9383c..05386931 100644 --- a/include/zcbor_encode.h +++ b/include/zcbor_encode.h @@ -40,9 +40,13 @@ void zcbor_new_encode_state(zcbor_state_t *state_array, size_t n_states, * @param[in] elem_count The starting elem_count (typically 1). */ #define ZCBOR_STATE_E(name, num_backups, payload, payload_size, elem_count) \ -zcbor_state_t name[((num_backups) + 2)]; \ +ZCBOR_ALIGNAS(zcbor_state_t) uint8_t name##_storage[ \ + (((num_backups) + 1 + ZCBOR_CONST_STATE_SLOTS) * sizeof(zcbor_state_t)) \ +]; \ +zcbor_state_t *name = (zcbor_state_t *)name##_storage; \ do { \ - zcbor_new_encode_state(name, ZCBOR_ARRAY_SIZE(name), payload, payload_size, elem_count); \ + zcbor_new_encode_state(name, ((num_backups) + 1 + ZCBOR_CONST_STATE_SLOTS), \ + payload, payload_size, elem_count); \ } while(0) @@ -188,6 +192,29 @@ bool zcbor_multi_encode_minmax(size_t min_encode, size_t max_encode, const size_t *num_encode, zcbor_encoder_t encoder, zcbor_state_t *state, const void *input, size_t input_len); +/** + * @brief Iterator callback for streaming encode of repeated fields. + * + * Return values: + * - 1: produced one element, stored at *elem_out + * - 0: done (normal end) + * - <0: error (negative errno-style) + * + * The pointer stored in *elem_out must remain valid until the next call to next() + * (or until encoding is finished), whichever comes first. + */ +typedef int (*zcbor_stream_iter)(void *ctx, const void **elem_out); + +/** + * @brief Encode a repeated field by iterating elements from a callback. + * + * This is intended for streaming encode where the caller does not want (or + * cannot afford) to materialize an array + count upfront. + */ +bool zcbor_multi_encode_iter_minmax(size_t min_encode, size_t max_encode, + zcbor_encoder_t encoder, zcbor_state_t *state, + zcbor_stream_iter next, void *ctx); + /* Supplementary string (bstr/tstr) encoding functions: */ @@ -231,6 +258,74 @@ bool zcbor_bstr_start_encode(zcbor_state_t *state); */ bool zcbor_bstr_end_encode(zcbor_state_t *state, struct zcbor_string *result); +/** + * @brief Chunk iterator for streaming encode of indefinite-length text/byte strings. + * + * Return values: + * - 1: produced one chunk (*ptr, *len) + * - 0: done (normal end) + * - <0: error (negative errno-style) + */ +typedef int (*zcbor_next_chunk_fn)(void *ctx, const uint8_t **ptr, size_t *len); + +/* Encode an indefinite-length tstr using a chunk iterator. */ +bool zcbor_tstr_encode_indefinite_chunks(zcbor_state_t *state, + zcbor_next_chunk_fn next_chunk, void *ctx); +bool zcbor_bstr_encode_indefinite_chunks(zcbor_state_t *state, + zcbor_next_chunk_fn next_chunk, void *ctx); + +/** + * @brief Push callback for streaming encode of indefinite-length text/byte strings. + * + * The callback should emit 0 or more chunks using zcbor_*str_encode_chunk(), + * and return true on success. + */ +typedef bool (*zcbor_stream_chunk_out)(void *ctx, zcbor_state_t *state); + +/* Indefinite-length tstr/bstr using a chunk callback. */ +bool zcbor_tstr_chunk_out(zcbor_state_t *state, + zcbor_stream_chunk_out call, void *ctx); +bool zcbor_bstr_chunk_out(zcbor_state_t *state, + zcbor_stream_chunk_out call, void *ctx); + +/* Push-oriented helpers for indefinite-length tstr/bstr. */ +bool zcbor_tstr_start_encode_indefinite(zcbor_state_t *state); +bool zcbor_bstr_start_encode_indefinite(zcbor_state_t *state); +bool zcbor_tstr_encode_chunk(zcbor_state_t *state, const uint8_t *ptr, size_t len); +bool zcbor_bstr_encode_chunk(zcbor_state_t *state, const uint8_t *ptr, size_t len); +bool zcbor_tstr_end_encode_indefinite(zcbor_state_t *state); +bool zcbor_bstr_end_encode_indefinite(zcbor_state_t *state); + +/** Stream write callback type. + * + * @param user_data User-provided context (e.g., UART device pointer) + * @param data Pointer to data to write + * @param len Number of bytes to write + * @return Number of bytes written (must equal len for success) + */ +typedef size_t (*zcbor_stream_write_fn)(void *user_data, const uint8_t *data, size_t len); + +/** Initialize encoding state for streaming mode. + * + * Bytes are written directly via the callback instead of buffering. + * In streaming mode, arrays and maps use indefinite-length encoding + * (0x9F/0xBF + items + 0xFF) to avoid backtracking. + * + * @param state_array Array of states (must have at least 2 elements for constant_state) + * @param n_states Number of states in array + * @param stream_write Callback function to write bytes + * @param stream_user_data User data passed to callback (e.g., UART device pointer) + * @param elem_count Starting element count (typically 1) + */ +void zcbor_new_encode_state_streaming(zcbor_state_t *state_array, size_t n_states, + zcbor_stream_write_fn stream_write, void *stream_user_data, size_t elem_count); + +size_t zcbor_stream_bytes_written(const zcbor_state_t *state); + +int zcbor_stream_entry_function(void *input, zcbor_state_t *states, size_t n_states, + zcbor_encoder_t func, zcbor_stream_write_fn stream_write, void *stream_user_data, + size_t elem_count, size_t *bytes_written_out); + #ifdef __cplusplus } #endif diff --git a/samples/pet/CMakeLists.txt b/samples/pet/CMakeLists.txt index c1e34bc5..0a1dcea5 100644 --- a/samples/pet/CMakeLists.txt +++ b/samples/pet/CMakeLists.txt @@ -14,6 +14,7 @@ if (REGENERATE_ZCBOR) zcbor code # Invoke code generation --decode --encode # Generate both encoding and decoding code --short-names # Attempt to make generated symbol names shorter (at the risk of collision) + --stream-encode # Generate streaming encode entrypoints -c ${CMAKE_CURRENT_LIST_DIR}/../../tests/cases/pet.cddl # Generate code for the data structures in pet.cddl -t Pet # Create a public API for decoding/encoding the "Pet" type from pet.cddl --output-cmake ${CMAKE_CURRENT_LIST_DIR}/pet.cmake # The generated cmake file will be placed here diff --git a/samples/pet/README.md b/samples/pet/README.md index 2e3bf591..bb69c9dd 100644 --- a/samples/pet/README.md +++ b/samples/pet/README.md @@ -7,6 +7,7 @@ structure. The 3 data structures are created in 3 different ways: 1. Converted from YAML using the zcbor script (See [CMakeLists.txt](CMakeLists.txt), [pet1.yml](pet1.yml), and pet1.h). 2. Encoded using the zcbor C API. 3. Encoded using zcbor-generated C code. +4. Encoded using zcbor-generated streaming entrypoints. The generated code is found in [src](src) and [include](include). To regenerate the files, invoke the [CMakeLists.txt](CMakeLists.txt) file with `-DREGENERATE_ZCBOR=Y`. @@ -41,3 +42,7 @@ build/app > Name: Gary Giraffe > Birthday: 0x010203040a0b0c0d > Species: Other +> +> Name: Sammy Streaming +> Birthday: 0xaabbccddeeff1122 +> Species: Cat diff --git a/samples/pet/include/pet_decode.h b/samples/pet/include/pet_decode.h index 94f31f13..f704c0ef 100644 --- a/samples/pet/include/pet_decode.h +++ b/samples/pet/include/pet_decode.h @@ -15,23 +15,32 @@ #include #include #include +#include "zcbor_encode.h" +#include "zcbor_decode.h" #include "pet_types.h" #ifdef __cplusplus extern "C" { #endif -#if DEFAULT_MAX_QTY != 3 +#if ZCBOR_GENERATED_DEFAULT_MAX_QTY != 3 #error "The type file was generated with a different default_max_qty than this file" #endif - int cbor_decode_Pet( const uint8_t *payload, size_t payload_len, struct Pet *result, size_t *payload_len_out); + + + + + + + + #ifdef __cplusplus } #endif diff --git a/samples/pet/include/pet_encode.h b/samples/pet/include/pet_encode.h index 70426105..d51eed76 100644 --- a/samples/pet/include/pet_encode.h +++ b/samples/pet/include/pet_encode.h @@ -15,22 +15,74 @@ #include #include #include +#include "zcbor_encode.h" +#include "zcbor_decode.h" #include "pet_types.h" #ifdef __cplusplus extern "C" { #endif -#if DEFAULT_MAX_QTY != 3 +#if ZCBOR_GENERATED_DEFAULT_MAX_QTY != 3 #error "The type file was generated with a different default_max_qty than this file" #endif - int cbor_encode_Pet( uint8_t *payload, size_t payload_len, const struct Pet *input, size_t *payload_len_out); +/* Streaming encode helpers */ +struct zcbor_stream_iter_io { + void *ctx; + zcbor_stream_iter next; +}; +#ifndef ZCBOR_CHUNK_OUT_DEFINED +#define ZCBOR_CHUNK_OUT_DEFINED +struct zcbor_chunk_out { + void *ctx; + zcbor_stream_chunk_out call; +}; +#endif +#ifndef ZCBOR_CHUNK_IN_DEFINED +#define ZCBOR_CHUNK_IN_DEFINED +struct zcbor_chunk_in { + void *ctx; + zcbor_stream_chunk_in call; +}; +#endif + +#ifndef CBOR_STREAM_IO_PET_DEFINED +#define CBOR_STREAM_IO_PET_DEFINED +struct cbor_stream_io_Pet { + /* Repeated fields (iterator) */ + struct zcbor_stream_iter_io Pet_name_names; + + /* Text string fields (chunk_out) */ + /* no tstr chunk_out io */ + + /* Byte string fields (chunk_out) */ + struct zcbor_chunk_out chunks_out_Timestamp; + + /* Text string fields (chunk_in) */ + /* no tstr chunk_in io */ + + /* Byte string fields (chunk_in) */ + struct zcbor_chunk_in chunks_in_Timestamp; +}; +#endif +typedef struct cbor_stream_io_Pet cbor_stream_io_Pet; + +int cbor_stream_encode_Pet( + zcbor_stream_write_fn stream_write, void *stream_user_data, + const struct Pet *input, + const cbor_stream_io_Pet *io, + size_t *bytes_written_out); + + + + + #ifdef __cplusplus } diff --git a/samples/pet/include/pet_types.h b/samples/pet/include/pet_types.h index 6268932d..1727e81e 100644 --- a/samples/pet/include/pet_types.h +++ b/samples/pet/include/pet_types.h @@ -27,7 +27,17 @@ extern "C" { * * See `zcbor --help` for more information about --default-max-qty */ -#define DEFAULT_MAX_QTY 3 +#define ZCBOR_GENERATED_DEFAULT_MAX_QTY 3 + +/* Allow build-system override. */ +#ifndef DEFAULT_MAX_QTY +#define DEFAULT_MAX_QTY ZCBOR_GENERATED_DEFAULT_MAX_QTY +#endif + +/* Allow build-system override for streaming state array size. */ +#ifndef ZCBOR_STREAM_STATE_ARRAY_SIZE +#define ZCBOR_STREAM_STATE_ARRAY_SIZE 8 +#endif struct Pet { struct zcbor_string names[3]; diff --git a/samples/pet/src/main.c b/samples/pet/src/main.c index eedabf8f..4687f2b8 100644 --- a/samples/pet/src/main.c +++ b/samples/pet/src/main.c @@ -6,10 +6,33 @@ #include #include +#include #include #include #include +struct stream_ctx { + uint8_t *buf; + size_t size; + size_t pos; +}; + +static size_t stream_write(void *user_data, const uint8_t *data, size_t len) +{ + struct stream_ctx *ctx = (struct stream_ctx *)user_data; + + if (!ctx || !data || len == 0) { + return 0; + } + if (ctx->pos + len > ctx->size) { + return 0; + } + + memcpy(&ctx->buf[ctx->pos], data, len); + ctx->pos += len; + return len; +} + static void print_pet(const struct Pet *pet) { printf("Name:"); @@ -120,10 +143,52 @@ static void get_pet3(void) print_pet(&decoded_pet); } +/** Fourth pet - encoded with zcbor-generated streaming entrypoint. */ +static void get_pet4_streaming(void) +{ + struct Pet decoded_pet; + struct Pet encoded_pet; + int err; + uint8_t pet4[30]; + size_t out_len = 0; + struct stream_ctx ctx = { + .buf = pet4, + .size = sizeof(pet4), + .pos = 0, + }; + const uint8_t first_name[] = "Sammy"; + const uint8_t last_name[] = "Streaming"; + const uint8_t timestamp4[] = { 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x11, 0x22 }; + + encoded_pet.names[0].value = first_name; + encoded_pet.names[0].len = sizeof(first_name) - 1; + encoded_pet.names[1].value = last_name; + encoded_pet.names[1].len = sizeof(last_name) - 1; + encoded_pet.names_count = 2; + encoded_pet.birthday.value = timestamp4; + encoded_pet.birthday.len = sizeof(timestamp4); + encoded_pet.species_choice = Pet_species_cat_c; + + err = cbor_stream_encode_Pet(stream_write, &ctx, &encoded_pet, NULL, &out_len); + if (err != ZCBOR_SUCCESS || out_len != ctx.pos) { + printf("Streaming encode failed for pet4: %d\r\n", err); + return; + } + + err = cbor_decode_Pet(pet4, out_len, &decoded_pet, NULL); + if (err != ZCBOR_SUCCESS) { + printf("Decoding failed for pet4: %d\r\n", err); + return; + } + + print_pet(&decoded_pet); +} + int main(void) { get_pet1(); get_pet2(); get_pet3(); + get_pet4_streaming(); return 0; } diff --git a/samples/pet/src/pet_decode.c b/samples/pet/src/pet_decode.c index 644593a5..c04bf656 100644 --- a/samples/pet/src/pet_decode.c +++ b/samples/pet/src/pet_decode.c @@ -16,7 +16,7 @@ #include "pet_decode.h" #include "zcbor_print.h" -#if DEFAULT_MAX_QTY != 3 +#if ZCBOR_GENERATED_DEFAULT_MAX_QTY != 3 #error "The type file was generated with a different default_max_qty than this file" #endif @@ -29,6 +29,8 @@ } \ } while(0) + + static bool decode_Pet(zcbor_state_t *state, struct Pet *result); @@ -55,14 +57,12 @@ static bool decode_Pet( return res; } - - int cbor_decode_Pet( const uint8_t *payload, size_t payload_len, struct Pet *result, size_t *payload_len_out) { - zcbor_state_t states[4]; + zcbor_state_t states[3 + ZCBOR_CONST_STATE_SLOTS]; return zcbor_entry_function(payload, payload_len, (void *)result, payload_len_out, states, (zcbor_decoder_t *)decode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1); diff --git a/samples/pet/src/pet_encode.c b/samples/pet/src/pet_encode.c index 9acee80f..0801d930 100644 --- a/samples/pet/src/pet_encode.c +++ b/samples/pet/src/pet_encode.c @@ -16,7 +16,7 @@ #include "pet_encode.h" #include "zcbor_print.h" -#if DEFAULT_MAX_QTY != 3 +#if ZCBOR_GENERATED_DEFAULT_MAX_QTY != 3 #error "The type file was generated with a different default_max_qty than this file" #endif @@ -29,6 +29,24 @@ } \ } while(0) +/* Streaming io struct (internal, schema-wide). */ +struct cbor_stream_io { + /* Repeated fields (iterator) */ + struct zcbor_stream_iter_io Pet_name_names; + + /* Text string fields (chunk_out) */ + /* no tstr chunk_out io */ + + /* Byte string fields (chunk_out) */ + struct zcbor_chunk_out chunks_out_Timestamp; + + /* Text string fields (chunk_in) */ + /* no tstr chunk_in io */ + + /* Byte string fields (chunk_in) */ + struct zcbor_chunk_in chunks_in_Timestamp; +}; + static bool encode_Pet(zcbor_state_t *state, const struct Pet *input); @@ -37,27 +55,53 @@ static bool encode_Pet( { zcbor_log("%s\r\n", __func__); - bool res = (((zcbor_list_start_encode(state, 3) && ((((zcbor_list_start_encode(state, 3) && ((zcbor_multi_encode_minmax(1, 3, &(*input).names_count, (zcbor_encoder_t *)zcbor_tstr_encode, state, (*&(*input).names), sizeof(struct zcbor_string))) || (zcbor_list_map_end_force_encode(state), false)) && zcbor_list_end_encode(state, 3))) + bool res = (((zcbor_list_start_encode(state, ZCBOR_VALUE_IS_INDEFINITE_LENGTH) && ((((zcbor_list_start_encode(state, ZCBOR_VALUE_IS_INDEFINITE_LENGTH) && ((((((const struct cbor_stream_io *)zcbor_get_stream_io(state)) && ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->Pet_name_names.next) ? (zcbor_multi_encode_iter_minmax(1, DEFAULT_MAX_QTY, (zcbor_encoder_t *)zcbor_tstr_encode, state, ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->Pet_name_names.next, ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->Pet_name_names.ctx)) : (zcbor_multi_encode_minmax(1, DEFAULT_MAX_QTY, &(*input).names_count, (zcbor_encoder_t *)zcbor_tstr_encode, state, (*&(*input).names), sizeof(struct zcbor_string))))) || (zcbor_list_map_end_force_encode(state), false)) && zcbor_list_end_encode(state, ZCBOR_VALUE_IS_INDEFINITE_LENGTH))) && (((((((*input).birthday.len == 8)) || (zcbor_error(state, ZCBOR_ERR_WRONG_RANGE), false))) || (zcbor_error(state, ZCBOR_ERR_WRONG_RANGE), false)) && (zcbor_bstr_encode(state, (&(*input).birthday)))) && ((((*input).species_choice == Pet_species_cat_c) ? ((zcbor_uint32_put(state, (1)))) : (((*input).species_choice == Pet_species_dog_c) ? ((zcbor_uint32_put(state, (2)))) : (((*input).species_choice == Pet_species_other_c) ? ((zcbor_uint32_put(state, (3)))) - : false))))) || (zcbor_list_map_end_force_encode(state), false)) && zcbor_list_end_encode(state, 3)))); + : false))))) || (zcbor_list_map_end_force_encode(state), false)) && zcbor_list_end_encode(state, ZCBOR_VALUE_IS_INDEFINITE_LENGTH)))); log_result(state, res, __func__); return res; } - - int cbor_encode_Pet( uint8_t *payload, size_t payload_len, const struct Pet *input, size_t *payload_len_out) { - zcbor_state_t states[4]; + zcbor_state_t states[3 + ZCBOR_CONST_STATE_SLOTS]; return zcbor_entry_function(payload, payload_len, (void *)input, payload_len_out, states, (zcbor_decoder_t *)encode_Pet, sizeof(states) / sizeof(zcbor_state_t), 1); } + +int cbor_stream_encode_Pet( + zcbor_stream_write_fn stream_write, void *stream_user_data, + const struct Pet *input, + const cbor_stream_io_Pet *io, + size_t *bytes_written_out) +{ + zcbor_state_t states[ZCBOR_STREAM_STATE_ARRAY_SIZE]; + zcbor_new_encode_state_streaming(states, sizeof(states) / sizeof(states[0]), + stream_write, stream_user_data, 1); + struct cbor_stream_io io_local = {0}; + if (io) { + io_local.Pet_name_names = io->Pet_name_names; + io_local.chunks_out_Timestamp = io->chunks_out_Timestamp; + zcbor_set_stream_io(&states[0], &io_local); + } else { + zcbor_set_stream_io(&states[0], NULL); + } + bool ok = encode_Pet(&states[0], input); + if (!ok) { + int err = zcbor_pop_error(&states[0]); + return (err == ZCBOR_SUCCESS) ? ZCBOR_ERR_UNKNOWN : err; + } + if (bytes_written_out) { + *bytes_written_out = zcbor_stream_bytes_written(&states[0]); + } + return ZCBOR_SUCCESS; +} diff --git a/samples/streaming_chunks/CMakeLists.txt b/samples/streaming_chunks/CMakeLists.txt new file mode 100644 index 00000000..98eff653 --- /dev/null +++ b/samples/streaming_chunks/CMakeLists.txt @@ -0,0 +1,28 @@ +# Copyright (c) 2026 +# +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.13.3) +project(sample_streaming_chunks) + +# Regenerate zcbor-generated files when requested. +if (REGENERATE_ZCBOR) + set(zcbor_command + zcbor code + --decode --encode + --stream-encode + --stream-decode + -c ${CMAKE_CURRENT_LIST_DIR}/streaming_chunks.cddl + -t StreamItem + --output-cmake ${CMAKE_CURRENT_LIST_DIR}/streaming_chunks.cmake + --file-header "Copyright (c) 2026\n\nSPDX-License-Identifier: Apache-2.0" + ) + execute_process(COMMAND ${zcbor_command}) +endif() + +include(${CMAKE_CURRENT_LIST_DIR}/streaming_chunks.cmake) + +add_executable(app src/main.c) + +# Link the library from streaming_chunks.cmake +target_link_libraries(app PRIVATE streaming_chunks) diff --git a/samples/streaming_chunks/README.md b/samples/streaming_chunks/README.md new file mode 100644 index 00000000..3d79bd22 --- /dev/null +++ b/samples/streaming_chunks/README.md @@ -0,0 +1,24 @@ +# Streaming chunks sample + +This sample demonstrates streaming encode and decode with chunk callbacks for +both `tstr` and `bstr` fields using zcbor-generated code. + +## To build + +```sh +cmake -S . -B build -DREGENERATE_ZCBOR=Y +cmake --build build +``` + +## To run + +```sh +build/app +``` + +## Expected output + +``` +Decoded name: Chunky Name +Decoded payload: de ad be ef ca fe ba be +``` diff --git a/samples/streaming_chunks/include/streaming_chunks_decode.h b/samples/streaming_chunks/include/streaming_chunks_decode.h new file mode 100644 index 00000000..f47cd7d7 --- /dev/null +++ b/samples/streaming_chunks/include/streaming_chunks_decode.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2026 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Generated using zcbor version 0.9.1 + * https://github.com/NordicSemiconductor/zcbor + * Generated with a --default-max-qty of 3 + */ + +#ifndef STREAMING_CHUNKS_DECODE_H__ +#define STREAMING_CHUNKS_DECODE_H__ + +#include +#include +#include +#include +#include "zcbor_encode.h" +#include "zcbor_decode.h" +#include "streaming_chunks_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if ZCBOR_GENERATED_DEFAULT_MAX_QTY != 3 +#error "The type file was generated with a different default_max_qty than this file" +#endif + +int cbor_decode_StreamItem( + const uint8_t *payload, size_t payload_len, + struct StreamItem *result, + size_t *payload_len_out); + + + + + +/* Streaming decode helpers */ +#ifndef ZCBOR_CHUNK_OUT_DEFINED +#define ZCBOR_CHUNK_OUT_DEFINED +struct zcbor_chunk_out { + void *ctx; + zcbor_stream_chunk_out call; +}; +#endif +#ifndef ZCBOR_CHUNK_IN_DEFINED +#define ZCBOR_CHUNK_IN_DEFINED +struct zcbor_chunk_in { + void *ctx; + zcbor_stream_chunk_in call; +}; +#endif + +#ifndef CBOR_STREAM_IO_STREAMITEM_DEFINED +#define CBOR_STREAM_IO_STREAMITEM_DEFINED +struct cbor_stream_io_StreamItem { + /* Text string fields (chunk_out) */ + struct zcbor_chunk_out chunks_out_StreamItem_name; + + /* Byte string fields (chunk_out) */ + struct zcbor_chunk_out chunks_out_StreamItem_payload; + + /* Text string fields (chunk_in) */ + struct zcbor_chunk_in chunks_in_StreamItem_name; + + /* Byte string fields (chunk_in) */ + struct zcbor_chunk_in chunks_in_StreamItem_payload; +}; +#endif +typedef struct cbor_stream_io_StreamItem cbor_stream_io_StreamItem; + +int cbor_stream_decode_StreamItem( + const uint8_t *payload, size_t payload_len, + struct StreamItem *result, + const cbor_stream_io_StreamItem *io, + size_t *payload_len_out); + + +#ifdef __cplusplus +} +#endif + +#endif /* STREAMING_CHUNKS_DECODE_H__ */ diff --git a/samples/streaming_chunks/include/streaming_chunks_encode.h b/samples/streaming_chunks/include/streaming_chunks_encode.h new file mode 100644 index 00000000..40ccf41e --- /dev/null +++ b/samples/streaming_chunks/include/streaming_chunks_encode.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2026 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Generated using zcbor version 0.9.1 + * https://github.com/NordicSemiconductor/zcbor + * Generated with a --default-max-qty of 3 + */ + +#ifndef STREAMING_CHUNKS_ENCODE_H__ +#define STREAMING_CHUNKS_ENCODE_H__ + +#include +#include +#include +#include +#include "zcbor_encode.h" +#include "zcbor_decode.h" +#include "streaming_chunks_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if ZCBOR_GENERATED_DEFAULT_MAX_QTY != 3 +#error "The type file was generated with a different default_max_qty than this file" +#endif + +int cbor_encode_StreamItem( + uint8_t *payload, size_t payload_len, + const struct StreamItem *input, + size_t *payload_len_out); + +/* Streaming encode helpers */ +struct zcbor_stream_iter_io { + void *ctx; + zcbor_stream_iter next; +}; +#ifndef ZCBOR_CHUNK_OUT_DEFINED +#define ZCBOR_CHUNK_OUT_DEFINED +struct zcbor_chunk_out { + void *ctx; + zcbor_stream_chunk_out call; +}; +#endif +#ifndef ZCBOR_CHUNK_IN_DEFINED +#define ZCBOR_CHUNK_IN_DEFINED +struct zcbor_chunk_in { + void *ctx; + zcbor_stream_chunk_in call; +}; +#endif + +#ifndef CBOR_STREAM_IO_STREAMITEM_DEFINED +#define CBOR_STREAM_IO_STREAMITEM_DEFINED +struct cbor_stream_io_StreamItem { + /* Repeated fields (iterator) */ + /* no repeated iters */ + + /* Text string fields (chunk_out) */ + struct zcbor_chunk_out chunks_out_StreamItem_name; + + /* Byte string fields (chunk_out) */ + struct zcbor_chunk_out chunks_out_StreamItem_payload; + + /* Text string fields (chunk_in) */ + struct zcbor_chunk_in chunks_in_StreamItem_name; + + /* Byte string fields (chunk_in) */ + struct zcbor_chunk_in chunks_in_StreamItem_payload; +}; +#endif +typedef struct cbor_stream_io_StreamItem cbor_stream_io_StreamItem; + +int cbor_stream_encode_StreamItem( + zcbor_stream_write_fn stream_write, void *stream_user_data, + const struct StreamItem *input, + const cbor_stream_io_StreamItem *io, + size_t *bytes_written_out); + + + + + + +#ifdef __cplusplus +} +#endif + +#endif /* STREAMING_CHUNKS_ENCODE_H__ */ diff --git a/samples/streaming_chunks/include/streaming_chunks_types.h b/samples/streaming_chunks/include/streaming_chunks_types.h new file mode 100644 index 00000000..c37d985a --- /dev/null +++ b/samples/streaming_chunks/include/streaming_chunks_types.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2026 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Generated using zcbor version 0.9.1 + * https://github.com/NordicSemiconductor/zcbor + * Generated with a --default-max-qty of 3 + */ + +#ifndef STREAMING_CHUNKS_TYPES_H__ +#define STREAMING_CHUNKS_TYPES_H__ + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Which value for --default-max-qty this file was created with. + * + * The define is used in the other generated file to do a build-time + * compatibility check. + * + * See `zcbor --help` for more information about --default-max-qty + */ +#define ZCBOR_GENERATED_DEFAULT_MAX_QTY 3 + +/* Allow build-system override. */ +#ifndef DEFAULT_MAX_QTY +#define DEFAULT_MAX_QTY ZCBOR_GENERATED_DEFAULT_MAX_QTY +#endif + +/* Allow build-system override for streaming state array size. */ +#ifndef ZCBOR_STREAM_STATE_ARRAY_SIZE +#define ZCBOR_STREAM_STATE_ARRAY_SIZE 8 +#endif + +struct StreamItem { + struct zcbor_string StreamItem_name; + struct zcbor_string StreamItem_payload; +}; + +#ifdef __cplusplus +} +#endif + +#endif /* STREAMING_CHUNKS_TYPES_H__ */ diff --git a/samples/streaming_chunks/src/main.c b/samples/streaming_chunks/src/main.c new file mode 100644 index 00000000..8b5792b1 --- /dev/null +++ b/samples/streaming_chunks/src/main.c @@ -0,0 +1,208 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include + +#include + +#include "streaming_chunks_decode.h" +#include "streaming_chunks_encode.h" + +struct stream_ctx { + uint8_t *buf; + size_t size; + size_t pos; +}; + +static size_t stream_write(void *user_data, const uint8_t *data, size_t len) +{ + struct stream_ctx *ctx = (struct stream_ctx *)user_data; + + if (!ctx || !data || len == 0) { + return 0; + } + if (ctx->pos + len > ctx->size) { + return 0; + } + + memcpy(&ctx->buf[ctx->pos], data, len); + ctx->pos += len; + return len; +} + +struct chunk_ctx { + const uint8_t *data; + size_t len; + size_t chunk_size; + size_t offset; +}; + +struct decode_chunk_ctx { + uint8_t *buf; + size_t size; + size_t len; +}; + +static bool tstr_chunks(void *user_ctx, zcbor_state_t *state) +{ + struct chunk_ctx *ctx = (struct chunk_ctx *)user_ctx; + + if (!ctx || !state) { + return false; + } + + while (ctx->offset < ctx->len) { + size_t remaining = ctx->len - ctx->offset; + size_t chunk_len = (remaining > ctx->chunk_size) ? ctx->chunk_size : remaining; + + if (!zcbor_tstr_encode_chunk(state, &ctx->data[ctx->offset], chunk_len)) { + return false; + } + ctx->offset += chunk_len; + } + + return true; +} + +static bool bstr_chunks(void *user_ctx, zcbor_state_t *state) +{ + struct chunk_ctx *ctx = (struct chunk_ctx *)user_ctx; + + if (!ctx || !state) { + return false; + } + + while (ctx->offset < ctx->len) { + size_t remaining = ctx->len - ctx->offset; + size_t chunk_len = (remaining > ctx->chunk_size) ? ctx->chunk_size : remaining; + + if (!zcbor_bstr_encode_chunk(state, &ctx->data[ctx->offset], chunk_len)) { + return false; + } + ctx->offset += chunk_len; + } + + return true; +} + +static bool decode_chunks(void *user_ctx, const uint8_t *data, size_t len) +{ + struct decode_chunk_ctx *ctx = (struct decode_chunk_ctx *)user_ctx; + + if (!ctx || !data) { + return false; + } + if (len == 0) { + return true; + } + if (ctx->len + len > ctx->size) { + return false; + } + + memcpy(&ctx->buf[ctx->len], data, len); + ctx->len += len; + return true; +} + +static void print_decoded(const struct StreamItem *decoded) +{ + printf("Decoded name: %.*s\r\n", + (int)decoded->StreamItem_name.len, + decoded->StreamItem_name.value); + printf("Decoded payload:"); + for (size_t i = 0; i < decoded->StreamItem_payload.len; i++) { + printf(" %02x", decoded->StreamItem_payload.value[i]); + } + printf("\r\n"); +} + +static int run_streaming(void) +{ + uint8_t buffer[128]; + size_t out_len = 0; + struct stream_ctx stream = { + .buf = buffer, + .size = sizeof(buffer), + .pos = 0, + }; + + const uint8_t name[] = "Long Chunky Name to be streamed with lots of characters"; + const uint8_t payload[] = { 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe, 0xba, 0xbe }; + + struct StreamItem item = { + .StreamItem_name = { .value = name, .len = sizeof(name) - 1 }, + .StreamItem_payload = { .value = payload, .len = sizeof(payload) }, + }; + + struct chunk_ctx name_ctx = { + .data = name, + .len = sizeof(name) - 1, + .chunk_size = 4, + .offset = 0, + }; + + struct chunk_ctx payload_ctx = { + .data = payload, + .len = sizeof(payload), + .chunk_size = 3, + .offset = 0, + }; + + struct cbor_stream_io_StreamItem io = {0}; + io.chunks_out_StreamItem_name.ctx = &name_ctx; + io.chunks_out_StreamItem_name.call = tstr_chunks; + io.chunks_out_StreamItem_payload.ctx = &payload_ctx; + io.chunks_out_StreamItem_payload.call = bstr_chunks; + + int err = cbor_stream_encode_StreamItem(stream_write, &stream, &item, &io, &out_len); + if (err != ZCBOR_SUCCESS || out_len != stream.pos) { + printf("Streaming encode failed: %d\r\n", err); + return 1; + } + + struct StreamItem decoded = {0}; + + uint8_t name_buf[64]; + uint8_t payload_buf[32]; + struct decode_chunk_ctx name_out = { + .buf = name_buf, + .size = sizeof(name_buf), + .len = 0, + }; + struct decode_chunk_ctx payload_out = { + .buf = payload_buf, + .size = sizeof(payload_buf), + .len = 0, + }; + + io.chunks_in_StreamItem_name.ctx = &name_out; + io.chunks_in_StreamItem_name.call = decode_chunks; + io.chunks_in_StreamItem_payload.ctx = &payload_out; + io.chunks_in_StreamItem_payload.call = decode_chunks; + + err = cbor_stream_decode_StreamItem(buffer, out_len, &decoded, &io, NULL); + if (err != ZCBOR_SUCCESS) { + printf("Decode failed: %d\r\n", err); + return 1; + } + + decoded.StreamItem_name.value = name_out.buf; + decoded.StreamItem_name.len = name_out.len; + decoded.StreamItem_payload.value = payload_out.buf; + decoded.StreamItem_payload.len = payload_out.len; + + print_decoded(&decoded); + return 0; +} + +int main(void) +{ + if (run_streaming() != 0) { + return 1; + } + return 0; +} diff --git a/samples/streaming_chunks/src/streaming_chunks_decode.c b/samples/streaming_chunks/src/streaming_chunks_decode.c new file mode 100644 index 00000000..3103ef71 --- /dev/null +++ b/samples/streaming_chunks/src/streaming_chunks_decode.c @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2026 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Generated using zcbor version 0.9.1 + * https://github.com/NordicSemiconductor/zcbor + * Generated with a --default-max-qty of 3 + */ + +#include +#include +#include +#include +#include "zcbor_decode.h" +#include "streaming_chunks_decode.h" +#include "zcbor_print.h" + +#if ZCBOR_GENERATED_DEFAULT_MAX_QTY != 3 +#error "The type file was generated with a different default_max_qty than this file" +#endif + +#define log_result(state, result, func) do { \ + if (!result) { \ + zcbor_trace_file(state); \ + zcbor_log("%s error: %s\r\n", func, zcbor_error_str(zcbor_peek_error(state))); \ + } else { \ + zcbor_log("%s success\r\n", func); \ + } \ +} while(0) + +/* Streaming io struct (internal, schema-wide). */ +struct cbor_stream_io { + /* Text string fields (chunk_out) */ + struct zcbor_chunk_out chunks_out_StreamItem_name; + + /* Byte string fields (chunk_out) */ + struct zcbor_chunk_out chunks_out_StreamItem_payload; + + /* Text string fields (chunk_in) */ + struct zcbor_chunk_in chunks_in_StreamItem_name; + + /* Byte string fields (chunk_in) */ + struct zcbor_chunk_in chunks_in_StreamItem_payload; +}; + +static bool decode_StreamItem(zcbor_state_t *state, struct StreamItem *result); + + +static bool decode_StreamItem( + zcbor_state_t *state, struct StreamItem *result) +{ + zcbor_log("%s\r\n", __func__); + struct zcbor_string tmp_str; + + bool res = (((zcbor_map_start_decode(state) && (((((zcbor_tstr_expect(state, ((tmp_str.value = (uint8_t *)"name", tmp_str.len = sizeof("name") - 1, &tmp_str))))) + && ((((const struct cbor_stream_io *)zcbor_get_stream_io(state)) && ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_in_StreamItem_name.call) ? (zcbor_tstr_chunk_in(state, ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_in_StreamItem_name.call, ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_in_StreamItem_name.ctx)) : ((zcbor_tstr_decode(state, (&(*result).StreamItem_name)))))) + && (((zcbor_tstr_expect(state, ((tmp_str.value = (uint8_t *)"payload", tmp_str.len = sizeof("payload") - 1, &tmp_str))))) + && ((((const struct cbor_stream_io *)zcbor_get_stream_io(state)) && ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_in_StreamItem_payload.call) ? (zcbor_bstr_chunk_in(state, ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_in_StreamItem_payload.call, ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_in_StreamItem_payload.ctx)) : ((zcbor_bstr_decode(state, (&(*result).StreamItem_payload))))))) || (zcbor_list_map_end_force_decode(state), false)) && zcbor_map_end_decode(state)))); + + log_result(state, res, __func__); + return res; +} + +int cbor_decode_StreamItem( + const uint8_t *payload, size_t payload_len, + struct StreamItem *result, + size_t *payload_len_out) +{ + zcbor_state_t states[2 + ZCBOR_CONST_STATE_SLOTS]; + + return zcbor_entry_function(payload, payload_len, (void *)result, payload_len_out, states, + (zcbor_decoder_t *)decode_StreamItem, sizeof(states) / sizeof(zcbor_state_t), 1); +} + + + +int cbor_stream_decode_StreamItem( + const uint8_t *payload, size_t payload_len, + struct StreamItem *result, + const cbor_stream_io_StreamItem *io, + size_t *payload_len_out) +{ + zcbor_state_t states[2 + ZCBOR_CONST_STATE_SLOTS]; + zcbor_new_state(states, sizeof(states) / sizeof(states[0]), payload, payload_len, + 1, NULL, 0); + struct cbor_stream_io io_local = {0}; + if (io) { + io_local.chunks_in_StreamItem_name = io->chunks_in_StreamItem_name; + io_local.chunks_in_StreamItem_payload = io->chunks_in_StreamItem_payload; + zcbor_set_stream_io(&states[0], &io_local); + } else { + zcbor_set_stream_io(&states[0], NULL); + } + bool ok = decode_StreamItem(&states[0], result); + if (!ok) { + int err = zcbor_pop_error(&states[0]); + return (err == ZCBOR_SUCCESS) ? ZCBOR_ERR_UNKNOWN : err; + } + if (payload_len_out != NULL) { + *payload_len_out = MIN(payload_len, + (size_t)states[0].payload - (size_t)payload); + } + return ZCBOR_SUCCESS; +} diff --git a/samples/streaming_chunks/src/streaming_chunks_encode.c b/samples/streaming_chunks/src/streaming_chunks_encode.c new file mode 100644 index 00000000..92ab0f6d --- /dev/null +++ b/samples/streaming_chunks/src/streaming_chunks_encode.c @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2026 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Generated using zcbor version 0.9.1 + * https://github.com/NordicSemiconductor/zcbor + * Generated with a --default-max-qty of 3 + */ + +#include +#include +#include +#include +#include "zcbor_encode.h" +#include "streaming_chunks_encode.h" +#include "zcbor_print.h" + +#if ZCBOR_GENERATED_DEFAULT_MAX_QTY != 3 +#error "The type file was generated with a different default_max_qty than this file" +#endif + +#define log_result(state, result, func) do { \ + if (!result) { \ + zcbor_trace_file(state); \ + zcbor_log("%s error: %s\r\n", func, zcbor_error_str(zcbor_peek_error(state))); \ + } else { \ + zcbor_log("%s success\r\n", func); \ + } \ +} while(0) + +/* Streaming io struct (internal, schema-wide). */ +struct cbor_stream_io { + /* Repeated fields (iterator) */ + /* no repeated iters */ + + /* Text string fields (chunk_out) */ + struct zcbor_chunk_out chunks_out_StreamItem_name; + + /* Byte string fields (chunk_out) */ + struct zcbor_chunk_out chunks_out_StreamItem_payload; + + /* Text string fields (chunk_in) */ + struct zcbor_chunk_in chunks_in_StreamItem_name; + + /* Byte string fields (chunk_in) */ + struct zcbor_chunk_in chunks_in_StreamItem_payload; +}; + +static bool encode_StreamItem(zcbor_state_t *state, const struct StreamItem *input); + + +static bool encode_StreamItem( + zcbor_state_t *state, const struct StreamItem *input) +{ + zcbor_log("%s\r\n", __func__); + struct zcbor_string tmp_str; + + bool res = (((zcbor_map_start_encode(state, ZCBOR_VALUE_IS_INDEFINITE_LENGTH) && (((((zcbor_tstr_encode(state, ((tmp_str.value = (uint8_t *)"name", tmp_str.len = sizeof("name") - 1, &tmp_str))))) + && ((((const struct cbor_stream_io *)zcbor_get_stream_io(state)) && ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_out_StreamItem_name.call) ? (zcbor_tstr_chunk_out(state, ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_out_StreamItem_name.call, ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_out_StreamItem_name.ctx)) : ((zcbor_tstr_encode(state, (&(*input).StreamItem_name)))))) + && (((zcbor_tstr_encode(state, ((tmp_str.value = (uint8_t *)"payload", tmp_str.len = sizeof("payload") - 1, &tmp_str))))) + && ((((const struct cbor_stream_io *)zcbor_get_stream_io(state)) && ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_out_StreamItem_payload.call) ? (zcbor_bstr_chunk_out(state, ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_out_StreamItem_payload.call, ((const struct cbor_stream_io *)zcbor_get_stream_io(state))->chunks_out_StreamItem_payload.ctx)) : ((zcbor_bstr_encode(state, (&(*input).StreamItem_payload))))))) || (zcbor_list_map_end_force_encode(state), false)) && zcbor_map_end_encode(state, ZCBOR_VALUE_IS_INDEFINITE_LENGTH)))); + + log_result(state, res, __func__); + return res; +} + +int cbor_encode_StreamItem( + uint8_t *payload, size_t payload_len, + const struct StreamItem *input, + size_t *payload_len_out) +{ + zcbor_state_t states[2 + ZCBOR_CONST_STATE_SLOTS]; + + return zcbor_entry_function(payload, payload_len, (void *)input, payload_len_out, states, + (zcbor_decoder_t *)encode_StreamItem, sizeof(states) / sizeof(zcbor_state_t), 1); +} + +int cbor_stream_encode_StreamItem( + zcbor_stream_write_fn stream_write, void *stream_user_data, + const struct StreamItem *input, + const cbor_stream_io_StreamItem *io, + size_t *bytes_written_out) +{ + zcbor_state_t states[ZCBOR_STREAM_STATE_ARRAY_SIZE]; + zcbor_new_encode_state_streaming(states, sizeof(states) / sizeof(states[0]), + stream_write, stream_user_data, 1); + struct cbor_stream_io io_local = {0}; + if (io) { + io_local.chunks_out_StreamItem_name = io->chunks_out_StreamItem_name; + io_local.chunks_out_StreamItem_payload = io->chunks_out_StreamItem_payload; + zcbor_set_stream_io(&states[0], &io_local); + } else { + zcbor_set_stream_io(&states[0], NULL); + } + bool ok = encode_StreamItem(&states[0], input); + if (!ok) { + int err = zcbor_pop_error(&states[0]); + return (err == ZCBOR_SUCCESS) ? ZCBOR_ERR_UNKNOWN : err; + } + if (bytes_written_out) { + *bytes_written_out = zcbor_stream_bytes_written(&states[0]); + } + return ZCBOR_SUCCESS; +} diff --git a/samples/streaming_chunks/streaming_chunks.cddl b/samples/streaming_chunks/streaming_chunks.cddl new file mode 100644 index 00000000..dae19786 --- /dev/null +++ b/samples/streaming_chunks/streaming_chunks.cddl @@ -0,0 +1,6 @@ +; SPDX-License-Identifier: Apache-2.0 + +StreamItem = { + "name": tstr, + "payload": bstr, +} diff --git a/samples/streaming_chunks/streaming_chunks.cmake b/samples/streaming_chunks/streaming_chunks.cmake new file mode 100644 index 00000000..61a6bfa7 --- /dev/null +++ b/samples/streaming_chunks/streaming_chunks.cmake @@ -0,0 +1,23 @@ +# +# Copyright (c) 2026 +# +# SPDX-License-Identifier: Apache-2.0 +# +# Generated using zcbor version 0.9.1 +# https://github.com/NordicSemiconductor/zcbor +# Generated with a --default-max-qty of 3 +# + +add_library(streaming_chunks) +target_sources(streaming_chunks PRIVATE + ${CMAKE_CURRENT_LIST_DIR}/../../src/zcbor_decode.c + ${CMAKE_CURRENT_LIST_DIR}/../../src/zcbor_encode.c + ${CMAKE_CURRENT_LIST_DIR}/../../src/zcbor_common.c + ${CMAKE_CURRENT_LIST_DIR}/../../src/zcbor_print.c + ${CMAKE_CURRENT_LIST_DIR}/src/streaming_chunks_decode.c + ${CMAKE_CURRENT_LIST_DIR}/src/streaming_chunks_encode.c + ) +target_include_directories(streaming_chunks PUBLIC + ${CMAKE_CURRENT_LIST_DIR}/../../include + ${CMAKE_CURRENT_LIST_DIR}/include + ) diff --git a/src/zcbor_common.c b/src/zcbor_common.c index 5bf959e6..516fdb0c 100644 --- a/src/zcbor_common.c +++ b/src/zcbor_common.c @@ -14,9 +14,6 @@ _Static_assert((sizeof(size_t) == sizeof(void *)), "This code needs size_t to be the same length as pointers."); -_Static_assert((sizeof(zcbor_state_t) >= sizeof(struct zcbor_state_constant)), - "This code needs zcbor_state_t to be at least as large as zcbor_backups_t."); - bool zcbor_new_backup(zcbor_state_t *state, size_t new_elem_count) { ZCBOR_CHECK_ERROR(); @@ -150,14 +147,41 @@ void zcbor_new_state(zcbor_state_t *state_array, size_t n_states, #endif state_array[0].constant_state = NULL; - if (n_states < 2) { + if (n_states < (1 + ZCBOR_CONST_STATE_SLOTS)) { return; } - /* Use the last state as a struct zcbor_state_constant object. */ - state_array[0].constant_state = (struct zcbor_state_constant *)&state_array[n_states - 1]; + /* Store the constant state object in the tail of the state array. + * + * The state array layout is: + * [0] : live state + * [1 .. backups] : backups + * [.. optional flags ..] : map smart search scratch (decode only, if flags points into the array) + * [tail] : struct zcbor_state_constant (may span multiple slots) + */ + const size_t const_state_slots = ZCBOR_CONST_STATE_SLOTS; + const size_t const_state_idx = n_states - const_state_slots; + + state_array[0].constant_state = (struct zcbor_state_constant *)&state_array[const_state_idx]; state_array[0].constant_state->backup_list = NULL; - state_array[0].constant_state->num_backups = n_states - 2; + + /* Backups live in state_array[1..num_backups]. If flags points inside the array, + * it indicates the start of the scratch region, and backups stop just before it. + */ + size_t backup_end_idx = const_state_idx - 1; + const uint8_t *const base = (const uint8_t *)state_array; + const uint8_t *const tail = (const uint8_t *)&state_array[const_state_idx]; + if (flags) { + const uint8_t *const f = flags; + if ((f >= base) && (f < tail)) { + size_t flags_idx = (size_t)(f - base) / sizeof(zcbor_state_t); + if (flags_idx > 0) { + backup_end_idx = flags_idx - 1; + } + } + } + + state_array[0].constant_state->num_backups = (backup_end_idx >= 1) ? backup_end_idx : 0; state_array[0].constant_state->current_backup = 0; state_array[0].constant_state->error = ZCBOR_SUCCESS; #ifdef ZCBOR_STOP_ON_ERROR @@ -168,7 +192,11 @@ void zcbor_new_state(zcbor_state_t *state_array, size_t n_states, #ifdef ZCBOR_MAP_SMART_SEARCH state_array[0].constant_state->map_search_elem_state_end = flags + flags_bytes; #endif - if (n_states > 2) { + state_array[0].constant_state->stream_write = NULL; + state_array[0].constant_state->stream_user_data = NULL; + state_array[0].constant_state->stream_bytes_written = 0; + state_array[0].constant_state->stream_io = NULL; + if (state_array[0].constant_state->num_backups > 0) { state_array[0].constant_state->backup_list = &state_array[1]; } } diff --git a/src/zcbor_decode.c b/src/zcbor_decode.c index 1ced3e0b..4d4dc5be 100644 --- a/src/zcbor_decode.c +++ b/src/zcbor_decode.c @@ -584,6 +584,72 @@ bool zcbor_is_last_fragment(const struct zcbor_string_fragment *fragment) } +static bool str_decode_chunks(zcbor_state_t *state, zcbor_stream_chunk_in call, void *ctx, + zcbor_major_type_t exp_major_type) +{ + INITIAL_CHECKS_WITH_TYPE(exp_major_type); + zcbor_assert_state(call != NULL, "chunk callback cannot be NULL.\r\n"); + + bool indefinite_length = false; + size_t str_len = 0; + + if (!value_extract(state, &str_len, exp_major_type, &indefinite_length)) { + ZCBOR_FAIL(); + } + + if (!indefinite_length) { + ZCBOR_ERR_IF(state->payload + str_len > state->payload_end, ZCBOR_ERR_NO_PAYLOAD); + if (!call(ctx, state->payload, str_len)) { + ZCBOR_FAIL(); + } + state->payload += str_len; + return true; + } + + while (true) { + ZCBOR_ERR_IF(state->payload >= state->payload_end, ZCBOR_ERR_NO_PAYLOAD); + + if (*state->payload == 0xFF) { + state->payload++; + break; + } + + uint8_t chunk_header = *state->payload; + ZCBOR_ERR_IF(ZCBOR_MAJOR_TYPE(chunk_header) != exp_major_type, ZCBOR_ERR_WRONG_TYPE); + + uint8_t additional = ZCBOR_ADDITIONAL(chunk_header); + ZCBOR_ERR_IF(additional == ZCBOR_VALUE_IS_INDEFINITE_LENGTH, + ZCBOR_ERR_ADDITIONAL_INVAL); + ZCBOR_ERR_IF(additional > ZCBOR_VALUE_IS_8_BYTES, ZCBOR_ERR_ADDITIONAL_INVAL); + + size_t len_len = additional_len(additional); + ZCBOR_ERR_IF((state->payload + 1 + len_len) > state->payload_end, + ZCBOR_ERR_NO_PAYLOAD); + + uint64_t chunk_len = 0; + if (len_len == 0) { + chunk_len = additional; + } else { + uint8_t *chunk_ptr = (uint8_t *)&chunk_len; + memset(chunk_ptr, 0, sizeof(chunk_len)); + endian_copy(chunk_ptr + ZCBOR_ECPY_OFFS(sizeof(chunk_len), len_len), + state->payload + 1, len_len); + } + + state->payload += 1 + len_len; + ZCBOR_ERR_IF((size_t)chunk_len > (size_t)(state->payload_end - state->payload), + ZCBOR_ERR_NO_PAYLOAD); + + if (!call(ctx, state->payload, (size_t)chunk_len)) { + ZCBOR_FAIL(); + } + + state->payload += (size_t)chunk_len; + } + + return true; +} + static bool str_decode(zcbor_state_t *state, struct zcbor_string *result, zcbor_major_type_t exp_major_type) { @@ -629,6 +695,12 @@ bool zcbor_bstr_decode(zcbor_state_t *state, struct zcbor_string *result) return str_decode(state, result, ZCBOR_MAJOR_TYPE_BSTR); } +bool zcbor_bstr_chunk_in(zcbor_state_t *state, zcbor_stream_chunk_in call, void *ctx) +{ + PRINT_FUNC(); + return str_decode_chunks(state, call, ctx, ZCBOR_MAJOR_TYPE_BSTR); +} + bool zcbor_bstr_decode_fragment(zcbor_state_t *state, struct zcbor_string_fragment *result) { @@ -650,6 +722,12 @@ bool zcbor_tstr_decode(zcbor_state_t *state, struct zcbor_string *result) return str_decode(state, result, ZCBOR_MAJOR_TYPE_TSTR); } +bool zcbor_tstr_chunk_in(zcbor_state_t *state, zcbor_stream_chunk_in call, void *ctx) +{ + PRINT_FUNC(); + return str_decode_chunks(state, call, ctx, ZCBOR_MAJOR_TYPE_TSTR); +} + bool zcbor_tstr_decode_fragment(zcbor_state_t *state, struct zcbor_string_fragment *result) { diff --git a/src/zcbor_encode.c b/src/zcbor_encode.c index 1240fd9b..b98bd6b7 100644 --- a/src/zcbor_encode.c +++ b/src/zcbor_encode.c @@ -15,6 +15,43 @@ _Static_assert((sizeof(size_t) == sizeof(void *)), "This code needs size_t to be the same length as pointers."); +#define ZCBOR_IS_STREAMING(state) ((state)->constant_state && (state)->constant_state->stream_write) + +#define ZCBOR_WRITE_BYTE(state, byte) \ + do { \ + if (ZCBOR_IS_STREAMING(state)) { \ + uint8_t b = (byte); \ + size_t ret = state->constant_state->stream_write( \ + state->constant_state->stream_user_data, &b, 1); \ + if (ret != 1) { \ + ZCBOR_ERR(ZCBOR_ERR_STREAM_WRITE_FAILED); \ + } \ + state->constant_state->stream_bytes_written++; \ + } else { \ + ZCBOR_CHECK_PAYLOAD(); \ + *(state->payload_mut) = (byte); \ + state->payload_mut++; \ + } \ + } while(0) + +#define ZCBOR_WRITE_BYTES(state, data, len) \ + do { \ + if (ZCBOR_IS_STREAMING(state)) { \ + size_t ret = state->constant_state->stream_write( \ + state->constant_state->stream_user_data, (data), (len)); \ + if (ret != (len)) { \ + ZCBOR_ERR(ZCBOR_ERR_STREAM_WRITE_FAILED); \ + } \ + state->constant_state->stream_bytes_written += (len); \ + } else { \ + if ((state->payload + (len)) > state->payload_end) { \ + ZCBOR_ERR(ZCBOR_ERR_NO_PAYLOAD); \ + } \ + memcpy(state->payload_mut, (data), (len)); \ + state->payload_mut += (len); \ + } \ + } while(0) + static uint8_t log2ceil(size_t val) { @@ -44,13 +81,11 @@ static bool encode_header_byte(zcbor_state_t *state, zcbor_major_type_t major_type, uint8_t additional) { ZCBOR_CHECK_ERROR(); - ZCBOR_CHECK_PAYLOAD(); zcbor_assert_state(additional < 32, "Unsupported additional value: %d\r\n", additional); - *(state->payload_mut) = (uint8_t)((major_type << 5) | (additional & 0x1F)); + ZCBOR_WRITE_BYTE(state, (uint8_t)((major_type << 5) | (additional & 0x1F))); zcbor_trace(state, "value_encode"); - state->payload_mut++; return true; } @@ -62,21 +97,16 @@ static bool value_encode_len(zcbor_state_t *state, zcbor_major_type_t major_type { uint8_t *u8_result = (uint8_t *)result; - if ((state->payload + 1 + result_len) > state->payload_end) { - ZCBOR_ERR(ZCBOR_ERR_NO_PAYLOAD); - } - if (!encode_header_byte(state, major_type, get_additional(result_len, u8_result[0]))) { ZCBOR_FAIL(); } #ifdef ZCBOR_BIG_ENDIAN - memcpy(state->payload_mut, u8_result, result_len); - state->payload_mut += result_len; + ZCBOR_WRITE_BYTES(state, u8_result, result_len); #else for (; result_len > 0; result_len--) { - *(state->payload_mut++) = u8_result[result_len - 1]; + ZCBOR_WRITE_BYTE(state, u8_result[result_len - 1]); } #endif /* ZCBOR_BIG_ENDIAN */ @@ -211,10 +241,12 @@ bool zcbor_size_encode(zcbor_state_t *state, const size_t *input) static bool str_start_encode(zcbor_state_t *state, const struct zcbor_string *input, zcbor_major_type_t major_type) { - if (input->value && ((zcbor_header_len_ptr(&input->len, sizeof(input->len)) - + input->len + (size_t)state->payload) - > (size_t)state->payload_end)) { - ZCBOR_ERR(ZCBOR_ERR_NO_PAYLOAD); + if (!ZCBOR_IS_STREAMING(state)) { + if (input->value && ((zcbor_header_len_ptr(&input->len, sizeof(input->len)) + + input->len + (size_t)state->payload) + > (size_t)state->payload_end)) { + ZCBOR_ERR(ZCBOR_ERR_NO_PAYLOAD); + } } if (!value_encode(state, major_type, &input->len, sizeof(input->len))) { ZCBOR_FAIL(); @@ -270,19 +302,25 @@ bool zcbor_bstr_end_encode(zcbor_state_t *state, struct zcbor_string *result) static bool str_encode(zcbor_state_t *state, const struct zcbor_string *input, zcbor_major_type_t major_type) { - ZCBOR_CHECK_PAYLOAD(); /* To make the size_t cast below safe. */ - if (input->len > (size_t)(state->payload_end - state->payload)) { - ZCBOR_ERR(ZCBOR_ERR_NO_PAYLOAD); + if (!ZCBOR_IS_STREAMING(state)) { + ZCBOR_CHECK_PAYLOAD(); /* To make the size_t cast below safe. */ + if (input->len > (size_t)(state->payload_end - state->payload)) { + ZCBOR_ERR(ZCBOR_ERR_NO_PAYLOAD); + } } if (!str_start_encode(state, input, major_type)) { ZCBOR_FAIL(); } - if (state->payload_mut != input->value) { - /* Use memmove since string might be encoded into the same space + if (ZCBOR_IS_STREAMING(state)) { + ZCBOR_WRITE_BYTES(state, input->value, input->len); + } else { + /* Buffer mode: use memmove since string might be encoded into the same space * because of bstrx_cbor_start_encode/bstrx_cbor_end_encode. */ - memmove(state->payload_mut, input->value, input->len); + if (state->payload_mut != input->value) { + memmove(state->payload_mut, input->value, input->len); + } + state->payload += input->len; } - state->payload += input->len; return true; } @@ -331,6 +369,14 @@ static bool list_map_start_encode(zcbor_state_t *state, size_t max_num, zcbor_major_type_t major_type) { #ifdef ZCBOR_CANONICAL + /* In streaming mode, always use indefinite-length to avoid backtracking. */ + if (ZCBOR_IS_STREAMING(state)) { + if (!encode_header_byte(state, major_type, ZCBOR_VALUE_IS_INDEFINITE_LENGTH)) { + ZCBOR_FAIL(); + } + return true; + } + if (!zcbor_new_backup(state, 0)) { ZCBOR_FAIL(); } @@ -367,6 +413,14 @@ static bool list_map_end_encode(zcbor_state_t *state, size_t max_num, zcbor_major_type_t major_type) { #ifdef ZCBOR_CANONICAL + /* In streaming mode, terminate indefinite-length containers with break (0xFF). */ + if (ZCBOR_IS_STREAMING(state)) { + if (!encode_header_byte(state, ZCBOR_MAJOR_TYPE_SIMPLE, ZCBOR_VALUE_IS_INDEFINITE_LENGTH)) { + ZCBOR_FAIL(); + } + return true; + } + size_t list_count = ((major_type == ZCBOR_MAJOR_TYPE_LIST) ? state->elem_count : (state->elem_count / 2)); @@ -581,6 +635,199 @@ bool zcbor_multi_encode_minmax(size_t min_encode, size_t max_encode, } } +bool zcbor_multi_encode_iter_minmax(size_t min_encode, size_t max_encode, + zcbor_encoder_t encoder, zcbor_state_t *state, + zcbor_stream_iter next, void *ctx) +{ + ZCBOR_CHECK_ERROR(); + + if (!next) { + ZCBOR_ERR(ZCBOR_ERR_STREAM_READ_FAILED); + } + + size_t count = 0; + for (; count < max_encode; count++) { + const void *elem = NULL; + int rc = next(ctx, &elem); + if (rc == 0) { + break; /* done */ + } + if (rc < 0 || !elem) { + ZCBOR_ERR(ZCBOR_ERR_STREAM_READ_FAILED); + } + + if (!encoder(state, elem)) { + ZCBOR_FAIL(); + } + } + + if (count < min_encode) { + ZCBOR_ERR(ZCBOR_ERR_ITERATIONS); + } + + zcbor_log("Encoded %zu elements (iter).\n", count); + return true; +} + +static bool encode_indefinite_chunks(zcbor_state_t *state, uint8_t major_type, + bool (*chunk_put)(zcbor_state_t *state, const struct zcbor_string *zs), + zcbor_next_chunk_fn next_chunk, void *ctx) +{ + ZCBOR_CHECK_ERROR(); + + if (!next_chunk || !chunk_put) { + ZCBOR_ERR(ZCBOR_ERR_STREAM_READ_FAILED); + } + + /* Start indefinite-length container. */ + if (!encode_header_byte(state, major_type, ZCBOR_VALUE_IS_INDEFINITE_LENGTH)) { + ZCBOR_FAIL(); + } + + while (true) { + const uint8_t *ptr = NULL; + size_t len = 0; + int rc = next_chunk(ctx, &ptr, &len); + if (rc == 0) { + break; /* done */ + } + if (rc < 0 || (!ptr && len != 0)) { + ZCBOR_ERR(ZCBOR_ERR_STREAM_READ_FAILED); + } + + const struct zcbor_string zs = { .value = ptr, .len = len }; + if (!chunk_put(state, &zs)) { + ZCBOR_FAIL(); + } + } + + /* Break (0xFF). */ + if (!encode_header_byte(state, ZCBOR_MAJOR_TYPE_SIMPLE, ZCBOR_VALUE_IS_INDEFINITE_LENGTH)) { + ZCBOR_FAIL(); + } + + return true; +} + +static bool encode_indefinite_start(zcbor_state_t *state, uint8_t major_type) +{ + ZCBOR_CHECK_ERROR(); + + if (!encode_header_byte(state, major_type, ZCBOR_VALUE_IS_INDEFINITE_LENGTH)) { + ZCBOR_FAIL(); + } + + return true; +} + +static bool encode_indefinite_end(zcbor_state_t *state) +{ + ZCBOR_CHECK_ERROR(); + + /* Break (0xFF). */ + if (!encode_header_byte(state, ZCBOR_MAJOR_TYPE_SIMPLE, ZCBOR_VALUE_IS_INDEFINITE_LENGTH)) { + ZCBOR_FAIL(); + } + + return true; +} + +bool zcbor_tstr_encode_indefinite_chunks(zcbor_state_t *state, + zcbor_next_chunk_fn next_chunk, void *ctx) +{ + return encode_indefinite_chunks(state, ZCBOR_MAJOR_TYPE_TSTR, + (bool (*)(zcbor_state_t *, const struct zcbor_string *))zcbor_tstr_encode, + next_chunk, ctx); +} + +bool zcbor_bstr_encode_indefinite_chunks(zcbor_state_t *state, + zcbor_next_chunk_fn next_chunk, void *ctx) +{ + return encode_indefinite_chunks(state, ZCBOR_MAJOR_TYPE_BSTR, + (bool (*)(zcbor_state_t *, const struct zcbor_string *))zcbor_bstr_encode, + next_chunk, ctx); +} + +bool zcbor_tstr_start_encode_indefinite(zcbor_state_t *state) +{ + return encode_indefinite_start(state, ZCBOR_MAJOR_TYPE_TSTR); +} + +bool zcbor_bstr_start_encode_indefinite(zcbor_state_t *state) +{ + return encode_indefinite_start(state, ZCBOR_MAJOR_TYPE_BSTR); +} + +bool zcbor_tstr_encode_chunk(zcbor_state_t *state, const uint8_t *ptr, size_t len) +{ + const struct zcbor_string zs = { .value = ptr, .len = len }; + return zcbor_tstr_encode(state, &zs); +} + +bool zcbor_bstr_encode_chunk(zcbor_state_t *state, const uint8_t *ptr, size_t len) +{ + const struct zcbor_string zs = { .value = ptr, .len = len }; + return zcbor_bstr_encode(state, &zs); +} + +bool zcbor_tstr_end_encode_indefinite(zcbor_state_t *state) +{ + return encode_indefinite_end(state); +} + +bool zcbor_bstr_end_encode_indefinite(zcbor_state_t *state) +{ + return encode_indefinite_end(state); +} + +bool zcbor_tstr_chunk_out(zcbor_state_t *state, + zcbor_stream_chunk_out call, void *ctx) +{ + ZCBOR_CHECK_ERROR(); + + if (!call) { + ZCBOR_ERR(ZCBOR_ERR_STREAM_READ_FAILED); + } + + if (!zcbor_tstr_start_encode_indefinite(state)) { + ZCBOR_FAIL(); + } + + if (!call(ctx, state)) { + ZCBOR_ERR(ZCBOR_ERR_STREAM_READ_FAILED); + } + + if (!zcbor_tstr_end_encode_indefinite(state)) { + ZCBOR_FAIL(); + } + + return true; +} + +bool zcbor_bstr_chunk_out(zcbor_state_t *state, + zcbor_stream_chunk_out call, void *ctx) +{ + ZCBOR_CHECK_ERROR(); + + if (!call) { + ZCBOR_ERR(ZCBOR_ERR_STREAM_READ_FAILED); + } + + if (!zcbor_bstr_start_encode_indefinite(state)) { + ZCBOR_FAIL(); + } + + if (!call(ctx, state)) { + ZCBOR_ERR(ZCBOR_ERR_STREAM_READ_FAILED); + } + + if (!zcbor_bstr_end_encode_indefinite(state)) { + ZCBOR_FAIL(); + } + + return true; +} + bool zcbor_multi_encode(const size_t num_encode, zcbor_encoder_t encoder, zcbor_state_t *state, const void *input, size_t result_len) @@ -601,3 +848,45 @@ void zcbor_new_encode_state(zcbor_state_t *state_array, size_t n_states, { zcbor_new_state(state_array, n_states, payload, payload_len, elem_count, NULL, 0); } + +void zcbor_new_encode_state_streaming(zcbor_state_t *state_array, size_t n_states, + zcbor_stream_write_fn stream_write, void *stream_user_data, size_t elem_count) +{ + /* Initialize with dummy buffer (not used in streaming mode) */ + uint8_t dummy_buffer[1] = { 0 }; + zcbor_new_state(state_array, n_states, dummy_buffer, sizeof(dummy_buffer), elem_count, NULL, 0); + + if (state_array[0].constant_state) { + state_array[0].constant_state->stream_write = stream_write; + state_array[0].constant_state->stream_user_data = stream_user_data; + state_array[0].constant_state->stream_bytes_written = 0; + state_array[0].constant_state->stream_io = NULL; + } +} + +size_t zcbor_stream_bytes_written(const zcbor_state_t *state) +{ + if (!state || !ZCBOR_IS_STREAMING(state)) { + return 0; + } + return state->constant_state->stream_bytes_written; +} + +int zcbor_stream_entry_function(void *input, zcbor_state_t *states, size_t n_states, + zcbor_encoder_t func, zcbor_stream_write_fn stream_write, void *stream_user_data, + size_t elem_count, size_t *bytes_written_out) +{ + zcbor_new_encode_state_streaming(states, n_states, stream_write, stream_user_data, elem_count); + + bool ret = func(states, input); + if (!ret) { + int err = zcbor_pop_error(states); + err = (err == ZCBOR_SUCCESS) ? ZCBOR_ERR_UNKNOWN : err; + return err; + } + + if (bytes_written_out) { + *bytes_written_out = zcbor_stream_bytes_written(&states[0]); + } + return ZCBOR_SUCCESS; +} diff --git a/src/zcbor_print.c b/src/zcbor_print.c index d62f2780..11be182a 100644 --- a/src/zcbor_print.c +++ b/src/zcbor_print.c @@ -93,6 +93,8 @@ const char *zcbor_error_str(int error) ZCBOR_ERR_CASE(ZCBOR_ERR_MAP_FLAGS_NOT_AVAILABLE) ZCBOR_ERR_CASE(ZCBOR_ERR_INVALID_VALUE_ENCODING) ZCBOR_ERR_CASE(ZCBOR_ERR_CONSTANT_STATE_MISSING) + ZCBOR_ERR_CASE(ZCBOR_ERR_STREAM_WRITE_FAILED) + ZCBOR_ERR_CASE(ZCBOR_ERR_STREAM_READ_FAILED) } #undef ZCBOR_ERR_CASE diff --git a/tests/cases/corner_cases.cddl b/tests/cases/corner_cases.cddl index baa46a96..a3d33f07 100644 --- a/tests/cases/corner_cases.cddl +++ b/tests/cases/corner_cases.cddl @@ -372,3 +372,17 @@ OptList = [ optlist_int: int, ?optlist: [uint], ] + +collision_foo = 2 + +CollisionList = [ + collision_foo: uint, +] + +CollisionMap = { + collision_foo: uint, +} + +CollisionTypedKeyMap = { + collision_foo => uint, +} diff --git a/tests/encode/test1_suit/CMakeLists.txt b/tests/encode/test1_suit/CMakeLists.txt index 65f93f68..8f862c79 100644 --- a/tests/encode/test1_suit/CMakeLists.txt +++ b/tests/encode/test1_suit/CMakeLists.txt @@ -23,6 +23,10 @@ set(py_command ${bit_arg} ) +if (STREAMING) + list(APPEND py_command --stream-encode) +endif() + execute_process( COMMAND ${py_command} COMMAND_ERROR_IS_FATAL ANY diff --git a/tests/encode/test1_suit/src/main.c b/tests/encode/test1_suit/src/main.c index b70cf803..3c9074d1 100644 --- a/tests/encode/test1_suit/src/main.c +++ b/tests/encode/test1_suit/src/main.c @@ -9,6 +9,70 @@ #include "manifest3_encode.h" #include "zcbor_print.h" +#ifdef STREAMING +#include + +struct stream_ctx { + uint8_t *buf; + size_t size; + size_t pos; +}; + +static size_t stream_write(void *user_data, const uint8_t *data, size_t len) +{ + struct stream_ctx *ctx = (struct stream_ctx *)user_data; + + if (!ctx || !data) { + return 0; + } + if (len == 0) { + return 0; + } + if (ctx->pos + len > ctx->size) { + return 0; + } + + memcpy(&ctx->buf[ctx->pos], data, len); + ctx->pos += len; + return len; +} + +static int stream_encode_SUIT_Command_Sequence(uint8_t *payload, size_t payload_len, + const struct SUIT_Command_Sequence *input, size_t *out_len) +{ + struct stream_ctx ctx = { + .buf = payload, + .size = payload_len, + .pos = 0, + }; + int rc = cbor_stream_encode_SUIT_Command_Sequence(stream_write, &ctx, input, NULL, out_len); + + if (rc == ZCBOR_SUCCESS) { + zassert_equal(*out_len, ctx.pos, NULL); + } + return rc; +} + +static int stream_encode_SUIT_Outer_Wrapper(uint8_t *payload, size_t payload_len, + const struct SUIT_Outer_Wrapper *input, size_t *out_len) +{ + struct stream_ctx ctx = { + .buf = payload, + .size = payload_len, + .pos = 0, + }; + int rc = cbor_stream_encode_SUIT_Outer_Wrapper(stream_write, &ctx, input, NULL, out_len); + + if (rc == ZCBOR_SUCCESS) { + zassert_equal(*out_len, ctx.pos, NULL); + } + return rc; +} + +#define cbor_encode_SUIT_Command_Sequence stream_encode_SUIT_Command_Sequence +#define cbor_encode_SUIT_Outer_Wrapper stream_encode_SUIT_Outer_Wrapper +#endif /* STREAMING */ + /* draft-ietf-suit-manifest-02 Example 0 */ uint8_t test_vector0_02[] = { 0xa2, 0x01, 0x58, 0x54, 0xd2, 0x84, 0x43, 0xa1, 0x01, diff --git a/tests/encode/test1_suit/testcase.yaml b/tests/encode/test1_suit/testcase.yaml index 0120b824..d8ecbb5e 100644 --- a/tests/encode/test1_suit/testcase.yaml +++ b/tests/encode/test1_suit/testcase.yaml @@ -7,3 +7,12 @@ tests: - qemu_malta/qemu_malta/be tags: zcbor encode canonical test1 extra_args: CANONICAL=CANONICAL + + zcbor.encode.test1_suit.streaming_canonical: + platform_allow: + - native_sim + - native_sim/native/64 + - mps2/an521/cpu0 + - qemu_malta/qemu_malta/be + tags: zcbor encode streaming canonical test1 + extra_args: STREAMING=1 CANONICAL=CANONICAL diff --git a/tests/encode/test2_simple/CMakeLists.txt b/tests/encode/test2_simple/CMakeLists.txt index 0ff39a35..974e238a 100644 --- a/tests/encode/test2_simple/CMakeLists.txt +++ b/tests/encode/test2_simple/CMakeLists.txt @@ -22,6 +22,11 @@ set(py_command --file-header ${CMAKE_CURRENT_LIST_DIR}/file_header_copyright.txt ) +if (STREAMING) + zephyr_compile_definitions(STREAMING) + list(APPEND py_command --stream-encode) +endif() + execute_process( COMMAND ${py_command} COMMAND_ERROR_IS_FATAL ANY @@ -31,4 +36,3 @@ include(${PROJECT_BINARY_DIR}/pet.cmake) target_link_libraries(pet PRIVATE zephyr_interface) target_link_libraries(app PRIVATE pet) - diff --git a/tests/encode/test2_simple/src/main.c b/tests/encode/test2_simple/src/main.c index 1a849965..c9260b91 100644 --- a/tests/encode/test2_simple/src/main.c +++ b/tests/encode/test2_simple/src/main.c @@ -8,11 +8,86 @@ #include #include -#ifndef ZCBOR_CANONICAL +#if !defined(ZCBOR_CANONICAL) || defined(STREAMING) #define TEST_INDEFINITE_LENGTH_ARRAYS #endif #include +#ifdef STREAMING +#include +#include + +struct stream_ctx { + uint8_t *buf; + size_t size; + size_t pos; +}; + +static size_t stream_write(void *user_data, const uint8_t *data, size_t len) +{ + struct stream_ctx *ctx = (struct stream_ctx *)user_data; + + if (!ctx) { + return 0; + } + if (len == 0) { + return 0; + } + if (!data) { + return 0; + } + if (ctx->pos + len > ctx->size) { + return 0; + } + + memcpy(&ctx->buf[ctx->pos], data, len); + ctx->pos += len; + return len; +} + +struct bstr_push_ctx { + const uint8_t *data; + size_t len; +}; + +static bool bstr_push(void *user_data, zcbor_state_t *state) +{ + struct bstr_push_ctx *ctx = (struct bstr_push_ctx *)user_data; + + if (!ctx) { + return false; + } + + return zcbor_bstr_encode_chunk(state, ctx->data, ctx->len); +} +#endif /* STREAMING */ + +#ifdef STREAMING +struct bstr_chunk_ctx { + int idx; + const uint8_t *chunks[2]; + size_t lens[2]; +}; + +static int bstr_next_chunk(void *user_ctx, const uint8_t **ptr, size_t *len) +{ + struct bstr_chunk_ctx *c = (struct bstr_chunk_ctx *)user_ctx; + + if (!c || !ptr || !len) { + return -EINVAL; + } + + if (c->idx >= 2) { + return 0; + } + + *ptr = c->chunks[c->idx]; + *len = c->lens[c->idx]; + c->idx++; + return 1; +} +#endif /* STREAMING */ + /* This test uses generated code to encode a 'Pet' instance. It populates the * generated struct, and runs the generated encoding function, then checks that @@ -41,12 +116,51 @@ ZTEST(cbor_encode_test2, test_pet) size_t out_len; /* Check that encoding succeeded. */ +#ifdef STREAMING + struct stream_ctx ctx = { + .buf = output, + .size = sizeof(output), + .pos = 0, + }; + + struct bstr_push_ctx push_ctx = { + .data = pet.birthday.value, + .len = pet.birthday.len, + }; + struct cbor_stream_io_Pet io = { 0 }; + + io.chunks_out_Timestamp.ctx = &push_ctx; + io.chunks_out_Timestamp.call = bstr_push; + + int rc = cbor_stream_encode_Pet(stream_write, &ctx, &pet, NULL, &out_len); + zassert_equal(ZCBOR_SUCCESS, rc, NULL); + zassert_equal(out_len, ctx.pos, NULL); +#else zassert_equal(ZCBOR_SUCCESS, cbor_encode_Pet(output, sizeof(output), &pet, &out_len), NULL); +#endif /* Check that the resulting length is correct. */ zassert_equal(sizeof(exp_output), out_len, NULL); /* Check the payload contents. */ zassert_mem_equal(exp_output, output, sizeof(exp_output), NULL); + +#ifdef STREAMING + const uint8_t *exp_output_push = exp_output; + size_t exp_output_push_len = sizeof(exp_output); + uint8_t output_push[32]; + struct stream_ctx push_out_ctx = { + .buf = output_push, + .size = sizeof(output_push), + .pos = 0, + }; + size_t out_len_push; + + rc = cbor_stream_encode_Pet(stream_write, &push_out_ctx, &pet, &io, &out_len_push); + zassert_equal(ZCBOR_SUCCESS, rc, NULL); + zassert_equal(out_len_push, push_out_ctx.pos, NULL); + zassert_equal(exp_output_push_len, out_len_push, NULL); + zassert_mem_equal(exp_output_push, output_push, exp_output_push_len, NULL); +#endif } @@ -59,7 +173,18 @@ ZTEST(cbor_encode_test2, test_pet) ZTEST(cbor_encode_test2, test_pet_raw) { uint8_t payload[100] = {0}; +#ifdef STREAMING + struct stream_ctx ctx = { + .buf = payload, + .size = sizeof(payload), + .pos = 0, + }; + zcbor_state_t states[4]; + zcbor_new_encode_state_streaming(states, 4, stream_write, &ctx, 1); + zcbor_state_t *state = states; +#else ZCBOR_STATE_E(state, 4, payload, sizeof(payload), 1); +#endif uint8_t exp_output[] = { LIST(3), @@ -97,10 +222,69 @@ ZTEST(cbor_encode_test2, test_pet_raw) /* Check that encoding succeeded. */ zassert_true(res, NULL); /* Check that the resulting length is correct. */ +#ifdef STREAMING + zassert_equal(sizeof(exp_output), ctx.pos, "%d != %d\r\n", + sizeof(exp_output), ctx.pos); +#else zassert_equal(sizeof(exp_output), state->payload - payload, "%d != %d\r\n", sizeof(exp_output), state->payload - payload); +#endif /* Check the payload contents. */ zassert_mem_equal(exp_output, payload, sizeof(exp_output), NULL); } +#ifdef STREAMING +ZTEST(cbor_encode_test2, test_bstr_indefinite_chunks) +{ + /* Expect: 0x5f (bstr indefinite), 0x43 01 02 03, 0x42 04 05, 0xff (break) */ + uint8_t output[16]; + struct stream_ctx ctx = { + .buf = output, + .size = sizeof(output), + .pos = 0, + }; + + const uint8_t c1[] = { 0x01, 0x02, 0x03 }; + const uint8_t c2[] = { 0x04, 0x05 }; + struct bstr_chunk_ctx chunks = { + .idx = 0, + .chunks = { c1, c2 }, + .lens = { sizeof(c1), sizeof(c2) }, + }; + + zcbor_state_t s[4]; + zcbor_new_encode_state_streaming(s, 4, stream_write, &ctx, 1); + + zassert_true(zcbor_bstr_encode_indefinite_chunks(s, bstr_next_chunk, &chunks), NULL); + zassert_equal(ctx.pos, 1 + 1 + sizeof(c1) + 1 + sizeof(c2) + 1, NULL); + + const uint8_t exp[] = { 0x5f, 0x43, 0x01, 0x02, 0x03, 0x42, 0x04, 0x05, 0xff }; + zassert_mem_equal(output, exp, ctx.pos, NULL); +} +#endif /* STREAMING */ + +#ifdef STREAMING +ZTEST(cbor_encode_test2, test_streaming_container_headers) +{ + /* In streaming mode, lists/maps must be indefinite-length (0x9f/0xbf ... 0xff). */ + uint8_t output[32]; + struct stream_ctx ctx = { + .buf = output, + .size = sizeof(output), + .pos = 0, + }; + + zcbor_state_t s[4]; + zcbor_new_encode_state_streaming(s, 4, stream_write, &ctx, 1); + + zassert_true(zcbor_list_start_encode(s, 3), NULL); + zassert_true(zcbor_uint32_put(s, 1), NULL); + zassert_true(zcbor_list_end_encode(s, 3), NULL); + + zassert_true(ctx.pos >= 2, NULL); + zassert_equal(output[0], 0x9f, NULL); + zassert_equal(output[ctx.pos - 1], 0xff, NULL); +} +#endif /* STREAMING */ + ZTEST_SUITE(cbor_encode_test2, NULL, NULL, NULL, NULL, NULL); diff --git a/tests/encode/test2_simple/testcase.yaml b/tests/encode/test2_simple/testcase.yaml index a373f987..d17dc522 100644 --- a/tests/encode/test2_simple/testcase.yaml +++ b/tests/encode/test2_simple/testcase.yaml @@ -6,6 +6,27 @@ tests: - mps2/an521/cpu0 - qemu_malta/qemu_malta/be tags: zcbor encode test + + zcbor.encode.test2_simple.streaming: + platform_allow: + - native_sim + - native_sim/native/64 + - mps2/an521/cpu0 + - qemu_malta/qemu_malta/be + tags: zcbor encode streaming test + extra_args: STREAMING=1 + + + zcbor.encode.test2_simple.streaming_canonical: + platform_allow: + - native_sim + - native_sim/native/64 + - mps2/an521/cpu0 + - qemu_malta/qemu_malta/be + tags: zcbor encode streaming canonical test + extra_args: STREAMING=1 CANONICAL=CANONICAL + + zcbor.encode.test2_simple.canonical: platform_allow: - native_sim diff --git a/tests/encode/test3_corner_cases/CMakeLists.txt b/tests/encode/test3_corner_cases/CMakeLists.txt index 6b6e7a0a..c55b8871 100644 --- a/tests/encode/test3_corner_cases/CMakeLists.txt +++ b/tests/encode/test3_corner_cases/CMakeLists.txt @@ -59,6 +59,10 @@ set(py_command --short-names ) +if (STREAMING) + list(APPEND py_command --stream-encode) +endif() + execute_process( COMMAND ${py_command} COMMAND_ERROR_IS_FATAL ANY diff --git a/tests/encode/test3_corner_cases/src/main.c b/tests/encode/test3_corner_cases/src/main.c index 61cf5a0c..4e47908c 100644 --- a/tests/encode/test3_corner_cases/src/main.c +++ b/tests/encode/test3_corner_cases/src/main.c @@ -12,6 +12,128 @@ #endif #include +#ifdef STREAMING +#include + +struct stream_ctx { + uint8_t *buf; + size_t size; + size_t pos; +}; + +static size_t stream_write(void *user_data, const uint8_t *data, size_t len) +{ + struct stream_ctx *ctx = (struct stream_ctx *)user_data; + + if (!ctx || !data) { + return 0; + } + if (len == 0) { + return 0; + } + if (ctx->pos + len > ctx->size) { + return 0; + } + + memcpy(&ctx->buf[ctx->pos], data, len); + ctx->pos += len; + return len; +} + +#define STREAM_ENCODE_WRAPPER(func, type) \ +static int stream_encode_##func(uint8_t *payload, size_t payload_len, \ + const type *input, size_t *out_len) \ +{ \ + struct stream_ctx ctx = { \ + .buf = payload, \ + .size = payload_len, \ + .pos = 0, \ + }; \ + int rc = cbor_stream_encode_##func(stream_write, &ctx, input, NULL, out_len); \ + if (rc == ZCBOR_SUCCESS) { \ + zassert_equal(*out_len, ctx.pos, NULL); \ + } \ + return rc; \ +} + +STREAM_ENCODE_WRAPPER(NestedListMap, struct NestedListMap) +STREAM_ENCODE_WRAPPER(NestedMapListMap, struct NestedMapListMap) +STREAM_ENCODE_WRAPPER(Numbers, struct Numbers) +STREAM_ENCODE_WRAPPER(Numbers2, struct Numbers2) +STREAM_ENCODE_WRAPPER(NumberMap, struct NumberMap) +STREAM_ENCODE_WRAPPER(TaggedUnion, struct TaggedUnion_r) +STREAM_ENCODE_WRAPPER(Strings, struct Strings) +STREAM_ENCODE_WRAPPER(Simple2, struct Simples) +STREAM_ENCODE_WRAPPER(Optional, struct Optional) +STREAM_ENCODE_WRAPPER(Union, struct Union_r) +STREAM_ENCODE_WRAPPER(Map, struct Map) +STREAM_ENCODE_WRAPPER(Level1, struct Level2) +STREAM_ENCODE_WRAPPER(Range, struct Range) +STREAM_ENCODE_WRAPPER(ValueRange, struct ValueRange) +STREAM_ENCODE_WRAPPER(SingleBstr, struct zcbor_string) +STREAM_ENCODE_WRAPPER(SingleInt_uint52, void) +STREAM_ENCODE_WRAPPER(SingleInt2, uint32_t) +STREAM_ENCODE_WRAPPER(Unabstracted, struct Unabstracted) +STREAM_ENCODE_WRAPPER(QuantityRange, struct QuantityRange) +STREAM_ENCODE_WRAPPER(DoubleMap, struct DoubleMap) +STREAM_ENCODE_WRAPPER(Floats, struct Floats) +STREAM_ENCODE_WRAPPER(Floats2, struct Floats2) +STREAM_ENCODE_WRAPPER(CBORBstr, struct CBORBstr) +STREAM_ENCODE_WRAPPER(MapLength, struct MapLength) +STREAM_ENCODE_WRAPPER(UnionInt2, struct UnionInt2) +STREAM_ENCODE_WRAPPER(Intmax1, void) +STREAM_ENCODE_WRAPPER(Intmax2, struct Intmax2) +STREAM_ENCODE_WRAPPER(InvalidIdentifiers, struct InvalidIdentifiers) +STREAM_ENCODE_WRAPPER(MapUnionPrimAlias, struct MapUnionPrimAlias) +STREAM_ENCODE_WRAPPER(EmptyContainer, struct EmptyContainer) +STREAM_ENCODE_WRAPPER(SingleElemList, struct SingleElemList) +STREAM_ENCODE_WRAPPER(Choice1, struct Choice1_r) +STREAM_ENCODE_WRAPPER(Choice2, struct Choice2_r) +STREAM_ENCODE_WRAPPER(Choice3, struct Choice3_r) +STREAM_ENCODE_WRAPPER(Choice4, struct Choice4_r) +STREAM_ENCODE_WRAPPER(Choice5, struct Choice5_r) +STREAM_ENCODE_WRAPPER(OptList, struct OptList) + +#undef STREAM_ENCODE_WRAPPER + +#define cbor_encode_NestedListMap stream_encode_NestedListMap +#define cbor_encode_NestedMapListMap stream_encode_NestedMapListMap +#define cbor_encode_Numbers stream_encode_Numbers +#define cbor_encode_Numbers2 stream_encode_Numbers2 +#define cbor_encode_NumberMap stream_encode_NumberMap +#define cbor_encode_TaggedUnion stream_encode_TaggedUnion +#define cbor_encode_Strings stream_encode_Strings +#define cbor_encode_Simple2 stream_encode_Simple2 +#define cbor_encode_Optional stream_encode_Optional +#define cbor_encode_Union stream_encode_Union +#define cbor_encode_Map stream_encode_Map +#define cbor_encode_Level1 stream_encode_Level1 +#define cbor_encode_Range stream_encode_Range +#define cbor_encode_ValueRange stream_encode_ValueRange +#define cbor_encode_SingleBstr stream_encode_SingleBstr +#define cbor_encode_SingleInt_uint52 stream_encode_SingleInt_uint52 +#define cbor_encode_SingleInt2 stream_encode_SingleInt2 +#define cbor_encode_Unabstracted stream_encode_Unabstracted +#define cbor_encode_QuantityRange stream_encode_QuantityRange +#define cbor_encode_DoubleMap stream_encode_DoubleMap +#define cbor_encode_Floats stream_encode_Floats +#define cbor_encode_Floats2 stream_encode_Floats2 +#define cbor_encode_CBORBstr stream_encode_CBORBstr +#define cbor_encode_MapLength stream_encode_MapLength +#define cbor_encode_UnionInt2 stream_encode_UnionInt2 +#define cbor_encode_Intmax1 stream_encode_Intmax1 +#define cbor_encode_Intmax2 stream_encode_Intmax2 +#define cbor_encode_InvalidIdentifiers stream_encode_InvalidIdentifiers +#define cbor_encode_MapUnionPrimAlias stream_encode_MapUnionPrimAlias +#define cbor_encode_EmptyContainer stream_encode_EmptyContainer +#define cbor_encode_SingleElemList stream_encode_SingleElemList +#define cbor_encode_Choice1 stream_encode_Choice1 +#define cbor_encode_Choice2 stream_encode_Choice2 +#define cbor_encode_Choice3 stream_encode_Choice3 +#define cbor_encode_Choice4 stream_encode_Choice4 +#define cbor_encode_Choice5 stream_encode_Choice5 +#define cbor_encode_OptList stream_encode_OptList +#endif /* STREAMING */ ZTEST(cbor_encode_test3, test_numbers) { diff --git a/tests/encode/test3_corner_cases/testcase.yaml b/tests/encode/test3_corner_cases/testcase.yaml index a49b690f..856197c9 100644 --- a/tests/encode/test3_corner_cases/testcase.yaml +++ b/tests/encode/test3_corner_cases/testcase.yaml @@ -6,6 +6,17 @@ tests: - mps2/an521/cpu0 - qemu_malta/qemu_malta/be tags: zcbor encode test3 + + zcbor.encode.test3_corner_cases.streaming: + platform_allow: + - native_sim + - native_sim/native/64 + - mps2/an521/cpu0 + - qemu_malta/qemu_malta/be + tags: zcbor encode streaming test3 + extra_args: STREAMING=1 + + zcbor.encode.test3_corner_cases.canonical: platform_allow: - native_sim @@ -14,3 +25,12 @@ tests: - qemu_malta/qemu_malta/be tags: zcbor encode canonical test3 extra_args: CANONICAL=CANONICAL + + zcbor.encode.test3_corner_cases.streaming_canonical: + platform_allow: + - native_sim + - native_sim/native/64 + - mps2/an521/cpu0 + - qemu_malta/qemu_malta/be + tags: zcbor encode streaming canonical test3 + extra_args: STREAMING=1 CANONICAL=CANONICAL diff --git a/tests/encode/test4_senml/CMakeLists.txt b/tests/encode/test4_senml/CMakeLists.txt index d1cf0961..9e5cb9a7 100644 --- a/tests/encode/test4_senml/CMakeLists.txt +++ b/tests/encode/test4_senml/CMakeLists.txt @@ -20,6 +20,10 @@ set(py_command ${bit_arg} ) +if (STREAMING) + list(APPEND py_command --stream-encode) +endif() + execute_process( COMMAND ${py_command} COMMAND_ERROR_IS_FATAL ANY diff --git a/tests/encode/test4_senml/src/main.c b/tests/encode/test4_senml/src/main.c index 1ba0f7a0..45452620 100644 --- a/tests/encode/test4_senml/src/main.c +++ b/tests/encode/test4_senml/src/main.c @@ -12,6 +12,53 @@ #endif #include +#ifdef STREAMING +#include + +struct stream_ctx { + uint8_t *buf; + size_t size; + size_t pos; +}; + +static size_t stream_write(void *user_data, const uint8_t *data, size_t len) +{ + struct stream_ctx *ctx = (struct stream_ctx *)user_data; + + if (!ctx || !data) { + return 0; + } + if (len == 0) { + return 0; + } + if (ctx->pos + len > ctx->size) { + return 0; + } + + memcpy(&ctx->buf[ctx->pos], data, len); + ctx->pos += len; + return len; +} + +static int stream_encode_lwm2m_senml(uint8_t *payload, size_t payload_len, + struct lwm2m_senml *input, size_t *out_len) +{ + struct stream_ctx ctx = { + .buf = payload, + .size = payload_len, + .pos = 0, + }; + int rc = cbor_stream_encode_lwm2m_senml(stream_write, &ctx, input, NULL, out_len); + + if (rc == ZCBOR_SUCCESS) { + zassert_equal(*out_len, ctx.pos, NULL); + } + return rc; +} + +#define cbor_encode_lwm2m_senml stream_encode_lwm2m_senml +#endif /* STREAMING */ + ZTEST(cbor_encode_test4, test_senml) { struct lwm2m_senml input = { diff --git a/tests/encode/test4_senml/testcase.yaml b/tests/encode/test4_senml/testcase.yaml index 57c91fcb..20e2b813 100644 --- a/tests/encode/test4_senml/testcase.yaml +++ b/tests/encode/test4_senml/testcase.yaml @@ -6,6 +6,17 @@ tests: - mps2/an521/cpu0 - qemu_malta/qemu_malta/be tags: zcbor encode test4 + + zcbor.encode.test4_senml.streaming: + platform_allow: + - native_sim + - native_sim/native/64 + - mps2/an521/cpu0 + - qemu_malta/qemu_malta/be + tags: zcbor encode streaming test4 + extra_args: STREAMING=1 + + zcbor.encode.test4_senml.canonical: platform_allow: - native_sim @@ -14,3 +25,12 @@ tests: - qemu_malta/qemu_malta/be tags: zcbor encode canonical test4 extra_args: CANONICAL=CANONICAL + + zcbor.encode.test4_senml.streaming_canonical: + platform_allow: + - native_sim + - native_sim/native/64 + - mps2/an521/cpu0 + - qemu_malta/qemu_malta/be + tags: zcbor encode streaming canonical test4 + extra_args: STREAMING=1 CANONICAL=CANONICAL diff --git a/tests/scripts/test_zcbor.py b/tests/scripts/test_zcbor.py index 4cc4433d..650cd850 100644 --- a/tests/scripts/test_zcbor.py +++ b/tests/scripts/test_zcbor.py @@ -764,6 +764,37 @@ def test_intmax2(self): self.assertEqual(decoded.UINT_64, 18446744073709551615) +class TestLabelKeyContext(TestCase): + def test_list_label_does_not_become_key_when_type_exists(self): + cddl_str = """ +foo = 2 + +my_list = [ + foo: uint +] +""" + cddl_res = zcbor.DataTranslator.from_cddl(cddl_str, 16) + cddl = cddl_res.my_types["my_list"] + self.assertEqual("LIST", cddl.type) + child = cddl.value[0] + self.assertEqual("foo", child.label) + self.assertIsNone(child.key) + + def test_map_member_still_uses_key_when_type_exists(self): + cddl_str = """ +foo = 2 + +my_map = { + foo: uint +} +""" + cddl_res = zcbor.DataTranslator.from_cddl(cddl_str, 16) + cddl = cddl_res.my_types["my_map"] + self.assertEqual("MAP", cddl.type) + child = cddl.value[0] + self.assertIsNotNone(child.key) + + class TestInvalidIdentifiers(TestCase): def test_invalid_identifiers0(self): cddl_res = zcbor.DataTranslator.from_cddl( diff --git a/tests/unit/test1_unit_tests/src/main.c b/tests/unit/test1_unit_tests/src/main.c index 990a0a25..ac3daf90 100644 --- a/tests/unit/test1_unit_tests/src/main.c +++ b/tests/unit/test1_unit_tests/src/main.c @@ -1024,10 +1024,12 @@ ZTEST(zcbor_unit_tests, test_error_str) test_str(ZCBOR_ERR_MAP_FLAGS_NOT_AVAILABLE); test_str(ZCBOR_ERR_INVALID_VALUE_ENCODING); test_str(ZCBOR_ERR_CONSTANT_STATE_MISSING); + test_str(ZCBOR_ERR_STREAM_WRITE_FAILED); + test_str(ZCBOR_ERR_STREAM_READ_FAILED); test_str(ZCBOR_ERR_UNKNOWN); zassert_mem_equal(zcbor_error_str(-1), "ZCBOR_ERR_UNKNOWN", sizeof("ZCBOR_ERR_UNKNOWN"), NULL); zassert_mem_equal(zcbor_error_str(-10), "ZCBOR_ERR_UNKNOWN", sizeof("ZCBOR_ERR_UNKNOWN"), NULL); - zassert_mem_equal(zcbor_error_str(ZCBOR_ERR_CONSTANT_STATE_MISSING + 1), "ZCBOR_ERR_UNKNOWN", sizeof("ZCBOR_ERR_UNKNOWN"), NULL); + zassert_mem_equal(zcbor_error_str(ZCBOR_ERR_STREAM_READ_FAILED + 1), "ZCBOR_ERR_UNKNOWN", sizeof("ZCBOR_ERR_UNKNOWN"), NULL); zassert_mem_equal(zcbor_error_str(100000), "ZCBOR_ERR_UNKNOWN", sizeof("ZCBOR_ERR_UNKNOWN"), NULL); } diff --git a/zcbor/zcbor.py b/zcbor/zcbor.py index 625ae2d5..86854495 100755 --- a/zcbor/zcbor.py +++ b/zcbor/zcbor.py @@ -215,7 +215,7 @@ class CddlParser: - For "GROUP" and "UNION" types, there is no separate data item for the instance. """ def __init__(self, default_max_qty, my_types, my_control_groups, base_name=None, - short_names=False, base_stem=''): + short_names=False, base_stem='', in_map=False): self.id_prefix = "temp_" + str(counter()) self.id_num = None # Unique ID number. Only populated if needed. # The value of the data item. Has different meaning for different @@ -263,6 +263,7 @@ def __init__(self, default_max_qty, my_types, my_control_groups, base_name=None, # Stem which can be used when generating an id. self.base_stem = base_stem.replace("-", "_") self.short_names = short_names + self.in_map = in_map if type(self) not in type(self).cddl_regexes: self.cddl_regexes_init() @@ -438,7 +439,7 @@ def init_kwargs(self): """Return the kwargs that should be used to initialize a new instance of this class.""" return { "my_types": self.my_types, "my_control_groups": self.my_control_groups, - "short_names": self.short_names} + "short_names": self.short_names, "in_map": self.in_map} def set_id_prefix(self, id_prefix=''): self.id_prefix = id_prefix @@ -699,6 +700,7 @@ def set_key(self, key): if key.type == "GROUP": raise TypeError("A key cannot be a group because it might represent more than 1 type.") self.key = key + key.is_key = True def set_key_or_label(self, key_or_label): """Set the self.label OR self.key of this element. @@ -707,7 +709,7 @@ def set_key_or_label(self, key_or_label): map. This code uses a slightly different method for choosing between label and key. If the string is recognized as a type, it is treated as a key. For use during CDDL parsing. """ - if key_or_label in self.my_types: + if self.in_map and key_or_label in self.my_types: self.set_key(self.parse(key_or_label)[0]) assert self.key.type == "OTHER", "This should only be able to produce an OTHER key." if self.label is None: @@ -784,13 +786,13 @@ def cddl_regexes_init(self): range_types = [ (r'(?P\[(?P(?>[^[\]]+|(?&bracket))*)\])', lambda m_self, list_str: m_self.type_and_value( - "LIST", lambda: m_self.parse(list_str))), + "LIST", lambda: m_self.parse(list_str, in_map=False))), (r'(?P\((?P(?>[^\(\)]+|(?&paren))*)\))', lambda m_self, group_str: m_self.type_and_value( - "GROUP", lambda: m_self.parse(group_str))), + "GROUP", lambda: m_self.parse(group_str, in_map=m_self.in_map))), (r'(?P{(?P(?>[^{}]+|(?&curly))*)})', lambda m_self, map_str: m_self.type_and_value( - "MAP", lambda: m_self.parse(map_str))), + "MAP", lambda: m_self.parse(map_str, in_map=True))), (r'\'(?P.*?)(?.*?)(?", "_").replace(".", "_") + access_name = getrp(r'[^a-zA-Z0-9_]').sub("_", access_name).strip("_") + if access_name and access_name not in base_name: + return f"{self.mode}_repeated_{access_name}_{base_name}{key_suffix}" + + return f"{self.mode}_repeated_{base_name}{key_suffix}" def single_func_prim_name(self, union_int=None, ptr_result=False): """Function name for xcoding this type, when it is a primitive type""" @@ -2347,6 +2400,60 @@ def xcode_single_func_prim(self, union_int=None): """Make a string from the list returned by single_func_prim()""" return xcode_statement(*self.single_func_prim(self.val_access(), union_int)) + def xcode_tstr_streaming(self): + if self.mode != "encode" or not self.stream_encode_functions: + return None + if self.value is not None or self.cbor: + return None + if getattr(self, "is_key", False): + return None + if self.count_var_condition(): + return None + + prov_access = "((const struct cbor_stream_io *)zcbor_get_stream_io(state))" + chunk_name = self.stream_chunk_field_name("out") + chunk_call = ( + f"zcbor_tstr_chunk_out(state, " + f"{prov_access}->{chunk_name}.call, {prov_access}->{chunk_name}.ctx)" + ) + chunk_check = f"({prov_access} && {prov_access}->{chunk_name}.call)" + + fallback = self.xcode_single_func_prim() + return f"({chunk_check} ? ({chunk_call}) : ({fallback}))" + + def xcode_tstr_streaming_decode(self): + if self.mode != "decode" or not self.stream_decode_functions: + return None + if self.value is not None or self.cbor: + return None + if getattr(self, "is_key", False): + return None + if self.count_var_condition(): + return None + + prov_access = ( + "((const struct cbor_stream_io *)" + "zcbor_get_stream_io(state))" + ) + chunk_name = self.stream_chunk_field_name("in") + chunk_call = ( + f"zcbor_tstr_chunk_in(state, " + f"{prov_access}->{chunk_name}.call, {prov_access}->{chunk_name}.ctx)" + ) + chunk_check = f"({prov_access} && {prov_access}->{chunk_name}.call)" + + fallback = self.xcode_single_func_prim() + return f"({chunk_check} ? ({chunk_call}) : ({fallback}))" + + def xcode_tstr(self): + """Encode TSTR, with optional streaming chunk io support.""" + stream_xcode = self.xcode_tstr_streaming() + if stream_xcode is None: + stream_xcode = self.xcode_tstr_streaming_decode() + if stream_xcode is not None: + return stream_xcode + return self.xcode_single_func_prim() + def list_counts(self): """Recursively sum the total minimum and maximum element count for this element.""" retval = ({ @@ -2386,9 +2493,18 @@ def xcode_list(self): "zcbor_map_end_decode", "zcbor_map_end_encode"] assert self.type in ["LIST", "MAP"], \ "Expected LIST or MAP type, was %s." % self.type - _, max_counts = zip( - *(child.list_counts() for child in self.value)) if self.value else ((0,), (0,)) - count_arg = f', {str(sum(max_counts))}' if self.mode == 'encode' else '' + + # Default: definite-length. When generating streaming encode entrypoints, use + # indefinite-length containers. + if self.mode == 'encode' and self.stream_encode_functions: + count_arg = ', ZCBOR_VALUE_IS_INDEFINITE_LENGTH' + elif self.mode == 'encode': + _, max_counts = zip( + *(child.list_counts() for child in self.value)) if self.value else ((0,), (0,)) + count_arg = f', {str(sum(max_counts))}' + else: + count_arg = '' + with_children = "(%s && ((%s) || (%s, false)) && %s)" % ( f"{start_func}(state{count_arg})", f"{newl_ind}&& ".join(child.full_xcode() for child in self.value), @@ -2448,7 +2564,57 @@ def xcode_union(self): [child.enum_var_name() for child in self.value], [child.full_xcode() for child in self.value]) + def xcode_bstr_streaming(self): + if self.mode != "encode" or not self.stream_encode_functions: + return None + if self.value is not None or self.cbor: + return None + if getattr(self, "is_key", False): + return None + if self.count_var_condition(): + return None + + prov_access = "((const struct cbor_stream_io *)zcbor_get_stream_io(state))" + chunk_name = self.stream_chunk_field_name("out") + chunk_call = ( + f"zcbor_bstr_chunk_out(state, " + f"{prov_access}->{chunk_name}.call, {prov_access}->{chunk_name}.ctx)" + ) + chunk_check = f"({prov_access} && {prov_access}->{chunk_name}.call)" + + fallback = self.xcode_single_func_prim() + return f"({chunk_check} ? ({chunk_call}) : ({fallback}))" + + def xcode_bstr_streaming_decode(self): + if self.mode != "decode" or not self.stream_decode_functions: + return None + if self.value is not None or self.cbor: + return None + if getattr(self, "is_key", False): + return None + if self.count_var_condition(): + return None + + prov_access = ( + "((const struct cbor_stream_io *)" + "zcbor_get_stream_io(state))" + ) + chunk_name = self.stream_chunk_field_name("in") + chunk_call = ( + f"zcbor_bstr_chunk_in(state, " + f"{prov_access}->{chunk_name}.call, {prov_access}->{chunk_name}.ctx)" + ) + chunk_check = f"({prov_access} && {prov_access}->{chunk_name}.call)" + + fallback = self.xcode_single_func_prim() + return f"({chunk_check} ? ({chunk_call}) : ({fallback}))" + def xcode_bstr(self): + stream_xcode = self.xcode_bstr_streaming() + if stream_xcode is None: + stream_xcode = self.xcode_bstr_streaming_decode() + if stream_xcode is not None: + return stream_xcode if self.cbor and not self.cbor.is_entry_type(): access_arg = f', {deref_if_not_null(self.val_access())}' if self.mode == 'decode' \ else '' @@ -2545,7 +2711,7 @@ def repeated_xcode(self, union_int=None): "NINT": lambda: self.xcode_single_func_prim(val_union_int), "FLOAT": self.xcode_single_func_prim, "BSTR": self.xcode_bstr, - "TSTR": self.xcode_single_func_prim, + "TSTR": self.xcode_tstr, "BOOL": self.xcode_single_func_prim, "NIL": self.xcode_single_func_prim, "UNDEF": self.xcode_single_func_prim, @@ -2604,10 +2770,47 @@ def full_xcode(self, union_int=None): minmax = "_minmax" if self.mode == "encode" else "" mode = self.mode + + # Keep encode max_qty overrideable (streaming write entrypoints only) when the CDDL + # omitted a max and we fell back to --default-max-qty. + if (self.mode == "encode" + and self.stream_encode_functions + and self.max_qty == self.default_max_qty): + max_qty = "DEFAULT_MAX_QTY" + else: + max_qty = self.max_qty + + # Optional streaming encode: stream io iterator overrides pointer+count. + if self.mode == "encode" and self.stream_encode_functions: + prov_name = self.var_name(with_prefix=True, observe_skipped=False) + prov_access = ( + "((const struct cbor_stream_io *)" + "zcbor_get_stream_io(state))" + ) + has_prov = f"({prov_access} && {prov_access}->{prov_name}.next)" + iter_call = ( + f"zcbor_multi_encode_iter_minmax({self.min_qty}, {max_qty}, " + f"(zcbor_encoder_t *){func}, state, {prov_access}->{prov_name}.next, " + f"{prov_access}->{prov_name}.ctx)" + ) + ptr_arg = ( + "*" + arg + if arg != "NULL" and self.result_len() != "0" + else arg + ) + ptr_call = ( + f"zcbor_multi_{mode}{minmax}(" + f"{self.min_qty}, {max_qty}, &{self.count_var_access()}, " + f"(zcbor_{mode}r_t *){func}, " + f"{xcode_args(ptr_arg)}, " + f"{self.result_len()})" + ) + return f"({has_prov} ? ({iter_call}) : ({ptr_call}))" + return ( f"zcbor_multi_{mode}{minmax}(%s, %s, &%s, (zcbor_{mode}r_t *)%s, %s, %s)" % (self.min_qty, - self.max_qty, + max_qty, self.count_var_access(), func, xcode_args("*" + arg if arg != "NULL" and self.result_len() != "0" else arg), @@ -2645,19 +2848,29 @@ def xcoders(self): def public_xcode_func_sig(self): type_name = self.type_name() if struct_ptr_name(self.mode) in self.full_xcode() else "void" - return f""" -int cbor_{self.xcode_func_name()}( + return f"""int cbor_{self.xcode_func_name()}( {"const " if self.mode == "decode" else ""}uint8_t *payload, size_t payload_len, {"" if self.mode == "decode" else "const "}{type_name} *{struct_ptr_name(self.mode)}, {"size_t *payload_len_out"})""" + def public_stream_decode_func_sig(self): + type_name = self.type_name() if struct_ptr_name("decode") in self.full_xcode() else "void" + io_name = f"cbor_stream_io_{self.var_name(with_prefix=True, observe_skipped=False)}" + return f"""int cbor_stream_decode_{self.var_name(with_prefix=True, observe_skipped=False)}( + const uint8_t *payload, size_t payload_len, + {type_name} *{struct_ptr_name("decode")}, + const struct {io_name} *io, + size_t *payload_len_out)""" + class CodeRenderer(): - def __init__(self, entry_types, modes, print_time, default_max_qty, git_sha='', file_header=''): + def __init__(self, entry_types, modes, print_time, default_max_qty, git_sha='', file_header='', + stream_encode_functions=False, stream_decode_functions=False): self.entry_types = entry_types self.print_time = print_time self.default_max_qty = default_max_qty - + self.stream_encode_functions = stream_encode_functions + self.stream_decode_functions = stream_decode_functions self.sorted_types = dict() self.functions = dict() self.type_defs = dict() @@ -2683,6 +2896,82 @@ def __init__(self, entry_types, modes, print_time, default_max_qty, git_sha='', at: ''' + datetime.now().strftime('%Y-%m-%d %H:%M:%S') if self.print_time else ''} Generated with a --default-max-qty of {self.default_max_qty}""" + def stream_io_type_name(self, xcoder): + entry_name = xcoder.var_name(with_prefix=True, observe_skipped=False) + return f"cbor_stream_io_{entry_name}" + + def collect_stream_iter_names_for_entry(self, entry): + if not self.stream_encode_functions: + return [] + + repeats = set() + visited = set() + + def walk(elem): + if id(elem) in visited: + return + visited.add(id(elem)) + + if elem.count_var_condition(): + repeats.add(elem.var_name(with_prefix=True, observe_skipped=False)) + + if elem.type in ["LIST", "MAP", "GROUP", "UNION"]: + for child in elem.value: + walk(child) + + if elem.cbor: + walk(elem.cbor) + + if elem.key: + walk(elem.key) + + if elem.type == "OTHER" and elem.value in elem.my_types: + walk(elem.my_types[elem.value]) + + walk(entry) + return sorted(repeats) + + def collect_stream_chunk_names_for_entry(self, entry, direction): + if not (self.stream_encode_functions or self.stream_decode_functions): + return ([], []) + + tstrs = set() + bstrs = set() + visited = set() + + def walk(elem): + if id(elem) in visited: + return + visited.add(id(elem)) + + if (not elem.count_var_condition() + and elem.type == "TSTR" + and elem.value is None + and not elem.cbor): + tstrs.add(elem.stream_chunk_field_name(direction)) + + if (not elem.count_var_condition() + and elem.type == "BSTR" + and elem.value is None + and not elem.cbor): + bstrs.add(elem.stream_chunk_field_name(direction)) + + if elem.type in ["LIST", "MAP", "GROUP", "UNION"]: + for child in elem.value: + walk(child) + + if elem.cbor: + walk(elem.cbor) + + if elem.key: + walk(elem.key) + + if elem.type == "OTHER" and elem.value in elem.my_types: + walk(elem.my_types[elem.value]) + + walk(entry) + return (sorted(tstrs), sorted(bstrs)) + def header_guard(self, file_name): return path.basename(file_name).replace(".", "_").replace("-", "_").upper() + "__" @@ -2803,10 +3092,9 @@ def render_function(self, xcoder, mode): def render_entry_function(self, xcoder, mode): """Render a single entry function (API function) with signature and body.""" func_name, func_arg = (xcoder.xcode_func_name(), struct_ptr_name(mode)) - return f""" -{xcoder.public_xcode_func_sig()} + return f"""{xcoder.public_xcode_func_sig()} {{ - zcbor_state_t states[{xcoder.num_backups() + 2}]; + zcbor_state_t states[{xcoder.num_backups() + 1} + ZCBOR_CONST_STATE_SLOTS]; return zcbor_entry_function(payload, payload_len, (void *){func_arg}, payload_len_out, states, (zcbor_decoder_t *){func_name}, sizeof(states) / sizeof(zcbor_state_t), { @@ -2828,7 +3116,19 @@ def render_c_file(self, header_file_name, mode): zcbor_log("%s success\\r\\n", func); \\ } \\ } while(0)""" - return f"""/*{self.render_file_header(" *")} + stream_io_internal = "" + if mode == "encode" and self.stream_encode_functions: + stream_io_internal = self.render_internal_stream_io_type() + if mode == "decode" and self.stream_decode_functions: + stream_io_internal = self.render_internal_stream_io_decode_type() + + default_max_qty_macro = ( + "ZCBOR_GENERATED_DEFAULT_MAX_QTY" + if (self.stream_encode_functions or self.stream_decode_functions) + else "DEFAULT_MAX_QTY" + ) + + return (f"""/*{self.render_file_header(" *")} */ #include @@ -2839,22 +3139,100 @@ def render_c_file(self, header_file_name, mode): #include "{header_file_name}" #include "zcbor_print.h" -#if DEFAULT_MAX_QTY != {self.default_max_qty} +#if {default_max_qty_macro} \ +!= {self.default_max_qty} #error "The type file was generated with a different default_max_qty than this file" #endif {log_result_define} +{stream_io_internal} + {linesep.join([self.render_forward_declaration(xcoder, mode) for xcoder in self.functions[mode]])} {linesep.join([self.render_function(xcoder, mode) for xcoder in self.functions[mode]])} {linesep.join([self.render_entry_function(xcoder, mode) for xcoder in self.entry_types[mode]])} -""" + +{self.render_write_impls(mode) if self.stream_encode_functions and mode == "encode" else ""} + +{self.render_stream_decode_impls(mode) if self.stream_decode_functions and mode == "decode" else ""} +""").rstrip() + "\n" + + def render_write_impls(self, mode): + if mode != "encode": + return "" + return (linesep * 2).join([ + f"""int cbor_stream_encode_{xcoder.var_name(with_prefix=True, observe_skipped=False)}( +\t\tzcbor_stream_write_fn stream_write, void *stream_user_data, +\t\tconst {xcoder.type_name() if struct_ptr_name(mode) in xcoder.full_xcode() else "void"} *input, +\t\tconst {self.stream_io_type_name(xcoder)} *io, +\t\tsize_t *bytes_written_out) +{{ +\tzcbor_state_t states[ZCBOR_STREAM_STATE_ARRAY_SIZE]; +\tzcbor_new_encode_state_streaming(states, sizeof(states) / sizeof(states[0]), +\t\tstream_write, stream_user_data, 1); +\tstruct cbor_stream_io io_local = {{0}}; +\tif (io) {{ +\t\t{self.render_stream_io_copy(xcoder)} +\t\tzcbor_set_stream_io(&states[0], &io_local); +\t}} else {{ +\t\tzcbor_set_stream_io(&states[0], NULL); +\t}} +\tbool ok = {xcoder.xcode_func_name()}(&states[0], input); +\tif (!ok) {{ +\t\tint err = zcbor_pop_error(&states[0]); +\t\treturn (err == ZCBOR_SUCCESS) ? ZCBOR_ERR_UNKNOWN : err; +\t}} +\tif (bytes_written_out) {{ +\t\t*bytes_written_out = zcbor_stream_bytes_written(&states[0]); +\t}} +\treturn ZCBOR_SUCCESS; +}}""" + for xcoder in self.entry_types[mode] + ]) + + def render_stream_decode_impls(self, mode): + if mode != "decode": + return "" + return (linesep * 2).join([ + f"""int cbor_stream_decode_{xcoder.var_name(with_prefix=True, observe_skipped=False)}( +\t\tconst uint8_t *payload, size_t payload_len, +\t\t{xcoder.type_name() if struct_ptr_name(mode) in xcoder.full_xcode() else "void"} *result, +\t\tconst {self.stream_io_type_name(xcoder)} *io, +\t\tsize_t *payload_len_out) +{{ +\tzcbor_state_t states[{xcoder.num_backups() + 1} + ZCBOR_CONST_STATE_SLOTS]; +\tzcbor_new_state(states, sizeof(states) / sizeof(states[0]), payload, payload_len, +\t\t{xcoder.list_counts()[1]}, NULL, 0); +\tstruct cbor_stream_io io_local = {{0}}; +\tif (io) {{ +\t\t{self.render_stream_io_decode_copy(xcoder)} +\t\tzcbor_set_stream_io(&states[0], &io_local); +\t}} else {{ +\t\tzcbor_set_stream_io(&states[0], NULL); +\t}} +\tbool ok = {xcoder.xcode_func_name()}(&states[0], result); +\tif (!ok) {{ +\t\tint err = zcbor_pop_error(&states[0]); +\t\treturn (err == ZCBOR_SUCCESS) ? ZCBOR_ERR_UNKNOWN : err; +\t}} +\tif (payload_len_out != NULL) {{ +\t\t*payload_len_out = MIN(payload_len, +\t\t\t(size_t)states[0].payload - (size_t)payload); +\t}} +\treturn ZCBOR_SUCCESS; +}}""" + for xcoder in self.entry_types[mode] + ]) def render_h_file(self, type_def_file, header_guard, mode): """Render the entire generated header file contents.""" - return \ + if self.stream_encode_functions or self.stream_decode_functions: + write_includes = '#include "zcbor_encode.h"\n#include "zcbor_decode.h"' + else: + write_includes = "" + return ( f"""/*{self.render_file_header(" *")} */ @@ -2865,32 +3243,458 @@ def render_h_file(self, type_def_file, header_guard, mode): #include #include #include +{write_includes} #include "{type_def_file}" #ifdef __cplusplus extern "C" {{ #endif -#if DEFAULT_MAX_QTY != {self.default_max_qty} +#if {"ZCBOR_GENERATED_DEFAULT_MAX_QTY" if (self.stream_encode_functions or self.stream_decode_functions) else "DEFAULT_MAX_QTY"} != {self.default_max_qty} #error "The type file was generated with a different default_max_qty than this file" #endif {(linesep * 2).join([f"{xcoder.public_xcode_func_sig()};" for xcoder in self.entry_types[mode]])} +{self.render_write_types(mode) if self.stream_encode_functions and mode == "encode" else ""} + +{self.render_write_decls(mode) if self.stream_encode_functions and mode == "encode" else ""} + +{self.render_decode_types(mode) if self.stream_decode_functions and mode == "decode" else ""} + +{self.render_stream_decode_decls(mode) if self.stream_decode_functions and mode == "decode" else ""} + #ifdef __cplusplus }} #endif #endif /* {header_guard} */ +""").rstrip() + "\n" + + def render_write_types(self, mode): + if mode != "encode": + return "" + + if not self.stream_encode_functions: + return "" + + iter_provider = """struct zcbor_stream_iter_io { + void *ctx; + zcbor_stream_iter next; +}; +""" + chunk_out_provider = """#ifndef ZCBOR_CHUNK_OUT_DEFINED +#define ZCBOR_CHUNK_OUT_DEFINED +struct zcbor_chunk_out { + void *ctx; + zcbor_stream_chunk_out call; +}; +#endif +""" + chunk_in_provider = """#ifndef ZCBOR_CHUNK_IN_DEFINED +#define ZCBOR_CHUNK_IN_DEFINED +struct zcbor_chunk_in { + void *ctx; + zcbor_stream_chunk_in call; +}; +#endif """ + bstr_push_provider = "" + + def render_entry_struct(entry): + repeats = self.collect_stream_iter_names_for_entry(entry) + tstr_out, bstr_out = self.collect_stream_chunk_names_for_entry(entry, "out") + tstr_in, bstr_in = self.collect_stream_chunk_names_for_entry(entry, "in") + + repeat_fields = linesep.join( + [f"\tstruct zcbor_stream_iter_io {name};" for name in repeats] + ) + tstr_out_fields = linesep.join( + [f"\tstruct zcbor_chunk_out {name};" for name in tstr_out] + ) + bstr_out_fields = linesep.join( + [f"\tstruct zcbor_chunk_out {name};" for name in bstr_out] + ) + tstr_in_fields = linesep.join( + [f"\tstruct zcbor_chunk_in {name};" for name in tstr_in] + ) + bstr_in_fields = linesep.join( + [f"\tstruct zcbor_chunk_in {name};" for name in bstr_in] + ) + + if not repeat_fields: + repeat_fields = "\t/* no repeated iters */" + if not tstr_out_fields: + tstr_out_fields = "\t/* no tstr chunk_out io */" + if not bstr_out_fields: + bstr_out_fields = "\t/* no bstr chunk_out io */" + if not tstr_in_fields: + tstr_in_fields = "\t/* no tstr chunk_in io */" + if not bstr_in_fields: + bstr_in_fields = "\t/* no bstr chunk_in io */" + + struct_name = self.stream_io_type_name(entry) + entry_name = entry.var_name(with_prefix=True, observe_skipped=False) + guard = f"CBOR_STREAM_IO_{entry_name.upper()}_DEFINED" + return f"""#ifndef {guard} +#define {guard} +struct {struct_name} {{ + /* Repeated fields (iterator) */ +{repeat_fields} + + /* Text string fields (chunk_out) */ +{tstr_out_fields} + + /* Byte string fields (chunk_out) */ +{bstr_out_fields} + + /* Text string fields (chunk_in) */ +{tstr_in_fields} + + /* Byte string fields (chunk_in) */ +{bstr_in_fields} +}}; +#endif +typedef struct {struct_name} {struct_name};""" + + stream_comment = "/* Streaming encode helpers */" + entry_structs = (linesep * 2).join( + [render_entry_struct(entry) for entry in self.entry_types["encode"]] + ) + + return f"""{stream_comment} +{iter_provider}{chunk_out_provider}{chunk_in_provider}{bstr_push_provider} +{entry_structs}""" + + def render_decode_types(self, mode): + if mode != "decode": + return "" + + if not self.stream_decode_functions: + return "" + + chunk_out_provider = """#ifndef ZCBOR_CHUNK_OUT_DEFINED +#define ZCBOR_CHUNK_OUT_DEFINED +struct zcbor_chunk_out { + void *ctx; + zcbor_stream_chunk_out call; +}; +#endif +""" + chunk_in_provider = """#ifndef ZCBOR_CHUNK_IN_DEFINED +#define ZCBOR_CHUNK_IN_DEFINED +struct zcbor_chunk_in { + void *ctx; + zcbor_stream_chunk_in call; +}; +#endif +""" + + def render_entry_struct(entry): + tstr_out, bstr_out = self.collect_stream_chunk_names_for_entry(entry, "out") + tstr_in, bstr_in = self.collect_stream_chunk_names_for_entry(entry, "in") + + tstr_out_fields = linesep.join( + [f"\tstruct zcbor_chunk_out {name};" for name in tstr_out] + ) + bstr_out_fields = linesep.join( + [f"\tstruct zcbor_chunk_out {name};" for name in bstr_out] + ) + tstr_in_fields = linesep.join( + [f"\tstruct zcbor_chunk_in {name};" for name in tstr_in] + ) + bstr_in_fields = linesep.join( + [f"\tstruct zcbor_chunk_in {name};" for name in bstr_in] + ) + + if not tstr_out_fields: + tstr_out_fields = "\t/* no tstr chunk_out io */" + if not bstr_out_fields: + bstr_out_fields = "\t/* no bstr chunk_out io */" + if not tstr_in_fields: + tstr_in_fields = "\t/* no tstr chunk_in io */" + if not bstr_in_fields: + bstr_in_fields = "\t/* no bstr chunk_in io */" + + struct_name = self.stream_io_type_name(entry) + entry_name = entry.var_name(with_prefix=True, observe_skipped=False) + guard = f"CBOR_STREAM_IO_{entry_name.upper()}_DEFINED" + return f"""#ifndef {guard} +#define {guard} +struct {struct_name} {{ + /* Text string fields (chunk_out) */ +{tstr_out_fields} + + /* Byte string fields (chunk_out) */ +{bstr_out_fields} + + /* Text string fields (chunk_in) */ +{tstr_in_fields} + + /* Byte string fields (chunk_in) */ +{bstr_in_fields} +}}; +#endif +typedef struct {struct_name} {struct_name};""" + + stream_comment = "/* Streaming decode helpers */" + entry_structs = (linesep * 2).join( + [render_entry_struct(entry) for entry in self.entry_types["decode"]] + ) + + return f"""{stream_comment} +{chunk_out_provider}{chunk_in_provider} +{entry_structs}""" + + def collect_stream_iter_names(self): + """Collect iterator field names for the whole schema (encode side only). + + These names are stable, unique (with_prefix=True), and map directly to the + var_name() used in generated encode code. + """ + if not self.stream_encode_functions: + return [] + + repeats = set() + + visited = set() + + def walk(elem): + # Avoid infinite recursion on self-referential types. + if id(elem) in visited: + return + visited.add(id(elem)) + + if elem.count_var_condition(): + repeats.add(elem.var_name(with_prefix=True, observe_skipped=False)) + + if elem.type in ["LIST", "MAP", "GROUP", "UNION"]: + for child in elem.value: + walk(child) + + if elem.cbor: + walk(elem.cbor) + + if elem.key: + walk(elem.key) + + # Follow named types. + if elem.type == "OTHER" and elem.value in elem.my_types: + walk(elem.my_types[elem.value]) + + # Walk from all top-level types; this makes the io struct schema-wide. + for root in self.sorted_types["encode"]: + walk(root) + + return sorted(repeats) + + def collect_stream_chunk_names(self, direction): + """Collect chunk io field names for tstr/bstr (streaming).""" + if not (self.stream_encode_functions or self.stream_decode_functions): + return ([], []) + + tstrs = set() + bstrs = set() + visited = set() + + def walk(elem): + if id(elem) in visited: + return + visited.add(id(elem)) + + if (not elem.count_var_condition() + and elem.type == "TSTR" + and elem.value is None + and not elem.cbor): + tstrs.add(elem.stream_chunk_field_name(direction)) + + if (not elem.count_var_condition() + and elem.type == "BSTR" + and elem.value is None + and not elem.cbor): + bstrs.add(elem.stream_chunk_field_name(direction)) + + if elem.type in ["LIST", "MAP", "GROUP", "UNION"]: + for child in elem.value: + walk(child) + + if elem.cbor: + walk(elem.cbor) + + if elem.key: + walk(elem.key) + + if elem.type == "OTHER" and elem.value in elem.my_types: + walk(elem.my_types[elem.value]) + + for root in self.sorted_types["encode"]: + walk(root) + + return (sorted(tstrs), sorted(bstrs)) + + def render_internal_stream_io_type(self): + repeats = self.collect_stream_iter_names() + tstr_out, bstr_out = self.collect_stream_chunk_names("out") + tstr_in, bstr_in = self.collect_stream_chunk_names("in") + + repeat_fields = linesep.join( + [f"\tstruct zcbor_stream_iter_io {name};" for name in repeats] + ) + tstr_out_fields = linesep.join( + [f"\tstruct zcbor_chunk_out {name};" for name in tstr_out] + ) + bstr_out_fields = linesep.join( + [f"\tstruct zcbor_chunk_out {name};" for name in bstr_out] + ) + tstr_in_fields = linesep.join( + [f"\tstruct zcbor_chunk_in {name};" for name in tstr_in] + ) + bstr_in_fields = linesep.join( + [f"\tstruct zcbor_chunk_in {name};" for name in bstr_in] + ) + + if not repeat_fields: + repeat_fields = "\t/* no repeated iters */" + if not tstr_out_fields: + tstr_out_fields = "\t/* no tstr chunk_out io */" + if not bstr_out_fields: + bstr_out_fields = "\t/* no bstr chunk_out io */" + if not tstr_in_fields: + tstr_in_fields = "\t/* no tstr chunk_in io */" + if not bstr_in_fields: + bstr_in_fields = "\t/* no bstr chunk_in io */" + + return f"""/* Streaming io struct (internal, schema-wide). */ +struct cbor_stream_io {{ + /* Repeated fields (iterator) */ +{repeat_fields} + + /* Text string fields (chunk_out) */ +{tstr_out_fields} + + /* Byte string fields (chunk_out) */ +{bstr_out_fields} + + /* Text string fields (chunk_in) */ +{tstr_in_fields} + + /* Byte string fields (chunk_in) */ +{bstr_in_fields} +}};""" + + def render_internal_stream_io_decode_type(self): + tstr_out, bstr_out = self.collect_stream_chunk_names("out") + tstr_in, bstr_in = self.collect_stream_chunk_names("in") + + tstr_out_fields = linesep.join( + [f"\tstruct zcbor_chunk_out {name};" for name in tstr_out] + ) + bstr_out_fields = linesep.join( + [f"\tstruct zcbor_chunk_out {name};" for name in bstr_out] + ) + tstr_in_fields = linesep.join( + [f"\tstruct zcbor_chunk_in {name};" for name in tstr_in] + ) + bstr_in_fields = linesep.join( + [f"\tstruct zcbor_chunk_in {name};" for name in bstr_in] + ) + + if not tstr_out_fields: + tstr_out_fields = "\t/* no tstr chunk_out io */" + if not bstr_out_fields: + bstr_out_fields = "\t/* no bstr chunk_out io */" + if not tstr_in_fields: + tstr_in_fields = "\t/* no tstr chunk_in io */" + if not bstr_in_fields: + bstr_in_fields = "\t/* no bstr chunk_in io */" + + return f"""/* Streaming io struct (internal, schema-wide). */ +struct cbor_stream_io {{ + /* Text string fields (chunk_out) */ +{tstr_out_fields} + + /* Byte string fields (chunk_out) */ +{bstr_out_fields} + + /* Text string fields (chunk_in) */ +{tstr_in_fields} + + /* Byte string fields (chunk_in) */ +{bstr_in_fields} +}};""" + + def render_stream_io_copy(self, entry): + repeats = self.collect_stream_iter_names_for_entry(entry) + tstr_pushes, bstr_pushes = self.collect_stream_chunk_names_for_entry(entry, "out") + fields = repeats + tstr_pushes + bstr_pushes + if not fields: + return "/* no streaming io for entry */" + return linesep.join([f"\t\tio_local.{name} = io->{name};" for name in fields]) + + def render_stream_io_decode_copy(self, entry): + tstr_pushes, bstr_pushes = self.collect_stream_chunk_names_for_entry(entry, "in") + fields = tstr_pushes + bstr_pushes + if not fields: + return "/* no streaming decode io for entry */" + return linesep.join([f"\t\tio_local.{name} = io->{name};" for name in fields]) + + def render_write_decls(self, mode): + if mode != "encode": + return "" + return (linesep * 2).join([ + f"""int cbor_stream_encode_{xcoder.var_name(with_prefix=True, observe_skipped=False)}( + zcbor_stream_write_fn stream_write, void *stream_user_data, + const {xcoder.type_name() if struct_ptr_name(mode) in xcoder.full_xcode() else "void"} *input, + const {self.stream_io_type_name(xcoder)} *io, + size_t *bytes_written_out);""" + for xcoder in self.entry_types[mode] + ]) + + def render_stream_decode_decls(self, mode): + if mode != "decode": + return "" + return (linesep * 2).join([ + f"""int cbor_stream_decode_{xcoder.var_name(with_prefix=True, observe_skipped=False)}( + const uint8_t *payload, size_t payload_len, + {xcoder.type_name() if struct_ptr_name(mode) in xcoder.full_xcode() else "void"} *result, + const {self.stream_io_type_name(xcoder)} *io, + size_t *payload_len_out);""" + for xcoder in self.entry_types[mode] + ]) def render_type_file(self, header_guard, mode): body = ( linesep + linesep).join( [f"{typedef[1]} {{{linesep}{linesep.join(typedef[0][1:])};" for typedef in self.type_defs[mode]]) - return \ + if self.stream_encode_functions or self.stream_decode_functions: + default_max_qty_define = ( + "#define ZCBOR_GENERATED_DEFAULT_MAX_QTY " + + str(self.default_max_qty) + + linesep + + linesep + + "/* Allow build-system override. */" + + linesep + + "#ifndef DEFAULT_MAX_QTY" + + linesep + + "#define DEFAULT_MAX_QTY ZCBOR_GENERATED_DEFAULT_MAX_QTY" + + linesep + + "#endif" + + linesep + + linesep + + "/* Allow build-system override for streaming state array size. */" + + linesep + + "#ifndef ZCBOR_STREAM_STATE_ARRAY_SIZE" + + linesep + + "#define ZCBOR_STREAM_STATE_ARRAY_SIZE 8" + + linesep + + "#endif" + ) + else: + default_max_qty_define = "#define DEFAULT_MAX_QTY " + str(self.default_max_qty) + indef_strings_define = "" + return ( f"""/*{self.render_file_header(" *")} */ @@ -2913,7 +3717,7 @@ def render_type_file(self, header_guard, mode): * * See `zcbor --help` for more information about --default-max-qty */ -#define DEFAULT_MAX_QTY {self.default_max_qty} +{default_max_qty_define}{indef_strings_define} {body} @@ -2922,7 +3726,7 @@ def render_type_file(self, header_guard, mode): #endif #endif /* {header_guard} */ -""" +""").rstrip() + "\n" def render_cmake_file(self, target_name, h_files, c_files, type_file, output_c_dir, output_h_dir, cmake_dir): @@ -2937,6 +3741,7 @@ def relativify(p): except ValueError: # On Windows, the above will fail if the paths are on different drives. return Path(p).absolute().as_posix() + cmake_defs = "" return \ f"""\ #{self.render_file_header("#")} @@ -2953,7 +3758,8 @@ def relativify(p): target_include_directories({target_name} PUBLIC {(linesep + " ").join(((str(relativify(f)) for f in include_dirs)))} ) -""" +{cmake_defs} +""".rstrip() + "\n" def render(self, modes, h_files, c_files, type_file, include_prefix, cmake_file=None, output_c_dir=None, output_h_dir=None): @@ -3045,6 +3851,30 @@ def parse_args(): to allow it to be configurable when building the code. This is not always possible, as sometimes the value is needed for internal computations. If so, the script will raise an exception.""") + code_parser.add_argument( + "--repeated-as-pointers", required=False, action="store_true", default=False, + help="""Represent repeated fields (max_qty > 1) as pointer + count instead of embedding a +fixed-size array in the generated types. + +This can significantly reduce the size of top-level unions/structs at the cost of requiring the +caller to provide storage for decode, and a readable array for encode.""") + code_parser.add_argument( + "--stream-encode", required=False, + action="store_true", default=False, dest="stream_encode_functions", + help="""Also generate streaming encode entrypoints (cbor_stream_encode_) for each +entry type. +These use zcbor_new_encode_state_streaming() and are intended for low-RAM UART write paths. + +Streaming encode entrypoints support: + - repeated fields via zcbor_multi_encode_iter_minmax() (iterator callback) + - tstr/bstr fields via chunk callbacks""") + code_parser.add_argument( + "--stream-decode", required=False, + action="store_true", default=False, dest="stream_decode_functions", + help="""Also generate streaming decode entrypoints (cbor_stream_decode_) for each +entry type. +These use zcbor_tstr_chunk_in()/zcbor_bstr_chunk_in() callbacks and allow +indefinite-length tstr/bstr values to be processed without reassembly.""") code_parser.add_argument( "--output-c", "--oc", required=False, type=str, help="""Path to output C file. If both --decode and --encode are specified, _decode and @@ -3205,6 +4035,8 @@ def process_code(args): if args.file_header and Path(args.file_header).exists(): args.file_header = Path(args.file_header).read_text(encoding="utf-8") + elif args.file_header: + args.file_header = args.file_header.replace("\\n", "\n") print("Parsing files: " + ", ".join((c.name for c in args.cddl))) @@ -3214,7 +4046,9 @@ def process_code(args): for mode in modes: cddl_res[mode] = CodeGenerator.from_cddl( mode, cddl_contents, args.default_max_qty, mode, args.entry_types, - args.default_bit_size, short_names=args.short_names) + args.default_bit_size, args.repeated_as_pointers, + args.stream_encode_functions, args.stream_decode_functions, + short_names=args.short_names) # Parsing is done, pretty print the result. verbose_print(args.verbose, "Parsed CDDL types:") @@ -3265,12 +4099,19 @@ def add_mode_to_fname(filename, mode): or (args.output_h and Path(args.output_h).with_name(Path(args.output_h).stem + "_types.h")) or Path(cmake_dir, 'include', filenames + '_types.h')) - renderer = CodeRenderer(entry_types={mode: [cddl_res[mode].my_types[entry] - for entry in args.entry_types] for mode in modes}, - modes=modes, print_time=args.time_header, - default_max_qty=args.default_max_qty, git_sha=git_sha, - file_header=args.file_header - ) + renderer = CodeRenderer( + entry_types={ + mode: [cddl_res[mode].my_types[entry] for entry in args.entry_types] + for mode in modes + }, + modes=modes, + print_time=args.time_header, + default_max_qty=args.default_max_qty, + git_sha=git_sha, + file_header=args.file_header, + stream_encode_functions=args.stream_encode_functions, + stream_decode_functions=args.stream_decode_functions, + ) c_code_dir = C_SRC_PATH h_code_dir = C_INCLUDE_PATH @@ -3289,7 +4130,6 @@ def add_mode_to_fname(filename, mode): copyfile(Path(h_code_dir, "zcbor_print.h"), Path(new_h_code_dir, "zcbor_print.h")) c_code_dir = new_c_code_dir h_code_dir = new_h_code_dir - renderer.render(modes, output_h, output_c, output_h_types, args.include_prefix, output_cmake, c_code_dir, h_code_dir)