Support variable-length structs as array elements

compiler/back_end/cpp/generated_code_templates

@@ -475,6 +475,12 @@ ${support_namespace}::GenericArrayView<
     ${addressable_unit_size} ${element_view_parameter_types}>
 
 
+// ** dynamic_size_array_view_adapter ** ///////////////////////////////////////
+${support_namespace}::GenericDynamicSizeArrayView<
+    typename ${element_view_type}, typename ${buffer_type},
+    ${addressable_unit_size}${element_view_parameter_types}>
+
+
 // ** structure_field_validator ** /////////////////////////////////////////////
 struct ${name} {
   template <typename ValueType>

compiler/back_end/cpp/header_generator.py

@@ -329,14 +329,6 @@ def _wrap_in_namespace(body, namespace):
     return body
 
 
-def _get_type_size(type_ir, ir):
-    size = ir_util.fixed_size_of_type_in_bits(type_ir, ir)
-    assert (
-        size is not None
-    ), "_get_type_size should only be called for constant-sized types."
-    return size
-
-
 def _offset_storage_adapter(buffer_type, alignment, static_offset):
     return "{}::template OffsetStorageType</**/{}, {}>".format(
         buffer_type, alignment, static_offset
@@ -497,42 +489,82 @@ def _get_cpp_view_type_for_physical_type(
     if ir_util.is_array(type_ir):
         # An array view is parameterized by the element's view type.
         base_type = type_ir.array_type.base_type
-        element_size_in_bits = _get_type_size(base_type, ir)
-        assert (
-            element_size_in_bits
-        ), "TODO(bolms): Implement arrays of dynamically-sized elements."
-        assert (
-            element_size_in_bits % parent_addressable_unit == 0
-        ), "Array elements must fall on byte boundaries."
-        element_size = element_size_in_bits // parent_addressable_unit
-        element_view_type, element_view_parameter_types, element_view_parameters = (
-            _get_cpp_view_type_for_physical_type(
-                base_type,
-                element_size_in_bits,
-                byte_order,
-                ir,
-                _offset_storage_adapter(buffer_type, element_size, 0),
-                parent_addressable_unit,
-                validator,
+        element_size_in_bits = ir_util.fixed_size_of_type_in_bits(base_type, ir)
+        if element_size_in_bits is not None:
+            # Fixed-size elements: use the standard GenericArrayView
+            assert (
+                element_size_in_bits % parent_addressable_unit == 0
+            ), "Array elements must fall on byte boundaries."
+            element_size = element_size_in_bits // parent_addressable_unit
+            element_view_type, element_view_parameter_types, element_view_parameters = (
+                _get_cpp_view_type_for_physical_type(
+                    base_type,
+                    element_size_in_bits,
+                    byte_order,
+                    ir,
+                    _offset_storage_adapter(buffer_type, element_size, 0),
+                    parent_addressable_unit,
+                    validator,
+                )
             )
-        )
-        return (
-            code_template.format_template(
-                _TEMPLATES.array_view_adapter,
-                support_namespace=_SUPPORT_NAMESPACE,
-                # TODO(bolms): The element size should be calculable from the field
-                # size and array length.
-                element_view_type=element_view_type,
-                element_view_parameter_types="".join(
-                    ", " + p for p in element_view_parameter_types
+            return (
+                code_template.format_template(
+                    _TEMPLATES.array_view_adapter,
+                    support_namespace=_SUPPORT_NAMESPACE,
+                    # TODO(bolms): The element size should be calculable from the field
+                    # size and array length.
+                    element_view_type=element_view_type,
+                    element_view_parameter_types="".join(
+                        ", " + p for p in element_view_parameter_types
+                    ),
+                    element_size=element_size,
+                    addressable_unit_size=int(parent_addressable_unit),
+                    buffer_type=buffer_type,
                 ),
-                element_size=element_size,
-                addressable_unit_size=int(parent_addressable_unit),
-                buffer_type=buffer_type,
-            ),
-            element_view_parameter_types,
-            element_view_parameters,
-        )
+                element_view_parameter_types,
+                element_view_parameters,
+            )
+        else:
+            # Dynamic-size elements: use GenericDynamicSizeArrayView
+            # Only byte-oriented types are supported for dynamic arrays
+            assert (
+                parent_addressable_unit == 8
+            ), "Dynamic-size arrays are only supported in byte-oriented structures."
+            element_view_type, element_view_parameter_types, element_view_parameters = (
+                _get_cpp_view_type_for_physical_type(
+                    base_type,
+                    None,  # Size is not known at compile time
+                    byte_order,
+                    ir,
+                    _offset_storage_adapter(buffer_type, 1, 0),
+                    parent_addressable_unit,
+                    validator,
+                )
+            )
+            # Add element count as an extra constructor argument (not a template param)
+            # The element_count goes after element view params but before the buffer
+            # We add it as an extra parameter expression
+            all_parameters = list(element_view_parameters) + [
+                type_ir.array_type.element_count
+            ]
+            # Also add the parameter type for proper rendering
+            all_parameter_types = list(element_view_parameter_types) + ["::std::size_t"]
+            return (
+                code_template.format_template(
+                    _TEMPLATES.dynamic_size_array_view_adapter,
+                    support_namespace=_SUPPORT_NAMESPACE,
+                    element_view_type=element_view_type,
+                    # Only element view parameter types go in template args
+                    element_view_parameter_types="".join(
+                        ", " + p for p in element_view_parameter_types
+                    ),
+                    addressable_unit_size=int(parent_addressable_unit),
+                    buffer_type=buffer_type,
+                ),
+                # But ALL parameter types (including element_count) for constructor
+                all_parameter_types,
+                all_parameters,
+            )
     else:
         assert type_ir.has_field("atomic_type")
         reference = type_ir.atomic_type.reference

compiler/front_end/constraints.py

@@ -77,7 +77,7 @@ def _check_that_inner_array_dimensions_are_constant(type_ir, source_file_name, e
 
 
 def _check_that_array_base_types_are_fixed_size(type_ir, source_file_name, errors, ir):
-    """Checks that the sizes of array elements are known at compile time."""
+    """Checks that the sizes of array elements are known at compile time for auto-sized arrays."""
     if type_ir.base_type.has_field("array_type"):
         # An array is fixed size if its base_type is fixed size and its array
         # dimension is constant.  This function will be called again on the inner
@@ -95,15 +95,18 @@ def _check_that_array_base_types_are_fixed_size(type_ir, source_file_name, error
         base_type.attribute, attributes.FIXED_SIZE
     )
     if base_type_fixed_size is None:
-        errors.append(
-            [
-                error.error(
-                    source_file_name,
-                    type_ir.base_type.atomic_type.source_location,
-                    "Array elements must be fixed size.",
-                )
-            ]
-        )
+        # If the array has an explicit element count, variable-sized elements are
+        # allowed. Only auto-sized arrays require fixed-size elements.
+        if type_ir.which_size == "automatic":
+            errors.append(
+                [
+                    error.error(
+                        source_file_name,
+                        type_ir.base_type.atomic_type.source_location,
+                        "Arrays with automatic size must have fixed-size elements.",
+                    )
+                ]
+            )
 
 
 def _check_that_array_base_types_in_structs_are_multiples_of_bytes(

compiler/front_end/constraints_test.py

@@ -120,7 +120,7 @@ def test_error_non_fixed_size_inner_array_dimension(self):
     def test_error_non_constant_inner_array_dimensions(self):
         ir = _make_ir_from_emb(
             "struct Foo:\n"
-            "  0 [+1]  Bar[1]  one_byte\n"
+            "  0 [+1]  Bar[]  one_byte\n"
             # There is no dynamically-sized byte-oriented type in
             # the Prelude, so this test has to make its own.
             "external Bar:\n"
@@ -134,18 +134,30 @@ def test_error_non_constant_inner_array_dimensions(self):
                     error.error(
                         "m.emb",
                         error_array.base_type.atomic_type.source_location,
-                        "Array elements must be fixed size.",
+                        "Arrays with automatic size must have fixed-size elements.",
                     )
                 ]
             ],
             error.filter_errors(constraints.check_constraints(ir)),
         )
 
-    def test_error_dynamically_sized_array_elements(self):
+    def test_no_error_dynamically_sized_array_elements_with_explicit_count(self):
+        # Arrays with explicit element count can have dynamically-sized elements.
         ir = _make_ir_from_emb(
             '[$default byte_order: "LittleEndian"]\n'
             "struct Foo:\n"
-            "  0 [+1]  Bar[1]  bar\n"
+            "  0 [+10]  Bar[1]  bar\n"
+            "struct Bar:\n"
+            "  0 [+1]     UInt      size\n"
+            "  1 [+size]  UInt:8[]  payload\n"
+        )
+        self.assertEqual([], error.filter_errors(constraints.check_constraints(ir)))
+
+    def test_error_dynamically_sized_array_elements_with_automatic_size(self):
+        ir = _make_ir_from_emb(
+            '[$default byte_order: "LittleEndian"]\n'
+            "struct Foo:\n"
+            "  0 [+10]  Bar[]  bar\n"
             "struct Bar:\n"
             "  0 [+1]     UInt      size\n"
             "  1 [+size]  UInt:8[]  payload\n"
@@ -157,7 +169,7 @@ def test_error_dynamically_sized_array_elements(self):
                     error.error(
                         "m.emb",
                         error_array.base_type.atomic_type.source_location,
-                        "Array elements must be fixed size.",
+                        "Arrays with automatic size must have fixed-size elements.",
                     )
                 ]
             ],

runtime/cpp/emboss_array_view.h

@@ -364,6 +364,159 @@ class GenericArrayView final {
   BufferType buffer_;
 };
 
+// View for an array in a structure where element sizes are not known at
+// compile time (dynamic-size arrays).
+//
+// ElementView should be the view class for a single array element. The element
+// view type must satisfy the following interface requirements:
+//   - SizeInBytes(): Returns the size of the element in bytes
+//   - SizeIsKnown(): Returns true if the element's size can be determined
+//   - Ok(): Returns true if the element is valid
+//
+// BufferType is the storage type that will be passed into the array.
+//
+// kAddressableUnitSize is the size of a single addressable unit.  It should be
+// 8 (one byte) for dynamic-size arrays.
+//
+// ElementViewParameterTypes is a list of the types of parameters which must be
+// passed down to each element of the array.  ElementViewParameterTypes can be
+// empty.
+template <class ElementView, class BufferType,
+          ::std::size_t kAddressableUnitSize,
+          typename... ElementViewParameterTypes>
+class GenericDynamicSizeArrayView final {
+ public:
+  using ViewType = ElementView;
+
+  GenericDynamicSizeArrayView() : buffer_(), element_count_(0) {}
+  explicit GenericDynamicSizeArrayView(
+      const ElementViewParameterTypes &...parameters,
+      ::std::size_t element_count, BufferType buffer)
+      : parameters_{parameters...},
+        buffer_{buffer},
+        element_count_(element_count) {}
+
+  // Returns the element at the given index by computing the offset from
+  // previous elements' sizes.
+  ElementView operator[](::std::size_t index) const {
+    ::std::size_t offset = 0;
+    for (::std::size_t i = 0; i < index; ++i) {
+      ElementView element = ConstructElementAtOffset(offset);
+      if (!element.SizeIsKnown()) {
+        // Return an invalid view if we can't determine previous element sizes
+        return ElementView();
+      }
+      offset += element.SizeInBytes();
+    }
+    return ConstructElementAtOffset(offset);
+  }
+
+  ::std::size_t ElementCount() const { return element_count_; }
+
+  template <int N = 0>
+  typename ::std::enable_if<((void)N, kAddressableUnitSize == 8),
+                            ::std::size_t>::type
+  SizeInBytes() const {
+    return buffer_.SizeInBytes();
+  }
+
+  bool Ok() const {
+    if (!buffer_.Ok()) return false;
+    ::std::size_t offset = 0;
+    for (::std::size_t i = 0; i < element_count_; ++i) {
+      ElementView element = ConstructElementAtOffset(offset);
+      if (!element.Ok()) return false;
+      if (!element.SizeIsKnown()) return false;
+      offset += element.SizeInBytes();
+    }
+    // Check that total element sizes don't exceed buffer size
+    return offset <= buffer_.SizeInBytes();
+  }
+
+  template <class OtherElementView, class OtherBufferType>
+  bool Equals(
+      const GenericDynamicSizeArrayView<OtherElementView, OtherBufferType,
+                                        kAddressableUnitSize> &other) const {
+    if (ElementCount() != other.ElementCount()) return false;
+    for (::std::size_t i = 0; i < ElementCount(); ++i) {
+      if (!(*this)[i].Equals(other[i])) return false;
+    }
+    return true;
+  }
+
+  template <class OtherElementView, class OtherBufferType>
+  bool UncheckedEquals(
+      const GenericDynamicSizeArrayView<OtherElementView, OtherBufferType,
+                                        kAddressableUnitSize> &other) const {
+    if (ElementCount() != other.ElementCount()) return false;
+    for (::std::size_t i = 0; i < ElementCount(); ++i) {
+      if (!(*this)[i].UncheckedEquals(other[i])) return false;
+    }
+    return true;
+  }
+
+  bool IsComplete() const { return buffer_.Ok(); }
+
+  template <class Stream>
+  bool UpdateFromTextStream(Stream *stream) const {
+    return ReadArrayFromTextStream(this, stream);
+  }
+
+  template <class Stream>
+  void WriteToTextStream(Stream *stream,
+                         const TextOutputOptions &options) const {
+    WriteArrayToTextStream(this, stream, options);
+  }
+
+  static constexpr bool IsAggregate() { return true; }
+
+  BufferType BackingStorage() const { return buffer_; }
+
+  bool operator==(const GenericDynamicSizeArrayView &other) const {
+    return parameters_ == other.parameters_ && buffer_ == other.buffer_ &&
+           element_count_ == other.element_count_;
+  }
+
+ private:
+  ElementView ConstructElementAtOffset(::std::size_t offset) const {
+    return ConstructElementAtOffsetHelper<(sizeof...(ElementViewParameterTypes)
+                                           == 0)>::Construct(parameters_,
+                                                             buffer_, offset);
+  }
+
+  // Helper to expand parameters tuple into individual constructor arguments.
+  template <bool, ::std::size_t... N>
+  struct ConstructElementAtOffsetHelper {
+    static ElementView Construct(
+        const ::std::tuple<ElementViewParameterTypes...> &parameters,
+        BufferType buffer, ::std::size_t offset) {
+      return ConstructElementAtOffsetHelper<
+          (sizeof...(ElementViewParameterTypes) == 1 + sizeof...(N)), N...,
+          sizeof...(N)>::Construct(parameters, buffer, offset);
+    }
+  };
+
+  template </**/ ::std::size_t... N>
+  struct ConstructElementAtOffsetHelper<true, N...> {
+    static ElementView Construct(
+        const ::std::tuple<ElementViewParameterTypes...> &parameters,
+        BufferType buffer, ::std::size_t offset) {
+      if (offset > buffer.SizeInBytes()) {
+        // Offset is past the end of the buffer
+        return ElementView();
+      }
+      ::std::size_t remaining = buffer.SizeInBytes() - offset;
+      return ElementView(
+          ::std::get<N>(parameters)...,
+          buffer.template GetOffsetStorage<1, 0>(offset, remaining));
+    }
+  };
+
+  ::std::tuple<ElementViewParameterTypes...> parameters_;
+  BufferType buffer_;
+  ::std::size_t element_count_;
+};
+
 // Optionally prints a shorthand representation of a BitArray in a comment.
 template <class ElementView, class BufferType, ::std::size_t kElementSize,
           ::std::size_t kAddressableUnitSize, class Stream>