From 6253aca9734db026598ac747782ed70fc7d01e33 Mon Sep 17 00:00:00 2001 From: Vandern Rodrigues Date: Mon, 11 Aug 2025 00:59:14 +0100 Subject: [PATCH 1/6] Define Encodable protocol --- lib/msgpack/encodable.ex | 46 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 lib/msgpack/encodable.ex diff --git a/lib/msgpack/encodable.ex b/lib/msgpack/encodable.ex new file mode 100644 index 0000000..07cf24b --- /dev/null +++ b/lib/msgpack/encodable.ex @@ -0,0 +1,46 @@ +defprotocol Msgpack.Encodable do + @moduledoc """ + A protocol for converting custom Elixir structs into a Msgpack-encodable + format. + + This protocol provides a hook into the `Msgpack.encode/2` function, allowing + developers to define custom serialization logic for their structs. + + ## Contract + + An implementation of `encode/1` for a struct must return a basic Elixir term + that the Msgpack library can encode directly. This includes: + - Maps (with string, integer, or atom keys that will be converted to strings) + - Lists + - Strings or Binaries + - Integers + - Floats + - Booleans + - `nil` + + Importantly, the returned term **must not** contain other custom structs that + themselves require an `Encodable` implementation. The purpose of this protocol + is to perform a single-level transformation from a custom struct into a + directly encodable term. Returning a nested custom struct will result in an + `{:error, {:unsupported_type, term}}` during encoding. + + ## Example + + ```elixir + defimpl Msgpack.Encodable, for: User do + def encode(%User{id: id, name: name}) do + # Transform the User struct into a map, which is directly encodable. + {:ok, %{"id" => id, "name" => name}} + end + end + ``` + """ + + @doc """ + Receives a custom struct and must return `{:ok, term}` or `{:error, reason}`. + + The `term` in a successful result must be a directly encodable Elixir type. + """ + @spec encode(struct()) :: {:ok, term()} | {:error, any()} + def encode(struct) +end From 2738eb11e0aceea077d3ac332561d9051ce50f31 Mon Sep 17 00:00:00 2001 From: Vandern Rodrigues Date: Mon, 11 Aug 2025 01:03:07 +0100 Subject: [PATCH 2/6] Add function clause to encoder for handling custom structs --- lib/msgpack/encoder.ex | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/msgpack/encoder.ex b/lib/msgpack/encoder.ex index 7830019..8da9d4e 100644 --- a/lib/msgpack/encoder.ex +++ b/lib/msgpack/encoder.ex @@ -3,6 +3,8 @@ defmodule Msgpack.Encoder do Handles the logic of encoding Elixir terms into iodata. """ + alias Msgpack.Encodable + @spec encode(term(), keyword()) :: {:ok, iodata()} | {:error, term()} def encode(term, opts \\ []) do merged_opts = Keyword.merge(default_opts(), opts) @@ -95,6 +97,20 @@ defmodule Msgpack.Encoder do {:ok, encoded_binary} end + # ==== Structs (Custom) ==== + defp do_encode(%_{} = struct, opts) when Keyword.get(opts, :protocol_dispatch_enabled, true) do + case Encodable.encode(struct) do + {:ok, term} -> + do_encode(term, Keyword.put(opts, :protocol_dispatch_enabled, false)) + + {:error, reason} -> + {:error, reason} + rescue + e in [Protocol.UndefinedError] -> + {:error, {:unsupported_type, e.struct}} + end + end + # ==== Structs (DateTime, NaiveDateTime and Ext) ==== defp do_encode(%DateTime{} = datetime, opts) do case DateTime.shift_zone(datetime, "Etc/UTC") do From 8ab6a67cb43aec293e24c56d3693b9526ba71d61 Mon Sep 17 00:00:00 2001 From: Vandern Rodrigues Date: Mon, 11 Aug 2025 01:54:43 +0100 Subject: [PATCH 3/6] Add tests for protocol and cleanup function clause in encoder --- lib/msgpack/encoder.ex | 35 ++++++++++++++++++------------- mix.exs | 3 ++- test/encodable_test.exs | 21 +++++++++++++++++++ test/support/encodable_structs.ex | 19 +++++++++++++++++ 4 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 test/encodable_test.exs create mode 100644 test/support/encodable_structs.ex diff --git a/lib/msgpack/encoder.ex b/lib/msgpack/encoder.ex index 8da9d4e..b1ee915 100644 --- a/lib/msgpack/encoder.ex +++ b/lib/msgpack/encoder.ex @@ -97,20 +97,6 @@ defmodule Msgpack.Encoder do {:ok, encoded_binary} end - # ==== Structs (Custom) ==== - defp do_encode(%_{} = struct, opts) when Keyword.get(opts, :protocol_dispatch_enabled, true) do - case Encodable.encode(struct) do - {:ok, term} -> - do_encode(term, Keyword.put(opts, :protocol_dispatch_enabled, false)) - - {:error, reason} -> - {:error, reason} - rescue - e in [Protocol.UndefinedError] -> - {:error, {:unsupported_type, e.struct}} - end - end - # ==== Structs (DateTime, NaiveDateTime and Ext) ==== defp do_encode(%DateTime{} = datetime, opts) do case DateTime.shift_zone(datetime, "Etc/UTC") do @@ -146,6 +132,20 @@ defmodule Msgpack.Encoder do {:ok, [header, data]} end + # ==== Structs (Custom via Protocol) ==== + defp do_encode(%_{} = struct, opts) do + with true <- Keyword.get(opts, :protocol_dispatch_enabled, true), + {:ok, term} <- try_protocol_encode(struct) do + do_encode(term, Keyword.put(opts, :protocol_dispatch_enabled, false)) + else + false -> + {:error, {:unsupported_type, struct.__struct__}} + + {:error, reason} -> + {:error, reason} + end + end + # ==== Lists ==== defp do_encode(list, opts) when is_list(list) do acc = {:ok, []} @@ -250,4 +250,11 @@ defmodule Msgpack.Encoder do [<<0xC7, 12, -1::signed-8>>, <>] end end + + defp try_protocol_encode(struct) do + Encodable.encode(struct) + rescue + e in [Protocol.UndefinedError] -> + {:error, {:unsupported_type, e.value.__struct__}} + end end diff --git a/mix.exs b/mix.exs index 9386637..c1efcbf 100644 --- a/mix.exs +++ b/mix.exs @@ -25,7 +25,8 @@ defmodule MsgpackElixir.MixProject do ] end - defp elixirc_paths(_env), do: ["lib"] + defp elixirc_paths(:test), do: ["lib", "test/support"] + defp elixirc_paths(_), do: ["lib"] defp deps do [ diff --git a/test/encodable_test.exs b/test/encodable_test.exs new file mode 100644 index 0000000..784a56d --- /dev/null +++ b/test/encodable_test.exs @@ -0,0 +1,21 @@ +defmodule Msgpack.EncodableTest do + use ExUnit.Case, async: true + + alias Msgpack.EncodableTest.User + alias Msgpack.EncodableTest.Product + alias Msgpack + + test "successfully encodes a custom struct with a protocol implementation" do + user = %User{id: 1, name: "Bob"} + expected_binary = <<0x82, 0xA2, "id", 1, 0xA4, "name", 0xA3, "Bob">> + + assert Msgpack.encode(user) == {:ok, expected_binary} + end + + test "returns an error when encoding a struct without a protocol implementation" do + product = %Product{id: 1234} + expected_error = {:error, {:unsupported_type, Product}} + + assert Msgpack.encode(product) == expected_error + end +end diff --git a/test/support/encodable_structs.ex b/test/support/encodable_structs.ex new file mode 100644 index 0000000..a31385e --- /dev/null +++ b/test/support/encodable_structs.ex @@ -0,0 +1,19 @@ +defmodule Msgpack.EncodableTest.User do + @moduledoc """ + A simple struct used for testing protocol implementations. + """ + defstruct [:id, :name] +end + +defmodule Msgpack.EncodableTest.Product do + @moduledoc """ + A simple struct with no protocol implementation. + """ + defstruct [:id] +end + +defimpl Msgpack.Encodable, for: Msgpack.EncodableTest.User do + def encode(%Msgpack.EncodableTest.User{id: id, name: name}) do + {:ok, %{"id" => id, "name" => name}} + end +end From 579fb912575b27663b534c5b83bac7035ab2a671 Mon Sep 17 00:00:00 2001 From: Vandern Rodrigues Date: Mon, 11 Aug 2025 01:56:11 +0100 Subject: [PATCH 4/6] Update lib version in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 892d3e3..daec755 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Add `msgpack_elixir` to your list of dependencies in `mix.exs`: ```elixir def deps do - [{:msgpack_elixir, "~> 1.0.0"}] + [{:msgpack_elixir, "~> 2.0.0"}] end ``` From 89d86db4d70eb50bb13034c48f39122fada12a81 Mon Sep 17 00:00:00 2001 From: Vandern Rodrigues Date: Mon, 11 Aug 2025 02:29:55 +0100 Subject: [PATCH 5/6] Update documentation --- CHANGELOG.md | 4 ++++ README.md | 38 ++++++++++++++++++++++++++++++++++++-- lib/msgpack.ex | 29 +++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f92cbcb..40f22f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a `:deterministic` option to `Msgpack.encode/2` - You can set this to `false` to disable key sorting for higher performance in contexts where deterministic output is not required. +- Added the `Msgpack.Encodable` protocol to allow for custom serialization logic + for any Elixir struct + - This allows users to encode their own data types, such as %Product{} or + %User{}, directly ## [v1.1.1] - 2025-08-09 diff --git a/README.md b/README.md index daec755..34b0a49 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,10 @@ types. ## Features - **Specification Compliance:** Implements the complete MessagePack type system. -- **Elixir Struct Support:** Encodes and decodes `DateTime` and `NaiveDateTime` - structs via the Timestamp extension type. +- **Extensible Struct Support:** + - Natively encodes and decodes `DateTime` and `NaiveDateTime` structs via the + Timestamp extension type. + - Allows any custom struct to be encoded via the `Msgpack.Encodable` protocol. - **Configurable Validation:** Provides an option to bypass UTF-8 validation on strings for performance-critical paths. - **Resource Limiting:** Includes configurable `:max_depth` and `:max_byte_size` @@ -104,6 +106,38 @@ determinism is not required, you can disable it: Msgpack.encode(map, deterministic: false) ``` +### Custom Struct Serialization + +You can add custom encoding logic for your own Elixir structs by implementing +the `Msgpack.Encodable` protocol. This allows `Msgpack.encode/2` to accept your +structs directly, centralizing conversion logic within the protocol +implementation. + + +```elixir +# 1. Define your application's struct +defmodule Product do + defstruct [:id, :name] +end + +# 2. Implement the `Msgpack.Encodable` protocol for that struct +defimpl Msgpack.Encodable, for: Product do + + # 3. Inside the protocol's `encode/1` function, transform your struct into a basic + # Elixir term that MessagePack can encode (e.g., a map or a list). + def encode(%Product{id: id, name: name}) do + {:ok, %{"id" => id, "name" => name}} + end +end + +iex> product = %Product{id: 1, name: "Elixir"} +iex> {:ok, binary} = Msgpack.encode(product) +<<130, 162, 105, 100, 1, 164, 110, 97, 109, 101, 166, 69, 108, 105, 120, 105, 114>> + +iex> Msgpack.decode(binary) +{:ok, %{"id" => 1, "name" => "Elixir"}} +``` + ## Full Documentation For detailed information on all features, options, and functions, see the [full diff --git a/lib/msgpack.ex b/lib/msgpack.ex index ddd0e8a..ef1320f 100644 --- a/lib/msgpack.ex +++ b/lib/msgpack.ex @@ -29,6 +29,8 @@ defmodule Msgpack do to limit resource allocation when decoding. - **Telemetry Integration:** Emits `:telemetry` events for monitoring and observability. + - **Extensible Structs:** Allows any custom Elixir struct to be encoded by + implementing the `Msgpack.Encodable` protocol. ## Options @@ -93,6 +95,33 @@ defmodule Msgpack do * `false` - Disables key sorting, which can provide a performance gain in cases where determinism is not required. + ## Custom Struct Support + + This function can encode any custom Elixir struct that implements the + `Msgpack.Encodable` protocol. This allows you to define custom serialization + logic for your application structs. + + For example, given a `Product` struct: + + ```elixir + # 1. Define your struct + defmodule Product do + defstruct [:id, :name] + end + + # 2. Implement the protocol + defimpl Msgpack.Encodable, for: Product do + def encode(%Product{id: id, name: name}) do + # Transform the struct into an encodable term (e.g., a map) + {:ok, %{"id" => id, "name" => name}} + end + end + + iex> product = %Product{id: 1, name: "Elixir"} + iex> {:ok, binary} = Msgpack.encode(product) + <<130, 162, 105, 100, 1, 164, 110, 97, 109, 101, 166, 69, 108, 105, 120, 105, 114>> + ``` + ## Examples ### Standard Encoding From a78fff25dee031f9e4641f86df274ae8fcbf334d Mon Sep 17 00:00:00 2001 From: Vandern Rodrigues Date: Mon, 11 Aug 2025 02:33:46 +0100 Subject: [PATCH 6/6] [skip ci] wording --- lib/msgpack/encodable.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/msgpack/encodable.ex b/lib/msgpack/encodable.ex index 07cf24b..a651091 100644 --- a/lib/msgpack/encodable.ex +++ b/lib/msgpack/encodable.ex @@ -18,11 +18,11 @@ defprotocol Msgpack.Encodable do - Booleans - `nil` - Importantly, the returned term **must not** contain other custom structs that - themselves require an `Encodable` implementation. The purpose of this protocol - is to perform a single-level transformation from a custom struct into a - directly encodable term. Returning a nested custom struct will result in an - `{:error, {:unsupported_type, term}}` during encoding. + It is important that the returned term **must not** contain other custom + structs that themselves require an `Encodable` implementation. The purpose of + this protocol is to perform a single-level transformation from a custom struct + into a directly encodable term. Returning a nested custom struct will result + in an `{:error, {:unsupported_type, term}}` during encoding. ## Example