From d91ea814a94ec1a6148da113f8dd9da049507b55 Mon Sep 17 00:00:00 2001 From: Julian Doherty Date: Tue, 16 Sep 2025 21:04:44 +1000 Subject: [PATCH 1/5] include input_name in Table.Reader fields --- lib/benchee/suite.ex | 13 +- test/benchee/suite_test.exs | 350 +++++++++++++++++++++++++++++++++++- 2 files changed, 360 insertions(+), 3 deletions(-) diff --git a/lib/benchee/suite.ex b/lib/benchee/suite.ex index 571079db..5a734c90 100644 --- a/lib/benchee/suite.ex +++ b/lib/benchee/suite.ex @@ -50,6 +50,7 @@ end if Code.ensure_loaded?(Table.Reader) do defimpl Table.Reader, for: Benchee.Suite do + alias Benchee.Benchmark alias Benchee.CollectionData alias Benchee.Scenario @@ -96,7 +97,7 @@ if Code.ensure_loaded?(Table.Reader) do Enum.map(fields, fn field -> "#{measurement_type}_#{field}" end) end) - ["job_name" | measurement_headers] + ["job_name", "input_name"] ++ measurement_headers end defp fields_for(:run_time), do: @run_time_fields @@ -113,7 +114,15 @@ if Code.ensure_loaded?(Table.Reader) do |> get_stats_from_collection_data(measurement_type, config_percentiles) end) - row = [scenario.job_name | secenario_data] + no_input = Benchmark.no_input() + + input_name = + case scenario.input_name do + name when name in [nil, no_input] -> "" + name -> name + end + + row = [scenario.job_name, input_name] ++ secenario_data {row, count + 1} end) diff --git a/test/benchee/suite_test.exs b/test/benchee/suite_test.exs index f6aaf17a..ce87feac 100644 --- a/test/benchee/suite_test.exs +++ b/test/benchee/suite_test.exs @@ -225,18 +225,361 @@ defmodule Benchee.SuiteTest do ] } + @suite_with_inputs %Suite{ + system: @system, + configuration: %Benchee.Configuration{ + inputs: [ + {"small", [1, 2, 3, 4]}, + {"large", [1, 2, 3, 4, 5, 6, 7, 8]} + ], + input_names: ["small", "large"], + percentiles: [50, 99] + }, + scenarios: [ + %Benchee.Scenario{ + input_name: "small", + input: [1, 2, 3, 4], + job_name: "Test 1", + memory_usage_data: %Benchee.CollectionData{ + samples: [1_792, 1_792, 1_792], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 1_792.0, + ips: nil, + maximum: 1_792, + median: 1_792.0, + minimum: 1_792, + mode: 1_792, + percentiles: %{50 => 1_792.0, 99 => 1_792.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: +0.0, + std_dev_ips: nil, + std_dev_ratio: +0.0 + } + }, + name: "Test 1", + reductions_data: %Benchee.CollectionData{ + samples: [], + statistics: %Benchee.Statistics{} + }, + run_time_data: %Benchee.CollectionData{ + samples: [21_580, 2_986, 11_502], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 2_854.02659820102, + ips: 350_382.1585371105, + maximum: 3_741_076, + median: 2_164.0, + minimum: 2_063, + mode: 2_124, + percentiles: %{50 => 2_164.0, 99 => 5881.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: 13_106.875011228927, + std_dev_ips: 1_609_100.3359973046, + std_dev_ratio: 4.592415158110506 + } + } + }, + %Benchee.Scenario{ + job_name: "Test 2", + input_name: "small", + input: [1, 2, 3, 4], + memory_usage_data: %Benchee.CollectionData{ + samples: [1_792, 1_792, 1_792], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 1_792.0, + ips: nil, + maximum: 1_792, + median: 1_792.0, + minimum: 1_792, + mode: 1_792, + percentiles: %{50 => 1_792.0, 99 => 1_792.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: +0.0, + std_dev_ips: nil, + std_dev_ratio: +0.0 + } + }, + name: "Test 2", + reductions_data: %Benchee.CollectionData{ + samples: [], + statistics: %Benchee.Statistics{} + }, + run_time_data: %Benchee.CollectionData{ + samples: [21_580, 2_986, 11_502], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 2_854.02659820102, + ips: 350_382.1585371105, + maximum: 3_741_076, + median: 2_164.0, + minimum: 2_063, + mode: 2_124, + percentiles: %{50 => 2_164.0, 99 => 5881.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: 13_106.875011228927, + std_dev_ips: 1_609_100.3359973046, + std_dev_ratio: 4.592415158110506 + } + } + }, + %Benchee.Scenario{ + job_name: "Test 1", + input_name: "large", + input: [1, 2, 3, 4, 5, 6, 7, 8], + memory_usage_data: %Benchee.CollectionData{ + samples: [1_792, 1_792, 1_792], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 1_792.0, + ips: nil, + maximum: 1_792, + median: 1_792.0, + minimum: 1_792, + mode: 1_792, + percentiles: %{50 => 1_792.0, 99 => 1_792.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: +0.0, + std_dev_ips: nil, + std_dev_ratio: +0.0 + } + }, + name: "Test 1", + reductions_data: %Benchee.CollectionData{ + samples: [], + statistics: %Benchee.Statistics{} + }, + run_time_data: %Benchee.CollectionData{ + samples: [21_580, 2_986, 11_502], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 2_854.02659820102, + ips: 350_382.1585371105, + maximum: 3_741_076, + median: 2_164.0, + minimum: 2_063, + mode: 2_124, + percentiles: %{50 => 2_164.0, 99 => 5881.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: 13_106.875011228927, + std_dev_ips: 1_609_100.3359973046, + std_dev_ratio: 4.592415158110506 + } + } + }, + %Benchee.Scenario{ + job_name: "Test 2", + input_name: "large", + input: [1, 2, 3, 4, 5, 6, 7, 8], + memory_usage_data: %Benchee.CollectionData{ + samples: [1_792, 1_792, 1_792], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 1_792.0, + ips: nil, + maximum: 1_792, + median: 1_792.0, + minimum: 1_792, + mode: 1_792, + percentiles: %{50 => 1_792.0, 99 => 1_792.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: +0.0, + std_dev_ips: nil, + std_dev_ratio: +0.0 + } + }, + name: "Test 2", + reductions_data: %Benchee.CollectionData{ + samples: [], + statistics: %Benchee.Statistics{} + }, + run_time_data: %Benchee.CollectionData{ + samples: [21_580, 2_986, 11_502], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 2_854.02659820102, + ips: 350_382.1585371105, + maximum: 3_741_076, + median: 2_164.0, + minimum: 2_063, + mode: 2_124, + percentiles: %{50 => 2_164.0, 99 => 5881.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: 13_106.875011228927, + std_dev_ips: 1_609_100.3359973046, + std_dev_ratio: 4.592415158110506 + } + } + } + ] + } + test "should return a table when no scenarios are in the suite" do table_results = Table.Reader.init(@empty_suite) assert {:rows, %{ columns: [ - "job_name" + "job_name", + "input_name" ], count: 0 }, []} = table_results end + test "should return a table with data when multiple scenarios and inputs are in the suite" do + table_results = Table.Reader.init(@suite_with_inputs) + + assert {:rows, + %{ + columns: [ + "job_name", + "input_name", + "run_time_samples", + "run_time_ips", + "run_time_average", + "run_time_std_dev", + "run_time_median", + "run_time_minimum", + "run_time_maximum", + "run_time_mode", + "run_time_sample_size", + "run_time_p_50", + "run_time_p_99", + "memory_samples", + "memory_average", + "memory_std_dev", + "memory_median", + "memory_minimum", + "memory_maximum", + "memory_mode", + "memory_sample_size", + "memory_p_50", + "memory_p_99" + ], + count: 4 + }, + [ + [ + "Test 1", + "small", + [21_580, 2_986, 11_502], + 350_382.1585371105, + 2_854.02659820102, + 13_106.875011228927, + 2_164.0, + 2_063, + 3_741_076, + 2_124, + 3, + 2_164.0, + 5881.0, + [1_792, 1_792, 1_792], + 1_792.0, + +0.0, + 1_792.0, + 1_792, + 1_792, + 1_792, + 3, + 1_792.0, + 1_792.0 + ], + [ + "Test 2", + "small", + [21_580, 2_986, 11_502], + 350_382.1585371105, + 2_854.02659820102, + 13_106.875011228927, + 2_164.0, + 2_063, + 3_741_076, + 2_124, + 3, + 2_164.0, + 5881.0, + [1_792, 1_792, 1_792], + 1_792.0, + +0.0, + 1_792.0, + 1_792, + 1_792, + 1_792, + 3, + 1_792.0, + 1_792.0 + ], + [ + "Test 1", + "large", + [21_580, 2_986, 11_502], + 350_382.1585371105, + 2_854.02659820102, + 13_106.875011228927, + 2_164.0, + 2_063, + 3_741_076, + 2_124, + 3, + 2_164.0, + 5881.0, + [1_792, 1_792, 1_792], + 1_792.0, + +0.0, + 1_792.0, + 1_792, + 1_792, + 1_792, + 3, + 1_792.0, + 1_792.0 + ], + [ + "Test 2", + "large", + [21_580, 2_986, 11_502], + 350_382.1585371105, + 2_854.02659820102, + 13_106.875011228927, + 2_164.0, + 2_063, + 3_741_076, + 2_124, + 3, + 2_164.0, + 5881.0, + [1_792, 1_792, 1_792], + 1_792.0, + +0.0, + 1_792.0, + 1_792, + 1_792, + 1_792, + 3, + 1_792.0, + 1_792.0 + ] + ]} = table_results + end + test "should return a table with data when multiple scenarios are in the suite" do table_results = Table.Reader.init(@suite_with_data) @@ -244,6 +587,7 @@ defmodule Benchee.SuiteTest do %{ columns: [ "job_name", + "input_name", "run_time_samples", "run_time_ips", "run_time_average", @@ -271,6 +615,7 @@ defmodule Benchee.SuiteTest do [ [ "Test 1", + "", [21_580, 2_986, 11_502], 350_382.1585371105, 2_854.02659820102, @@ -295,6 +640,7 @@ defmodule Benchee.SuiteTest do ], [ "Test 2", + "", [21_580, 2_986, 11_502], 350_382.1585371105, 2_854.02659820102, @@ -327,6 +673,7 @@ defmodule Benchee.SuiteTest do %{ columns: [ "job_name", + "input_name", "run_time_samples", "run_time_ips", "run_time_average", @@ -364,6 +711,7 @@ defmodule Benchee.SuiteTest do [ [ "Test 1", + "", [21_580, 2_986, 11_502], 350_382.1585371105, 2_854.02659820102, From 6d32a8e96831cad87c5d32c33ab7466c74f85c1c Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Tue, 21 Oct 2025 21:26:05 +0200 Subject: [PATCH 2/5] Be more explicit about `no_input` handling * Make sure we only treat the special no input marker as no input, not `nil` * Make that clear in the type spec which was wrong and definitely got this value even before but why would dialyzer complain? --- lib/benchee/benchmark.ex | 2 ++ lib/benchee/scenario.ex | 4 ++-- lib/benchee/suite.ex | 2 +- test/benchee/suite_test.exs | 12 ++++++++++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/benchee/benchmark.ex b/lib/benchee/benchmark.ex index 2b843e29..3ed49aa1 100644 --- a/lib/benchee/benchmark.ex +++ b/lib/benchee/benchmark.ex @@ -11,10 +11,12 @@ defmodule Benchee.Benchmark do alias Benchee.Utility.DeepConvert @no_input :__no_input + @type no_input :: :__no_input @doc """ Public access for the special key representing no input for a scenario. """ + @spec no_input() :: no_input() def no_input, do: @no_input @doc """ diff --git a/lib/benchee/scenario.ex b/lib/benchee/scenario.ex index 1c5a1e16..bdd7337d 100644 --- a/lib/benchee/scenario.ex +++ b/lib/benchee/scenario.ex @@ -60,8 +60,8 @@ defmodule Benchee.Scenario do name: String.t(), job_name: String.t(), function: benchmarking_function, - input_name: String.t() | nil, - input: any | nil, + input_name: String.t() | Benchee.Benchmark.no_input(), + input: any | nil | Benchee.Benchmark.no_input(), run_time_data: CollectionData.t(), memory_usage_data: CollectionData.t(), reductions_data: CollectionData.t(), diff --git a/lib/benchee/suite.ex b/lib/benchee/suite.ex index 5a734c90..73aa3482 100644 --- a/lib/benchee/suite.ex +++ b/lib/benchee/suite.ex @@ -118,7 +118,7 @@ if Code.ensure_loaded?(Table.Reader) do input_name = case scenario.input_name do - name when name in [nil, no_input] -> "" + ^no_input -> "" name -> name end diff --git a/test/benchee/suite_test.exs b/test/benchee/suite_test.exs index ce87feac..d5445821 100644 --- a/test/benchee/suite_test.exs +++ b/test/benchee/suite_test.exs @@ -56,6 +56,7 @@ defmodule Benchee.SuiteTest do if Code.ensure_loaded?(Table.Reader) do describe "Table.Reader protocol" do + @no_input Benchee.Benchmark.no_input() @suite_with_data %Suite{ system: @system, configuration: %Benchee.Configuration{ @@ -64,6 +65,9 @@ defmodule Benchee.SuiteTest do scenarios: [ %Benchee.Scenario{ job_name: "Test 1", + name: "Test 1", + input_name: @no_input, + input: @no_input, memory_usage_data: %Benchee.CollectionData{ samples: [1_792, 1_792, 1_792], statistics: %Benchee.Statistics{ @@ -83,7 +87,6 @@ defmodule Benchee.SuiteTest do std_dev_ratio: +0.0 } }, - name: "Test 1", reductions_data: %Benchee.CollectionData{ samples: [], statistics: %Benchee.Statistics{} @@ -110,6 +113,9 @@ defmodule Benchee.SuiteTest do }, %Benchee.Scenario{ job_name: "Test 2", + name: "Test 2", + input_name: @no_input, + input: @no_input, memory_usage_data: %Benchee.CollectionData{ samples: [1_792, 1_792, 1_792], statistics: %Benchee.Statistics{ @@ -129,7 +135,6 @@ defmodule Benchee.SuiteTest do std_dev_ratio: +0.0 } }, - name: "Test 2", reductions_data: %Benchee.CollectionData{ samples: [], statistics: %Benchee.Statistics{} @@ -165,6 +170,9 @@ defmodule Benchee.SuiteTest do scenarios: [ %Benchee.Scenario{ job_name: "Test 1", + name: "Test 1", + input_name: @no_input, + input: @no_input, memory_usage_data: %Benchee.CollectionData{ samples: [1_792, 1_792, 1_792], statistics: %Benchee.Statistics{ From ccc52f27646a57e808ad4922b1dc635816fcd68e Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Tue, 21 Oct 2025 21:29:55 +0200 Subject: [PATCH 3/5] Actually use `name` instead of `job_name`, which my past self says is the correct thing to use now --- lib/benchee/suite.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/benchee/suite.ex b/lib/benchee/suite.ex index 73aa3482..7ea42d68 100644 --- a/lib/benchee/suite.ex +++ b/lib/benchee/suite.ex @@ -122,7 +122,7 @@ if Code.ensure_loaded?(Table.Reader) do name -> name end - row = [scenario.job_name, input_name] ++ secenario_data + row = [scenario.name, input_name] ++ secenario_data {row, count + 1} end) From 3f15d7b22d32ce0329595b3792261483d75af96b Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Tue, 21 Oct 2025 21:30:24 +0200 Subject: [PATCH 4/5] Fix scenario typo --- lib/benchee/suite.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/benchee/suite.ex b/lib/benchee/suite.ex index 7ea42d68..7e57c310 100644 --- a/lib/benchee/suite.ex +++ b/lib/benchee/suite.ex @@ -107,7 +107,7 @@ if Code.ensure_loaded?(Table.Reader) do config_percentiles = suite.configuration.percentiles Enum.map_reduce(suite.scenarios, 0, fn %Scenario{} = scenario, count -> - secenario_data = + scenario_data = Enum.flat_map(measurements_processed, fn measurement_type -> scenario |> Scenario.measurement_data(measurement_type) @@ -122,7 +122,7 @@ if Code.ensure_loaded?(Table.Reader) do name -> name end - row = [scenario.name, input_name] ++ secenario_data + row = [scenario.name, input_name] ++ scenario_data {row, count + 1} end) From 0b9f247c07647b4cc36a0a221e3c58ee7f0cc3de Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Tue, 21 Oct 2025 21:36:14 +0200 Subject: [PATCH 5/5] Add changelog entry for Table.Reader fix --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac8f8c38..01e68c88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Defaults to `1_000_000`, setting it to `nil` gathers unlimited samples again (be Especially important for run time, you can remove samples caused by garbage collection or external factors. Defaults to `false`. Shout out to [@NickNeck](https://github.com/NickNeck) who implemented this long wished for feature over in `Statistex`. +* Display `input_name` entries in Livebook/`Table.Reader` protocol. Thanks [@madlep](https://github.com/madlep)! ### Bugfixes (User Facing) * fixed a bug where if times were supplied as `0` instead of `0.0` we'd sometimes gather a single measurement