diff --git a/CHANGELOG.md b/CHANGELOG.md index ac8f8c38..01e68c88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Defaults to `1_000_000`, setting it to `nil` gathers unlimited samples again (be Especially important for run time, you can remove samples caused by garbage collection or external factors. Defaults to `false`. Shout out to [@NickNeck](https://github.com/NickNeck) who implemented this long wished for feature over in `Statistex`. +* Display `input_name` entries in Livebook/`Table.Reader` protocol. Thanks [@madlep](https://github.com/madlep)! ### Bugfixes (User Facing) * fixed a bug where if times were supplied as `0` instead of `0.0` we'd sometimes gather a single measurement diff --git a/lib/benchee/benchmark.ex b/lib/benchee/benchmark.ex index 2b843e29..3ed49aa1 100644 --- a/lib/benchee/benchmark.ex +++ b/lib/benchee/benchmark.ex @@ -11,10 +11,12 @@ defmodule Benchee.Benchmark do alias Benchee.Utility.DeepConvert @no_input :__no_input + @type no_input :: :__no_input @doc """ Public access for the special key representing no input for a scenario. """ + @spec no_input() :: no_input() def no_input, do: @no_input @doc """ diff --git a/lib/benchee/scenario.ex b/lib/benchee/scenario.ex index 1c5a1e16..bdd7337d 100644 --- a/lib/benchee/scenario.ex +++ b/lib/benchee/scenario.ex @@ -60,8 +60,8 @@ defmodule Benchee.Scenario do name: String.t(), job_name: String.t(), function: benchmarking_function, - input_name: String.t() | nil, - input: any | nil, + input_name: String.t() | Benchee.Benchmark.no_input(), + input: any | nil | Benchee.Benchmark.no_input(), run_time_data: CollectionData.t(), memory_usage_data: CollectionData.t(), reductions_data: CollectionData.t(), diff --git a/lib/benchee/suite.ex b/lib/benchee/suite.ex index 571079db..7e57c310 100644 --- a/lib/benchee/suite.ex +++ b/lib/benchee/suite.ex @@ -50,6 +50,7 @@ end if Code.ensure_loaded?(Table.Reader) do defimpl Table.Reader, for: Benchee.Suite do + alias Benchee.Benchmark alias Benchee.CollectionData alias Benchee.Scenario @@ -96,7 +97,7 @@ if Code.ensure_loaded?(Table.Reader) do Enum.map(fields, fn field -> "#{measurement_type}_#{field}" end) end) - ["job_name" | measurement_headers] + ["job_name", "input_name"] ++ measurement_headers end defp fields_for(:run_time), do: @run_time_fields @@ -106,14 +107,22 @@ if Code.ensure_loaded?(Table.Reader) do config_percentiles = suite.configuration.percentiles Enum.map_reduce(suite.scenarios, 0, fn %Scenario{} = scenario, count -> - secenario_data = + scenario_data = Enum.flat_map(measurements_processed, fn measurement_type -> scenario |> Scenario.measurement_data(measurement_type) |> get_stats_from_collection_data(measurement_type, config_percentiles) end) - row = [scenario.job_name | secenario_data] + no_input = Benchmark.no_input() + + input_name = + case scenario.input_name do + ^no_input -> "" + name -> name + end + + row = [scenario.name, input_name] ++ scenario_data {row, count + 1} end) diff --git a/test/benchee/suite_test.exs b/test/benchee/suite_test.exs index f6aaf17a..d5445821 100644 --- a/test/benchee/suite_test.exs +++ b/test/benchee/suite_test.exs @@ -56,6 +56,7 @@ defmodule Benchee.SuiteTest do if Code.ensure_loaded?(Table.Reader) do describe "Table.Reader protocol" do + @no_input Benchee.Benchmark.no_input() @suite_with_data %Suite{ system: @system, configuration: %Benchee.Configuration{ @@ -64,6 +65,9 @@ defmodule Benchee.SuiteTest do scenarios: [ %Benchee.Scenario{ job_name: "Test 1", + name: "Test 1", + input_name: @no_input, + input: @no_input, memory_usage_data: %Benchee.CollectionData{ samples: [1_792, 1_792, 1_792], statistics: %Benchee.Statistics{ @@ -83,7 +87,6 @@ defmodule Benchee.SuiteTest do std_dev_ratio: +0.0 } }, - name: "Test 1", reductions_data: %Benchee.CollectionData{ samples: [], statistics: %Benchee.Statistics{} @@ -110,6 +113,9 @@ defmodule Benchee.SuiteTest do }, %Benchee.Scenario{ job_name: "Test 2", + name: "Test 2", + input_name: @no_input, + input: @no_input, memory_usage_data: %Benchee.CollectionData{ samples: [1_792, 1_792, 1_792], statistics: %Benchee.Statistics{ @@ -129,7 +135,6 @@ defmodule Benchee.SuiteTest do std_dev_ratio: +0.0 } }, - name: "Test 2", reductions_data: %Benchee.CollectionData{ samples: [], statistics: %Benchee.Statistics{} @@ -165,6 +170,9 @@ defmodule Benchee.SuiteTest do scenarios: [ %Benchee.Scenario{ job_name: "Test 1", + name: "Test 1", + input_name: @no_input, + input: @no_input, memory_usage_data: %Benchee.CollectionData{ samples: [1_792, 1_792, 1_792], statistics: %Benchee.Statistics{ @@ -225,18 +233,361 @@ defmodule Benchee.SuiteTest do ] } + @suite_with_inputs %Suite{ + system: @system, + configuration: %Benchee.Configuration{ + inputs: [ + {"small", [1, 2, 3, 4]}, + {"large", [1, 2, 3, 4, 5, 6, 7, 8]} + ], + input_names: ["small", "large"], + percentiles: [50, 99] + }, + scenarios: [ + %Benchee.Scenario{ + input_name: "small", + input: [1, 2, 3, 4], + job_name: "Test 1", + memory_usage_data: %Benchee.CollectionData{ + samples: [1_792, 1_792, 1_792], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 1_792.0, + ips: nil, + maximum: 1_792, + median: 1_792.0, + minimum: 1_792, + mode: 1_792, + percentiles: %{50 => 1_792.0, 99 => 1_792.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: +0.0, + std_dev_ips: nil, + std_dev_ratio: +0.0 + } + }, + name: "Test 1", + reductions_data: %Benchee.CollectionData{ + samples: [], + statistics: %Benchee.Statistics{} + }, + run_time_data: %Benchee.CollectionData{ + samples: [21_580, 2_986, 11_502], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 2_854.02659820102, + ips: 350_382.1585371105, + maximum: 3_741_076, + median: 2_164.0, + minimum: 2_063, + mode: 2_124, + percentiles: %{50 => 2_164.0, 99 => 5881.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: 13_106.875011228927, + std_dev_ips: 1_609_100.3359973046, + std_dev_ratio: 4.592415158110506 + } + } + }, + %Benchee.Scenario{ + job_name: "Test 2", + input_name: "small", + input: [1, 2, 3, 4], + memory_usage_data: %Benchee.CollectionData{ + samples: [1_792, 1_792, 1_792], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 1_792.0, + ips: nil, + maximum: 1_792, + median: 1_792.0, + minimum: 1_792, + mode: 1_792, + percentiles: %{50 => 1_792.0, 99 => 1_792.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: +0.0, + std_dev_ips: nil, + std_dev_ratio: +0.0 + } + }, + name: "Test 2", + reductions_data: %Benchee.CollectionData{ + samples: [], + statistics: %Benchee.Statistics{} + }, + run_time_data: %Benchee.CollectionData{ + samples: [21_580, 2_986, 11_502], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 2_854.02659820102, + ips: 350_382.1585371105, + maximum: 3_741_076, + median: 2_164.0, + minimum: 2_063, + mode: 2_124, + percentiles: %{50 => 2_164.0, 99 => 5881.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: 13_106.875011228927, + std_dev_ips: 1_609_100.3359973046, + std_dev_ratio: 4.592415158110506 + } + } + }, + %Benchee.Scenario{ + job_name: "Test 1", + input_name: "large", + input: [1, 2, 3, 4, 5, 6, 7, 8], + memory_usage_data: %Benchee.CollectionData{ + samples: [1_792, 1_792, 1_792], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 1_792.0, + ips: nil, + maximum: 1_792, + median: 1_792.0, + minimum: 1_792, + mode: 1_792, + percentiles: %{50 => 1_792.0, 99 => 1_792.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: +0.0, + std_dev_ips: nil, + std_dev_ratio: +0.0 + } + }, + name: "Test 1", + reductions_data: %Benchee.CollectionData{ + samples: [], + statistics: %Benchee.Statistics{} + }, + run_time_data: %Benchee.CollectionData{ + samples: [21_580, 2_986, 11_502], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 2_854.02659820102, + ips: 350_382.1585371105, + maximum: 3_741_076, + median: 2_164.0, + minimum: 2_063, + mode: 2_124, + percentiles: %{50 => 2_164.0, 99 => 5881.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: 13_106.875011228927, + std_dev_ips: 1_609_100.3359973046, + std_dev_ratio: 4.592415158110506 + } + } + }, + %Benchee.Scenario{ + job_name: "Test 2", + input_name: "large", + input: [1, 2, 3, 4, 5, 6, 7, 8], + memory_usage_data: %Benchee.CollectionData{ + samples: [1_792, 1_792, 1_792], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 1_792.0, + ips: nil, + maximum: 1_792, + median: 1_792.0, + minimum: 1_792, + mode: 1_792, + percentiles: %{50 => 1_792.0, 99 => 1_792.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: +0.0, + std_dev_ips: nil, + std_dev_ratio: +0.0 + } + }, + name: "Test 2", + reductions_data: %Benchee.CollectionData{ + samples: [], + statistics: %Benchee.Statistics{} + }, + run_time_data: %Benchee.CollectionData{ + samples: [21_580, 2_986, 11_502], + statistics: %Benchee.Statistics{ + absolute_difference: nil, + average: 2_854.02659820102, + ips: 350_382.1585371105, + maximum: 3_741_076, + median: 2_164.0, + minimum: 2_063, + mode: 2_124, + percentiles: %{50 => 2_164.0, 99 => 5881.0}, + relative_less: nil, + relative_more: nil, + sample_size: 3, + std_dev: 13_106.875011228927, + std_dev_ips: 1_609_100.3359973046, + std_dev_ratio: 4.592415158110506 + } + } + } + ] + } + test "should return a table when no scenarios are in the suite" do table_results = Table.Reader.init(@empty_suite) assert {:rows, %{ columns: [ - "job_name" + "job_name", + "input_name" ], count: 0 }, []} = table_results end + test "should return a table with data when multiple scenarios and inputs are in the suite" do + table_results = Table.Reader.init(@suite_with_inputs) + + assert {:rows, + %{ + columns: [ + "job_name", + "input_name", + "run_time_samples", + "run_time_ips", + "run_time_average", + "run_time_std_dev", + "run_time_median", + "run_time_minimum", + "run_time_maximum", + "run_time_mode", + "run_time_sample_size", + "run_time_p_50", + "run_time_p_99", + "memory_samples", + "memory_average", + "memory_std_dev", + "memory_median", + "memory_minimum", + "memory_maximum", + "memory_mode", + "memory_sample_size", + "memory_p_50", + "memory_p_99" + ], + count: 4 + }, + [ + [ + "Test 1", + "small", + [21_580, 2_986, 11_502], + 350_382.1585371105, + 2_854.02659820102, + 13_106.875011228927, + 2_164.0, + 2_063, + 3_741_076, + 2_124, + 3, + 2_164.0, + 5881.0, + [1_792, 1_792, 1_792], + 1_792.0, + +0.0, + 1_792.0, + 1_792, + 1_792, + 1_792, + 3, + 1_792.0, + 1_792.0 + ], + [ + "Test 2", + "small", + [21_580, 2_986, 11_502], + 350_382.1585371105, + 2_854.02659820102, + 13_106.875011228927, + 2_164.0, + 2_063, + 3_741_076, + 2_124, + 3, + 2_164.0, + 5881.0, + [1_792, 1_792, 1_792], + 1_792.0, + +0.0, + 1_792.0, + 1_792, + 1_792, + 1_792, + 3, + 1_792.0, + 1_792.0 + ], + [ + "Test 1", + "large", + [21_580, 2_986, 11_502], + 350_382.1585371105, + 2_854.02659820102, + 13_106.875011228927, + 2_164.0, + 2_063, + 3_741_076, + 2_124, + 3, + 2_164.0, + 5881.0, + [1_792, 1_792, 1_792], + 1_792.0, + +0.0, + 1_792.0, + 1_792, + 1_792, + 1_792, + 3, + 1_792.0, + 1_792.0 + ], + [ + "Test 2", + "large", + [21_580, 2_986, 11_502], + 350_382.1585371105, + 2_854.02659820102, + 13_106.875011228927, + 2_164.0, + 2_063, + 3_741_076, + 2_124, + 3, + 2_164.0, + 5881.0, + [1_792, 1_792, 1_792], + 1_792.0, + +0.0, + 1_792.0, + 1_792, + 1_792, + 1_792, + 3, + 1_792.0, + 1_792.0 + ] + ]} = table_results + end + test "should return a table with data when multiple scenarios are in the suite" do table_results = Table.Reader.init(@suite_with_data) @@ -244,6 +595,7 @@ defmodule Benchee.SuiteTest do %{ columns: [ "job_name", + "input_name", "run_time_samples", "run_time_ips", "run_time_average", @@ -271,6 +623,7 @@ defmodule Benchee.SuiteTest do [ [ "Test 1", + "", [21_580, 2_986, 11_502], 350_382.1585371105, 2_854.02659820102, @@ -295,6 +648,7 @@ defmodule Benchee.SuiteTest do ], [ "Test 2", + "", [21_580, 2_986, 11_502], 350_382.1585371105, 2_854.02659820102, @@ -327,6 +681,7 @@ defmodule Benchee.SuiteTest do %{ columns: [ "job_name", + "input_name", "run_time_samples", "run_time_ips", "run_time_average", @@ -364,6 +719,7 @@ defmodule Benchee.SuiteTest do [ [ "Test 1", + "", [21_580, 2_986, 11_502], 350_382.1585371105, 2_854.02659820102,