From 1a1bd59ef7cd8cbf8c0606d55be29d2e10e80422 Mon Sep 17 00:00:00 2001 From: Szymon Mentel Date: Wed, 28 Aug 2013 18:12:19 +0200 Subject: [PATCH 1/2] Add memory tests along with config generator --- scripts/config_generator.erl | 76 +++++++++++ scripts/memory_usage_test | 240 +++++++++++++++++++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 scripts/config_generator.erl create mode 100755 scripts/memory_usage_test diff --git a/scripts/config_generator.erl b/scripts/config_generator.erl new file mode 100644 index 0000000..02b7725 --- /dev/null +++ b/scripts/config_generator.erl @@ -0,0 +1,76 @@ +-module(config_generator). + +-export([generate/2]). + +-spec generate(PortsCnt :: non_neg_integer(), + LogicalSwitchesCnt :: non_neg_integer()) -> + Result :: term(). +generate(PortsCnt, LogicalSwitchesCnt) -> + [ + {linc, + [ + {of_config, enabled}, + {capable_switch_ports, generate_ports(PortsCnt * LogicalSwitchesCnt)}, + {capable_switch_queues, []}, + {logical_switches, + generate_logical_switches(PortsCnt, LogicalSwitchesCnt)} + ]}, + {enetconf, + [ + {capabilities, [{base, {1, 1}}, + {startup, {1, 0}}, + {'writable-running', {1, 0}}]}, + {callback_module, linc_ofconfig}, + {sshd_ip, any}, + {sshd_port, 1830}, + {sshd_user_passwords, + [ + {"linc", "linc"} + ]} + ]}, + {lager, + [ + {handlers, + [ + {lager_console_backend, info}, + {lager_file_backend, + [ + {"log/error.log", error, 10485760, "$D0", 5}, + {"log/console.log", info, 10485760, "$D0", 5} + ]} + ]} + ]}, + {sasl, + [ + {sasl_error_logger, {file, "log/sasl-error.log"}}, + {errlog_type, error}, + {error_logger_mf_dir, "log/sasl"}, % Log directory + {error_logger_mf_maxbytes, 10485760}, % 10 MB max file size + {error_logger_mf_maxfiles, 5} % 5 files max + ]}, + {sync, + [ + {excluded_modules, [procket]} + ]} + ]. + +generate_ports(PortsCnt) -> + [{port, N, [{interface, "tap" ++ integer_to_list(N)}]} + || N <- lists:seq(1, PortsCnt)]. + +generate_logical_switches(_, 0) -> + []; +generate_logical_switches(PortsCnt, LogicalSwitchesCnt) -> + [ + {switch, N, + [ + {backend, linc_us4}, + {controllers,[]}, + {queues_status, disabled}, + {ports, generate_logical_switch_ports(PortsCnt, (N - 1) * PortsCnt + 1)} + ]} + || N <- lists:seq(1, LogicalSwitchesCnt)]. + +generate_logical_switch_ports(PortsCnt, StartPortNo) -> + [{port, N, {queues, []}} || N <- lists:seq(StartPortNo, + StartPortNo + PortsCnt - 1)]. diff --git a/scripts/memory_usage_test b/scripts/memory_usage_test new file mode 100755 index 0000000..4a0dab2 --- /dev/null +++ b/scripts/memory_usage_test @@ -0,0 +1,240 @@ +#!/usr/bin/env escript +%%! -sname linc_tester@localhost + +-define(CONFIG_FILE, "tmp.sys.config"). +-define(COOKIE, cookie). +-define(LINC_NODE_NAME_BASE, "linc_under_test"). +-define(MNESIA_DIR_BASE, "Mnesia." ++ ?LINC_NODE_NAME_BASE). +-define(LOG_DIR, "log/"). + +main([Type, Arg1, Arg2]) -> + process_flag(trap_exit, true), + compile:file("config_generator.erl"), + register(linc_tester_proc, self()), + erlang:set_cookie(node(), ?COOKIE), + case Type of + "ports" -> + increasing_ports(get_linc_node_name(), + 0, + list_to_integer(Arg1), + list_to_integer(Arg2), + {0, []}); + "switches" -> + increasing_switches(get_linc_node_name(), + list_to_integer(Arg1), + 0, + list_to_integer(Arg2), {0, 0}); + _ -> + io:format("Unknown test.~n"), + usage() + end, + clean_up(); +main(_) -> + usage(). + +usage() -> + io:format("~s [ports " + ++ " | switches ]~n", + [escript:script_name()]). + + + +increasing_switches(_ , Ports, Switches, MaxSwitches, {_, MemPerSwitchAcc}) + when Switches > MaxSwitches -> + file:write_file(result_file(switches), + io_lib:fwrite("On avarage a logical switch with ~p ports" + ++ " consumes ~p kb.~n", + [Ports, MemPerSwitchAcc/(Switches - 1)]), + [append]); +increasing_switches(LincNode, Ports, Switches, MaxSwitches, Data) -> + ok = generate_config(Ports, Switches), + LincNodePort = start_linc_node(LincNode), + Mem = test_memory(LincNode), + ok = stop_linc_node(LincNode, LincNodePort), + NewData = case Switches of + 0 -> + write_result(switches, Switches, Mem, undef), + {Mem, 0}; + _ -> + {LastMem, MemPerSwitchAcc} = Data, + Diff = Mem - LastMem, + write_result(switches, Switches, Mem, Diff), + {Mem, MemPerSwitchAcc + Diff} + end, + increasing_switches( + get_linc_node_name(), Ports, Switches + 1, MaxSwitches, NewData). + +increasing_ports(LincNode, PortsCnt, Interval, MaxPortsCnt, TestData) + when PortsCnt > MaxPortsCnt -> + increasing_ports( + LincNode, + MaxPortsCnt, + MaxPortsCnt rem Interval, + MaxPortsCnt, + TestData); +increasing_ports(LincNode, PortsCnt, Interval, MaxPortsCnt, TestData) -> + ok = generate_config(PortsCnt, 1), + LincNodePort = start_linc_node(LincNode), + NewTestData = case PortsCnt of + 0 -> + Mem = test_memory(LincNode), + write_result(ports, PortsCnt, Mem, undef), + {Mem, []}; + _ -> + Mem = test_memory(LincNode), + {LastMem, PortAvgMem} = TestData, + write_result(ports, PortsCnt, Mem, + Diff = Mem - LastMem), + {Mem, [Diff div Interval | PortAvgMem]} + end, + ok = stop_linc_node(LincNode, LincNodePort), + case PortsCnt == MaxPortsCnt of + true -> + {_, PortAvgMem2} = NewTestData, + file:write_file(result_file(ports), + io_lib:fwrite("On average a port consumes ~p kb.~n", + [lists:foldl(fun(X, Acc) -> + X + Acc end, + 0, PortAvgMem2) + / length(PortAvgMem2)]), + [append]); + _ -> + increasing_ports(get_linc_node_name(), + PortsCnt + Interval, + Interval, + MaxPortsCnt, + NewTestData) + end. + +test_memory(LincNode) -> + Pid = self(), + spawn(LincNode, fun() -> + Pid ! {mem_test, erlang:memory(total)} + end), + receive + {mem_test, Memory} -> + io:format("Got memory usage: ~p kb.~n", [Memory]), + Memory div 1024 + after + 5000 -> + io:format("No response from ~p.~n", [LincNode]), + 0 + end. + +start_linc_node(LincNode) -> + Cmd = "erl -env ERL_MAX_ETS_TABLES 6000 " + ++ code_paths() + ++ " -config " ++ ?CONFIG_FILE + ++ " -setcookie " ++ atom_to_list(?COOKIE) + ++ " -sname " ++ atom_to_list(LincNode) + ++ " -detached" + ++ " -eval \"" + ++ eval(Msg = hello) + ++ "\"", + LincNodePort = erlang:open_port({spawn, Cmd}, []), + io:format("Waiting for test node to connect..."), + receive + Msg -> + io:format("connected~n") + after + 20000 -> + io:format("Error while starting test node~n") + end, + LincNodePort. + +stop_linc_node(LincNode, LincNodePort) -> + %% port_close(LincNodePort). + spawn(LincNode, fun() -> + init:stop() + end), + receive + {'EXIT', LincNodePort, normal} -> + ok + after + 10000 -> + io:format("LINC node: not exitted~n"), + error + end. + +code_paths() -> + Paths = lists:foldl(fun(Path, Acc) -> + Acc ++ filelib:wildcard(Path) + end, [], ["../apps/*/ebin", "../deps/*/ebin"]), + lists:foldl(fun(Path, Acc) -> + Acc ++ " -pa " ++ Path + end, [], Paths). + +eval(Msg) -> + {registered_name, ProcName} = process_info(self(), registered_name), + "lists:map(fun application:start/1, + [kernel, stdlib, public_key, crypto, ssl, + compiler, syntax_tools, runtime_tools, + xmerl, mnesia, lager, linc])," + ++ "true = net_kernel:connect_node(" ++ atom_to_list(node()) ++ ")," + ++ "{" ++ atom_to_list(ProcName) ++ "," ++ atom_to_list(node()) ++ "}" + ++ " ! " ++ atom_to_list(Msg) ++ ".". + + +ports_result_header() -> + "Ports | Memory | Difference\n". + +switches_result_header() -> + "Switches | Memory | Difference\n". + +result_line(0, Mem, _) -> + "0 " ++ integer_to_list(Mem) ++ " none\n"; +result_line(PortsCnt, Mem, Diff) -> + integer_to_list(PortsCnt) + ++ " " + ++ integer_to_list(Mem) + ++ " " + ++ integer_to_list(Diff) + ++ "\n". + +generate_config(Ports, Switches) -> + file:write_file(?CONFIG_FILE, + io_lib:fwrite( + "~p.~n", + [config_generator:generate(Ports, Switches)])). + +write_result(Type, Cnt, Mem, Diff) -> + Content = case Cnt of + 0 -> + result_header(Type) ++ result_line(0, Mem, undef); + _ -> + result_line(Cnt, Mem, Diff) + end, + ok = file:write_file(result_file(Type), Content, case Cnt of + 0 -> [write]; + _ -> [append] + end). +result_file(Type) -> + case Type of + ports -> "ports.test"; + switches -> "switches.test" + end. + +result_header(Type) -> + case Type of + ports -> + ports_result_header(); + switches -> + switches_result_header() + end. + +get_linc_node_name() -> + list_to_atom(?LINC_NODE_NAME_BASE + ++ integer_to_list(element(3, now())) + ++ "@localhost"). + +clean_up() -> + file:delete(?CONFIG_FILE), + lists:foreach(fun(Dir) -> + ok = delete_directory(Dir) + end, [ ?LOG_DIR | filelib:wildcard(?MNESIA_DIR_BASE ++ "*")]). + +delete_directory(Dir) -> + lists:foreach(fun(File) -> + ok = file:delete(File) + end, filelib:wildcard(Dir ++ "/*")), + ok = file:del_dir(Dir). From 00838f26c75b472604eaffa4fc75c8478efc5ee9 Mon Sep 17 00:00:00 2001 From: Szymon Mentel Date: Thu, 29 Aug 2013 14:57:10 +0200 Subject: [PATCH 2/2] Add quantification wiki page along with improved comments in the config file Quantification wiki page describes LINC's resources consumption and provides information on how to test it. It also explains how to add more resources to the LINC and how to tune Erlang VM. --- docs/quantification.md | 47 ++++++++++++++++++++++++++++++++++++++++++ rel/files/sys.config | 5 +++++ 2 files changed, 52 insertions(+) create mode 100644 docs/quantification.md diff --git a/docs/quantification.md b/docs/quantification.md new file mode 100644 index 0000000..92060cc --- /dev/null +++ b/docs/quantification.md @@ -0,0 +1,47 @@ +# Quantification # +This page describes LINC's resources consumption. + +## Adding resources ## +To add ports and logical switches to the LINC instance you have to edit `rel/files/sys.config` file. You will there comments on how to do this. + +## Memory consumptions ## +LINC has three major components that make up memory consumption: +* ports and logical switches, +* flow entries. + +### Average values ### +On average a LINC port consumes ~ 11 kb. +Memory consumed by logical switches is not linear. For example a logical switch with 20 ports consumes: +* ~ 1200 kb if we have 10 such switches +* ~ 2200 kb if we have 20 such switches +* ~ 5100 kb if we have 30 such switches + +### Measuring ### +There is a script `scripts/mem_usage_test` that helps with estimating memory requirements. + +To measure how much memory one port will take use it as follows: +```bash +./scripts/mem_usage_test ports +``` +In the test amount of ports indicated by the `interval` will be added to the LINC instance in each iteration until reaching the `max_port_number`. The script produces output file `ports.test` that has three columns: +* number of ports in the LINC instance, +* memory consumed by the LINC instance, +* memory consumed by this LINC instance minus the amount of memory consumed in the previous test. +At the bottom of the file there's a summary line. + +To memory how much memory one logical switch witch fixed number of ports invoke the script as follows: + +```bash +./scripts/mem_usage_test switches +``` +In this test an additional logical switch will be added to the LINC instance in each iteration until reaching the `max_switches`. The test produces an output file that has three columns: +* number of logical switches started in a LINC instance, +* memory consumed by the LINC instance, +* memory consumed by this LINC instance minus the amount of memory consumed in the previous test. +At the bottom of the file there's a summary line. + +## Tuning Erlang VM ## +Each logical switch allocates approximately 280 ETS tables. To change this value you have to edit `rel/files/vm.args` file and change the value of ERL_MAX_ETS_TABLES. You find details in the file. + +## TODO ## +Measure flow entries. diff --git a/rel/files/sys.config b/rel/files/sys.config index da637e8..ef1881c 100644 --- a/rel/files/sys.config +++ b/rel/files/sys.config @@ -41,6 +41,11 @@ %% Configuration of the logical switches. {logical_switches, [ + %% To add a new logical switch add a new entry to this list similar to + %% the one below. Keep in mind that: + %% a. logical switches need uniquie interger id (second element + %% of the tuple), + %% b. you have to assign ports to the logical switch manually. {switch, 0, [ %% Configuration of switch backend implementation used by ofs_logic