From 18e5bbe3d0da1697b1df4d3b50b138859d6d638a Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Thu, 30 Jan 2025 15:44:44 +0000 Subject: [PATCH 1/4] Add a yotal_iodepth option to be slit over total volumes Signed-off-by: Chris Harris(harriscr@uk.ibm.com --- benchmark/librbdfio.py | 55 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/benchmark/librbdfio.py b/benchmark/librbdfio.py index 44b9b660..8fffaee7 100644 --- a/benchmark/librbdfio.py +++ b/benchmark/librbdfio.py @@ -9,6 +9,7 @@ import settings import monitoring +from typing import Optional from .benchmark import Benchmark logger = logging.getLogger("cbt") @@ -50,6 +51,14 @@ def __init__(self, archive_dir, cluster, config): self.rate_iops = config.get('rate_iops', None) self.fio_out_format = config.get('fio_out_format', 'json,normal') self.data_pool = None + + self._ioddepth_per_volume: dict[int, int] = {} + total_iodepth: Optional[str] = config.get("total_iodepth", None) + if total_iodepth is not None: + self._ioddepth_per_volume = self._calculate_iodepth_per_volume( + int(self.volumes_per_client), int(total_iodepth) + ) + # use_existing_volumes needs to be true to set the pool and rbd names self.use_existing_volumes = bool(config.get('use_existing_volumes', False)) self.no_sudo = bool(config.get('no_sudo', False)) @@ -244,7 +253,7 @@ def run(self): self.analyze(self.out_dir) - def mkfiocmd(self, volnum): + def mkfiocmd(self, volnum: int) -> str: """ Construct a FIO cmd (note the shell interpolation for the host executing FIO). @@ -257,7 +266,7 @@ def mkfiocmd(self, volnum): logger.debug('Using rbdname %s', rbdname) out_file = f'{self.run_dir}/output.{volnum:d}' - fio_cmd = '' + fio_cmd: str = '' if not self.no_sudo: fio_cmd = 'sudo ' fio_cmd += '%s --ioengine=rbd --clientname=admin --pool=%s --rbdname=%s --invalidate=0' % (self.cmd_path, self.pool_name, rbdname) @@ -274,7 +283,12 @@ def mkfiocmd(self, volnum): fio_cmd += ' --numjobs=%s' % self.numjobs fio_cmd += ' --direct=1' fio_cmd += ' --bs=%dB' % self.op_size - fio_cmd += ' --iodepth=%d' % self.iodepth + + iodepth: str = f"{self.iodepth}" + if self._ioddepth_per_volume != {}: + iodepth = f"{self._ioddepth_per_volume[volnum]}" + + fio_cmd += ' --iodepth=%s' % iodepth fio_cmd += ' --end_fsync=%d' % self.end_fsync # if self.vol_size: # fio_cmd += ' -- size=%dM' % self.vol_size @@ -401,6 +415,41 @@ def analyze(self, out_dir): logger.info('Convert results to json format.') self.parse(out_dir) + def _calculate_iodepth_per_volume(self, number_of_volumes: int, total_desired_iodepth: int) -> dict[int, int]: + """ + Given the total desired iodepth and the number of volumes from the + configuration yaml file, calculate the iodepth for each volume + + If the iodepth specified in total_iodepth is too small to allow + an iodepth of 1 per volume, then reduce the number of volumes + used to allow an iodepth of 1 per volume. + """ + queue_depths: dict[int, int] = {} + + if number_of_volumes > total_desired_iodepth: + logger.warning( + "The total iodepth requested: %s is less than 1 per volume (%s)", + total_desired_iodepth, + number_of_volumes, + ) + logger.warning( + "Number of volumes per client will be reduced from %s to %s", number_of_volumes, total_desired_iodepth + ) + number_of_volumes = total_desired_iodepth + self.volumes_per_client = number_of_volumes + + iodepth_per_volume: int = total_desired_iodepth // number_of_volumes + remainder: int = total_desired_iodepth % number_of_volumes + + for volume_id in range(number_of_volumes): + iodepth: int = iodepth_per_volume + + if remainder > 0: + iodepth += 1 + remainder -= 1 + queue_depths[volume_id] = iodepth + + return queue_depths def __str__(self): return "%s\n%s\n%s" % (self.run_dir, self.out_dir, super(LibrbdFio, self).__str__()) From cee02b9067802de08344a9ce0157cde9e23e425f Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Thu, 6 Feb 2025 14:25:53 +0000 Subject: [PATCH 2/4] Workload fixes for total_iodepth Signed-off-by: CHris Harris(harriscr@uk.ibm.com) --- benchmark/librbdfio.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/benchmark/librbdfio.py b/benchmark/librbdfio.py index 8fffaee7..0ee13343 100644 --- a/benchmark/librbdfio.py +++ b/benchmark/librbdfio.py @@ -172,7 +172,18 @@ def run_workloads(self): enable_monitor = bool(test['monitor']) # TODO: simplify this loop to have a single iterator for general queu depth for job in test['numjobs']: - for iod in test['iodepth']: + iodepth: list[str] = [] + use_total_iodepth: bool = False + if "total_iodepth" in test.keys(): + iodepth = test["total_iodepth"] + use_total_iodepth = True + else: + iodepth = test["iodepth"] + for iod in iodepth: + if use_total_iodepth: + self._ioddepth_per_volume = self._calculate_iodepth_per_volume( + int(self.volumes_per_client), int(iod) + ) self.mode = test['mode'] if 'op_size' in test: self.op_size = test['op_size'] @@ -183,7 +194,10 @@ def run_workloads(self): f'iodepth-{int(self.iodepth):03d}/numjobs-{int(self.numjobs):03d}' ) common.make_remote_dir(self.run_dir) - for i in range(self.volumes_per_client): + number_of_volumes: int = int(self.volumes_per_client) + if use_total_iodepth: + number_of_volumes = len(self._ioddepth_per_volume.keys()) + for i in range(number_of_volumes): fio_cmd = self.mkfiocmd(i) p = common.pdsh(settings.getnodes('clients'), fio_cmd) ps.append(p) @@ -235,7 +249,10 @@ def run(self): monitoring.start(self.run_dir) logger.info('Running rbd fio %s test.', self.mode) ps = [] - for i in range(self.volumes_per_client): + number_of_volumes: int = int(self.volumes_per_client) + if self._ioddepth_per_volume != {}: + number_of_volumes = len(self._ioddepth_per_volume.keys()) + for i in range(number_of_volumes): fio_cmd = self.mkfiocmd(i) p = common.pdsh(settings.getnodes('clients'), fio_cmd) ps.append(p) @@ -436,7 +453,6 @@ def _calculate_iodepth_per_volume(self, number_of_volumes: int, total_desired_io "Number of volumes per client will be reduced from %s to %s", number_of_volumes, total_desired_iodepth ) number_of_volumes = total_desired_iodepth - self.volumes_per_client = number_of_volumes iodepth_per_volume: int = total_desired_iodepth // number_of_volumes remainder: int = total_desired_iodepth % number_of_volumes From 83f637c72d29cd23f4509f252007bbd73190d75b Mon Sep 17 00:00:00 2001 From: lee-j-sanders Date: Tue, 11 Feb 2025 16:25:34 +0000 Subject: [PATCH 3/4] This commit addresses the following - specify a precond_time in the preconditioning workload of a RBD YAML that is using the workloads feature. So you can have a different length of preconditioning time compared to a regular runtime (time in the YAML) - Fixes an issue where if you specified more than one workload with the same block size, but with different rwmixread ratios, the 2nd workload would overwrite the 1st workloads results log file. Signed-off-by: lee-j-sanders --- benchmark/librbdfio.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/benchmark/librbdfio.py b/benchmark/librbdfio.py index 0ee13343..eaa9f63d 100644 --- a/benchmark/librbdfio.py +++ b/benchmark/librbdfio.py @@ -29,6 +29,8 @@ def __init__(self, archive_dir, cluster, config): self.recov_test_type = config.get('recov_test_type', 'blocking') self.data_pool_profile = config.get('data_pool_profile', None) self.time = config.get('time', None) + self.precond_time = config.get('precond_time',None ) + # Global FIO options can be overwritten for specific workload options # would be nice to have them as a separate class -- future PR self.time_based = bool(config.get('time_based', False)) @@ -187,18 +189,36 @@ def run_workloads(self): self.mode = test['mode'] if 'op_size' in test: self.op_size = test['op_size'] + if 'precond' in test: + fioruntime = self.precond_time + else: + fioruntime = self.time + self.mode = test['mode'] self.numjobs = job self.iodepth = iod - self.run_dir = ( f'{self.base_run_dir}/{self.mode}_{int(self.op_size)}/' - f'iodepth-{int(self.iodepth):03d}/numjobs-{int(self.numjobs):03d}' ) + + # Needed to allow for different mixed ratio results with the same block size, we + # store the ratio within the directory name. Otherwise workloads would only support + # 1 mixed workload for a given block size. For 100% read, 100% write don't need to + # store the read/write ratio. + + if self.mode == 'randrw': + self.rwmixread = test['rwmixread'] + self.rwmixwrite = 100 - self.rwmixread + self.run_dir = ( f'{self.base_run_dir}/{self.mode}{self.rwmixread}{self.rwmixwrite}_{int(self.op_size)}/' + f'iodepth-{int(self.iodepth):03d}/numjobs-{int(self.numjobs):03d}' ) + else: + self.run_dir = ( f'{self.base_run_dir}/{self.mode}_{int(self.op_size)}/' + f'iodepth-{int(self.iodepth):03d}/numjobs-{int(self.numjobs):03d}' ) + common.make_remote_dir(self.run_dir) number_of_volumes: int = int(self.volumes_per_client) if use_total_iodepth: number_of_volumes = len(self._ioddepth_per_volume.keys()) for i in range(number_of_volumes): - fio_cmd = self.mkfiocmd(i) + fio_cmd = self.mkfiocmd(i,fioruntime) p = common.pdsh(settings.getnodes('clients'), fio_cmd) ps.append(p) if enable_monitor: @@ -253,7 +273,7 @@ def run(self): if self._ioddepth_per_volume != {}: number_of_volumes = len(self._ioddepth_per_volume.keys()) for i in range(number_of_volumes): - fio_cmd = self.mkfiocmd(i) + fio_cmd = self.mkfiocmd(i,self.time) p = common.pdsh(settings.getnodes('clients'), fio_cmd) ps.append(p) for p in ps: @@ -269,8 +289,7 @@ def run(self): common.sync_files(f'{self.run_dir}/*', self.out_dir) self.analyze(self.out_dir) - - def mkfiocmd(self, volnum: int) -> str: + def mkfiocmd(self, volnum: int, time) -> str: """ Construct a FIO cmd (note the shell interpolation for the host executing FIO). @@ -291,8 +310,8 @@ def mkfiocmd(self, volnum: int) -> str: fio_cmd += ' --output-format=%s' % self.fio_out_format if (self.mode == 'readwrite' or self.mode == 'randrw'): fio_cmd += ' --rwmixread=%s --rwmixwrite=%s' % (self.rwmixread, self.rwmixwrite) - if self.time is not None: - fio_cmd += ' --runtime=%d' % self.time + if time is not None: + fio_cmd += ' --runtime=%d' % time if self.time_based is True: fio_cmd += ' --time_based' if self.ramp is not None: From b0c45ba0bbb0ee7a97a7df78e7fecf062ad4c5af Mon Sep 17 00:00:00 2001 From: lee-j-sanders Date: Wed, 12 Feb 2025 14:58:43 +0000 Subject: [PATCH 4/4] Add precond_time to the test_bm_librbdfio.py Signed-off-by: lee-j-sanders --- tests/test_bm_librbdfio.py | 14 +++++++++++++- tools/baseline.json | 2 ++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_bm_librbdfio.py b/tests/test_bm_librbdfio.py index fb410261..c0da60f3 100644 --- a/tests/test_bm_librbdfio.py +++ b/tests/test_bm_librbdfio.py @@ -16,7 +16,7 @@ class TestBenchmarklibrbdfio(unittest.TestCase): cl_name = "tools/invariant.yaml" bl_name = "tools/baseline.json" bl_json = {} - bl_md5 = 'e6b6fcd2be74bd08939c64a249ab2125' + bl_md5 = '84dd2f3a66eab442cc3825e0d57a9e3f' md5_returned = None @classmethod @@ -40,6 +40,12 @@ def test_valid_baseline(self): """ Verify the baseline has not been compromised """ self.assertEqual( self.bl_md5, str(self.md5_returned) ) + def test_valid__ioddepth_per_volume(self): + """ Basic sanity attribute identity _ioddepth_per_volume check""" + b = benchmarkfactory.get_object(self.archive_dir, + self.cluster, 'librbdfio', self.iteration) + self.assertEqual(self.bl_json['librbdfio']['_ioddepth_per_volume'], b.__dict__['_ioddepth_per_volume']) + def test_valid_archive_dir(self): """ Basic sanity attribute identity archive_dir check""" b = benchmarkfactory.get_object(self.archive_dir, @@ -208,6 +214,12 @@ def test_valid_pool_profile(self): self.cluster, 'librbdfio', self.iteration) self.assertEqual(self.bl_json['librbdfio']['pool_profile'], b.__dict__['pool_profile']) + def test_valid_precond_time(self): + """ Basic sanity attribute identity precond_time check""" + b = benchmarkfactory.get_object(self.archive_dir, + self.cluster, 'librbdfio', self.iteration) + self.assertEqual(self.bl_json['librbdfio']['precond_time'], b.__dict__['precond_time']) + def test_valid_prefill_vols(self): """ Basic sanity attribute identity prefill_vols check""" b = benchmarkfactory.get_object(self.archive_dir, diff --git a/tools/baseline.json b/tools/baseline.json index ae5db2b0..461b315b 100644 --- a/tools/baseline.json +++ b/tools/baseline.json @@ -727,6 +727,7 @@ "vol_size": 58982.4 }, "librbdfio": { + "_ioddepth_per_volume": {}, "acceptable": {}, "archive_dir": "/tmp/results/00000000/id-83a653b5", "base_run_dir": "/tmp/cbt.XYZ/00000000/LibrbdFio", @@ -865,6 +866,7 @@ "pgs": 2048, "pool_name": "cbt-librbdfio", "pool_profile": "default", + "precond_time": null, "prefill_vols": { "blocksize": "4M", "numjobs": "1"