diff --git a/benchmarking/mod/bblock/fiorbd.py b/benchmarking/mod/bblock/fiorbd.py index f8299a8..bcbdeac 100644 --- a/benchmarking/mod/bblock/fiorbd.py +++ b/benchmarking/mod/bblock/fiorbd.py @@ -1,9 +1,11 @@ from ..benchmark import * from collections import OrderedDict import itertools +import sys class FioRbd(Benchmark): def load_parameter(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).load_parameter() self.cluster["rbdlist"] = self.get_rbd_list(self.benchmark["poolname"]) if len(self.cluster["rbdlist"]) < int(self.all_conf_data.get("rbd_volume_count")): @@ -15,6 +17,7 @@ def load_parameter(self): self.testjob_distribution(disk_num_per_client, instance_list) def prepare_images(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] controller = self.cluster["head"] rbd_count = self.all_conf_data.get("rbd_volume_count") @@ -23,7 +26,7 @@ def prepare_images(self): if rbd_count and rbd_size: super(self.__class__, self).create_image(rbd_count, rbd_size, self.benchmark["poolname"]) else: - common.printout("ERROR","need to set rbd_volume_count and volune_size in all.conf") + common.printout("ERROR","need to set rbd_volume_count and volune_size in all.conf",log_level="LVL1") #start to init dest_dir = self.cluster["tmp_dir"] disk_num_per_client = self.cluster["disk_num_per_client"] @@ -40,11 +43,11 @@ def prepare_images(self): common.printout("LOG","%d FIO Jobs starts on %s" % (len(self.cluster["testjob_distribution"][client]), client)) time.sleep(1) if not self.check_fio_pgrep(clients, fio_job_num_total): - common.printout("ERROR","Failed to start FIO process") + common.printout("ERROR","Failed to start FIO process",log_level="LVL1") common.pdsh(user, clients, "killall -9 fio", option = "check_return") raise KeyboardInterrupt if not fio_job_num_total: - common.printout("ERROR","Planed to run 0 Fio Job, please check all.conf") + common.printout("ERROR","Planed to run 0 Fio Job, please check all.conf",log_level="LVL1") raise KeyboardInterrupt common.printout("LOG","Wait rbd initialization stop") @@ -58,6 +61,7 @@ def prepare_images(self): common.printout("LOG","rbd initialization finished") def prepare_result_dir(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") #1. prepare result dir self.get_runid() self.benchmark["section_name"] = "fiorbd-%s-%s-qd%s-%s-%s-%s-%s" % (self.benchmark["iopattern"], self.benchmark["block_size"], self.benchmark["qd"], self.benchmark["volume_size"],self.benchmark["rampup"], self.benchmark["runtime"], self.benchmark["poolname"]) @@ -66,6 +70,7 @@ def prepare_result_dir(self): super(self.__class__, self).prepare_result_dir() def prerun_check(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).prerun_check() #1. check is vclient alive user = self.cluster["user"] @@ -74,15 +79,16 @@ def prerun_check(self): common.printout("LOG","check if FIO rbd engine installed") res = common.pdsh(user, nodes, "fio -enghelp | grep rbd", option = "check_return") if res and not res[0]: - common.printout("ERROR","FIO rbd engine not installed") + common.printout("ERROR","FIO rbd engine not installed",log_level="LVL1") sys.exit() planed_space = str(len(self.cluster["rbdlist"]) * int(self.volume_size)) + "MB" common.printout("LOG","check if rbd volume fully initialized") if not self.check_rbd_init_completed(planed_space): - common.printout("WARNING","rbd volume initialization has not be done") + common.printout("WARNING","rbd volume initialization has not be done",log_level="LVL1") self.prepare_images() def run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).run() user = self.cluster["user"] waittime = int(self.benchmark["runtime"]) + int(self.benchmark["rampup"]) @@ -98,16 +104,17 @@ def run(self): self.chkpoint_to_log("fio start") time.sleep(1) if not self.check_fio_pgrep(self.benchmark["distribution"].keys(), fio_job_num_total): - common.printout("ERROR","Failed to start FIO process") + common.printout("ERROR","Failed to start FIO process",log_level="LVL1") raise KeyboardInterrupt if not fio_job_num_total: - common.printout("ERROR","Planned to start 0 FIO process, seems to be an error") + common.printout("ERROR","Planned to start 0 FIO process, seems to be an error",log_level="LVL1") raise KeyboardInterrupt common.printout("LOG","%d FIO Jobs starts on %s" % ( fio_job_num_total, str(self.benchmark["distribution"].keys()))) while self.check_fio_pgrep(self.benchmark["distribution"].keys()): time.sleep(5) def prepare_run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).prepare_run() user = self.cluster["user"] dest_dir = self.cluster["tmp_dir"] @@ -117,6 +124,7 @@ def prepare_run(self): self.cleanup() def wait_workload_to_stop(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") common.printout("LOG","Waiting Workload to complete its work") user = self.cluster["user"] stop_flag = 0 @@ -136,12 +144,14 @@ def wait_workload_to_stop(self): common.printout("LOG","Workload completed") def stop_workload(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] nodes = self.benchmark["distribution"].keys() common.pdsh(user, nodes, "killall -9 fio", option = "check_return") self.chkpoint_to_log("fio stop") def generate_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") fio_capping = self.all_conf_data.get('fio_capping') enable_zipf = self.all_conf_data.get('enable_zipf') fio_randrepeat = self.all_conf_data.get('fio_randrepeat') @@ -201,7 +211,8 @@ def generate_benchmark_cases(self, testcase): try: rwmixread = self.all_conf_data.get(key_name) fio_template.append(" rwmixread=%s" % rwmixread) - except: + except Exception,e: + common.printout("LOG"," ERR_MSG:%s"%(self.__class__.__name__,sys._getframe().f_code.co_name,e),log_level="LVL2") pass fio_list.extend(fio_template) @@ -210,6 +221,7 @@ def generate_benchmark_cases(self, testcase): return True def parse_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") p = testcase testcase_dict = { "instance_number":p[0], "volume_size":p[1], "iopattern":p[2], @@ -223,6 +235,7 @@ def parse_benchmark_cases(self, testcase): return testcase_dict def archive(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).archive() user = self.cluster["user"] head = self.cluster["head"] diff --git a/benchmarking/mod/bblock/qemurbd.py b/benchmarking/mod/bblock/qemurbd.py index 76f2f1f..98b7047 100644 --- a/benchmarking/mod/bblock/qemurbd.py +++ b/benchmarking/mod/bblock/qemurbd.py @@ -1,9 +1,11 @@ from ..benchmark import * from collections import OrderedDict import itertools +import sys class QemuRbd(Benchmark): def load_parameter(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).load_parameter() self.cluster["vclient"] = self.all_conf_data.get_list("list_vclient") @@ -13,6 +15,7 @@ def load_parameter(self): self.testjob_distribution(disk_num_per_client, self.instance_list) def prepare_result_dir(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") #1. prepare result dir self.get_runid() vdisk = self.benchmark["vdisk"].split('/')[-1] @@ -21,12 +24,13 @@ def prepare_result_dir(self): self.cluster["dest_dir"] = "/%s/%s" % (self.cluster["dest_dir"], self.benchmark["dirname"]) if common.remote_dir_exist( self.cluster["user"], self.cluster["head"], self.cluster["dest_dir"] ): - common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"])) + common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"]),log_level="LVL1") sys.exit() common.pdsh(self.cluster["user"] ,["%s" % (self.cluster["head"])], "mkdir -p %s" % (self.cluster["dest_dir"])) def prepare_images(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] dest_dir = self.cluster["tmp_dir"] controller = self.cluster["head"] @@ -35,7 +39,7 @@ def prepare_images(self): if rbd_count and rbd_size: super(self.__class__, self).create_image(rbd_count, rbd_size, 'rbd') else: - common.printout("ERROR","need to set rbd_volume_count and volune_size in all.conf") + common.printout("ERROR","need to set rbd_volume_count and volune_size in all.conf",log_level="LVL1") #create image xml common.printout("LOG","create rbd volume vm attach xml") @@ -62,11 +66,11 @@ def prepare_images(self): nodes.extend(vclients) time.sleep(1) if not self.check_fio_pgrep(nodes, fio_job_num_total, check_type = "nodenum"): - common.printout("ERROR","Failed to start FIO process") + common.printout("ERROR","Failed to start FIO process",log_level="LVL1") common.pdsh(user, nodes, "killall -9 fio", option = "check_return") raise KeyboardInterrupt if not fio_job_num_total: - common.printout("ERROR","Planed to run 0 Fio Job, please check all.conf") + common.printout("ERROR","Planed to run 0 Fio Job, please check all.conf",log_level="LVL1") raise KeyboardInterrupt common.printout("LOG","FIO Jobs starts on %s" % (nodes)) @@ -76,10 +80,12 @@ def prepare_images(self): while self.check_fio_pgrep(nodes): time.sleep(5) except KeyboardInterrupt: + common.printout("WARNING","Caught KeyboardInterrupt, stop check fio pgrep.",log_level="LVL1") common.pdsh(user, nodes, "killall -9 fio", option = "check_return") common.printout("LOG","rbd initialization finished") def prerun_check(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).prerun_check() #1. check is vclient alive user = self.cluster["user"] @@ -110,6 +116,7 @@ def prerun_check(self): common.pdsh(user, nodes, "mpstat") def attach_images(self, to_attach_dict = None): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] vdisk = self.benchmark["vdisk"] dest_dir = self.cluster["tmp_dir"] @@ -125,6 +132,7 @@ def attach_images(self, to_attach_dict = None): common.pdsh(user, [client], "cd %s/vdbs; virsh attach-device %s %s.xml" % (dest_dir, node, node), except_returncode=1) def detach_images(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] vdisk = self.benchmark["vdisk"] tmp_vdisk = re.search('/dev/(\w+)',vdisk) @@ -139,6 +147,7 @@ def detach_images(self): common.pdsh(user, [client], "virsh detach-disk %s %s" % (node, vdisk_suffix), except_returncode=1) def run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).run() user = self.cluster["user"] waittime = int(self.benchmark["runtime"]) + int(self.benchmark["rampup"]) @@ -164,10 +173,10 @@ def run(self): self.chkpoint_to_log("fio start") time.sleep(5) if not self.check_fio_pgrep(nodes, fio_job_num_total, check_type = "nodenum"): - common.printout("ERROR","Failed to start FIO process") + common.printout("ERROR","Failed to start FIO process",log_level="LVL1") raise KeyboardInterrupt if not fio_job_num_total: - common.printout("ERROR","Planned to start 0 FIO process, seems to be an error") + common.printout("ERROR","Planned to start 0 FIO process, seems to be an error",log_level="LVL1") raise KeyboardInterrupt common.printout("LOG","FIO Jobs starts on %s" % str(nodes)) @@ -176,6 +185,7 @@ def run(self): time.sleep(5) def chkpoint_to_log(self, log_str): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).chkpoint_to_log(log_str) dest_dir = self.cluster["tmp_dir"] user = self.cluster["user"] @@ -185,6 +195,7 @@ def chkpoint_to_log(self, log_str): common.pdsh(user, nodes, "echo `date +%s`' %s' >> %s/`hostname`_process_log.txt" % ('%s', log_str, dest_dir)) def cleanup(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).cleanup() #1. clean the tmp res dir user = self.cluster["user"] @@ -195,6 +206,7 @@ def cleanup(self): common.pdsh(user, nodes, "rm -f %s/*.log" % self.cluster["tmp_dir"]) def prepare_run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).prepare_run() user = self.cluster["user"] dest_dir = self.cluster["tmp_dir"] @@ -205,6 +217,7 @@ def prepare_run(self): self.cleanup() def wait_workload_to_stop(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") common.printout("LOG","Waiting Workload to complete its work") nodes = [] for client in self.benchmark["distribution"]: @@ -219,6 +232,7 @@ def wait_workload_to_stop(self): common.printout("LOG","Workload completed") def stop_data_collecters(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).stop_data_collecters() user = self.cluster["user"] dest_dir = self.cluster["tmp_dir"] @@ -230,6 +244,7 @@ def stop_data_collecters(self): common.pdsh(user, nodes, "killall -9 top; echo `date +%s`' top stop' >> %s/`hostname`_process_log.txt" % ('%s', dest_dir), option = "check_return") def stop_workload(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] for client in self.benchmark["distribution"]: nodes = self.benchmark["distribution"][client] @@ -239,9 +254,10 @@ def stop_workload(self): try: self.detach_images() except KeyboardInterrupt: - common.printout("WARNING","Caught KeyboardInterrupt, stop detaching") + common.printout("WARNING","Caught KeyboardInterrupt, stop detaching",log_level="LVL1") def archive(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).archive() user = self.cluster["user"] head = self.cluster["head"] @@ -256,6 +272,7 @@ def archive(self): common.bash("mkdir -p %s/conf/fio_errorlog/;find %s/raw/ -name '*_fio_errorlog.txt' | while read file; do cp $file %s/conf/fio_errorlog/;done" % (dest_dir, dest_dir, dest_dir)) def generate_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") fio_capping = self.all_conf_data.get('fio_capping') enable_zipf = self.all_conf_data.get ('enable_zipf') fio_randrepeat = self.all_conf_data.get('fio_randrepeat') @@ -314,7 +331,8 @@ def generate_benchmark_cases(self, testcase): try: rwmixread = self.all_conf_data.get(key_name) fio_template.append(" rwmixread=%s" % rwmixread) - except: + except Exception,e: + common.printout("LOG"," ERR_MSG:%s"%(self.__class__.__name__,sys._getframe().f_code.co_name,e),log_level="LVL2") pass fio_list.extend(fio_template) @@ -323,6 +341,7 @@ def generate_benchmark_cases(self, testcase): return True def parse_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") p = testcase testcase_dict = { "instance_number":p[0], "volume_size":p[1], "iopattern":p[2], diff --git a/benchmarking/mod/bblock/vdbench.py b/benchmarking/mod/bblock/vdbench.py index a83c9ca..8b8b7e6 100644 --- a/benchmarking/mod/bblock/vdbench.py +++ b/benchmarking/mod/bblock/vdbench.py @@ -1,9 +1,11 @@ from ..benchmark import * from collections import OrderedDict import itertools +import sys class VdBench(Benchmark): def __init__(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") print "try try" self.bench_type = "vdbench" super(self.__class__, self).__init__() @@ -15,6 +17,7 @@ def __init__(self): common.printout("LOG","bench dir: %s, format output dir: %s, result dir: %s" % (self.cluster["bench_dir"], self.cluster["format_output_dir"], self.cluster["result_dir"])) def load_parameter(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).load_parameter() self.custom_script = self.all_conf_data.get("custom_script", True ) self.cluster["vclient"] = self.all_conf_data.get_list("list_vclient") @@ -25,6 +28,7 @@ def load_parameter(self): self.testjob_distribution(disk_num_per_client, self.instance_list) def cal_run_job_distribution(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") number = int(self.benchmark["instance_number"]) client_total = len(self.cluster["client"]) # Assume number is always 50 here @@ -40,6 +44,7 @@ def cal_run_job_distribution(self): self.cluster["nodes_distribution"] = nodes def prepare_result_dir(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") #1. prepare result dir self.get_runid() vdisk = self.benchmark["vdisk"].split('/')[-1] @@ -49,12 +54,13 @@ def prepare_result_dir(self): res = common.pdsh(self.cluster["user"],["%s"%(self.cluster["head"])],"test -d %s" % (self.cluster["dest_dir"]), option = "check_return") if not res[1]: - common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"])) + common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"]),log_level="LVL1") sys.exit() common.pdsh(self.cluster["user"] ,["%s" % (self.cluster["head"])], "mkdir -p %s" % (self.cluster["dest_dir"])) def cleanup(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).cleanup() #1. clean the tmp res dir user = self.cluster["user"] @@ -63,6 +69,7 @@ def cleanup(self): common.pdsh(user, nodes, "rm -rf %s/*" % self.cluster["result_dir"]) def check_run_success(self, check_file, max_time, run_type="format"): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] nodes = self.cluster["nodes_distribution"] cur_check = 0 @@ -78,11 +85,12 @@ def check_run_success(self, check_file, max_time, run_type="format"): common.printout("LOG", "checking done,all nodes execute %s completely" % run_type) return cur_check += 1 - common.printout("ERROR","Checking run in %s failed" % check_file) + common.printout("ERROR","Checking run in %s failed" % check_file,log_level="LVL1") stdout, stderr = common.pdsh(user, nodes, "grep -q completed %s; if [ $? -ne 0 ]; then echo Run is not completed successfully; fi" % check_file, option="check_return") sys.exit() def format_run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") common.printout("LOG", "Start Formatting!") user = self.cluster["user"] nodes = self.cluster["nodes_distribution"] @@ -91,6 +99,7 @@ def format_run(self): self.check_run_success(check_file, 100) def prepare_run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).prepare_run() user = self.cluster["user"] dest_dir = self.cluster["tmp_dir"] @@ -100,6 +109,7 @@ def prepare_run(self): #Add new method to check vdbench def check_vdbench_pgrep(self, nodes, vdbench_node_num = 1, check_type="jobnum"): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] stdout, stderr = common.pdsh(user, nodes, "pgrep -x java", option = "check_return") res = common.format_pdsh_return(stdout) @@ -123,6 +133,7 @@ def check_vdbench_pgrep(self, nodes, vdbench_node_num = 1, check_type="jobnum"): #Updated wait_workload_to_stop and stop_workload def wait_workload_to_stop(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") common.printout("LOG","Waiting Workload to complete its work") nodes = self.cluster["nodes_distribution"] max_check_times = 30 @@ -136,6 +147,7 @@ def wait_workload_to_stop(self): #Add Stop_data_collecters def stop_data_collecters(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).stop_data_collecters() user = self.cluster["user"] nodes = self.cluster["nodes_distribution"] @@ -145,6 +157,7 @@ def stop_data_collecters(self): common.pdsh(user, nodes, "killall -9 top", option = "check_return") def chkpoint_to_log(self, log_str): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).chkpoint_to_log(log_str) dest_dir = self.cluster["tmp_dir"] user = self.cluster["user"] @@ -152,6 +165,7 @@ def chkpoint_to_log(self, log_str): common.pdsh(user, nodes, "echo `date +%s`' %s' >> %s/`hostname`_process_log.txt" % ('%s', log_str, dest_dir)) def stop_workload(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] nodes = self.cluster["nodes_distribution"] common.pdsh(user, nodes, "killall -9 java", option = "check_return") @@ -160,10 +174,11 @@ def stop_workload(self): try: self.detach_images() except KeyboardInterrupt: - common.printout("WARNING","Caught KeyboardInterrupt, stop detaching") + common.printout("WARNING","Caught KeyboardInterrupt, stop detaching",log_level="LVL1") #end def generate_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") io_pattern = testcase["iopattern"] block_size = testcase["block_size"] queue_depth = testcase["qd"] @@ -188,7 +203,7 @@ def generate_benchmark_cases(self, testcase): read_percentage = int(str2[1]); if int(re.findall(r"\d", block_size)[0]) * depth * width > int(re.findall(r"\d", rbd_volume_size)[0]) * 1024 * 1024: - common.printout("ERROR","Files total size is too big, bigger than volume size!") + common.printout("ERROR","Files total size is too big, bigger than volume size!",log_level="LVL1") raise KeyboardInterrupt if io_pattern in ["randread", "randwrite", "randrw"]: @@ -223,6 +238,7 @@ def generate_benchmark_cases(self, testcase): return True def parse_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") p = testcase testcase_dict = { "instance_number":p[0], "volume_size":p[1], "iopattern":p[2], @@ -232,7 +248,7 @@ def parse_benchmark_cases(self, testcase): if len(p) == 10: testcase_dict["description"] = p[9] elif len(p) > 10: - common.printout("ERROR","Too much columns found for test case ") + common.printout("ERROR","Too much columns found for test case ",log_level="LVL1") sys.exit() else: testcase_dict["description"] = "" @@ -240,6 +256,7 @@ def parse_benchmark_cases(self, testcase): return testcase_dict def run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).run() user = self.cluster["user"] nodes = self.cluster["nodes_distribution"] @@ -259,6 +276,7 @@ def run(self): self.check_run_success(check_file, 100, "test") def archive(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).archive() user = self.cluster["user"] nodes = self.cluster["nodes_distribution"] @@ -272,6 +290,7 @@ def archive(self): def prepare_images(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] dest_dir = self.cluster["tmp_dir"] controller = self.cluster["head"] @@ -280,7 +299,7 @@ def prepare_images(self): if rbd_count and rbd_size: super(self.__class__, self).create_image(rbd_count, rbd_size, 'rbd') else: - common.printout("ERROR","need to set rbd_volume_count and volune_size in all.conf") + common.printout("ERROR","need to set rbd_volume_count and volune_size in all.conf",log_level="LVL1") #create image xml common.printout("LOG","create rbd volume vm attach xml") @@ -299,6 +318,7 @@ def prepare_images(self): common.printout("LOG","rbd initialization finished") def prepare_case(self, user, nodes): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") stdout, stderr = common.pdsh(user, nodes, "test -d %s" % self.cluster["bench_dir"], option="check_return") if stderr: common.printout("LOG","Distribute vdbench benchmark execution file") @@ -312,6 +332,7 @@ def prepare_case(self, user, nodes): common.scp(user, node, "../conf/vdbench_test.cfg", "%s" % self.cluster["bench_dir"]) def prerun_check(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).prerun_check() #1. check is vclient alive user = self.cluster["user"] @@ -341,6 +362,7 @@ def prerun_check(self): common.pdsh(user, nodes, "killall -9 java", option = "check_return") def attach_images(self, to_attach_dict = None): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] vdisk = self.benchmark["vdisk"] dest_dir = self.cluster["tmp_dir"] @@ -357,6 +379,7 @@ def attach_images(self, to_attach_dict = None): common.pdsh(user, [node], "mount | grep /dev/vdb1; if [ $? ne 0]; then parted -s -a optimal /dev/vdb mklabel gpt -- mkpart primary ext4 1 100%; mkfs -t ext4 /dev/vdb1; mount /dev/vdb1 /mnt; fi") def detach_images(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] vdisk = self.benchmark["vdisk"] tmp_vdisk = re.search('/dev/(\w+)',vdisk) diff --git a/benchmarking/mod/bcephfs/fiocephfs.py b/benchmarking/mod/bcephfs/fiocephfs.py index 075b38f..909c002 100644 --- a/benchmarking/mod/bcephfs/fiocephfs.py +++ b/benchmarking/mod/bcephfs/fiocephfs.py @@ -1,13 +1,16 @@ from ..benchmark import * from collections import OrderedDict import itertools +import sys class FioCephFS(Benchmark): def load_parameter(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).load_parameter() self.cluster["fiocephfs_dir"] = self.all_conf_data.get("fio_for_libcephfs_dir") def prepare_result_dir(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") #1. prepare result dir self.get_runid() self.benchmark["section_name"] = "%s-%s-qd%s-%s-%s-%s-fiocephfs" % (self.benchmark["iopattern"], self.benchmark["block_size"], self.benchmark["qd"], self.benchmark["volume_size"],self.benchmark["rampup"], self.benchmark["runtime"]) @@ -16,11 +19,12 @@ def prepare_result_dir(self): res = common.pdsh(self.cluster["user"],["%s"%(self.cluster["head"])],"test -d %s" % (self.cluster["dest_dir"]), option = "check_return") if not res[1]: - common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"])) + common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"]),log_level="LVL1") sys.exit() common.pdsh(self.cluster["user"] ,["%s" % (self.cluster["head"])], "mkdir -p %s" % (self.cluster["dest_dir"])) def prerun_check(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") #1. check is vclient alive user = self.cluster["user"] nodes = self.benchmark["distribution"].keys() @@ -28,11 +32,12 @@ def prerun_check(self): common.pdsh(user, nodes, "%s/fio -v" % fio_dir ) res = common.pdsh(user, nodes, "%s/fio -enghelp | grep cephfs" % fio_dir, option = "check_return") if res and not res[0]: - common.printout("ERROR","FIO cephfs engine not installed") + common.printout("ERROR","FIO cephfs engine not installed",log_level="LVL1") print "You can get the source code of fiocephfs from: https://github.com/noahdesu/fio.git" sys.exit() def run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).run() user = self.cluster["user"] waittime = int(self.benchmark["runtime"]) + int(self.benchmark["rampup"]) @@ -46,12 +51,13 @@ def run(self): time.sleep(1) res = common.pdsh(user, [client], "pgrep -x fio", option = "check_return") if res and not len(res[0].split('\n')) >= len(self.benchmark["distribution"][client]): - common.printout("ERROR","Failed to start FIO process") + common.printout("ERROR","Failed to start FIO process",log_level="LVL1") raise KeyboardInterrupt common.printout("LOG","%d FIO Jobs starts on %s" % (len(self.benchmark["distribution"][client]), client)) time.sleep(waittime) def prepare_run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).prepare_run() user = self.cluster["user"] dest_dir = self.cluster["tmp_dir"] @@ -59,6 +65,7 @@ def prepare_run(self): common.scp(user, client, "../conf/fio.conf", dest_dir) def wait_workload_to_stop(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") common.printout("LOG","Waiting Workload to complete its work") user = self.cluster["user"] stop_flag = 0 @@ -78,14 +85,17 @@ def wait_workload_to_stop(self): common.printout("LOG","Workload completed") def stop_workload(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] nodes = self.benchmark["distribution"].keys() common.pdsh(user, nodes, "killall -9 fio", option = "check_return") def testjob_distribution(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") pass def cal_run_job_distribution(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") number = int(self.benchmark["instance_number"]) client_total = len(self.cluster["client"]) instance_per_client = number/client_total + (number % client_total > 0 ) @@ -94,6 +104,7 @@ def cal_run_job_distribution(self): self.benchmark["distribution"][client] = range(instance_per_client) def generate_benchmark_cases(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") engine = self.all_conf_data.get_list('benchmark_engine') if "fiocephfs" not in engine: return [[],[]] @@ -139,12 +150,14 @@ def generate_benchmark_cases(self): try: rwmixread = self.all_conf_data.get('rwmixread') fio_template.append(" rwmixread=%s" % rwmixread) - except: + except Exception,e: + common.printout("LOG"," ERR_MSG:%s"%(self.__class__.__name__,sys._getframe().f_code.co_name,e),log_level="LVL2") pass fio_list.extend(fio_template) return [testcase_list, fio_list] def parse_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") p = testcase testcase_dict = { "instance_number":p[0], "volume_size":p[1], "iopattern":p[2], diff --git a/benchmarking/mod/benchmark.py b/benchmarking/mod/benchmark.py index 410779e..efffd4e 100644 --- a/benchmarking/mod/benchmark.py +++ b/benchmarking/mod/benchmark.py @@ -10,6 +10,7 @@ class Benchmark(object): def __init__(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") self.runid = 0 self.all_conf_data = config.Config(lib_path+"/conf/all.conf") self.benchmark = {} @@ -17,7 +18,9 @@ def __init__(self): self.pwd = os.path.abspath(os.path.join('..')) def go(self, testcase, tuning): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") try: + common.clean_process_log(os.path.join(self.pwd,'log')) cancel_file = open("../conf/execute_op_type.conf","r") execute_op_type = cancel_file.read().strip("\n") if execute_op_type != "cancel_all": @@ -32,7 +35,7 @@ def go(self, testcase, tuning): self.set_runid() if not self.generate_benchmark_cases(self.benchmark): - common.printout("ERROR", "Failed to generate Fio/cosbench configuration file.") + common.printout("ERROR", "Failed to generate Fio/cosbench configuration file.",log_level="LVL1") sys.exit() self.benchmark["tuning_section"] = tuning @@ -50,7 +53,7 @@ def go(self, testcase, tuning): except KeyboardInterrupt: interrupted_flag = True self.setStatus("Interrupted") - common.printout("WARNING","Caught Signal to Cancel this run, killing Workload now, pls wait") + common.printout("WARNING","Caught Signal to Cancel this run, killing Workload now, pls wait",log_level="LVL1") self.real_runtime = time.time() - test_start_time self.stop_workload() self.stop_data_collecters() @@ -66,11 +69,12 @@ def go(self, testcase, tuning): try: analyzer.main(['--path', self.cluster["dest_dir"], 'process_data']) except: - common.printout("ERROR","analyzer failed, pls try cd analyzer; python analyzer.py --path %s process_data " % self.cluster["dest_dir"]) + common.printout("ERROR","analyzer failed, pls try cd analyzer; python analyzer.py --path %s process_data " % self.cluster["dest_dir"],log_level="LVL1") except: - common.printout("ERROR","The test has been stopped.") + common.printout("ERROR","The test has been stopped.",log_level="LVL1") def create_image(self, volume_count, volume_size, poolname): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] controller = self.cluster["head"] rbd_list = self.get_rbd_list(poolname) @@ -84,11 +88,12 @@ def create_image(self, volume_count, volume_size, poolname): common.printout("LOG","%d RBD Image Created" % need_to_create) def get_rbd_list(self, poolname): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] controller = self.cluster["head"] stdout, stderr = common.pdsh(user, [controller], "rbd ls -p %s" % poolname, option="check_return") if stderr: - common.printout("ERROR","unable get rbd list, return msg: %s" % stderr) + common.printout("ERROR","unable get rbd list, return msg: %s" % stderr,log_level="LVL1") #sys.exit() res = common.format_pdsh_return(stdout) if res != {}: @@ -98,6 +103,7 @@ def get_rbd_list(self, poolname): return rbd_list_tmp def after_run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") #1. check workload stoped self.wait_workload_to_stop() @@ -114,10 +120,12 @@ def after_run(self): common.pdsh(user, nodes, "cat /proc/interrupts > %s/`hostname`_interrupts_end.txt" % (dest_dir)) def prepare_run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") # self.stop_workload() self.stop_data_collecters() def cleanup(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] nodes = self.cluster["osd"] dest_dir = self.cluster["tmp_dir"] @@ -127,6 +135,7 @@ def cleanup(self): common.printout("LOG","Cleaned original data under %s " % dest_dir) def prerun_check(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] nodes = self.cluster["osd"] common.printout("LOG","Prerun_check: check if sysstat installed " % nodes) @@ -149,6 +158,7 @@ def prerun_check(self): common.pdsh(user, nodes, "lttng -V") def run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") waittime = int(self.benchmark["runtime"]) + int(self.benchmark["rampup"]) common.printout("LOG","This test will run %d secs until finish." % waittime) @@ -238,18 +248,21 @@ def run(self): common.pdsh(user, nodes, "echo `date +%s`' perfcounter start' >> %s/`hostname`_process_log.txt; bash %s/ceph_admin.bash; echo `date +%s`' perfcounter stop' >> %s/`hostname`_process_log.txt; rm -rf %s/ceph_admin.bash" % ('%s', dest_dir, dest_dir, '%s', dest_dir, dest_dir), option="force") def archive(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] head = self.cluster["head"] dest_dir = self.cluster["dest_dir"] #collect all.conf try: common.bash("mkdir -p %s/conf" % (dest_dir)) + common.bash("mkdir -p %s/log" % (dest_dir)) common.cp("%s/conf/all.conf" % self.pwd, "%s/conf/" % dest_dir) common.cp("%s/conf/%s" % (self.pwd, common.cetune_log_file), "%s/conf/" % dest_dir) common.cp("%s/conf/%s" % (self.pwd, common.cetune_error_file), "%s/conf/" % dest_dir) common.bash("rm -f %s/conf/%s" % (self.pwd, common.cetune_log_file)) common.bash("rm -f %s/conf/%s" % (self.pwd, common.cetune_error_file)) - except: + except Exception,e: + common.printout("LOG"," ERR_MSG:%s"%(self.__class__.__name__,sys._getframe().f_code.co_name,e),log_level="LVL2") pass #collect tuner.yaml worksheet = common.load_yaml_conf("%s/conf/tuner.yaml" % self.pwd) @@ -283,7 +296,11 @@ def archive(self): with open("%s/real_runtime.txt" % dest_dir, "w") as f: f.write(str(int(self.real_runtime))) + common.bash("cp %s/log/* %s/log/" %(self.pwd,dest_dir)) + common.bash("rm -f %s/log/*" %self.pwd) + def stop_data_collecters(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") #2. clean running process user = self.cluster["user"] nodes = self.cluster["osd"] @@ -315,24 +332,29 @@ def stop_data_collecters(self): common.pdsh(user, nodes, "cat /proc/interrupts > %s/`hostname`_interrupts_end.txt; echo `date +%s`' interrupt stop' >> %s/`hostname`_process_log.txt" % (dest_dir, '%s', dest_dir)) def tuning(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") pass def get_runid(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") if self.runid != 0: return try: with open("%s/.run_number" % lib_path, "r") as f: self.runid = int(f.read()) - except: + except Exception,e: + common.printout("LOG"," ERR_MSG:%s"%(self.__class__.__name__,sys._getframe().f_code.co_name,e),log_level="LVL2") pass def set_runid(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") if not self.runid: self.get_runid() with open("%s/.run_number" % lib_path, "w") as f: f.write(str(self.runid+1)) def testjob_distribution(self, disk_num_per_client, instance_list): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") start_vclient_num = 0 client_num = 0 self.cluster["testjob_distribution"] = {} @@ -344,6 +366,7 @@ def testjob_distribution(self, disk_num_per_client, instance_list): client_num += 1 def cal_run_job_distribution(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") number = int(self.benchmark["instance_number"]) client_total = len(self.cluster["client"]) if (number % client_total) > 0: @@ -364,6 +387,7 @@ def cal_run_job_distribution(self): remained_instance_num = remained_instance_num - volume_num_upper_bound def check_fio_pgrep(self, nodes, fio_node_num = 1, check_type="jobnum"): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] stdout, stderr = common.pdsh(user, nodes, "pgrep -x fio", option = "check_return") res = common.format_pdsh_return(stdout) @@ -382,16 +406,17 @@ def check_fio_pgrep(self, nodes, fio_node_num = 1, check_type="jobnum"): if check_type == "jobnum": common.printout("WARNING","Expecting %d nodes run fio, detect %d node runing" % (fio_node_num, fio_running_job_num)) return False - common.printout("WARNING","Detect no fio job runing" % (fio_node_num, fio_running_node_num)) + common.printout("WARNING","Detect no fio job runing" % (fio_node_num, fio_running_node_num),log_level="LVL1") return False def check_rbd_init_completed(self, planed_space, pool_name="rbd"): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] controller = self.cluster["head"] stdout, stderr = common.pdsh(user, [controller], "ceph df | grep %s | awk '{print $3}'" % pool_name, option = "check_return") res = common.format_pdsh_return(stdout) if controller not in res: - common.printout("ERROR","cannot get ceph space, seems to be a dead error") + common.printout("ERROR","cannot get ceph space, seems to be a dead error",log_level="LVL1") #sys.exit() cur_space = common.size_to_Kbytes(res[controller]) planned_space = common.size_to_Kbytes(planed_space) @@ -402,16 +427,20 @@ def check_rbd_init_completed(self, planed_space, pool_name="rbd"): return True def create_admin_daemon_dump_script(self, dest_dir, total_count, monitor_interval): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") with open('ceph_admin.bash', 'w') as f: f.write("for i in `seq 1 %d`; do find /var/run/ceph -name '*osd*asok' | while read path; do filename=`echo $path | awk -F/ '{print $NF}'`;res_file=%s/`hostname`_${filename}.txt; echo `ceph --admin-daemon $path perf dump`, >> ${res_file} & done; sleep %s; done;" % (total_count, dest_dir, monitor_interval)) def generate_benchmark_cases(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") return [[],[]] def parse_benchmark_cases(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") pass def load_parameter(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") self.cluster["user"] = self.all_conf_data.get("user") self.cluster["head"] = self.all_conf_data.get("head") self.cluster["tmp_dir"] = self.all_conf_data.get("tmp_dir") @@ -432,6 +461,7 @@ def load_parameter(self): self.cluster["collector"] = self.all_conf_data.get_list("collector") def chkpoint_to_log(self, log_str): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") dest_dir = self.cluster["tmp_dir"] user = self.cluster["user"] nodes = [] @@ -440,6 +470,7 @@ def chkpoint_to_log(self, log_str): common.pdsh(user, nodes, "echo `date +%s`' %s' >> %s/`hostname`_process_log.txt" % ('%s', log_str, dest_dir)) def setStatus(self, status): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] head = self.cluster["head"] dest_dir = self.cluster["dest_dir"] @@ -447,9 +478,10 @@ def setStatus(self, status): common.bash("echo %s > %s/conf/status" % (status, dest_dir)) def prepare_result_dir(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") res = common.pdsh(self.cluster["user"],["%s"%(self.cluster["head"])],"test -d %s" % (self.cluster["dest_dir"]), option = "check_return") if not res[1]: - common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"])) + common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"]),log_level="LVL1") sys.exit() common.pdsh(self.cluster["user"] ,["%s" % (self.cluster["head"])], "mkdir -p %s" % (self.cluster["dest_dir"])) diff --git a/benchmarking/mod/bobject/cosbench.py b/benchmarking/mod/bobject/cosbench.py index fe91dc1..63bb1cc 100644 --- a/benchmarking/mod/bobject/cosbench.py +++ b/benchmarking/mod/bobject/cosbench.py @@ -8,9 +8,11 @@ import itertools from collections import OrderedDict from deploy import * +import sys class Cosbench(Benchmark): def load_parameter(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__,self).load_parameter() self.rgw={} self.rgw["rgw_server"]=self.all_conf_data.get_list("rgw_server") @@ -33,6 +35,7 @@ def load_parameter(self): self.cluster["client"] = self.cosbench["cosbench_driver"] def parse_conobj_script(self, string): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") m = re.findall("(\w{1})\((\d+),(\d+)\)", string) result = {} if m: @@ -43,6 +46,7 @@ def parse_conobj_script(self, string): return result def prepare_result_dir(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") self.benchmark["container"] = self.parse_conobj_script( self.benchmark["container"] ) self.benchmark["objecter"] = self.parse_conobj_script( self.benchmark["objecter"] ) @@ -52,11 +56,12 @@ def prepare_result_dir(self): self.cluster["dest_dir"] = "/%s/%s" % (self.cluster["dest_dir"], self.benchmark["dirname"]) if common.remote_dir_exist( self.cluster["user"], self.cluster["head"], self.cluster["dest_dir"] ): - common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"])) + common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"]),log_level="LVL1") sys.exit() common.pdsh(self.cluster["user"] ,["%s" % (self.cluster["head"])], "mkdir -p %s" % (self.cluster["dest_dir"])) def print_all_attributes(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") print "self.cosbench:" print self.cosbench print "self.rgw" @@ -65,6 +70,7 @@ def print_all_attributes(self): print self.cluster def produce_cosbench_config(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") controller_config = [] driver_config = [] conf = controller_config @@ -115,6 +121,7 @@ def produce_cosbench_config(self): common.scp(self.cluster["user"],node,file_path,self.cosbench["cosbench_folder"]+"/conf/driver.conf") def deploy_cosbench(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") common.printout('LOG', "Start to install cosbench on controllers and clients") cosbench_nodes = copy.deepcopy([self.cosbench["cosbench_controller"]]) cosbench_nodes = common.unique_extend(cosbench_nodes, self.cosbench["cosbench_driver"]) @@ -137,7 +144,7 @@ def deploy_cosbench(self): need_to_install_java_nodes.append(node) if not ( len(need_to_install_curl_nodes) + len(need_to_install_java_nodes) ): - common.printout("ERROR","Please install curl and openjdk-7-jre on below nodes, curl:%s, java:%s" % (str(need_to_install_curl_nodes), str(need_to_install_java_nodes))) + common.printout("ERROR","Please install curl and openjdk-7-jre on below nodes, curl:%s, java:%s" % (str(need_to_install_curl_nodes), str(need_to_install_java_nodes)),log_level="LVL1") sys.exit() count = 0 @@ -152,7 +159,7 @@ def deploy_cosbench(self): if int(stdout) == 0: common.printout("LOG", "Cosbench version %s downloaded successfully" % version) else: - common.printout("ERROR", "Cosbench version %s downloaded failed" % version) + common.printout("ERROR", "Cosbench version %s downloaded failed" % version,log_level="LVL1") sys.exit() for node in cosbench_nodes: @@ -165,6 +172,7 @@ def deploy_cosbench(self): common.printout('LOG', "Succeeded in installing cosbench on controllers and clients") def restart_cosbench_daemon(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") # distribute hosts file cosbench nodes nodes = [] nodes.append(self.cosbench["cosbench_controller"]) @@ -177,6 +185,7 @@ def restart_cosbench_daemon(self): stdout,stderr = common.pdsh(self.cluster["user"],[self.cosbench["cosbench_controller"]],"cd %s;chmod +x *.sh; ./stop-controller.sh; http_proxy=%s ./start-controller.sh" %(self.cosbench["cosbench_folder"],self.cosbench["proxy"]),'check_return') def prerun_check(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).prerun_check() cosbench_server = [] cosbench_server.append(self.cosbench["cosbench_controller"]) @@ -207,6 +216,7 @@ def prerun_check(self): self.restart_cosbench_daemon() def check_cosbench_runing(self, cosbench_controller, cosbench_driver): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") error_type = [] stdout,stderr = common.pdsh(self.cluster["user"], [cosbench_controller], "http_proxy=%s sh %s/cli.sh info 2>/dev/null | grep driver" % (self.cosbench["proxy"], self.cosbench["cosbench_folder"]),'check_return') res = common.format_pdsh_return(stdout) @@ -245,26 +255,28 @@ def check_cosbench_runing(self, cosbench_controller, cosbench_driver): return True def check_rgw_runing(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] stdout, stderr = common.pdsh( user, [self.cosbench["cosbench_controller"]], "http_proxy=%s curl -D - -H 'X-Auth-User: %s' -H 'X-Auth-Key: %s' %s" % (self.cosbench["proxy"], self.cosbench["auth_username"], self.cosbench["auth_password"], self.cosbench["auth_url"]), option = "check_return|console") if re.search('(refused|error)', stderr): - common.printout("ERROR","Cosbench connect to Radosgw Connection Failed") + common.printout("ERROR","Cosbench connect to Radosgw Connection Failed",log_level="LVL1") return False if re.search('Failed', stderr): - common.printout("ERROR","Cosbench connect to Radosgw Connection Failed") + common.printout("ERROR","Cosbench connect to Radosgw Connection Failed",log_level="LVL1") return False if re.search('Service Unavailable', stdout): - common.printout("ERROR","Cosbench connect to Radosgw Connection Failed") + common.printout("ERROR","Cosbench connect to Radosgw Connection Failed",log_level="LVL1") return False if re.search('Error', stdout): - common.printout("ERROR","Cosbench connect to Radosgw Connection Failed") + common.printout("ERROR","Cosbench connect to Radosgw Connection Failed",log_level="LVL1") return False if re.search("AccessDenied", stdout): - common.printout("[ERROR]","Cosbench connect to Radosgw Auth Failed") + common.printout("[ERROR]","Cosbench connect to Radosgw Auth Failed",log_level="LVL1") return False return True def check_cosbench_installed(self, cosbench_server): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] # check if cosbench installed installed = True @@ -276,6 +288,7 @@ def check_cosbench_installed(self, cosbench_server): return installed def stop_data_collectors(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).stop_data_collecters() user = self.cluster["user"] dest_dir = self.cluster["tmp_dir"] @@ -288,6 +301,7 @@ def stop_data_collectors(self): common.pdsh(user, nodes, "cat /proc/interrupts > %s/`hostname`_interrupts_stop.txt; echo `date +%s`' interrupt stop' >> %s/`hostname`_process_log.txt;" % (dest_dir, '%s', dest_dir)) def prepare_run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).prepare_run() self.stop_data_collectors() @@ -299,6 +313,7 @@ def prepare_run(self): self.cleanup() def run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).run() user = self.cluster["user"] waittime = int(self.benchmark["runtime"]) + int(self.benchmark["rampup"]) @@ -321,7 +336,7 @@ def run(self): stdout, stderr = common.pdsh( user, [self.cosbench["cosbench_controller"]], run_command, option="check_return") m = re.search('Accepted with ID:\s*(\w+)', stdout) if not m: - common.printout("ERROR",'Cosbench controller and driver run failed!') + common.printout("ERROR",'Cosbench controller and driver run failed!',log_level="LVL1") raise KeyboardInterrupt common.printout("LOG", "Cosbench job start, in cosbench scope the job num will be %s" % m.group(1)) @@ -332,6 +347,7 @@ def run(self): time.sleep(5) def chkpoint_to_log(self, log_str): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).chkpoint_to_log(log_str) dest_dir = self.cluster["tmp_dir"] user = self.cluster["user"] @@ -340,6 +356,7 @@ def chkpoint_to_log(self, log_str): common.pdsh(user, nodes, "echo `date +%s`' %s' >> %s/`hostname`_process_log.txt" % ('%s', log_str, dest_dir)) def check_cosbench_testjob_running(self, node, runid ): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] stdout, stderr = common.pdsh(user, [node], "http_proxy=%s sh %s/cli.sh info 2>/dev/null | grep PROCESSING | awk '{print $1}'" % (self.cosbench["proxy"], self.cosbench["cosbench_folder"]), option="check_return") res = common.format_pdsh_return(stdout) @@ -350,12 +367,14 @@ def check_cosbench_testjob_running(self, node, runid ): return False def stop_workload(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] controller = self.cosbench["cosbench_controller"] common.pdsh( user, [controller], 'http_proxy=%s sh %s/cli.sh cancel %s' % (self.cosbench["proxy"], self.cosbench["cosbench_folder"], self.cosbench["cosbench_job_id"]), option="console") self.chkpoint_to_log("cosbench stop") def wait_workload_to_stop(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") common.printout("LOG","Waiting Workload to complete its work") max_check_times = 30 cur_check = 0 @@ -367,6 +386,7 @@ def wait_workload_to_stop(self): common.printout("LOG","Workload completed") def cleanup(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).cleanup() cosbench_server = copy.deepcopy(self.cosbench["cosbench_driver"]) cosbench_server.append(self.cosbench["cosbench_controller"]) @@ -378,14 +398,17 @@ def cleanup(self): common.pdsh(user, cosbench_server, "rm -rf %s/*.txt; rm -rf %s/*.log" % (dest_dir, dest_dir)) def testjob_distribution(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") pass def cal_run_job_distribution(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") self.benchmark["distribution"] = {} for driver in self.cosbench["cosbench_driver"]: self.benchmark["distribution"][driver] = driver def archive(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).archive() user = self.cluster["user"] head = self.cluster["head"] @@ -401,6 +424,7 @@ def archive(self): common.rscp(user, cosbench_controller, "%s/raw/%s/"%(dest_dir, cosbench_controller), "%s/archive/%s-*"%(self.cosbench["cosbench_folder"], self.cosbench["cosbench_job_id"])) def parse_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") p = testcase testcase_dict = { "worker":p[0], "container":p[1], "iopattern":p[2], @@ -414,6 +438,7 @@ def parse_benchmark_cases(self, testcase): return testcase_dict def generate_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") benchmark = {} benchmark["cosbench_config_dir"]=self.all_conf_data.get("cosbench_config_dir") benchmark["cosbench_controller"]=self.all_conf_data.get("cosbench_controller") @@ -467,6 +492,7 @@ def generate_benchmark_cases(self, testcase): return True def replace_conf_xml(self, benchmark): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") with open(lib_path+"/benchmarking/mod/bobject/.template_config.xml",'r') as infile: with open(benchmark["configfile"],'w+') as outfile: line = infile.read() diff --git a/benchmarking/mod/generic/generic.py b/benchmarking/mod/generic/generic.py index 92b49ba..872c31f 100644 --- a/benchmarking/mod/generic/generic.py +++ b/benchmarking/mod/generic/generic.py @@ -1,9 +1,11 @@ from ..benchmark import * from collections import OrderedDict import itertools +import sys class Generic(Benchmark): def load_parameter(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).load_parameter() self.cluster["test_disks"] = self.all_conf_data.get_list("test_disks") @@ -12,6 +14,7 @@ def load_parameter(self): self.testjob_distribution(disk_num_per_client, instance_list) def prepare_images(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] controller = self.cluster["head"] fio_job_num_total = 0 @@ -22,11 +25,11 @@ def prepare_images(self): fio_job_num_total += len(self.cluster["testjob_distribution"][client]) time.sleep(1) if not self.check_fio_pgrep(clients, fio_job_num_total): - common.printout("ERROR","Failed to start FIO process") + common.printout("ERROR","Failed to start FIO process",log_level="LVL1") common.pdsh(user, clients, "killall -9 fio", option = "check_return") raise KeyboardInterrupt if not fio_job_num_total: - common.printout("ERROR","Planed to run 0 Fio Job, please check all.conf") + common.printout("ERROR","Planed to run 0 Fio Job, please check all.conf",log_level="LVL1") raise KeyboardInterrupt common.printout("LOG","%d FIO Jobs starts on %s" % (len(self.cluster["testjob_distribution"][client]), client)) @@ -41,6 +44,7 @@ def prepare_images(self): common.printout("LOG","rbd initialization finished") def prepare_result_dir(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") #1. prepare result dir self.get_runid() self.benchmark["section_name"] = "generic-%s-%s-qd%s-%s-%s-%s-generic" % (self.benchmark["iopattern"], self.benchmark["block_size"], self.benchmark["qd"], self.benchmark["volume_size"],self.benchmark["rampup"], self.benchmark["runtime"]) @@ -49,12 +53,13 @@ def prepare_result_dir(self): res = common.pdsh(self.cluster["user"],["%s"%(self.cluster["head"])],"test -d %s" % (self.cluster["dest_dir"]), option = "check_return") if not res[1]: - common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"])) + common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"]),log_level="LVL1") sys.exit() common.pdsh(self.cluster["user"] ,["%s" % (self.cluster["head"])], "mkdir -p %s" % (self.cluster["dest_dir"])) def prerun_check(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") #1. check is vclient alive user = self.cluster["user"] nodes = self.benchmark["distribution"].keys() @@ -62,10 +67,11 @@ def prerun_check(self): common.printout("LOG","check if FIO rbd engine installed") res = common.pdsh(user, nodes, "fio -enghelp | grep libaio", option = "check_return") if res and not res[0]: - common.printout("ERROR","FIO libaio engine not installed") + common.printout("ERROR","FIO libaio engine not installed",log_level="LVL1") sys.exit() def run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] dest_dir = self.cluster["tmp_dir"] @@ -102,16 +108,17 @@ def run(self): self.chkpoint_to_log("fio start") time.sleep(1) if not self.check_fio_pgrep(self.benchmark["distribution"].keys(), fio_job_num_total): - common.printout("ERROR","Failed to start FIO process") + common.printout("ERROR","Failed to start FIO process",log_level="LVL1") raise KeyboardInterrupt if not fio_job_num_total: - common.printout("ERROR","Planned to start 0 FIO process, seems to be an error") + common.printout("ERROR","Planned to start 0 FIO process, seems to be an error",log_level="LVL1") raise KeyboardInterrupt common.printout("LOG","%d FIO Jobs starts on %s" % ( fio_job_num_total, str(self.benchmark["distribution"].keys()))) while self.check_fio_pgrep(self.benchmark["distribution"].keys()): time.sleep(5) def prepare_run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).prepare_run() user = self.cluster["user"] dest_dir = self.cluster["tmp_dir"] @@ -121,6 +128,7 @@ def prepare_run(self): self.cleanup() def wait_workload_to_stop(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") common.printout("LOG","Waiting Workload to complete its work") user = self.cluster["user"] stop_flag = 0 @@ -140,12 +148,14 @@ def wait_workload_to_stop(self): common.printout("LOG","Workload completed") def stop_workload(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] nodes = self.benchmark["distribution"].keys() common.pdsh(user, nodes, "killall -9 fio", option = "check_return") self.chkpoint_to_log("fio stop") def generate_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") fio_capping = self.all_conf_data.get('fio_capping') io_pattern = testcase["iopattern"] @@ -191,7 +201,8 @@ def generate_benchmark_cases(self, testcase): try: rwmixread = self.all_conf_data.get('rwmixread') fio_template.append(" rwmixread=%s" % rwmixread) - except: + except Exception,e: + common.printout("LOG"," ERR_MSG:%s"%(self.__class__.__name__,sys._getframe().f_code.co_name,e),log_level="LVL2") pass fio_list.extend(fio_template) with open("../conf/fio.conf", "w+") as f: @@ -199,6 +210,7 @@ def generate_benchmark_cases(self, testcase): return True def parse_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") p = testcase testcase_dict = { "instance_number":p[0], "volume_size":p[1], "iopattern":p[2], @@ -212,6 +224,7 @@ def parse_benchmark_cases(self, testcase): return testcase_dict def archive(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).archive() user = self.cluster["user"] head = self.cluster["head"] diff --git a/benchmarking/mod/generic/hook.py b/benchmarking/mod/generic/hook.py index b4811d1..f59eafe 100644 --- a/benchmarking/mod/generic/hook.py +++ b/benchmarking/mod/generic/hook.py @@ -2,13 +2,16 @@ from collections import OrderedDict import itertools from plugin import * +import sys class Hook(Benchmark): def load_parameter(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).load_parameter() self.custom_script = self.all_conf_data.get("custom_script", True ) def prepare_result_dir(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") #1. prepare result dir self.get_runid() self.benchmark["section_name"] = "hook-%s-%s-qd%s-%s-%s-%s-hook" % (self.benchmark["iopattern"], self.benchmark["block_size"], self.benchmark["qd"], self.benchmark["volume_size"],self.benchmark["rampup"], self.benchmark["runtime"]) @@ -17,30 +20,35 @@ def prepare_result_dir(self): res = common.pdsh(self.cluster["user"],["%s"%(self.cluster["head"])],"test -d %s" % (self.cluster["dest_dir"]), option = "check_return") if not res[1]: - common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"])) + common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"]),log_level="LVL1") sys.exit() common.pdsh(self.cluster["user"] ,["%s" % (self.cluster["head"])], "mkdir -p %s" % (self.cluster["dest_dir"])) def prepare_run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).prepare_run() user = self.cluster["user"] dest_dir = self.cluster["tmp_dir"] self.cleanup() def wait_workload_to_stop(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") pass def stop_workload(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") pass def generate_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") if self.benchmark["description"] != "": custom_script = self.all_conf_data.get("%s|custom_script" % self.benchmark["description"], True ) self.custom_script = custom_script return True def parse_benchmark_cases(self, testcase): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") p = testcase testcase_dict = { "instance_number":p[0], "volume_size":p[1], "iopattern":p[2], @@ -54,6 +62,7 @@ def parse_benchmark_cases(self, testcase): return testcase_dict def run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).run() common.printout("LOG", "custom_script: %s" % self.custom_script) waittime = int(self.benchmark["runtime"]) + int(self.benchmark["rampup"]) + self.cluster["run_time_extend"] @@ -64,6 +73,7 @@ def run(self): time.sleep(1) def archive(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") super(self.__class__, self).archive() user = self.cluster["user"] head = self.cluster["head"] @@ -74,6 +84,7 @@ def archive(self): common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.log" % self.cluster["tmp_dir"]) def cal_run_job_distribution(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") self.cluster["testjob_distribution"] = {} self.benchmark["distribution"] = {} for node in self.cluster["client"]: diff --git a/benchmarking/run_cases.py b/benchmarking/run_cases.py index e8a78c6..8bad0e7 100644 --- a/benchmarking/run_cases.py +++ b/benchmarking/run_cases.py @@ -84,7 +84,7 @@ def main(args): if testcase["engine"] == "vdbench": benchmark = vdbench.VdBench() if not benchmark: - common.printout("ERROR","Unknown benchmark engine") + common.printout("ERROR","Unknown benchmark engine",log_level="LVL1") try: additional_option = '' for i in testcase["parameter"]: @@ -101,7 +101,7 @@ def main(args): benchmark.go(testcase["parameter"], tuning_section) except KeyboardInterrupt: - common.printout("WARNING","Caught KeyboardInterrupt Interruption") + common.printout("WARNING","Caught KeyboardInterrupt Interruption",log_level="LVL1") if __name__ == '__main__': import sys diff --git a/conf/common.py b/conf/common.py index c1ef073..793e4b0 100644 --- a/conf/common.py +++ b/conf/common.py @@ -20,6 +20,13 @@ cetune_log_file = "../conf/cetune_process.log" cetune_error_file = "../conf/cetune_error.log" cetune_console_file = "../conf/cetune_console.log" + +cetune_python_log_file = "../log/cetune_python_log_file.log" +cetune_python_error_log_file = "../log/cetune_python_error_log_file.log" +cetune_console_log_file = "../log/cetune_console_log_file.log" +cetune_process_log_file = "../log/cetune_process_log_file.log" +cetune_error_log_file = "../log/cetune_error_log_file.log" +cetune_operate_log_file = "../log/cetune_operate_log_file.log" no_die = False class bcolors: @@ -57,9 +64,9 @@ def addressToNetwork(self,ip,net): def getIpByHostInSubnet(self, hostname, subnet ): "Get IP by hostname and filter with subnet" - stdout, stderr = pdsh('root', [hostname] ,"ifconfig", option = "check_return") + stdout, stderr = pdsh('root', [hostname] ,"ifconfig", option = "check_return",loglevel="LVL6") if len(stderr): - printout("ERROR", 'Error to get ips: %s' % stderr) + printout("ERROR", 'Error to get ips: %s' % stderr,log_level="LVL1") sys.exit() ipaddrlist = [] res = re.findall("inet addr:\d+\.\d+\.\d+\.\d+",stdout) @@ -71,7 +78,7 @@ def getIpByHostInSubnet(self, hostname, subnet ): if b != "127.0.0.1": ipaddrlist.append(b) if len(ipaddrlist) == 0: - printout("ERROR", "No IP found") + printout("ERROR", "No IP found",log_level="LVL1") sys.exit() try: network, netmask = self.networkMask(subnet) @@ -108,6 +115,38 @@ def get_largest_list_len( data ): def clean_console(): bash("echo > %s" % cetune_console_file) +def cetune_log_collecter(func): + def wrapper(level, content, screen = True,log_level = "LVL3"): + #if not os.path.exists("../log/"): + # bash("mkdir -p ../log") + if log_level in ["LVL1"]: + output = "[%s][%s]: %s" % (log_level,level,content) + with open(cetune_error_log_file, "a+") as f: + f.write("[%s]%s\n" % (datetime.datetime.now().isoformat(),output)) + if log_level in ["LVL2"]: + output = "[%s][%s]: %s" % (log_level,level,content) + with open(cetune_python_error_log_file, "a+") as f: + f.write("[%s]%s\n" % (datetime.datetime.now().isoformat(),output)) + if log_level in ["LVL1","LVL3"]: + output = "[%s][%s]: %s" % (log_level,level,content) + with open(cetune_console_log_file, "a+") as f: + f.write("[%s]%s\n" % (datetime.datetime.now().isoformat(),output)) + if log_level in ["LVL2","LVL4"]: + output = "[%s][%s]: %s" % (log_level,level,content) + with open(cetune_python_log_file, "a+") as f: + f.write("[%s]%s\n" % (datetime.datetime.now().isoformat(),output)) + if log_level in ["LVL1","LVL2","LVL3"]: + output = "[%s][%s]: %s" % (log_level,level,content) + with open(cetune_process_log_file, "a+") as f: + f.write("[%s]%s\n" % (datetime.datetime.now().isoformat(),output)) + if log_level in ["LVL6"]: + output = "[%s][%s]: %s" % (log_level,level,content) + with open(cetune_operate_log_file, "a+") as f: + f.write("[%s]%s\n" % (datetime.datetime.now().isoformat(),output)) + return func(level, content, screen) + return wrapper + +@cetune_log_collecter def printout(level, content, screen = True): if level == "ERROR": output = "[ERROR]: %s" % content @@ -143,6 +182,22 @@ def printout(level, content, screen = True): f.write("[%s]%s\n" % (datetime.datetime.now().isoformat(),content)) print content +def clean_process_log(file_path): + start_line = 0 + with open(os.path.join(file_path,'cetune_process_log_file.log'),'rw') as f: + data = f.readlines() + for i in range(len(data)): + if data[i].strip('\n').find("============start deploy============") > 0: + start_line = int(i)+1 + if start_line != 0: + old_name = os.path.join(file_path,'cetune_process_log_file.log') + new_name = os.path.join(file_path,'cetune_process_log_file.log') + '.new' + bash("tail -n +%d %s > %s" % (start_line,old_name,new_name)) + bash("mv %s %s"%(new_name,old_name)) + else: + bash("rm %s/*" % (file_path)) + printout("LOG", "Clean process log file.",log_level="LVL3") + def remote_dir_exist( user, node, path ): stdout, stderr = pdsh(user, [node] ,"test -d %s; echo $?" % path, option = "check_return") res = format_pdsh_return(stdout) @@ -155,14 +210,14 @@ def remote_file_exist( user, node ,path): for node, returncode in res.items(): return int(returncode) == 0 -def pdsh(user, nodes, command, option="error_check", except_returncode=0, nodie=False): +def pdsh(user, nodes, command, option="error_check", except_returncode=0, nodie=False,loglevel="LVL3"): _nodes = [] for node in nodes: _nodes.append("%s@%s" % (user, node)) _nodes = ",".join(_nodes) args = ['pdsh', '-R', 'exec', '-w', _nodes, '-f', str(len(nodes)), 'ssh', '%h', '-oConnectTimeout=15', command] # args = ['pdsh', '-w', _nodes, command] - printout("CONSOLE", args, screen=False) + printout("CONSOLE", args, screen=False,log_level=loglevel) _subp = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if "force" in option: @@ -179,9 +234,9 @@ def pdsh(user, nodes, command, option="error_check", except_returncode=0, nodie= stderr = _subp.stderr.read() stdout = "".join(stdout) if stdout: - printout("CONSOLE", stdout, screen=False) + printout("CONSOLE", stdout, screen=False,log_level=loglevel) if stderr: - printout("CONSOLE", stderr, screen=False) + printout("CONSOLE", stderr, screen=False,log_level=loglevel) if stderr: returncode_re = re.search('ssh exited with exit code (\d+)', stderr) @@ -201,7 +256,7 @@ def pdsh(user, nodes, command, option="error_check", except_returncode=0, nodie= for line in stderr_tmp: if "ssh exited with exit code 255" not in line: stderr_print.append(line) - printout("ERROR",'\n'.join(stderr_print), screen=False) + printout("ERROR",'\n'.join(stderr_print), screen=False,log_level="LVL1") return [stdout, stderr] else: if returncode or "Connection timed out" in stderr: @@ -212,11 +267,11 @@ def pdsh(user, nodes, command, option="error_check", except_returncode=0, nodie= if "ssh exited with exit code 255" not in line: stderr_print.append(line) print('pdsh: %s' % args) - printout("ERROR",'\n'.join(stderr_print)) + printout("ERROR",'\n'.join(stderr_print),log_level="LVL1") if not nodie: sys.exit() -def bash(command, force=False, option="", nodie=False): +def bash(command, force=False, option="", nodie=False,loglevel = "LVL3"): args = ['bash', '-c', command] printout("CONSOLE", args, screen=False) @@ -231,16 +286,16 @@ def bash(command, force=False, option="", nodie=False): stderr = _subp.stderr.read() stdout = "".join(stdout) if stdout: - printout("CONSOLE", stdout, screen=False) + printout("CONSOLE", stdout, screen=False,log_level=loglevel) if stderr: - printout("CONSOLE", stderr, screen=False) + printout("CONSOLE", stderr, screen=False,log_level=loglevel) if force: return [stdout, stderr] if returncode: if stderr: print('bash: %s' % args) - printout("ERROR",stderr+"\n") + printout("ERROR",stderr+"\n",log_level="LVL1") if not nodie: sys.exit() return stdout @@ -539,7 +594,7 @@ def wait_ceph_to_health( user, controller ): if waitcount < 300: printout("LOG","Tuning has applied to ceph cluster, ceph is Healthy now") else: - printout("ERROR","ceph is unHealthy after 300sec waiting, please fix the issue manually") + printout("ERROR","ceph is unHealthy after 300sec waiting, please fix the issue manually",log_level="LVL1") sys.exit() def check_health( user, controller ): @@ -553,7 +608,7 @@ def check_health( user, controller ): def get_ceph_health(user, node): check_count = 0 output = {} - stdout, stderr = pdsh(user, [node], "timeout 3 ceph -s", option = "check_return") + stdout, stderr = pdsh(user, [node], "timeout 3 ceph -s", option = "check_return",loglevel="LVL5") res = format_pdsh_return(stdout) if len(res): stdout = res[node] @@ -598,7 +653,7 @@ def parse_device_name(dev_name): res = nvme_pattern.search(dev_name) if res: return res.group() - printout("ERROR", "device path error!\n") + printout("ERROR", "device path error!\n",log_level="LVL1") return None def parse_disk_format( disk_format_str ): diff --git a/conf/config.py b/conf/config.py index c2457e2..51ee9b8 100644 --- a/conf/config.py +++ b/conf/config.py @@ -227,12 +227,12 @@ def check_config(self, key, value): output = helper._check_config( key, value ) return output - def get(self, key, dotry=False): + def get(self, key, dotry=False,loglevel="LVL3"): if key in self.conf_data: return self.conf_data[key] else: if not dotry: - common.printout("WARNING","%s not defined in all.conf" % key) + common.printout("WARNING","%s not defined in all.conf" % key,log_level=loglevel) sys.exit() else: return "" diff --git a/deploy/mod/deploy.py b/deploy/mod/deploy.py index 9870825..8d05c86 100644 --- a/deploy/mod/deploy.py +++ b/deploy/mod/deploy.py @@ -15,6 +15,7 @@ pp = pprint.PrettyPrinter(indent=4) class Deploy(object): def __init__(self, tunings=""): + common.printout("LOG", "============start deploy============",log_level="LVL3") self.all_conf_data = config.Config("../conf/all.conf") self.cluster = {} self.cluster["clean_build"] = self.all_conf_data.get("clean_build") @@ -181,7 +182,7 @@ def install_binary(self, version=""): if code in installed_list[0]: version = version_name elif len(installed_list) >= 2: - common.printout("ERROR", "More than two versions of ceph installed, %s" % installed_list) + common.printout("ERROR", "More than two versions of ceph installed, %s" % installed_list,log_level="LVL1") sys.exit() if len(uninstall_nodes): self.uninstall_binary(uninstall_nodes) @@ -784,12 +785,12 @@ def get_daemon_info_from_ceph_conf(self, daemon): if not daemon: common.printout("ERROR", - "please select your daemon[osd, mon or mds]") + "please select your daemon[osd, mon or mds]",log_level="LVL1") sys.exit(1) if daemon not in ["osd", "mon", "mds"]: common.printout("ERROR", - "the daemon is not one of osd, mon or mds") + "the daemon is not one of osd, mon or mds",log_level="LVL1") sys.exit(1) ceph_conf = "" @@ -802,7 +803,7 @@ def get_daemon_info_from_ceph_conf(self, daemon): ceph_conf = f.readlines() except: common.printout("ERROR", - "Current Cluster ceph_current_status file not exists under CeTune/conf/") + "Current Cluster ceph_current_status file not exists under CeTune/conf/",log_level="LVL1") sys.exit(1) num = 0 diff --git a/tuner/tuner.py b/tuner/tuner.py index 3e102cd..f7a6852 100644 --- a/tuner/tuner.py +++ b/tuner/tuner.py @@ -14,6 +14,7 @@ pp = pprint.PrettyPrinter(indent=4) class Tuner: def __init__(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") self.cur_tuning = {} self.all_conf_data = config.Config("../conf/all.conf") self.worksheet = common.load_yaml_conf("../conf/tuner.yaml") @@ -36,10 +37,12 @@ def __init__(self): self.cluster[osd].append( osd_journal[1] ) def default_all_conf(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") self.cluster = {} self.cluster["user"] = self.all_conf_data.get("user") def handle_disk(self, option="get", param={'read_ahead_kb':2048, 'max_sectors_kb':512, 'scheduler':'deadline'}, fs_params=""): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] osds = self.cluster["osds"] @@ -70,6 +73,7 @@ def handle_disk(self, option="get", param={'read_ahead_kb':2048, 'max_sectors_kb stdout, stderr = common.pdsh(user, [osd], 'sh -c "echo %s > /sys/block/%s/queue/%s"' % (str(value), parsed_device_name, key), option="check_return") def get_version(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] osds = self.cluster["osds"] clients = self.cluster["client"] @@ -109,10 +113,11 @@ def get_version(self): return ceph_version.get() def get_osd_config(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] osds = self.cluster["osds"] - stdout, stderr = common.pdsh(user, osds, 'path=`find /var/run/ceph -name "*osd*asok" | head -1`; timeout 5 ceph --admin-daemon $path config show', option="check_return") + stdout, stderr = common.pdsh(user, osds, 'path=`find /var/run/ceph -name "*osd*asok" | head -1`; timeout 5 ceph --admin-daemon $path config show', option="check_return",loglevel="LVL6") res = common.format_pdsh_return(stdout) # merge config diff osd_config = common.MergableDict() @@ -121,10 +126,11 @@ def get_osd_config(self): return osd_config.get() def get_mon_config(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] mons = self.cluster["mons"] - stdout, stderr = common.pdsh(user, mons, 'path=`find /var/run/ceph -name "*mon*asok" | head -1`; timeout 5 ceph --admin-daemon $path config show', option="check_return") + stdout, stderr = common.pdsh(user, mons, 'path=`find /var/run/ceph -name "*mon*asok" | head -1`; timeout 5 ceph --admin-daemon $path config show', option="check_return",loglevel="LVL6") res = common.format_pdsh_return(stdout) mon_config = common.MergableDict() for node in res: @@ -132,6 +138,7 @@ def get_mon_config(self): return mon_config.get() def get_pool_config(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] controller = self.cluster["head"] @@ -147,11 +154,13 @@ def get_pool_config(self): for index in range(4, len(raw_res),2): try: pool_config[name][raw_res[index]] = raw_res[index+1] - except: + except Exception,e: + common.printout("LOG"," ERR_MSG:%s"%(self.__class__.__name__,sys._getframe().f_code.co_name,e),log_level="LVL2") pass return pool_config def dump_config(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") # check ceph config and os config meet request user = self.cluster["user"] controller = self.cluster["head"] @@ -163,7 +172,8 @@ def dump_config(self): #get [system] config try: config["disk"] = self.handle_disk(option="get") - except: + except Exception,e: + common.printout("LOG"," ERR_MSG:%s"%(self.__class__.__name__,sys._getframe().f_code.co_name,e),log_level="LVL2") pass #get [ceph version] @@ -180,6 +190,7 @@ def dump_config(self): return config def apply_version(self, jobname): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] controller = self.cluster["head"] pwd = os.path.abspath(os.path.join('..')) @@ -213,6 +224,7 @@ def apply_version(self, jobname): run_deploy.main(['install_binary', '--version', planed_version]) def check_tuning(self, jobname): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") if not self.cur_tuning: self.cur_tuning = self.dump_config() tuning_diff = [] @@ -250,6 +262,7 @@ def check_tuning(self, jobname): return tuning_diff def apply_tuning(self, jobname, no_check = False): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") #check the diff between worksheet tuning and cur system if not no_check: common.printout("LOG","Calculate Difference between Current Ceph Cluster Configuration with tuning") @@ -266,7 +279,8 @@ def apply_tuning(self, jobname, no_check = False): self.handle_disk( option="set", param=param ) else: self.handle_disk( option="set" ) - except: + except Exception,e: + common.printout("LOG"," ERR_MSG:%s"%(self.__class__.__name__,sys._getframe().f_code.co_name,e),log_level="LVL2") pass if 'global' in tmp_tuning_diff or 'osd' in tmp_tuning_diff or 'mon' in tmp_tuning_diff: if self.cluster["rgw_enable"]=="true" and len(self.cluster["rgw"]): @@ -332,6 +346,7 @@ def apply_tuning(self, jobname, no_check = False): common.wait_ceph_to_health( user, controller ) def handle_pool(self, option="set", param = {}): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] controller = self.cluster["head"] if option == "create": diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index b0164d5..11c4031 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -52,7 +52,7 @@ def generate_summary_page(self): for node_type, node_data in self.result.items(): if not isinstance(node_data, dict): continue - common.printout("LOG","Generating %s view" % node_type) + common.printout("LOG","Generating %s view" % node_type,log_level="LVL5") output.extend(self.generate_node_view(node_type)) output.append("") @@ -71,13 +71,13 @@ def generate_summary_page(self): return # Copy local result to remote dir - common.printout("LOG","Session result generated, copy to remote") + common.printout("LOG","Session result generated, copy to remote",log_level="LVL5") common.bash("scp -r %s %s" % (self.path, self.dest_dir_remote_bak)) remote_bak, remote_dir = self.dest_dir_remote_bak.split(':') output = self.generate_history_view(remote_bak, remote_dir, self.user) - common.printout("LOG","History view generated, copy to remote") + common.printout("LOG","History view generated, copy to remote",log_level="LVL5") with open("%s/cetune_history.html" % self.path, 'w') as f: f.write(output) common.bash("scp -r %s/cetune_history.html %s" % (self.path, self.dest_dir_remote_bak)) @@ -198,7 +198,7 @@ def check_DB_case_list(self,re_dir,dbpath): return False def generate_history_view(self, remote_host="127.0.0.1", remote_dir="/mnt/data/", user='root', html_format=True): - common.printout("LOG","Generating history view") + common.printout("LOG","Generating history view",log_level="LVL5") dbpath = os.path.join(self.db_path,"cetune_report.db") if not self.check_DB_case_list(self.db_path,dbpath): #stdout, stderr = common.pdsh(user, [remote_host], "find %s -name '*.html' | grep -v 'cetune_history'|sort -u | while read file;do session=`echo $file | awk -F/ {'print $(NF-1)'}`; awk -v session=\"$session\" 'BEGIN{find=0;}{if(match($1,\"tbody\")&&find==2){find=0;}if(find==2){if(match($1,\"\");else print ;};if(match($1,\"div\")&&match($2,\"summary\"))find=1;if(match($1,\"tbody\")&&find==1){find+=1}}' $file; done" % remote_dir, option="check_return") @@ -207,7 +207,7 @@ def generate_history_view(self, remote_host="127.0.0.1", remote_dir="/mnt/data/" # common.printout("ERROR","Generating history view failed") # return False # some modification in greped trs - stdout = common.bash("find %s -name '*.html' | grep -v 'cetune_history'|sort -u | while read file;do session=`echo $file | awk -F/ {'print $(NF-1)'}`; awk -v session=\"$session\" 'BEGIN{find=0;}{if(match($1,\"tbody\")&&find==2){find=0;}if(find==2){if(match($1,\"\");else print ;};if(match($1,\"div\")&&match($2,\"summary\"))find=1;if(match($1,\"tbody\")&&find==1){find+=1}}' $file; done" % remote_dir) + stdout = common.bash("find %s -name '*.html' | grep -v 'cetune_history'|sort -u | while read file;do session=`echo $file | awk -F/ {'print $(NF-1)'}`; awk -v session=\"$session\" 'BEGIN{find=0;}{if(match($1,\"tbody\")&&find==2){find=0;}if(find==2){if(match($1,\"\");else print ;};if(match($1,\"div\")&&match($2,\"summary\"))find=1;if(match($1,\"tbody\")&&find==1){find+=1}}' $file; done" % remote_dir,loglevel="LVL6") res_tmp = stdout; formated_report = {} report_lines = re.findall('()',res_tmp,re.S) @@ -352,7 +352,7 @@ def generate_line_chart(self, data, node_type, field, append_table=True): output = [] common.bash("mkdir -p ../visualizer/include/pic") common.bash("mkdir -p ../visualizer/include/csv") - common.printout("LOG","generate %s line chart" % node_type) + common.printout("LOG","generate %s line chart" % node_type,log_level="LVL5") for field_column, field_data in data.items(): pyplot.figure(figsize=(9, 4)) for node, node_data in field_data.items(): diff --git a/webui/webui.py b/webui/webui.py index 25660d5..72bc267 100644 --- a/webui/webui.py +++ b/webui/webui.py @@ -254,7 +254,7 @@ def delete_result(self, request_type,key): def get_summary(self): view = visualizer.Visualizer({}) conf = config.Config("../conf/all.conf") - dest_dir = conf.get("dest_dir") + dest_dir = conf.get("dest_dir",loglevel="LVL6") output = view.generate_history_view("127.0.0.1",dest_dir,"root",False) if not output: return "" diff --git a/workflow/workflow.py b/workflow/workflow.py index 05a1cf0..b9d55f9 100644 --- a/workflow/workflow.py +++ b/workflow/workflow.py @@ -16,6 +16,7 @@ pp = pprint.PrettyPrinter(indent=4) class Workflow: def __init__(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") self.cur_tuning = {} self.all_conf_data = config.Config("../conf/all.conf") self.worksheet = common.load_yaml_conf("../conf/tuner.yaml") @@ -38,10 +39,12 @@ def __init__(self): self.cluster[osd].append( osd_journal[1] ) def default_all_conf(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") self.cluster = {} self.cluster["user"] = self.all_conf_data.get("user") def run(self): + common.printout("LOG"," Test start running function : %s"%(self.__class__.__name__,sys._getframe().f_code.co_name),screen=False,log_level="LVL4") user = self.cluster["user"] controller = self.cluster["head"] osds = self.cluster["osds"] @@ -71,7 +74,7 @@ def run(self): time.sleep(3) run_cases.main(['--tuning', section]) else: - common.printout("ERROR","Unknown tuner workstage %s" % work) + common.printout("ERROR","Unknown tuner workstage %s" % work,log_level="LVL1") def main(args): parser = argparse.ArgumentParser(description='workflow')