From 0b1c2ffb09de12cf9be7fe9c411a7eaf531b6e65 Mon Sep 17 00:00:00 2001 From: Emlyn Corrin Date: Wed, 14 Feb 2018 12:22:04 +0000 Subject: [PATCH 1/2] Add --apps option for `aztk spark job get` to list apps --- aztk_cli/spark/endpoints/job/get.py | 5 +++- aztk_cli/utils.py | 37 +++++++++++++++++------------ 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/aztk_cli/spark/endpoints/job/get.py b/aztk_cli/spark/endpoints/job/get.py index 026a3cc7..9cf2050c 100644 --- a/aztk_cli/spark/endpoints/job/get.py +++ b/aztk_cli/spark/endpoints/job/get.py @@ -11,9 +11,12 @@ def setup_parser(parser: argparse.ArgumentParser): dest='job_id', required=True, help='The unique id of your AZTK job') + parser.add_argument('--apps', + action='store_true', + help='Also list applications') def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) - utils.print_job(spark_client, spark_client.get_job(args.job_id)) + utils.print_job(spark_client, spark_client.get_job(args.job_id), args.apps) diff --git a/aztk_cli/utils.py b/aztk_cli/utils.py index 57b281aa..32fe8945 100644 --- a/aztk_cli/utils.py +++ b/aztk_cli/utils.py @@ -5,7 +5,7 @@ import time import yaml from subprocess import call -from typing import List +from typing import Dict, List import azure.batch.models as batch_models from aztk import error, utils from aztk.utils import get_ssh_key, helpers @@ -254,7 +254,14 @@ def print_jobs(jobs: List[models.Job]): ) -def print_job(client, job: models.Job): +def get_applications(client, job: models.Job): + if job.applications: + return {j.name: j for j in job.applications} + else: + return client.list_applications(job.id) + + +def print_job(client, job: models.Job, apps: bool): print_format = '{:<36}| {:<15}' log.info("") @@ -274,11 +281,12 @@ def print_job(client, job: models.Job): log.info(print_format.format("Cluster", "Provisioning")) log.info("") - if job.applications: - application_summary(job.applications) - else: - application_summary(client.list_applications(job.id)) + applications = get_applications(client, job) + application_summary(applications) log.info("") + if apps: + print_applications(applications) + log.info("") def node_state_count(cluster: models.Cluster): @@ -303,22 +311,22 @@ def print_cluster_summary(cluster: models.Cluster): log.info("") -def application_summary(applications): +def application_summary(applications: Dict[str, models.Application]): states = {"scheduling": 0} for state in batch_models.TaskState: states[state.name] = 0 warn_scheduling = False - for application in applications: - if type(application) == str: + for name, application in applications.items(): + if application is None: states["scheduling"] += 1 warn_scheduling = True else: states[application.state] += 1 print_format = '{:<17} {:<14}' - log.info("Applications") + log.info("Application States") log.info("-"*42) for state in states: if states[state] > 0: @@ -327,15 +335,15 @@ def application_summary(applications): if warn_scheduling: log.warning("\nNo Spark applications will be scheduled until the master is selected.") -def print_applications(applications): +def print_applications(applications: Dict[str, models.Application]): print_format = '{:<36}| {:<15}| {:<16} | {:^9} |' print_format_underline = '{:-<36}|{:-<16}|{:-<18}|{:-<11}|' log.info(print_format.format("Applications", "State", "Transition Time", "Exit Code")) log.info(print_format_underline.format('', '', '', '')) warn_scheduling = False - for name in applications: - if applications[name] is None: + for name, application in applications.items(): + if application is None: log.info( print_format.format( name, @@ -346,10 +354,9 @@ def print_applications(applications): ) warn_scheduling = True else: - application = applications[name] log.info( print_format.format( - application.name, + name, application.state, utc_to_local(application.state_transition_time), application.exit_code if application.exit_code is not None else "-" From af70b2ea610a243d5629f695debd48086cd6b1da Mon Sep 17 00:00:00 2001 From: Emlyn Corrin Date: Sat, 17 Feb 2018 18:16:57 +0000 Subject: [PATCH 2/2] Only warn about scheduling once in verbose job output --- aztk_cli/utils.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/aztk_cli/utils.py b/aztk_cli/utils.py index 32fe8945..6cafb272 100644 --- a/aztk_cli/utils.py +++ b/aztk_cli/utils.py @@ -287,6 +287,8 @@ def print_job(client, job: models.Job, apps: bool): if apps: print_applications(applications) log.info("") + else: + warn_scheduling(applications) def node_state_count(cluster: models.Cluster): @@ -316,12 +318,9 @@ def application_summary(applications: Dict[str, models.Application]): for state in batch_models.TaskState: states[state.name] = 0 - warn_scheduling = False - for name, application in applications.items(): if application is None: states["scheduling"] += 1 - warn_scheduling = True else: states[application.state] += 1 @@ -332,7 +331,14 @@ def application_summary(applications: Dict[str, models.Application]): if states[state] > 0: log.info(print_format.format(state + ":", states[state])) - if warn_scheduling: + +def warn_scheduling(applications: Dict[str, models.Application]): + warn = False + for application in applications.values(): + if application is None: + warn = True + break + if warn: log.warning("\nNo Spark applications will be scheduled until the master is selected.") def print_applications(applications: Dict[str, models.Application]): @@ -341,7 +347,6 @@ def print_applications(applications: Dict[str, models.Application]): log.info(print_format.format("Applications", "State", "Transition Time", "Exit Code")) log.info(print_format_underline.format('', '', '', '')) - warn_scheduling = False for name, application in applications.items(): if application is None: log.info( @@ -352,7 +357,6 @@ def print_applications(applications: Dict[str, models.Application]): "-" ) ) - warn_scheduling = True else: log.info( print_format.format( @@ -362,8 +366,7 @@ def print_applications(applications: Dict[str, models.Application]): application.exit_code if application.exit_code is not None else "-" ) ) - if warn_scheduling: - log.warning("\nNo Spark applications will be scheduled until the master is selected.") + warn_scheduling(applications) def print_application(application: models.Application): print_format = '{:<30}| {:<15}'