From 0a81af9806c297e555bc35cf3c3ca5c7886050c7 Mon Sep 17 00:00:00 2001 From: Gregory Jordan Date: Thu, 9 Dec 2021 12:05:03 -0500 Subject: [PATCH 1/4] WIP logger edits --- src/main/java/bio/terra/cli/utils/Logger.java | 45 +++++-------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/src/main/java/bio/terra/cli/utils/Logger.java b/src/main/java/bio/terra/cli/utils/Logger.java index 55d8ec2b3..b170f1d92 100644 --- a/src/main/java/bio/terra/cli/utils/Logger.java +++ b/src/main/java/bio/terra/cli/utils/Logger.java @@ -9,9 +9,8 @@ import ch.qos.logback.classic.filter.ThresholdFilter; import ch.qos.logback.core.ConsoleAppender; import ch.qos.logback.core.OutputStreamAppender; -import ch.qos.logback.core.rolling.FixedWindowRollingPolicy; import ch.qos.logback.core.rolling.RollingFileAppender; -import ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy; +import ch.qos.logback.core.rolling.TimeBasedRollingPolicy; import ch.qos.logback.core.util.FileSize; import ch.qos.logback.core.util.StatusPrinter; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -27,7 +26,7 @@ public class Logger { private static final String LOG_FORMAT = "%d{yyyy-MM-dd HH:mm:ss.SSS zz} [%thread] %-5level %logger{50} - %msg%n"; - private static final long MAX_FILE_SIZE = 5 * FileSize.MB_COEFFICIENT; // 5 MB + private static final long MAX_FILE_SIZE = 50 * FileSize.MB_COEFFICIENT; // 5 MB private static final int MAX_NUM_FILES = 5; /** @@ -53,38 +52,18 @@ public static void setupLogging(LogLevel consoleLoggingLevel, LogLevel fileLoggi rollingFileAppender.setContext(loggerContext); rollingFileAppender.setFile(Context.getLogFile().toString()); - // trigger a file rollover based on the log file size - SizeBasedTriggeringPolicy triggeringPolicy = new SizeBasedTriggeringPolicy(); - triggeringPolicy.setContext(loggerContext); - triggeringPolicy.setMaxFileSize(new FileSize(MAX_FILE_SIZE)); - rollingFileAppender.setTriggeringPolicy(triggeringPolicy); - - // maintain a window of logs (e.g. the last 5 files worth) - // this means that once the maximum amount of logs are stored (i.e. max # of files, max size - // each), the oldest logs will be overwritten - FixedWindowRollingPolicy rollingPolicy = new FixedWindowRollingPolicy(); - rollingPolicy.setContext(loggerContext); - rollingPolicy.setFileNamePattern( - Context.getLogFile() - .getParent() - .resolve("%i." + Context.getLogFile().getFileName()) - .toString()); - rollingPolicy.setParent(rollingFileAppender); - - // the base file name (e.g. terra.log) is not included in rollingPolicy's count, so subtract one - // here to account for that - // e.g. MAX_NUM_FILES=3: terra.log, 1.terra.log, 2.terra.log - if (MAX_NUM_FILES < 1) { - throw new IllegalArgumentException("Maximum number of log files must be >= 1"); - } - rollingPolicy.setMinIndex(1); - rollingPolicy.setMaxIndex(MAX_NUM_FILES - 1); - rollingFileAppender.setRollingPolicy(rollingPolicy); + TimeBasedRollingPolicy timeBasedRollingPolicy = new TimeBasedRollingPolicy(); + timeBasedRollingPolicy.setContext(loggerContext); + timeBasedRollingPolicy.setMaxHistory(MAX_NUM_FILES); + timeBasedRollingPolicy.setTotalSizeCap(new FileSize(MAX_FILE_SIZE)); + timeBasedRollingPolicy.setFileNamePattern("%d-terra.log"); + // Context.getLogFile().getParent().resolve("%d{yyyy-MM-dd}-terra.log").toString()); + rollingFileAppender.setRollingPolicy(timeBasedRollingPolicy); + rollingFileAppender.setTriggeringPolicy(timeBasedRollingPolicy); // make sure to start the policies after the cross-references between the policy and appender // have been set - triggeringPolicy.start(); - rollingPolicy.start(); + timeBasedRollingPolicy.start(); setupEncoderAndFilter(rollingFileAppender, loggerContext, fileLoggingLevel.getLogLevelImpl()); rollingFileAppender.start(); @@ -110,7 +89,7 @@ public static void setupLogging(LogLevel consoleLoggingLevel, LogLevel fileLoggi // each CLI command invocation is a new process (i.e. new JVM) and may be very short-lived. // so, include an additional manual check here on startup, to see if we should roll over the // file based on its size. - File activeFile = new File(rollingPolicy.getActiveFileName()); + File activeFile = new File(timeBasedRollingPolicy.getActiveFileName()); if (activeFile.length() > MAX_FILE_SIZE) { rollingFileAppender.rollover(); } From 16695b82be33d12b812b9472de2ec84cec9c51e1 Mon Sep 17 00:00:00 2001 From: Gregory Jordan Date: Thu, 9 Dec 2021 14:07:05 -0500 Subject: [PATCH 2/4] Clean up and add size-and-date rolling policy. --- src/main/java/bio/terra/cli/utils/Logger.java | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/main/java/bio/terra/cli/utils/Logger.java b/src/main/java/bio/terra/cli/utils/Logger.java index b170f1d92..eedaba68d 100644 --- a/src/main/java/bio/terra/cli/utils/Logger.java +++ b/src/main/java/bio/terra/cli/utils/Logger.java @@ -10,7 +10,7 @@ import ch.qos.logback.core.ConsoleAppender; import ch.qos.logback.core.OutputStreamAppender; import ch.qos.logback.core.rolling.RollingFileAppender; -import ch.qos.logback.core.rolling.TimeBasedRollingPolicy; +import ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy; import ch.qos.logback.core.util.FileSize; import ch.qos.logback.core.util.StatusPrinter; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -26,8 +26,9 @@ public class Logger { private static final String LOG_FORMAT = "%d{yyyy-MM-dd HH:mm:ss.SSS zz} [%thread] %-5level %logger{50} - %msg%n"; - private static final long MAX_FILE_SIZE = 50 * FileSize.MB_COEFFICIENT; // 5 MB - private static final int MAX_NUM_FILES = 5; + private static final long MAX_PER_FILE_SIZE = 10 * FileSize.MB_COEFFICIENT; + private static final long MAX_TOTAL_LOG_SIZE = 100 * FileSize.MB_COEFFICIENT; + private static final int MAX_HISTORY_FILES = 30; // Keep up to 30 archived log files /** * Setup a file and console appender for the root logger. Each may use a different logging level, @@ -47,26 +48,27 @@ public static void setupLogging(LogLevel consoleLoggingLevel, LogLevel fileLoggi StatusPrinter.printIfErrorsOccured(loggerContext); // build the rolling file appender - RollingFileAppender rollingFileAppender = new RollingFileAppender(); - rollingFileAppender.setName("RollingFileAppender"); - rollingFileAppender.setContext(loggerContext); - rollingFileAppender.setFile(Context.getLogFile().toString()); - - TimeBasedRollingPolicy timeBasedRollingPolicy = new TimeBasedRollingPolicy(); - timeBasedRollingPolicy.setContext(loggerContext); - timeBasedRollingPolicy.setMaxHistory(MAX_NUM_FILES); - timeBasedRollingPolicy.setTotalSizeCap(new FileSize(MAX_FILE_SIZE)); - timeBasedRollingPolicy.setFileNamePattern("%d-terra.log"); - // Context.getLogFile().getParent().resolve("%d{yyyy-MM-dd}-terra.log").toString()); - rollingFileAppender.setRollingPolicy(timeBasedRollingPolicy); - rollingFileAppender.setTriggeringPolicy(timeBasedRollingPolicy); + RollingFileAppender fileAppender = new RollingFileAppender(); + fileAppender.setName("RollingFileAppender"); + fileAppender.setContext(loggerContext); + + SizeAndTimeBasedRollingPolicy rollingPolicy = new SizeAndTimeBasedRollingPolicy(); + rollingPolicy.setContext(loggerContext); + rollingPolicy.setParent(fileAppender); + rollingPolicy.setFileNamePattern( + Context.getLogFile().getParent().resolve("terra-%d{yyyy-MM-dd}.%i.log").toString()); + rollingPolicy.setMaxHistory(MAX_HISTORY_FILES); + rollingPolicy.setTotalSizeCap(new FileSize(MAX_TOTAL_LOG_SIZE)); + rollingPolicy.setMaxFileSize(new FileSize(MAX_PER_FILE_SIZE)); + fileAppender.setRollingPolicy(rollingPolicy); + fileAppender.setTriggeringPolicy(rollingPolicy); // make sure to start the policies after the cross-references between the policy and appender // have been set - timeBasedRollingPolicy.start(); + rollingPolicy.start(); - setupEncoderAndFilter(rollingFileAppender, loggerContext, fileLoggingLevel.getLogLevelImpl()); - rollingFileAppender.start(); + setupEncoderAndFilter(fileAppender, loggerContext, fileLoggingLevel.getLogLevelImpl()); + fileAppender.start(); // build the console appender ConsoleAppender consoleAppender = new ConsoleAppender(); @@ -82,19 +84,19 @@ public static void setupLogging(LogLevel consoleLoggingLevel, LogLevel fileLoggi ch.qos.logback.classic.Logger rootLogger = loggerContext.getLogger(ROOT_LOGGER_NAME); rootLogger.setLevel(Level.ALL); rootLogger.detachAndStopAllAppenders(); - rootLogger.addAppender(rollingFileAppender); + rootLogger.addAppender(fileAppender); rootLogger.addAppender(consoleAppender); // if a process is too short-lived, then the policy may not check if it should rollover. // each CLI command invocation is a new process (i.e. new JVM) and may be very short-lived. // so, include an additional manual check here on startup, to see if we should roll over the // file based on its size. - File activeFile = new File(timeBasedRollingPolicy.getActiveFileName()); - if (activeFile.length() > MAX_FILE_SIZE) { - rollingFileAppender.rollover(); + File activeFile = new File(rollingPolicy.getActiveFileName()); + if (activeFile.length() > MAX_PER_FILE_SIZE) { + fileAppender.rollover(); } - // StatusPrinter.print(loggerContext); // helpful for debugging + StatusPrinter.print(loggerContext); // helpful for debugging } /** From 6dcb0d1dc4ae055c90c29600e4c501c51446a17c Mon Sep 17 00:00:00 2001 From: Gregory Jordan Date: Mon, 13 Dec 2021 16:24:19 -0500 Subject: [PATCH 3/4] Adjust comments --- src/main/java/bio/terra/cli/utils/Logger.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/bio/terra/cli/utils/Logger.java b/src/main/java/bio/terra/cli/utils/Logger.java index eedaba68d..05a2ae77e 100644 --- a/src/main/java/bio/terra/cli/utils/Logger.java +++ b/src/main/java/bio/terra/cli/utils/Logger.java @@ -28,7 +28,7 @@ public class Logger { private static final long MAX_PER_FILE_SIZE = 10 * FileSize.MB_COEFFICIENT; private static final long MAX_TOTAL_LOG_SIZE = 100 * FileSize.MB_COEFFICIENT; - private static final int MAX_HISTORY_FILES = 30; // Keep up to 30 archived log files + private static final int MAX_HISTORY_FILES = 30; /** * Setup a file and console appender for the root logger. Each may use a different logging level, @@ -55,6 +55,7 @@ public static void setupLogging(LogLevel consoleLoggingLevel, LogLevel fileLoggi SizeAndTimeBasedRollingPolicy rollingPolicy = new SizeAndTimeBasedRollingPolicy(); rollingPolicy.setContext(loggerContext); rollingPolicy.setParent(fileAppender); + // Log files will be stored at "~/.terra/logs/terra-2021-01-01.0.log", etc. rollingPolicy.setFileNamePattern( Context.getLogFile().getParent().resolve("terra-%d{yyyy-MM-dd}.%i.log").toString()); rollingPolicy.setMaxHistory(MAX_HISTORY_FILES); From 04f2c486bce0a74beb064a9cc3f2b1d9cc5af260 Mon Sep 17 00:00:00 2001 From: Gregory Jordan Date: Mon, 13 Dec 2021 16:26:18 -0500 Subject: [PATCH 4/4] Remove debug println command --- src/main/java/bio/terra/cli/utils/Logger.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/bio/terra/cli/utils/Logger.java b/src/main/java/bio/terra/cli/utils/Logger.java index 05a2ae77e..5428994e5 100644 --- a/src/main/java/bio/terra/cli/utils/Logger.java +++ b/src/main/java/bio/terra/cli/utils/Logger.java @@ -97,7 +97,7 @@ public static void setupLogging(LogLevel consoleLoggingLevel, LogLevel fileLoggi fileAppender.rollover(); } - StatusPrinter.print(loggerContext); // helpful for debugging + // StatusPrinter.print(loggerContext); // helpful for debugging } /**