From 22de82041ac079b939d0e456631524dc535a92a2 Mon Sep 17 00:00:00 2001 From: Ponnuvel Palaniyappan Date: Thu, 22 Jan 2026 15:41:01 +0000 Subject: [PATCH 1/3] tests: enhance functional test with more checks. Fixes SET-1807. Fixes #24. Signed-off-by: Ponnuvel Palaniyappan --- tests/functional-test-microceph.sh | 77 +++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 23 deletions(-) diff --git a/tests/functional-test-microceph.sh b/tests/functional-test-microceph.sh index 472bc88..6d9aa7f 100755 --- a/tests/functional-test-microceph.sh +++ b/tests/functional-test-microceph.sh @@ -22,6 +22,7 @@ cleanup() { pkill -f "rbd bench" || true # Remove test files + cat /tmp/osdtrace.log /tmp/radostrace.log rm -f /tmp/osdtrace.log /tmp/radostrace.log # Remove test RBD resources @@ -33,6 +34,9 @@ cleanup() { trap cleanup EXIT +OSDTRACE_LOG="/tmp/osdtrace.log" +RADOSTRACE_LOG="/tmp/radostrace.log" + # Check if running as root or with sudo if [ "$EUID" -ne 0 ]; then echo "Error: This test must be run as root or with sudo" @@ -93,18 +97,17 @@ while [ $ELAPSED -lt $TIMEOUT ]; do done microceph.ceph status +microceph --version echo "=== Step 5: Get Ceph version from snap manifest ===" -CEPH_VERSION=$(cat /snap/microceph/current/snap/manifest.yaml | grep "^ceph-osd:" | awk '{print $2}') -LIBRADOS_VERSION=$(cat /snap/microceph/current/snap/manifest.yaml | grep "^librados2:" | awk '{print $2}') +CEPH_VERSION=$(microceph --version | awk '{gsub(";", "", $2); print $2}') -echo "Ceph OSD version: $CEPH_VERSION" -echo "Librados version: $LIBRADOS_VERSION" +echo "Ceph version: $CEPH_VERSION" echo "=== Step 6: Locate DWARF JSON files in repository ===" # Look for matching DWARF files in the repository OSD_DWARF="$PROJECT_ROOT/files/ubuntu/osdtrace/osd-${CEPH_VERSION}_dwarf.json" -RADOS_DWARF="$PROJECT_ROOT/files/ubuntu/radostrace/${LIBRADOS_VERSION}_dwarf.json" +RADOS_DWARF="$PROJECT_ROOT/files/ubuntu/radostrace/${CEPH_VERSION}_dwarf.json" if [ ! -f "$OSD_DWARF" ]; then echo "Warning: OSD DWARF file not found at $OSD_DWARF" @@ -151,14 +154,14 @@ fi microceph.rbd create test_pool/testimage --size 1G || true echo "=== Step 9: Start osdtrace in background ===" -timeout 30 $PROJECT_ROOT/osdtrace -i $OSD_DWARF -p $OSD_PID --skip-version-check -x > /tmp/osdtrace.log 2>&1 & -OSDTRACE_PID=$! +timeout 30 $PROJECT_ROOT/osdtrace -i $OSD_DWARF -p $OSD_PID --skip-version-check -x >$OSDTRACE_LOG 2>&1 & +OSDTRACE_PID=(pidof osdtrace) echo "Started osdtrace with PID $OSDTRACE_PID" sleep 3 echo "=== Step 10: Start radostrace in background ===" # radostrace will trace all librados clients, including the rbd bench command -timeout 30 $PROJECT_ROOT/radostrace -i $RADOS_DWARF --skip-version-check > /tmp/radostrace.log 2>&1 & +timeout 30 $PROJECT_ROOT/radostrace -i $RADOS_DWARF --skip-version-check >$RADOSTRACE_LOG 2>&1 & RADOSTRACE_PID=$! echo "Started radostrace with PID $RADOSTRACE_PID" sleep 3 @@ -191,36 +194,63 @@ wait $OSDTRACE_PID 2>/dev/null || true wait $RADOSTRACE_PID 2>/dev/null || true echo "=== Step 14: Verify osdtrace output ===" -if [ ! -f /tmp/osdtrace.log ]; then - echo "Error: osdtrace log file not found" - exit 1 -fi -OSD_LINE_COUNT=$(wc -l < /tmp/osdtrace.log) +# 14a. Check trace exists +OSD_LINE_COUNT=$(wc -l < $OSDTRACE_LOG) echo "osdtrace captured $OSD_LINE_COUNT lines" - if [ $OSD_LINE_COUNT -lt 5 ]; then echo "Error: osdtrace did not capture enough trace data (expected at least 5 lines)" echo "osdtrace output:" - cat /tmp/osdtrace.log + cat $OSDTRACE_LOG + exit 1 +fi + +# 14b. Check OSD IDs range is within the expected limit +MAX_OSD_ID=$(microceph.ceph osd stat | grep -oP '\d+(?= osds:)' || echo "0") +MAX_OSD_ID=$((MAX_OSD_ID - 1)) # Convert count to max ID (0-indexed) +echo "Max OSD ID in cluster: $MAX_OSD_ID" + +osd_id_err=$(awk -v max_osd=$MAX_OSD_ID '$1=="osd" && ($2 < 0 || $2 > max_osd) {print $2; exit}' $OSDTRACE_LOG) +if [ -n "$osd_id_err" ]; then + echo "ERROR: Found OSD id outside the expected range, $osd_id_err" exit 1 fi -echo "✓ osdtrace successfully captured trace data" +# 14c. Check the correct pool id is used +TEST_POOL_ID=$(microceph.ceph osd pool ls detail | grep "^pool.*'test_pool'" | grep -oP "pool \K\d+") +pool_id_err=$(awk -v p_id=$TEST_POOL_ID '$1=="osd" && $2=="pg"{split($4, a, "."); if (a[0] != p_id) {print a[0]; exit}}' $OSDTRACE_LOG) +if [ -n "$pool_id_err" ]; then + echo "ERROR: Unexpected pool id found in osdtrace, $pool_id_err" + exit 1 +fi -echo "=== Step 15: Verify radostrace output ===" -if [ ! -f /tmp/radostrace.log ]; then - echo "Error: radostrace log file not found" +# 14d. Check PG ranges in the test pool +TOT_PG=$(microceph.ceph osd pool get test_pool pg_num | awk '{print $2}') +pg_range_err=$(awk -v tot=$TOT_PG '$1=="osd" && $2=="pg"{split($4, a, "."); pg=strtonum(a[1]); if (pg < 0 || pg >= tot)print a[1]}' $OSDTRACE_LOG) +if [[ -n $pg_range_err ]]; then + echo "ERROR: Found PGs outside the expected range: $pg_range_err" exit 1 fi -RADOS_LINE_COUNT=$(wc -l < /tmp/radostrace.log) +# 14e. Check for high latencies +# Maximum acceptable latency value (in microseconds) = 100ms +MAX_LATENCY=100000000 +high_lat=$(awk -v lmax=$MAX_LATENCY '$1=="osd" && $2=="pg" && $NF > lmax' $OSDTRACE_LOG) +if [[ -n $high_lat ]]; then + echo "ERROR: Found latencies over $MAX_LATENCY μs" + exit 1 +fi + +echo "✓ All osdtrace output fields validated successfully" + +echo "===Step 15: Verify radostrace output ===" +RADOS_LINE_COUNT=$(wc -l < $RADOSTRACE_LOG) echo "radostrace captured $RADOS_LINE_COUNT lines" if [ $RADOS_LINE_COUNT -lt 3 ]; then echo "Error: radostrace did not capture enough trace data (expected at least 3 lines)" echo "radostrace output:" - cat /tmp/radostrace.log + cat $RADOSTRACE_LOG exit 1 fi @@ -230,13 +260,14 @@ echo "" echo "=== Test Summary ===" echo "✓ MicroCeph cluster deployed successfully" echo "✓ osdtrace captured $OSD_LINE_COUNT lines of trace data" +echo "✓ osdtrace output validated: $LINES_VALIDATED lines checked, 0 errors" echo "✓ radostrace captured $RADOS_LINE_COUNT lines of trace data" echo "✓ All functional tests passed!" echo "" echo "Sample osdtrace output (first 10 lines):" -head -10 /tmp/osdtrace.log +head -10 $OSDTRACE_LOG echo "" echo "Sample radostrace output (first 10 lines):" -head -10 /tmp/radostrace.log +head -10 $RADOSTRACE_LOG exit 0 From 5589ba981f03ab308f43fe56a4be7f985478cfa5 Mon Sep 17 00:00:00 2001 From: Ponnuvel Palaniyappan Date: Wed, 28 Jan 2026 16:00:40 +0000 Subject: [PATCH 2/3] fix bugs and address comments from Dongdong Signed-off-by: Ponnuvel Palaniyappan --- tests/functional-test-microceph.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/functional-test-microceph.sh b/tests/functional-test-microceph.sh index 6d9aa7f..bfee644 100755 --- a/tests/functional-test-microceph.sh +++ b/tests/functional-test-microceph.sh @@ -22,7 +22,15 @@ cleanup() { pkill -f "rbd bench" || true # Remove test files - cat /tmp/osdtrace.log /tmp/radostrace.log + echo "OSD trace output:" + cat /tmp/osdtrace.log + echo " === END of OSD trace === " + echo "RADOS trace output:" + cat /tmp/radostrace.log + echo " === END of RADOS trace === " + + # Remove test files + echo "RADOS trace output" rm -f /tmp/osdtrace.log /tmp/radostrace.log # Remove test RBD resources @@ -155,7 +163,7 @@ microceph.rbd create test_pool/testimage --size 1G || true echo "=== Step 9: Start osdtrace in background ===" timeout 30 $PROJECT_ROOT/osdtrace -i $OSD_DWARF -p $OSD_PID --skip-version-check -x >$OSDTRACE_LOG 2>&1 & -OSDTRACE_PID=(pidof osdtrace) +OSDTRACE_PID=$(pidof osdtrace) echo "Started osdtrace with PID $OSDTRACE_PID" sleep 3 @@ -200,8 +208,6 @@ OSD_LINE_COUNT=$(wc -l < $OSDTRACE_LOG) echo "osdtrace captured $OSD_LINE_COUNT lines" if [ $OSD_LINE_COUNT -lt 5 ]; then echo "Error: osdtrace did not capture enough trace data (expected at least 5 lines)" - echo "osdtrace output:" - cat $OSDTRACE_LOG exit 1 fi @@ -233,7 +239,7 @@ if [[ -n $pg_range_err ]]; then fi # 14e. Check for high latencies -# Maximum acceptable latency value (in microseconds) = 100ms +# Maximum acceptable latency value (in microseconds) = 100s MAX_LATENCY=100000000 high_lat=$(awk -v lmax=$MAX_LATENCY '$1=="osd" && $2=="pg" && $NF > lmax' $OSDTRACE_LOG) if [[ -n $high_lat ]]; then @@ -249,8 +255,6 @@ echo "radostrace captured $RADOS_LINE_COUNT lines" if [ $RADOS_LINE_COUNT -lt 3 ]; then echo "Error: radostrace did not capture enough trace data (expected at least 3 lines)" - echo "radostrace output:" - cat $RADOSTRACE_LOG exit 1 fi From ea8715d7d20093f8ec8f26b38acf74e4a8310113 Mon Sep 17 00:00:00 2001 From: Ponnuvel Palaniyappan Date: Wed, 28 Jan 2026 16:17:01 +0000 Subject: [PATCH 3/3] Add helpers to print messages in consistent format Signed-off-by: Ponnuvel Palaniyappan --- tests/functional-test-microceph.sh | 170 +++++++++++++++-------------- 1 file changed, 86 insertions(+), 84 deletions(-) diff --git a/tests/functional-test-microceph.sh b/tests/functional-test-microceph.sh index bfee644..eb8a3c6 100755 --- a/tests/functional-test-microceph.sh +++ b/tests/functional-test-microceph.sh @@ -13,31 +13,38 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" echo "=== MicroCeph Functional Test for osdtrace and radostrace ===" echo "Project root: $PROJECT_ROOT" +info() { + echo "INFO: $@" +} + +err() { + echo "ERROR: $@" +} + # Cleanup function cleanup() { - echo "=== Cleanup ===" + info "=== Cleanup ===" # Kill any running trace processes pkill -f osdtrace || true pkill -f radostrace || true pkill -f "rbd bench" || true - # Remove test files - echo "OSD trace output:" + info "OSD trace output:" cat /tmp/osdtrace.log - echo " === END of OSD trace === " - echo "RADOS trace output:" + info " === END of OSD trace === " + info "RADOS trace output:" cat /tmp/radostrace.log - echo " === END of RADOS trace === " + info " === END of RADOS trace === " # Remove test files - echo "RADOS trace output" + info "RADOS trace output" rm -f /tmp/osdtrace.log /tmp/radostrace.log # Remove test RBD resources microceph.rbd rm test_pool/testimage 2>/dev/null || true microceph.ceph osd pool delete test_pool test_pool --yes-i-really-really-mean-it 2>/dev/null || true - echo "Cleanup completed" + info "Cleanup completed" } trap cleanup EXIT @@ -47,59 +54,59 @@ RADOSTRACE_LOG="/tmp/radostrace.log" # Check if running as root or with sudo if [ "$EUID" -ne 0 ]; then - echo "Error: This test must be run as root or with sudo" + err "This test must be run as root or with sudo" exit 1 fi # Check if osdtrace and radostrace binaries exist if [ ! -f "$PROJECT_ROOT/osdtrace" ]; then - echo "Error: osdtrace binary not found at $PROJECT_ROOT/osdtrace" - echo "Please build the project first with 'make osdtrace'" + err "osdtrace binary not found at $PROJECT_ROOT/osdtrace" + err "Please build the project first with 'make osdtrace'" exit 1 fi if [ ! -f "$PROJECT_ROOT/radostrace" ]; then - echo "Error: radostrace binary not found at $PROJECT_ROOT/radostrace" - echo "Please build the project first with 'make radostrace'" + err "radostrace binary not found at $PROJECT_ROOT/radostrace" + err "Please build the project first with 'make radostrace'" exit 1 fi -echo "=== Step 1: Install MicroCeph ===" +info "=== Step 1: Install MicroCeph ===" if ! snap list | grep -q microceph; then - echo "Installing MicroCeph snap..." + info "Installing MicroCeph snap..." snap install microceph snap refresh --hold microceph else - echo "MicroCeph already installed" + info "MicroCeph already installed" fi -echo "=== Step 2: Bootstrap MicroCeph cluster ===" +info "=== Step 2: Bootstrap MicroCeph cluster ===" if ! microceph cluster list 2>/dev/null | grep -q "$(hostname)"; then - echo "Bootstrapping MicroCeph cluster..." + info "Bootstrapping MicroCeph cluster..." microceph cluster bootstrap else - echo "MicroCeph cluster already bootstrapped" + info "MicroCeph cluster already bootstrapped" fi -echo "=== Step 3: Add OSDs ===" +info "=== Step 3: Add OSDs ===" # Check if we already have OSDs OSD_COUNT=$(microceph.ceph osd stat | grep -oP '\d+(?= osds:)' || echo "0") if [ "$OSD_COUNT" -lt 3 ]; then - echo "Adding 3 loop-backed OSDs (1GB each)..." + info "Adding 3 loop-backed OSDs (1GB each)..." microceph disk add loop,1G,3 else - echo "Already have $OSD_COUNT OSDs" + info "Already have $OSD_COUNT OSDs" fi -echo "=== Step 4: Wait for cluster to be healthy ===" +info "=== Step 4: Wait for cluster to be healthy ===" TIMEOUT=120 ELAPSED=0 while [ $ELAPSED -lt $TIMEOUT ]; do if microceph.ceph status | grep -q "HEALTH_OK\|HEALTH_WARN"; then - echo "Cluster is ready" + info "Cluster is ready" break fi - echo "Waiting for cluster to be ready... ($ELAPSED/$TIMEOUT seconds)" + info "Waiting for cluster to be ready... ($ELAPSED/$TIMEOUT seconds)" sleep 5 ELAPSED=$((ELAPSED + 5)) done @@ -107,51 +114,51 @@ done microceph.ceph status microceph --version -echo "=== Step 5: Get Ceph version from snap manifest ===" +info "=== Step 5: Get Ceph version from snap manifest ===" CEPH_VERSION=$(microceph --version | awk '{gsub(";", "", $2); print $2}') -echo "Ceph version: $CEPH_VERSION" +info "Ceph version: $CEPH_VERSION" -echo "=== Step 6: Locate DWARF JSON files in repository ===" +info "=== Step 6: Locate DWARF JSON files in repository ===" # Look for matching DWARF files in the repository OSD_DWARF="$PROJECT_ROOT/files/ubuntu/osdtrace/osd-${CEPH_VERSION}_dwarf.json" RADOS_DWARF="$PROJECT_ROOT/files/ubuntu/radostrace/${CEPH_VERSION}_dwarf.json" if [ ! -f "$OSD_DWARF" ]; then - echo "Warning: OSD DWARF file not found at $OSD_DWARF" - echo "Looking for any available OSD DWARF files..." + warn "OSD DWARF file not found at $OSD_DWARF" + info "Looking for any available OSD DWARF files..." OSD_DWARF=$(find "$PROJECT_ROOT/files/ubuntu/osdtrace/" -name "*_dwarf.json" | head -1) if [ -z "$OSD_DWARF" ]; then - echo "Error: No OSD DWARF files found in repository" + err "No OSD DWARF files found in repository" exit 1 fi - echo "Using: $OSD_DWARF" + info "Using: $OSD_DWARF" fi if [ ! -f "$RADOS_DWARF" ]; then - echo "Warning: Rados DWARF file not found at $RADOS_DWARF" - echo "Looking for any available radostrace DWARF files..." + warn "Rados DWARF file not found at $RADOS_DWARF" + info "Looking for any available radostrace DWARF files..." RADOS_DWARF=$(find "$PROJECT_ROOT/files/ubuntu/radostrace/" -name "*_dwarf.json" | head -1) if [ -z "$RADOS_DWARF" ]; then - echo "Error: No radostrace DWARF files found in repository" + err "No radostrace DWARF files found in repository" exit 1 fi - echo "Using: $RADOS_DWARF" + info "Using: $RADOS_DWARF" fi -echo "Using OSD DWARF file: $OSD_DWARF" -echo "Using Rados DWARF file: $RADOS_DWARF" +info "Using OSD DWARF file: $OSD_DWARF" +info "Using Rados DWARF file: $RADOS_DWARF" -echo "=== Step 7: Find OSD process PID ===" +info "=== Step 7: Find OSD process PID ===" OSD_PID=$(pgrep -f "ceph-osd.*--id 1" | head -1) if [ -z "$OSD_PID" ]; then - echo "Error: Could not find ceph-osd process" + err "Could not find ceph-osd process" ps aux | grep ceph-osd exit 1 fi -echo "Found OSD process: PID $OSD_PID" +info "Found OSD process: PID $OSD_PID" -echo "=== Step 8: Create RBD pool and image for testing ===" +info "=== Step 8: Create RBD pool and image for testing ===" # Create RBD pool if it doesn't exist if ! microceph.ceph osd pool ls | grep -q "^test_pool$"; then microceph.ceph osd pool create test_pool 32 @@ -161,22 +168,24 @@ fi # Create RBD image microceph.rbd create test_pool/testimage --size 1G || true -echo "=== Step 9: Start osdtrace in background ===" +info "=== Step 9: Start osdtrace in background ===" timeout 30 $PROJECT_ROOT/osdtrace -i $OSD_DWARF -p $OSD_PID --skip-version-check -x >$OSDTRACE_LOG 2>&1 & +sleep 2 # ensure osdtrace starts before we get its PID OSDTRACE_PID=$(pidof osdtrace) -echo "Started osdtrace with PID $OSDTRACE_PID" +info "Started osdtrace with PID $OSDTRACE_PID" sleep 3 -echo "=== Step 10: Start radostrace in background ===" +info "=== Step 10: Start radostrace in background ===" # radostrace will trace all librados clients, including the rbd bench command timeout 30 $PROJECT_ROOT/radostrace -i $RADOS_DWARF --skip-version-check >$RADOSTRACE_LOG 2>&1 & -RADOSTRACE_PID=$! -echo "Started radostrace with PID $RADOSTRACE_PID" +sleep 2 # ensure radostrace starts before we get its PID +RADOSTRACE_PID=$(pidof radostrace) +info "Started radostrace with PID $RADOSTRACE_PID" sleep 3 -echo "=== Step 11: Generate I/O traffic using rbd bench ===" +info "=== Step 11: Generate I/O traffic using rbd bench ===" # Run rbd bench for write operations -echo "Running rbd bench write..." +info "Running rbd bench write..." microceph.rbd bench --io-type write --io-size 4M --io-threads 4 --io-total 100M test_pool/testimage & RBD_BENCH_PID=$! @@ -184,15 +193,15 @@ RBD_BENCH_PID=$! sleep 10 # Run some rados operations to generate more librados traffic -echo "Performing rados operations..." +info "Performing rados operations..." microceph.rados -p test_pool put testobj /etc/hostname || true microceph.rados -p test_pool get testobj /tmp/testobj || true microceph.rados -p test_pool rm testobj || true -echo "=== Step 12: Wait for rbd bench to complete ===" +info "=== Step 12: Wait for rbd bench to complete ===" wait $RBD_BENCH_PID 2>/dev/null || true -echo "=== Step 13: Wait for traces to complete ===" +info "=== Step 13: Wait for traces to complete ===" sleep 5 # Kill trace processes gracefully @@ -201,77 +210,70 @@ kill $RADOSTRACE_PID 2>/dev/null || true wait $OSDTRACE_PID 2>/dev/null || true wait $RADOSTRACE_PID 2>/dev/null || true -echo "=== Step 14: Verify osdtrace output ===" +info "=== Step 14: Verify osdtrace output ===" -# 14a. Check trace exists +# 14.1 Check trace exists OSD_LINE_COUNT=$(wc -l < $OSDTRACE_LOG) -echo "osdtrace captured $OSD_LINE_COUNT lines" +info "osdtrace captured $OSD_LINE_COUNT lines" if [ $OSD_LINE_COUNT -lt 5 ]; then - echo "Error: osdtrace did not capture enough trace data (expected at least 5 lines)" + err "osdtrace did not capture enough trace data (expected at least 5 lines)" exit 1 fi -# 14b. Check OSD IDs range is within the expected limit +# 14.2 Check OSD IDs range is within the expected limit MAX_OSD_ID=$(microceph.ceph osd stat | grep -oP '\d+(?= osds:)' || echo "0") MAX_OSD_ID=$((MAX_OSD_ID - 1)) # Convert count to max ID (0-indexed) -echo "Max OSD ID in cluster: $MAX_OSD_ID" +info "Max OSD ID in cluster: $MAX_OSD_ID" osd_id_err=$(awk -v max_osd=$MAX_OSD_ID '$1=="osd" && ($2 < 0 || $2 > max_osd) {print $2; exit}' $OSDTRACE_LOG) if [ -n "$osd_id_err" ]; then - echo "ERROR: Found OSD id outside the expected range, $osd_id_err" + err "Found OSD id outside the expected range, $osd_id_err" exit 1 fi -# 14c. Check the correct pool id is used +# 14.3 Check the correct pool id is used TEST_POOL_ID=$(microceph.ceph osd pool ls detail | grep "^pool.*'test_pool'" | grep -oP "pool \K\d+") pool_id_err=$(awk -v p_id=$TEST_POOL_ID '$1=="osd" && $2=="pg"{split($4, a, "."); if (a[0] != p_id) {print a[0]; exit}}' $OSDTRACE_LOG) if [ -n "$pool_id_err" ]; then - echo "ERROR: Unexpected pool id found in osdtrace, $pool_id_err" + err "Unexpected pool id found in osdtrace, $pool_id_err" exit 1 fi -# 14d. Check PG ranges in the test pool +# 14.4 Check PG ranges in the test pool TOT_PG=$(microceph.ceph osd pool get test_pool pg_num | awk '{print $2}') pg_range_err=$(awk -v tot=$TOT_PG '$1=="osd" && $2=="pg"{split($4, a, "."); pg=strtonum(a[1]); if (pg < 0 || pg >= tot)print a[1]}' $OSDTRACE_LOG) if [[ -n $pg_range_err ]]; then - echo "ERROR: Found PGs outside the expected range: $pg_range_err" + err "Found PGs outside the expected range: $pg_range_err" exit 1 fi -# 14e. Check for high latencies +# 14.5 Check for high latencies # Maximum acceptable latency value (in microseconds) = 100s MAX_LATENCY=100000000 high_lat=$(awk -v lmax=$MAX_LATENCY '$1=="osd" && $2=="pg" && $NF > lmax' $OSDTRACE_LOG) if [[ -n $high_lat ]]; then - echo "ERROR: Found latencies over $MAX_LATENCY μs" + err "Found latencies over $MAX_LATENCY μs" exit 1 fi -echo "✓ All osdtrace output fields validated successfully" +info "✓ All osdtrace output fields validated successfully" -echo "===Step 15: Verify radostrace output ===" +info "===Step 15: Verify radostrace output ===" RADOS_LINE_COUNT=$(wc -l < $RADOSTRACE_LOG) -echo "radostrace captured $RADOS_LINE_COUNT lines" +info "radostrace captured $RADOS_LINE_COUNT lines" if [ $RADOS_LINE_COUNT -lt 3 ]; then - echo "Error: radostrace did not capture enough trace data (expected at least 3 lines)" + err "radostrace did not capture enough trace data (expected at least 3 lines)" exit 1 fi -echo "✓ radostrace successfully captured trace data" - -echo "" -echo "=== Test Summary ===" -echo "✓ MicroCeph cluster deployed successfully" -echo "✓ osdtrace captured $OSD_LINE_COUNT lines of trace data" -echo "✓ osdtrace output validated: $LINES_VALIDATED lines checked, 0 errors" -echo "✓ radostrace captured $RADOS_LINE_COUNT lines of trace data" -echo "✓ All functional tests passed!" -echo "" -echo "Sample osdtrace output (first 10 lines):" -head -10 $OSDTRACE_LOG -echo "" -echo "Sample radostrace output (first 10 lines):" -head -10 $RADOSTRACE_LOG +info "✓ radostrace successfully captured trace data" + +info "=== Test Summary ===" +info "✓ MicroCeph cluster deployed successfully" +info "✓ osdtrace captured $OSD_LINE_COUNT lines of trace data" +info "✓ osdtrace output validated: $LINES_VALIDATED lines checked, 0 errors" +info "✓ radostrace captured $RADOS_LINE_COUNT lines of trace data" +info "✓ All functional tests passed!" exit 0