From 74fb1a0cd12c9b3abe38c7c10f12211789f535e2 Mon Sep 17 00:00:00 2001 From: George Sittas Date: Mon, 28 Jul 2025 12:45:27 +0300 Subject: [PATCH 1/2] Chore: fail if >1 migration scripts start with the same version --- .pre-commit-config.yaml | 6 ++++++ scripts/check_migration_duplicates.sh | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100755 scripts/check_migration_duplicates.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 96eca9b6a4..3a9a978fed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,3 +23,9 @@ repos: files: *files require_serial: true exclude: ^(tests/fixtures) + - id: check-migration-duplicates + name: Check migration duplicates + language: script + entry: ./scripts/check_migration_duplicates.sh + files: ^sqlmesh/migrations/v\d{4}_.*\.py$ + pass_filenames: false diff --git a/scripts/check_migration_duplicates.sh b/scripts/check_migration_duplicates.sh new file mode 100755 index 0000000000..13afd4b50f --- /dev/null +++ b/scripts/check_migration_duplicates.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Check for duplicate migration version numbers in SQLMesh + +# Extract version numbers from migration files and check for duplicates +duplicates=$(ls sqlmesh/migrations/v[0-9][0-9][0-9][0-9]_*.py 2>/dev/null | \ + sed 's/.*\/v\([0-9]\{4\}\)_.*/\1/' | \ + sort | uniq -d) + +if [ -n "$duplicates" ]; then + echo "Error: Duplicate migration version(s) found:" >&2 + for version in $duplicates; do + echo " Version v$version appears in:" >&2 + ls sqlmesh/migrations/v${version}_*.py | sed 's/^/ - /' >&2 + done + exit 1 +fi + +exit 0 From 5a9cc0d8aaf0ee2746d16c6e4f711a77e201841a Mon Sep 17 00:00:00 2001 From: George Sittas Date: Mon, 28 Jul 2025 13:09:34 +0300 Subject: [PATCH 2/2] Refactor script to make it more comprehensive, move under tooling/ --- .pre-commit-config.yaml | 9 ++--- scripts/check_migration_duplicates.sh | 18 --------- tooling/validating_migration_numbers.sh | 53 +++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 23 deletions(-) delete mode 100755 scripts/check_migration_duplicates.sh create mode 100755 tooling/validating_migration_numbers.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3a9a978fed..e4b61cb0f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,9 +23,8 @@ repos: files: *files require_serial: true exclude: ^(tests/fixtures) - - id: check-migration-duplicates - name: Check migration duplicates - language: script - entry: ./scripts/check_migration_duplicates.sh - files: ^sqlmesh/migrations/v\d{4}_.*\.py$ + - id: valid migrations + name: valid migrations + entry: tooling/validating_migration_numbers.sh + language: system pass_filenames: false diff --git a/scripts/check_migration_duplicates.sh b/scripts/check_migration_duplicates.sh deleted file mode 100755 index 13afd4b50f..0000000000 --- a/scripts/check_migration_duplicates.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Check for duplicate migration version numbers in SQLMesh - -# Extract version numbers from migration files and check for duplicates -duplicates=$(ls sqlmesh/migrations/v[0-9][0-9][0-9][0-9]_*.py 2>/dev/null | \ - sed 's/.*\/v\([0-9]\{4\}\)_.*/\1/' | \ - sort | uniq -d) - -if [ -n "$duplicates" ]; then - echo "Error: Duplicate migration version(s) found:" >&2 - for version in $duplicates; do - echo " Version v$version appears in:" >&2 - ls sqlmesh/migrations/v${version}_*.py | sed 's/^/ - /' >&2 - done - exit 1 -fi - -exit 0 diff --git a/tooling/validating_migration_numbers.sh b/tooling/validating_migration_numbers.sh new file mode 100755 index 0000000000..6dbb597dc1 --- /dev/null +++ b/tooling/validating_migration_numbers.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Navigate to the migrations directory (modify the path if necessary) +cd "sqlmesh/migrations" || exit 1 + + +# Collect all migration files matching the pattern (e.g., v0001_initial.py) +migration_files=(v*.py) + +# Initialize an array to hold migration numbers +numbers=() + +# Extract migration numbers from filenames +for file in "${migration_files[@]}"; do + if [[ $file =~ ^v0*([0-9]+)_ ]]; then + num=${BASH_REMATCH[1]} + numbers+=("$num") + fi +done + +# Check if any migration files were found +if [[ ${#numbers[@]} -eq 0 ]]; then + echo "No migration files found matching the pattern 'v_.py'." + exit 1 +fi + +# Check for duplicate migration numbers +duplicates=$(printf "%s\n" "${numbers[@]}" | sort | uniq -d) +if [[ -n $duplicates ]]; then + echo "Error: Duplicate migration numbers found: $duplicates" + exit 1 +fi + +# Sort the migration numbers +sorted_numbers=($(printf "%s\n" "${numbers[@]}" | sort -n)) + +# Get the first and last migration numbers +first_number="${sorted_numbers[0]}" +last_index=$((${#sorted_numbers[@]} - 1)) +last_number="${sorted_numbers[$last_index]}" + +# Check for gaps in the migration sequence +expected_numbers=($(seq "$first_number" "$last_number")) + +if [[ "${sorted_numbers[*]}" != "${expected_numbers[*]}" ]]; then + echo "Error: Missing migration numbers in sequence." + echo "Expected sequence: ${expected_numbers[*]}" + echo "Found sequence: ${sorted_numbers[*]}" + exit 1 +fi + +echo "All migration numbers are sequential and without overlaps." +exit 0