chore: publish sqlmesh tests (#2270)

Author: eakmanrq

.circleci/config.yml

@@ -24,7 +24,9 @@ jobs:
       - run:
           name: Publish Python package
           command: make publish
-
+      - run:
+          name: Publish Python Tests package
+          command: make publish-tests
   gh-release:
     docker:
       - image: cimg/node:16.14

.circleci/update-pypirc.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+echo "[distutils]" >> ~/.pypirc
+echo "index-servers = tobiko-private" >> ~/.pypirc
+echo "[tobiko-private]" >> ~/.pypirc
+echo "repository = $TOBIKO_PRIVATE_PYPI_URL" >> ~/.pypirc
+echo "username = _json_key_base64" >> ~/.pypirc
+echo "password = $TOBIKO_PRIVATE_PYPI_KEY" >> ~/.pypirc

Makefile

@@ -33,6 +33,12 @@ package:
 publish: package
 	pip3 install twine && python3 -m twine upload dist/*
 
+package-tests:
+	pip3 install wheel && python3 tests/setup.py sdist bdist_wheel
+
+publish-tests: package-tests
+	pip3 install twine && python3 -m twine upload -r tobiko-private tests/dist/*
+
 develop:
 	python3 setup.py develop
 

tests/core/test_state_sync.py

@@ -28,6 +28,7 @@
     Snapshot,
     SnapshotChangeCategory,
     SnapshotId,
+    SnapshotIntervals,
     SnapshotTableCleanupTask,
     missing_intervals,
 )
@@ -211,12 +212,21 @@ def test_snapshots_exists(state_sync: EngineAdapterStateSync, snapshots: t.List[
     assert state_sync.snapshots_exist(snapshot_ids) == snapshot_ids
 
 
-def get_snapshot_intervals(state_sync, snapshot):
-    intervals = state_sync._get_snapshot_intervals([snapshot])[-1]
-    return intervals[0] if intervals else None
+@pytest.fixture
+def get_snapshot_intervals(state_sync) -> t.Callable[[Snapshot], t.Optional[SnapshotIntervals]]:
+
+    def _get_snapshot_intervals(snapshot: Snapshot) -> t.Optional[SnapshotIntervals]:
+        intervals = state_sync._get_snapshot_intervals([snapshot])[-1]
+        return intervals[0] if intervals else None
+
+    return _get_snapshot_intervals
 
 
-def test_add_interval(state_sync: EngineAdapterStateSync, make_snapshot: t.Callable) -> None:
+def test_add_interval(
+    state_sync: EngineAdapterStateSync,
+    make_snapshot: t.Callable,
+    get_snapshot_intervals: t.Callable,
+) -> None:
     snapshot = make_snapshot(
         SqlModel(
             name="a",
@@ -229,24 +239,24 @@ def test_add_interval(state_sync: EngineAdapterStateSync, make_snapshot: t.Calla
     state_sync.push_snapshots([snapshot])
 
     state_sync.add_interval(snapshot, "2020-01-01", "20200101")
-    assert get_snapshot_intervals(state_sync, snapshot).intervals == [
+    assert get_snapshot_intervals(snapshot).intervals == [
         (to_timestamp("2020-01-01"), to_timestamp("2020-01-02")),
     ]
 
     state_sync.add_interval(snapshot, "20200101", to_datetime("2020-01-04"))
-    assert get_snapshot_intervals(state_sync, snapshot).intervals == [
+    assert get_snapshot_intervals(snapshot).intervals == [
         (to_timestamp("2020-01-01"), to_timestamp("2020-01-04")),
     ]
 
     state_sync.add_interval(snapshot, to_datetime("2020-01-05"), "2020-01-10")
-    assert get_snapshot_intervals(state_sync, snapshot).intervals == [
+    assert get_snapshot_intervals(snapshot).intervals == [
         (to_timestamp("2020-01-01"), to_timestamp("2020-01-04")),
         (to_timestamp("2020-01-05"), to_timestamp("2020-01-11")),
     ]
 
     snapshot.change_category = SnapshotChangeCategory.FORWARD_ONLY
     state_sync.add_interval(snapshot, to_datetime("2020-01-16"), "2020-01-20", is_dev=True)
-    intervals = get_snapshot_intervals(state_sync, snapshot)
+    intervals = get_snapshot_intervals(snapshot)
     assert intervals.intervals == [
         (to_timestamp("2020-01-01"), to_timestamp("2020-01-04")),
         (to_timestamp("2020-01-05"), to_timestamp("2020-01-11")),
@@ -257,7 +267,9 @@ def test_add_interval(state_sync: EngineAdapterStateSync, make_snapshot: t.Calla
 
 
 def test_add_interval_partial(
-    state_sync: EngineAdapterStateSync, make_snapshot: t.Callable
+    state_sync: EngineAdapterStateSync,
+    make_snapshot: t.Callable,
+    get_snapshot_intervals: t.Callable,
 ) -> None:
     snapshot = make_snapshot(
         SqlModel(
@@ -271,10 +283,10 @@ def test_add_interval_partial(
     state_sync.push_snapshots([snapshot])
 
     state_sync.add_interval(snapshot, "2023-01-01", to_timestamp("2023-01-01") + 1000)
-    assert get_snapshot_intervals(state_sync, snapshot) is None
+    assert get_snapshot_intervals(snapshot) is None
 
     state_sync.add_interval(snapshot, "2023-01-01", to_timestamp("2023-01-02") + 1000)
-    assert get_snapshot_intervals(state_sync, snapshot).intervals == [
+    assert get_snapshot_intervals(snapshot).intervals == [
         (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")),
     ]
 
@@ -338,7 +350,7 @@ def test_refresh_snapshot_intervals(
 
 
 def test_get_snapshot_intervals(
-    state_sync: EngineAdapterStateSync, make_snapshot: t.Callable
+    state_sync: EngineAdapterStateSync, make_snapshot: t.Callable, get_snapshot_intervals
 ) -> None:
     state_sync.SNAPSHOT_BATCH_SIZE = 1
 
@@ -372,16 +384,20 @@ def test_get_snapshot_intervals(
         ),
         version="c",
     )
-    state_sync.add_interval(snapshot_c, "2020-01-03", "2020-01-03")
     state_sync.push_snapshots([snapshot_c])
+    state_sync.add_interval(snapshot_c, "2020-01-03", "2020-01-03")
 
-    _, intervals = state_sync._get_snapshot_intervals([snapshot_b, snapshot_c])
-    assert len(intervals) == 2
-    assert intervals[0].intervals == [(to_timestamp("2020-01-01"), to_timestamp("2020-01-02"))]
-    assert intervals[1].intervals == [(to_timestamp("2020-01-03"), to_timestamp("2020-01-04"))]
+    a_intervals = get_snapshot_intervals(snapshot_a)
+    c_intervals = get_snapshot_intervals(snapshot_c)
+    assert a_intervals.intervals == [(to_timestamp("2020-01-01"), to_timestamp("2020-01-02"))]
+    assert c_intervals.intervals == [(to_timestamp("2020-01-03"), to_timestamp("2020-01-04"))]
 
 
-def test_compact_intervals(state_sync: EngineAdapterStateSync, make_snapshot: t.Callable) -> None:
+def test_compact_intervals(
+    state_sync: EngineAdapterStateSync,
+    make_snapshot: t.Callable,
+    get_snapshot_intervals: t.Callable,
+) -> None:
     snapshot = make_snapshot(
         SqlModel(
             name="a",
@@ -408,14 +424,14 @@ def test_compact_intervals(state_sync: EngineAdapterStateSync, make_snapshot: t.
         (to_timestamp("2020-01-12"), to_timestamp("2020-01-14")),
     ]
 
-    assert get_snapshot_intervals(state_sync, snapshot).intervals == expected_intervals
+    assert get_snapshot_intervals(snapshot).intervals == expected_intervals
 
     state_sync.compact_intervals()
-    assert get_snapshot_intervals(state_sync, snapshot).intervals == expected_intervals
+    assert get_snapshot_intervals(snapshot).intervals == expected_intervals
 
     # Make sure compaction is idempotent.
     state_sync.compact_intervals()
-    assert get_snapshot_intervals(state_sync, snapshot).intervals == expected_intervals
+    assert get_snapshot_intervals(snapshot).intervals == expected_intervals
 
 
 def test_promote_snapshots(state_sync: EngineAdapterStateSync, make_snapshot: t.Callable):
@@ -1248,13 +1264,17 @@ def test_unpause_snapshots_remove_intervals(
         ),
         version="a",
     )
+    snapshot.categorize_as(SnapshotChangeCategory.BREAKING)
+    snapshot.version = "a"
     state_sync.push_snapshots([snapshot])
     state_sync.add_interval(snapshot, "2023-01-01", "2023-01-05")
 
     new_snapshot = make_snapshot(
         SqlModel(name="test_snapshot", query=parse_one("select 2, ds"), cron="@daily"),
         version="a",
     )
+    new_snapshot.categorize_as(SnapshotChangeCategory.FORWARD_ONLY)
+    new_snapshot.version = "a"
     new_snapshot.effective_from = "2023-01-03"
     state_sync.push_snapshots([new_snapshot])
     state_sync.add_interval(snapshot, "2023-01-06", "2023-01-06")

tests/setup.py

@@ -0,0 +1,25 @@
+import os
+from distutils.core import run_setup, setup
+
+os.chdir(os.path.join(os.path.dirname(__file__), ".."))
+sqlmesh_dist = run_setup("setup.py", stop_after="init")
+requirements = sqlmesh_dist.install_requires + sqlmesh_dist.extras_require["dev"]  # type: ignore
+os.chdir(os.path.dirname(__file__))
+
+setup(
+    name="sqlmesh-tests",
+    description="Tests for SQLMesh",
+    url="https://github.com/TobikoData/sqlmesh",
+    author="TobikoData Inc.",
+    author_email="engineering@tobikodata.com",
+    license="Apache License 2.0",
+    package_dir={"sqlmesh_tests": ""},
+    package_data={"": ["fixtures/**"]},
+    use_scm_version={
+        "write_to": "_version.py",
+        "fallback_version": "0.0.0",
+        "local_scheme": "no-local-version",
+    },
+    setup_requires=["setuptools_scm"],
+    install_requires=requirements,
+)