Skip to content

Commit aa41dee

Browse files
mykalmaxmykalmax
authored
Fix: switch from pooling to default method to detect changes (#3113)
1 parent d6e6d07 commit aa41dee

File tree

4 files changed

+31
-161
lines changed

4 files changed

+31
-161
lines changed

‎web/client/openapi.json‎

Lines changed: 3 additions & 145 deletions
Original file line numberDiff line numberDiff line change
@@ -1163,11 +1163,7 @@
11631163
"title": "Normalize Name",
11641164
"default": true
11651165
},
1166-
"snapshots": {
1167-
"items": { "$ref": "#/components/schemas/SnapshotTableInfo" },
1168-
"type": "array",
1169-
"title": "Snapshots"
1170-
},
1166+
"snapshots": { "items": {}, "type": "array", "title": "Snapshots" },
11711167
"start_at": {
11721168
"anyOf": [
11731169
{ "type": "string", "format": "date" },
@@ -1203,23 +1199,11 @@
12031199
"title": "Finalized Ts"
12041200
},
12051201
"promoted_snapshot_ids": {
1206-
"anyOf": [
1207-
{
1208-
"items": { "$ref": "#/components/schemas/SnapshotId" },
1209-
"type": "array"
1210-
},
1211-
{ "type": "null" }
1212-
],
1202+
"anyOf": [{ "items": {}, "type": "array" }, { "type": "null" }],
12131203
"title": "Promoted Snapshot Ids"
12141204
},
12151205
"previous_finalized_snapshots": {
1216-
"anyOf": [
1217-
{
1218-
"items": { "$ref": "#/components/schemas/SnapshotTableInfo" },
1219-
"type": "array"
1220-
},
1221-
{ "type": "null" }
1222-
],
1206+
"anyOf": [{ "items": {}, "type": "array" }, { "type": "null" }],
12231207
"title": "Previous Finalized Snapshots"
12241208
}
12251209
},
@@ -1550,25 +1534,6 @@
15501534
"type": "object",
15511535
"title": "ModelDetails"
15521536
},
1553-
"ModelKindName": {
1554-
"type": "string",
1555-
"enum": [
1556-
"INCREMENTAL_BY_TIME_RANGE",
1557-
"INCREMENTAL_BY_UNIQUE_KEY",
1558-
"INCREMENTAL_BY_PARTITION",
1559-
"INCREMENTAL_UNMANAGED",
1560-
"FULL",
1561-
"SCD_TYPE_2",
1562-
"SCD_TYPE_2_BY_TIME",
1563-
"SCD_TYPE_2_BY_COLUMN",
1564-
"VIEW",
1565-
"EMBEDDED",
1566-
"SEED",
1567-
"EXTERNAL"
1568-
],
1569-
"title": "ModelKindName",
1570-
"description": "The kind of model, determining how this data is computed and stored in the warehouse."
1571-
},
15721537
"ModelType": {
15731538
"type": "string",
15741539
"enum": ["python", "sql", "seed", "external"],
@@ -2140,113 +2105,6 @@
21402105
"title": "SnapshotChangeCategory",
21412106
"description": "Values are ordered by decreasing severity and that ordering is required.\n\nBREAKING: The change requires that snapshot modified and downstream dependencies be rebuilt\nNON_BREAKING: The change requires that only the snapshot modified be rebuilt\nFORWARD_ONLY: The change requires no rebuilding\nINDIRECT_BREAKING: The change was caused indirectly and is breaking.\nINDIRECT_NON_BREAKING: The change was caused indirectly by a non-breaking change.\nMETADATA: The change was caused by a metadata update."
21422107
},
2143-
"SnapshotDataVersion": {
2144-
"properties": {
2145-
"fingerprint": { "$ref": "#/components/schemas/SnapshotFingerprint" },
2146-
"version": { "type": "string", "title": "Version" },
2147-
"temp_version": {
2148-
"anyOf": [{ "type": "string" }, { "type": "null" }],
2149-
"title": "Temp Version"
2150-
},
2151-
"change_category": {
2152-
"anyOf": [
2153-
{ "$ref": "#/components/schemas/SnapshotChangeCategory" },
2154-
{ "type": "null" }
2155-
]
2156-
},
2157-
"physical_schema": {
2158-
"anyOf": [{ "type": "string" }, { "type": "null" }],
2159-
"title": "Physical Schema"
2160-
}
2161-
},
2162-
"additionalProperties": false,
2163-
"type": "object",
2164-
"required": ["fingerprint", "version"],
2165-
"title": "SnapshotDataVersion"
2166-
},
2167-
"SnapshotFingerprint": {
2168-
"properties": {
2169-
"data_hash": { "type": "string", "title": "Data Hash" },
2170-
"metadata_hash": { "type": "string", "title": "Metadata Hash" },
2171-
"parent_data_hash": {
2172-
"type": "string",
2173-
"title": "Parent Data Hash",
2174-
"default": "0"
2175-
},
2176-
"parent_metadata_hash": {
2177-
"type": "string",
2178-
"title": "Parent Metadata Hash",
2179-
"default": "0"
2180-
}
2181-
},
2182-
"additionalProperties": false,
2183-
"type": "object",
2184-
"required": ["data_hash", "metadata_hash"],
2185-
"title": "SnapshotFingerprint"
2186-
},
2187-
"SnapshotId": {
2188-
"properties": {
2189-
"name": { "type": "string", "title": "Name" },
2190-
"identifier": { "type": "string", "title": "Identifier" }
2191-
},
2192-
"additionalProperties": false,
2193-
"type": "object",
2194-
"required": ["name", "identifier"],
2195-
"title": "SnapshotId"
2196-
},
2197-
"SnapshotTableInfo": {
2198-
"properties": {
2199-
"name": { "type": "string", "title": "Name" },
2200-
"temp_version": {
2201-
"anyOf": [{ "type": "string" }, { "type": "null" }],
2202-
"title": "Temp Version"
2203-
},
2204-
"change_category": {
2205-
"anyOf": [
2206-
{ "$ref": "#/components/schemas/SnapshotChangeCategory" },
2207-
{ "type": "null" }
2208-
]
2209-
},
2210-
"fingerprint": { "$ref": "#/components/schemas/SnapshotFingerprint" },
2211-
"previous_versions": {
2212-
"items": { "$ref": "#/components/schemas/SnapshotDataVersion" },
2213-
"type": "array",
2214-
"title": "Previous Versions",
2215-
"default": []
2216-
},
2217-
"base_table_name_override": {
2218-
"anyOf": [{ "type": "string" }, { "type": "null" }],
2219-
"title": "Base Table Name Override"
2220-
},
2221-
"version": { "type": "string", "title": "Version" },
2222-
"physical_schema": { "type": "string", "title": "Physical Schema" },
2223-
"parents": {
2224-
"items": { "$ref": "#/components/schemas/SnapshotId" },
2225-
"type": "array",
2226-
"title": "Parents"
2227-
},
2228-
"kind_name": {
2229-
"anyOf": [
2230-
{ "$ref": "#/components/schemas/ModelKindName" },
2231-
{ "type": "null" }
2232-
]
2233-
},
2234-
"node_type": {
2235-
"allOf": [{ "$ref": "#/components/schemas/NodeType" }],
2236-
"default": "model"
2237-
}
2238-
},
2239-
"additionalProperties": false,
2240-
"type": "object",
2241-
"required": [
2242-
"name",
2243-
"fingerprint",
2244-
"version",
2245-
"physical_schema",
2246-
"parents"
2247-
],
2248-
"title": "SnapshotTableInfo"
2249-
},
22502108
"Status": {
22512109
"type": "string",
22522110
"enum": ["init", "success", "fail"],

‎web/client/src/library/components/fileExplorer/context.tsx‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ export default function FileExplorerProvider({
158158
}
159159

160160
function renameArtifact(artifact: ModelArtifact, newName?: string): void {
161-
newName = newName?.trim()
161+
newName = newName?.replaceAll('/', '').trim()
162162

163163
const parentArtifacts = artifact.parent?.artifacts ?? []
164164
const isDuplicate = parentArtifacts.some(a => a.name === newName)
@@ -300,7 +300,11 @@ export default function FileExplorerProvider({
300300
artifact.rename(artifact.copyName())
301301
}
302302

303-
const new_path = [target.path, artifact.name].join('/')
303+
let new_path = [target.path, artifact.name].join('/')
304+
305+
while (new_path.startsWith('/')) {
306+
new_path = new_path.slice(1).trim()
307+
}
304308

305309
if (artifact instanceof ModelDirectory) {
306310
moveArtifactCallbacks.push(() => {

‎web/client/src/library/pages/root/Navigation.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ export default function PageNavigation(): JSX.Element {
1414
return (
1515
<div
1616
className={clsx(
17-
'relative min-w-[10rem] px-2 min-h-8 max-h-8 w-full flex items-center',
17+
'relative min-w-[10rem] px-2 min-h-[2rem] max-h-[2rem] w-full flex items-center',
1818
modules.showHistoryNavigation ? 'justify-between' : 'justify-end',
1919
isFetchingModels && 'overflow-hidden',
2020
)}

‎web/server/watcher.py‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
invalidate_context_cache,
1717
)
1818
from web.server.utils import is_relative_to
19+
from sqlglot.helper import ensure_list
1920

2021

2122
async def watch_project() -> None:
@@ -28,33 +29,40 @@ async def watch_project() -> None:
2829
(settings.project_path / c.METRICS).resolve(),
2930
(settings.project_path / c.SEEDS).resolve(),
3031
]
32+
ignore_dirs = [".env"]
33+
ignore_paths: t.List[t.Union[str, Path]] = [(settings.project_path / c.CACHE).resolve()]
3134
ignore_entity_patterns = context.config.ignore_patterns if context else c.IGNORE_PATTERNS
32-
ignore_entity_patterns.append("^\\.DS_Store$")
3335
ignore_entity_patterns.append("^.*\\.db(\\.wal)?$")
34-
ignore_paths = [str((settings.project_path / c.CACHE).resolve())]
35-
watch_filter = DefaultFilter(
36-
ignore_paths=ignore_paths, ignore_entity_patterns=ignore_entity_patterns
37-
)
36+
3837
async for entries in awatch(
39-
settings.project_path, watch_filter=watch_filter, force_polling=True
38+
settings.project_path,
39+
watch_filter=DefaultFilter(
40+
ignore_paths=ensure_list(DefaultFilter.ignore_paths) + ignore_paths,
41+
ignore_entity_patterns=ensure_list(DefaultFilter.ignore_entity_patterns)
42+
+ ignore_entity_patterns,
43+
ignore_dirs=ensure_list(DefaultFilter.ignore_dirs) + ignore_dirs,
44+
),
4045
):
4146
changes: t.List[models.ArtifactChange] = []
4247
directories: t.Dict[str, models.Directory] = {}
4348
for change, path_str in entries:
4449
path = Path(path_str)
50+
relative_path = path.relative_to(settings.project_path)
4551
try:
46-
relative_path = path.relative_to(settings.project_path)
47-
if change == Change.modified and path.is_dir():
48-
directory = await _get_directory(path, settings)
49-
directories[directory.path] = directory
50-
elif change == Change.deleted or not path.exists():
52+
if change == Change.deleted or not path.exists():
5153
changes.append(
5254
models.ArtifactChange(
5355
change=Change.deleted,
5456
path=str(relative_path),
5557
)
5658
)
57-
elif change == Change.modified and path.is_file():
59+
elif change == Change.added:
60+
directory = await _get_directory(path.parent, settings)
61+
directories[directory.path] = directory
62+
elif path.is_dir() and change == Change.modified:
63+
directory = await _get_directory(path, settings)
64+
directories[directory.path] = directory
65+
elif path.is_file() and change == Change.modified:
5866
changes.append(
5967
models.ArtifactChange(
6068
type=models.ArtifactType.file,
@@ -90,7 +98,7 @@ async def watch_project() -> None:
9098
models.Modules.PLANS,
9199
models.Modules.LINEAGE,
92100
}
93-
):
101+
) and (changes or directories):
94102
api_console.log_event(
95103
event=models.EventName.FILE,
96104
data={"changes": changes, "directories": directories},

0 commit comments

Comments
 (0)