feat: making multi repo

Author: benfdking

vscode/extension/package.json

@@ -32,10 +32,13 @@
       "type": "object",
       "title": "SQLMesh",
       "properties": {
-        "sqlmesh.projectPath": {
-          "type": "string",
-          "default": "",
-          "markdownDescription": "The path to the SQLMesh project. If not set, the extension will try to find the project root automatically. If set, the extension will use the project root as the workspace path, e.g. it will run `sqlmesh` and `sqlmesh_lsp` in the project root. The path can be absolute `/Users/sqlmesh_user/sqlmesh_project/sushi` or relative `./project_folder/sushi` to the workspace root."
+        "sqlmesh.projectPaths": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "default": [],
+          "description": "The path to the SQLMesh project. If not set, the extension will try to find the project root automatically. If set, the extension will use the project root as the workspace path, e.g. it will run `sqlmesh` and `sqlmesh_lsp` in the project root. The path can be absolute `/Users/sqlmesh_user/sqlmesh_project/sushi` or relative `./project_folder/sushi` to the workspace root. Multiple paths can be used for multi-project setups."
         },
         "sqlmesh.lspEntrypoint": {
           "type": "string",

vscode/extension/src/utilities/config.ts

@@ -1,15 +1,17 @@
 import { workspace, WorkspaceFolder } from 'vscode'
 import path from 'path'
 import fs from 'fs'
-import { Result, err, ok } from '@bus/result'
+import { Result, err, isErr, ok } from '@bus/result'
 import { traceVerbose, traceInfo } from './common/log'
 import { parse } from 'shell-quote'
 import { z } from 'zod'
 
-export interface SqlmeshConfiguration {
-  projectPath: string
-  lspEntryPoint: string
-}
+const sqlmeshConfigurationSchema = z.object({
+  projectPaths: z.array(z.string()),
+  lspEntryPoint: z.string(),
+})
+
+export type SqlmeshConfiguration = z.infer<typeof sqlmeshConfigurationSchema>
 
 /**
  * Get the SQLMesh configuration from VS Code settings.
@@ -18,12 +20,18 @@ export interface SqlmeshConfiguration {
  */
 function getSqlmeshConfiguration(): SqlmeshConfiguration {
   const config = workspace.getConfiguration('sqlmesh')
-  const projectPath = config.get<string>('projectPath', '')
+  const projectPaths = config.get<string[]>('projectPaths', [])
   const lspEntryPoint = config.get<string>('lspEntrypoint', '')
-  return {
-    projectPath,
+  const parsed = sqlmeshConfigurationSchema.safeParse({
+    projectPaths,
     lspEntryPoint,
+  })
+  if (!parsed.success) {
+    throw new Error(
+      `Invalid SQLMesh configuration: ${JSON.stringify(parsed.error)}`,
+    )
   }
+  return parsed.data
 }
 
 const stringsArray = z.array(z.string())
@@ -57,31 +65,47 @@ export function getSqlmeshLspEntryPoint():
 }
 
 /**
- * Validate and resolve the project path from configuration.
+ * Validate and resolve the project paths from configuration.
  * If no project path is configured, use the workspace folder.
  * If the project path is configured, it must be a directory that contains a SQLMesh project.
  *
  * @param workspaceFolder The current workspace folder
- * @returns A Result containing the resolved project path or an error
+ * @returns A Result containing the resolved project paths or an error
  */
 export function resolveProjectPath(
   workspaceFolder: WorkspaceFolder,
-): Result<string, string> {
+): Result<string[], string> {
   const config = getSqlmeshConfiguration()
 
-  if (!config.projectPath) {
+  if (config.projectPaths.length === 0) {
     // If no project path is configured, use the workspace folder
     traceVerbose('No project path configured, using workspace folder')
-    return ok(workspaceFolder.uri.fsPath)
+    return ok([workspaceFolder.uri.fsPath])
   }
+
+  const resolvedPaths: string[] = []
+  for (const projectPath of config.projectPaths) {
+    const result = resolveSingleProjectPath(workspaceFolder, projectPath)
+    if (isErr(result)) {
+      return result
+    }
+    resolvedPaths.push(result.value)
+  }
+  return ok(resolvedPaths)
+}
+
+function resolveSingleProjectPath(
+  workspaceFolder: WorkspaceFolder,
+  projectPath: string,
+): Result<string, string> {
   let resolvedPath: string
 
   // Check if the path is absolute
-  if (path.isAbsolute(config.projectPath)) {
-    resolvedPath = config.projectPath
+  if (path.isAbsolute(projectPath)) {
+    resolvedPath = projectPath
   } else {
     // Resolve relative path from workspace root
-    resolvedPath = path.join(workspaceFolder.uri.fsPath, config.projectPath)
+    resolvedPath = path.join(workspaceFolder.uri.fsPath, projectPath)
   }
 
   // Normalize the path

vscode/extension/src/utilities/sqlmesh/sqlmesh.ts

@@ -69,8 +69,8 @@ export const isTcloudProject = async (): Promise<Result<boolean, string>> => {
   if (isErr(resolvedPath)) {
     return err(resolvedPath.error)
   }
-  const tcloudYamlPath = path.join(resolvedPath.value, 'tcloud.yaml')
-  const tcloudYmlPath = path.join(resolvedPath.value, 'tcloud.yml')
+  const tcloudYamlPath = path.join(resolvedPath.value[0], 'tcloud.yaml')
+  const tcloudYmlPath = path.join(resolvedPath.value[0], 'tcloud.yml')
   const isTcloudYamlFilePresent = fs.existsSync(tcloudYamlPath)
   const isTcloudYmlFilePresent = fs.existsSync(tcloudYmlPath)
   if (isTcloudYamlFilePresent || isTcloudYmlFilePresent) {
@@ -144,7 +144,7 @@ export const isSqlmeshEnterpriseInstalled = async (): Promise<
     })
   }
   const called = await execAsync(tcloudBin.value.bin, ['is_sqlmesh_installed'], {
-    cwd: resolvedPath.value,
+    cwd: resolvedPath.value[0],
     env: tcloudBin.value.env,
   })
   if (called.exitCode !== 0) {
@@ -185,7 +185,7 @@ export const installSqlmeshEnterprise = async (
   }
   const called = await execAsync(tcloudBin.value.bin, ['install_sqlmesh'], {
     signal: abortController.signal,
-    cwd: resolvedPath.value,
+    cwd: resolvedPath.value[0],
     env: tcloudBin.value.env,
   })
   if (called.exitCode !== 0) {
@@ -325,7 +325,7 @@ export const sqlmeshLspExec = async (): Promise<
     traceLog(`Using configured SQLMesh LSP entry point: ${configuredLSPExec.entrypoint} ${configuredLSPExec.args.join(' ')}`)
     return ok({
       bin: configuredLSPExec.entrypoint,
-      workspacePath,
+      workspacePath: workspacePath[0],
       env: process.env,
       args: configuredLSPExec.args,
     })
@@ -381,7 +381,7 @@ export const sqlmeshLspExec = async (): Promise<
       if (isSemVerGreaterThanOrEqual(tcloudBinVersion.value, [2, 10, 1])) {
         return ok ({
           bin: tcloudBin.value.bin,
-          workspacePath,
+          workspacePath: workspacePath[0],
           env: tcloudBin.value.env,
           args: ['sqlmesh_lsp'],
         })
@@ -407,7 +407,7 @@ export const sqlmeshLspExec = async (): Promise<
     }
     return ok({
       bin: binPath,
-      workspacePath,
+      workspacePath: workspacePath[0],
       env: env.value,
       args: [],
     })
@@ -427,7 +427,7 @@ export const sqlmeshLspExec = async (): Promise<
     }
     return ok({
       bin: sqlmeshLSP,
-      workspacePath,
+      workspacePath: workspacePath[0],
       env: env.value,
       args: [],
     })

vscode/extension/tests/mulit_project.spec.ts

@@ -0,0 +1,31 @@
+import path from 'path'
+import { test, expect } from './fixtures'
+import { MULTI_SOURCE_PATH, openLineageView, openServerPage, SUSHI_SOURCE_PATH, waitForLoadedSQLMesh } from './utils'
+import fs from 'fs-extra'
+
+test('should work with multi-project setups', async ({
+  page,
+  sharedCodeServer,
+  tempDir,
+}) => {
+  await fs.copy(MULTI_SOURCE_PATH, tempDir)
+
+  // Create the multi project config file
+  const settingsFilePath = path.join(tempDir, '.vscode','settings.json')
+  const settings = {
+    "sqlmesh.projectPaths": [
+        "./repo_1",
+        "./repo_2"
+    ]
+  }
+  await fs.writeJson(settingsFilePath, settings)
+
+  // Open the server
+  await openServerPage(page, tempDir, sharedCodeServer)
+
+  // Wait for for the project to be loaded
+  await waitForLoadedSQLMesh(page)
+
+  // Open the linage view
+  await openLineageView(page)
+})

vscode/extension/tests/utils.ts

@@ -14,6 +14,14 @@ export const SUSHI_SOURCE_PATH = path.join(
   'examples',
   'sushi',
 )
+export const MULTI_SOURCE_PATH = path.join(
+  __dirname,
+  '..',
+  '..',
+  '..',
+  'examples',
+  'multi',
+)
 export const REPO_ROOT = path.join(__dirname, '..', '..', '..')
 
 /**