Add improvements to local build / zip deploy

package.json

@@ -128,6 +128,7 @@
     "cross-spawn": "^7.0.5",
     "csv-parse": "^5.0.4",
     "deep-equal-in-any-order": "^2.0.6",
+    "es2020": "^1.1.9",
     "exegesis": "^4.2.0",
     "exegesis-express": "^4.0.0",
     "express": "^4.16.4",

src/api.ts

@@ -38,8 +38,7 @@ export const appDistributionOrigin = () =>
     "https://firebaseappdistribution.googleapis.com",
   );
 export const apphostingOrigin = () =>
-  utils.envOverride("FIREBASE_APPHOSTING_URL", "https://staging-firebaseapphosting.sandbox.googleapis.com");
-//    firebaseapphosting.googleapis.com");
+  utils.envOverride("FIREBASE_APPHOSTING_URL", "https://firebaseapphosting.googleapis.com");
 export const apphostingP4SADomain = () =>
   utils.envOverride(
     "FIREBASE_APPHOSTING_P4SA_DOMAIN",

src/apphosting/backend.ts

@@ -13,6 +13,7 @@ import {
   iamOrigin,
   secretManagerOrigin,
 } from "../api";
+import { logger } from "../logger";
 import { Backend, BackendOutputOnlyFields, API_VERSION } from "../gcp/apphosting";
 import { addServiceAccountToRoles } from "../gcp/resourceManager";
 import * as iam from "../gcp/iam";
@@ -272,6 +273,32 @@ export async function createGitRepoLink(
   await githubConnections.linkGitHubRepository(projectId, location, connectionId);
 }
 
+/**
+ * Ensures that the App Hosting service agent has the necessary permissions to
+ * manage resources in the project.
+ */
+export async function ensureAppHostingServiceAgentRoles(
+  projectId: string,
+  projectNumber: string,
+): Promise<void> {
+  const p4saEmail = apphosting.serviceAgentEmail(projectNumber);
+  try {
+    await addServiceAccountToRoles(
+      projectId,
+      p4saEmail,
+      ["roles/storage.objectViewer"],
+      /* skipAccountLookup= */ true,
+    );
+  } catch (err: unknown) {
+    logger.debug(`Failed to grant storage.objectViewer to ${p4saEmail}: ${err}`);
+    // We don't want to fail the entire prepare step if this fails, as it might
+    // be due to insufficient permissions to grant roles.
+    logWarning(
+      `Unable to verify App Hosting service agent permissions for ${p4saEmail}. If you encounter a PERMISSION_DENIED error during rollout, please ensure the service agent has the "Storage Object Viewer" role.`,
+    );
+  }
+}
+
 /**
  * Ensures the service account is present the user has permissions to use it by
  * checking the `iam.serviceAccounts.actAs` permission. If the permissions
@@ -356,7 +383,7 @@ export async function createBackend(
   const defaultServiceAccount = defaultComputeServiceAccountEmail(projectId);
   const backendReqBody: Omit<Backend, BackendOutputOnlyFields> = {
     servingLocality: "GLOBAL_ACCESS",
-    runtime: {value: "nodejs22"},
+    runtime: { value: "nodejs22" },
     codebase: repository
       ? {
           repository: `${repository.name}`,

src/apphosting/config.spec.ts

@@ -472,4 +472,25 @@ env:
       );
     });
   });
+
+  describe("splitEnvVars", () => {
+    it("should stringify numeric values", () => {
+      const env: AppHostingYamlConfig["env"] = {
+        STR: { value: "string" },
+        NUM: { value: 12345 as any },
+        BUILD_AND_RUNTIME_NUM: { value: 67890 as any, availability: ["BUILD", "RUNTIME"] },
+      };
+
+      const { build, runtime } = config.splitEnvVars(env);
+
+      expect(build["BUILD_AND_RUNTIME_NUM"].value).to.equal("67890");
+      expect(runtime).to.deep.include({ variable: "STR", value: "string" });
+      expect(runtime).to.deep.include({ variable: "NUM", value: "12345" });
+      expect(runtime).to.deep.include({
+        variable: "BUILD_AND_RUNTIME_NUM",
+        value: "67890",
+        availability: ["BUILD", "RUNTIME"],
+      });
+    });
+  });
 });

src/apphosting/config.ts

@@ -1,4 +1,4 @@
-import { join, dirname } from "path";
+import { join, dirname, basename } from "path";
 import { writeFileSync } from "fs";
 import * as yaml from "yaml";
 import * as clc from "colorette";
@@ -159,6 +159,7 @@ const dynamicDispatch = exports as {
   upsertEnv: typeof upsertEnv;
   store: typeof store;
   overrideChosenEnv: typeof overrideChosenEnv;
+  listAppHostingFilesInPath: typeof listAppHostingFilesInPath;
 };
 
 /**
@@ -349,3 +350,71 @@ export async function overrideChosenEnv(
 export function suggestedTestKeyName(variable: string): string {
   return "test-" + variable.replace(/_/g, "-").toLowerCase();
 }
+
+/**
+ * Split a set of environment variables into build and runtime variables.
+ */
+export function splitEnvVars(env: EnvMap): { build: EnvMap; runtime: Env[] } {
+  const build: EnvMap = {};
+  const runtime: Env[] = [];
+
+  for (const [key, val] of Object.entries(env)) {
+    const envVal = { ...val };
+    if (envVal.value !== undefined) {
+      envVal.value = String(envVal.value);
+    }
+
+    if (val.availability?.includes("BUILD")) {
+      build[key] = envVal;
+    }
+    if (val.availability?.includes("RUNTIME") || !val.availability) {
+      runtime.push({ variable: key, ...envVal });
+    }
+  }
+
+  return { build, runtime };
+}
+
+interface GetConfigOptions {
+  allowEmulator?: boolean;
+  allowLocal?: boolean;
+}
+
+/**
+ * Loads in apphosting.yaml, apphosting.emulator.yaml & apphosting.local.yaml as an
+ * overriding union.
+ *
+ * @param backendDir The directory containing the apphosting.yaml.
+ * @param options Options to control which files to load.
+ */
+export async function getAppHostingConfiguration(
+  backendDir: string,
+  options: GetConfigOptions = {},
+): Promise<AppHostingYamlConfig> {
+  const appHostingConfigPaths = dynamicDispatch.listAppHostingFilesInPath(backendDir);
+  const fileNameToPathMap = Object.fromEntries(
+    appHostingConfigPaths.map((path) => [basename(path), path]),
+  );
+  let output = AppHostingYamlConfig.empty();
+
+  const baseFilePath = fileNameToPathMap[APPHOSTING_BASE_YAML_FILE];
+  const emulatorsFilePath = fileNameToPathMap[APPHOSTING_EMULATORS_YAML_FILE];
+  const localFilePath = fileNameToPathMap[APPHOSTING_LOCAL_YAML_FILE];
+
+  if (baseFilePath) {
+    const baseFile = await AppHostingYamlConfig.loadFromFile(baseFilePath);
+    output = baseFile;
+  }
+
+  if (options.allowEmulator && emulatorsFilePath) {
+    const emulatorsConfig = await AppHostingYamlConfig.loadFromFile(emulatorsFilePath);
+    output.merge(emulatorsConfig, /* allowSecretsToBecomePlaintext= */ false);
+  }
+
+  if (options.allowLocal && localFilePath) {
+    const localYamlConfig = await AppHostingYamlConfig.loadFromFile(localFilePath);
+    output.merge(localYamlConfig, /* allowSecretsToBecomePlaintext= */ true);
+  }
+
+  return output;
+}

src/apphosting/localbuilds.ts

@@ -1,37 +1,78 @@
+import * as path from "path";
 import { BuildConfig, Env } from "../gcp/apphosting";
 import { localBuild as localAppHostingBuild } from "@apphosting/build";
+import { EnvMap } from "./yaml";
 
 /**
- * Triggers a local apphosting build.
+ * Triggers a local build of your App Hosting codebase.
+ *
+ * This function orchestrates the build process using the App Hosting build adapter.
+ * It detects the framework (though currently defaults/assumes 'nextjs' in some contexts),
+ * generates the necessary build artifacts, and returns metadata about the build.
+ *
+ * @param projectRoot - The root directory of the project to build.
+ * @param framework - The framework to use for the build (e.g., 'nextjs').
+ * @returns A promise that resolves to the build output, including:
+ *          - `outputFiles`: Paths to the generated build artifacts.
+ *          - `annotations`: Metadata annotations relating to the build.
+ *          - `buildConfig`: Configuration derived from the build process (e.g. run commands, environment variables).
  */
 export async function localBuild(
   projectRoot: string,
   framework: string,
+  env: EnvMap = {},
 ): Promise<{
   outputFiles: string[];
   annotations: Record<string, string>;
   buildConfig: BuildConfig;
 }> {
-  const apphostingBuildOutput = await localAppHostingBuild(projectRoot, framework);
+  // We need to inject the environment variables into the process.env
+  // because the build adapter uses them to build the app.
+  // We'll restore the original process.env after the build is done.
+  const originalEnv = process.env;
+  const projectNodeModules = path.join(projectRoot, "node_modules");
+  const newNodePath = originalEnv.NODE_PATH
+    ? `${originalEnv.NODE_PATH}${path.delimiter}${projectNodeModules}`
+    : projectNodeModules;
+
+  process.env = {
+    ...originalEnv,
+    ...toProcessEnv(env),
+    NODE_PATH: newNodePath,
+  };
+
+  let apphostingBuildOutput;
+  try {
+    apphostingBuildOutput = await localAppHostingBuild(projectRoot, framework);
+  } finally {
+    process.env = originalEnv;
+  }
 
   const annotations: Record<string, string> = Object.fromEntries(
     Object.entries(apphostingBuildOutput.metadata).map(([key, value]) => [key, String(value)]),
   );
 
-  const env: Env[] | undefined = apphostingBuildOutput.runConfig.environmentVariables?.map(
-    ({ variable, value, availability }) => ({
-      variable,
-      value,
-      availability,
-    }),
-  );
+  const discoveredEnv: Env[] | undefined =
+    apphostingBuildOutput.runConfig.environmentVariables?.map(
+      ({ variable, value, availability }) => ({
+        variable,
+        value,
+        availability,
+      }),
+    );
 
   return {
     outputFiles: apphostingBuildOutput.outputFiles?.serverApp.include ?? [],
     annotations,
     buildConfig: {
       runCommand: apphostingBuildOutput.runConfig.runCommand,
-      env: env ?? [],
+      env: discoveredEnv ?? [],
     },
   };
 }
+
+function toProcessEnv(env: EnvMap): NodeJS.ProcessEnv {
+  return Object.fromEntries(
+    Object.entries(env).map(([key, value]) => [key, value.value || ""]),
+  ) as NodeJS.ProcessEnv;
+}

src/apphosting/utils.ts

@@ -1,4 +1,5 @@
 import { FirebaseError } from "../error";
+import { WebConfig } from "../fetchWebSetup";
 import { APPHOSTING_BASE_YAML_FILE, APPHOSTING_YAML_FILE_REGEX } from "./config";
 import * as prompt from "../prompt";
 
@@ -53,3 +54,17 @@ export async function promptForAppHostingYaml(
 
   return fileToExportPath;
 }
+
+/**
+ * Returns the environment variables that are needed for the Firebase JS SDK auto-init.
+ */
+export function getAutoinitEnvVars(webappConfig: WebConfig): Record<string, string> {
+  return {
+    FIREBASE_WEBAPP_CONFIG: JSON.stringify(webappConfig),
+    FIREBASE_CONFIG: JSON.stringify({
+      databaseURL: webappConfig.databaseURL,
+      storageBucket: webappConfig.storageBucket,
+      projectId: webappConfig.projectId,
+    }),
+  };
+}

src/deploy/apphosting/args.ts

@@ -1,10 +1,11 @@
 import { AppHostingSingle } from "../../firebaseConfig";
-import { BuildConfig } from "../../gcp/apphosting";
+import { BuildConfig, Env } from "../../gcp/apphosting";
 
 export interface LocalBuild {
   buildConfig: BuildConfig;
   buildDir: string;
   annotations: Record<string, string>;
+  env: Env[];
 }
 
 export interface Context {

src/deploy/apphosting/deploy.spec.ts

@@ -42,6 +42,7 @@ function initializeContext(): Context {
         buildDir: "./nextjs/standalone",
         buildConfig: {},
         annotations: {},
+        env: [],
       },
     },
   };
@@ -50,7 +51,7 @@ function initializeContext(): Context {
 describe("apphosting", () => {
   let upsertBucketStub: sinon.SinonStub;
   let uploadObjectStub: sinon.SinonStub;
-  let createArchiveStub: sinon.SinonStub;
+  let createTarArchiveStub: sinon.SinonStub;
   let createReadStreamStub: sinon.SinonStub;
   let getProjectNumberStub: sinon.SinonStub;
 
@@ -60,7 +61,9 @@ describe("apphosting", () => {
       .throws("Unexpected getProjectNumber call");
     upsertBucketStub = sinon.stub(gcs, "upsertBucket").throws("Unexpected upsertBucket call");
     uploadObjectStub = sinon.stub(gcs, "uploadObject").throws("Unexpected uploadObject call");
-    createArchiveStub = sinon.stub(util, "createArchive").throws("Unexpected createArchive call");
+    createTarArchiveStub = sinon
+      .stub(util, "createTarArchive")
+      .throws("Unexpected createTarArchive call");
     createReadStreamStub = sinon
       .stub(fs, "createReadStream")
       .throws("Unexpected createReadStream call");
@@ -99,8 +102,8 @@ describe("apphosting", () => {
       const bucketName = `firebaseapphosting-sources-${projectNumber}-${location}`;
       getProjectNumberStub.resolves(projectNumber);
       upsertBucketStub.resolves(bucketName);
-      createArchiveStub.onFirstCall().resolves("path/to/foo-1234.zip");
-      createArchiveStub.onSecondCall().resolves("path/to/foo-local-build-1234.zip");
+      createTarArchiveStub.onFirstCall().resolves("path/to/foo-1234.tar.gz");
+      createTarArchiveStub.onSecondCall().resolves("path/to/foo-local-build-1234.tar.gz");
 
       uploadObjectStub.onFirstCall().resolves({
         bucket: bucketName,
@@ -156,7 +159,7 @@ describe("apphosting", () => {
           },
         },
       });
-      expect(createArchiveStub).to.be.calledWithExactly(
+      expect(createTarArchiveStub).to.be.calledWithExactly(
         context.backendConfigs["fooLocalBuild"],
         process.cwd(),
         "./nextjs/standalone",
@@ -174,8 +177,8 @@ describe("apphosting", () => {
       const bucketName = `firebaseapphosting-sources-${projectNumber}-${location}`;
       getProjectNumberStub.resolves(projectNumber);
       upsertBucketStub.resolves(bucketName);
-      createArchiveStub.onFirstCall().resolves("path/to/foo-1234.zip");
-      createArchiveStub.onSecondCall().resolves("path/to/foo-local-build-1234.zip");
+      createTarArchiveStub.onFirstCall().resolves("path/to/foo-1234.tar.gz");
+      createTarArchiveStub.onSecondCall().resolves("path/to/foo-local-build-1234.tar.gz");
 
       uploadObjectStub.onFirstCall().resolves({
         bucket: bucketName,
@@ -190,9 +193,9 @@ describe("apphosting", () => {
 
       await deploy(context, opts);
 
-      expect(context.backendStorageUris["foo"]).to.equal(`gs://${bucketName}/foo-1234.zip`);
+      expect(context.backendStorageUris["foo"]).to.equal(`gs://${bucketName}/foo-1234.tar.gz`);
       expect(context.backendStorageUris["fooLocalBuild"]).to.equal(
-        `gs://${bucketName}/foo-local-build-1234.zip`,
+        `gs://${bucketName}/foo-local-build-1234.tar.gz`,
       );
     });
   });