Luke/feat/backup

api/ENVIRONMENT.md

@@ -75,10 +75,15 @@ To perform the unit tests against an Azure server:
 To enable generating diffs for releases:
 
 - `ENABLE_PACKAGE_DIFFING`: Set to 'true'
+- `DIFF_PACKAGE_COUNT`: Set to the number of releases to generate diffs for. Defaults to 5 if unspecified.
 
 To enable KeyVault credential resolution, set:
 
 - `AZURE_KEYVAULT_ACCOUNT`: The name of your hosted Azure KeyVault account
 - `CLIENT_ID`: The client ID of an Active Directory app that has access to your KeyVault account
 - `CERTIFICATE_THUMBPRINT`: The thumbprint of the certificate associated with your Active Directory app (for which a .pfx has been uploaded to your certificate store)
 - `REFRESH_CREDENTIALS_INTERVAL` (Optional): The frequency, in milliseconds, to re-retrieve credentials from Key Vault (defaults to one day, currently only storage keys are supported)
+
+To proxy update check URLs through a caching or proxy service:
+
+- `UPDATE_CHECK_PROXY_URL`: The base URL to use for proxying update check responses. For example, if set to "https://yourcdn.com/" Azure blob storage URLs will be transformed to use this domain instead.

api/script/environment.ts

@@ -6,3 +6,7 @@ import * as os from 'os';
 export function getTempDirectory(): string {
   return process.env.TEMP || process.env.TMPDIR || os.tmpdir();
 }
+
+export function getUpdateCheckProxyUrl(): string {
+  return process.env.UPDATE_CHECK_PROXY_URL;
+}

api/script/routes/acquisition.ts

@@ -29,6 +29,7 @@ export interface AcquisitionConfig {
 function getUrlKey(originalUrl: string): string {
   const obj: any = URL.parse(originalUrl, /*parseQueryString*/ true);
   delete obj.query.clientUniqueId;
+  delete obj.query.client_unique_id;
   return obj.pathname + "?" + queryString.stringify(obj.query);
 }
 

api/script/routes/management.ts

@@ -722,9 +722,9 @@ export function getManagementRouter(config: ManagementConfig): Router {
         const newRolloutValue: number = info.rollout;
         if (validationUtils.isDefined(newRolloutValue)) {
           let errorMessage: string;
-          if (!isUnfinishedRollout(packageToUpdate.rollout)) {
+          if (!isUnfinishedRollout(packageToUpdate.rollout) && newRolloutValue !== 100) {
             errorMessage = "Cannot update rollout value for a completed rollout release.";
-          } else if (packageToUpdate.rollout >= newRolloutValue) {
+          } else if (packageToUpdate.rollout > newRolloutValue) {
             errorMessage = `Rollout value must be greater than "${packageToUpdate.rollout}", the existing value.`;
           }
 
@@ -1261,54 +1261,75 @@ export function getManagementRouter(config: ManagementConfig): Router {
     appPackage: storageTypes.Package,
     diffPackageMap: storageTypes.PackageHashToBlobInfoMap
   ) {
-    let updateHistory: boolean = false;
-
-    return storage
-      .getApp(accountId, appId)
-      .then((storageApp: storageTypes.App) => {
-        throwIfInvalidPermissions(storageApp, storageTypes.Permissions.Collaborator);
-        return storage.getPackageHistory(accountId, appId, deploymentId);
-      })
-      .then((history: storageTypes.Package[]) => {
-        if (history) {
-          for (let i = history.length - 1; i >= 0; i--) {
-            if (history[i].label === appPackage.label && !history[i].diffPackageMap) {
-              history[i].diffPackageMap = diffPackageMap;
-              updateHistory = true;
+    console.log(`[addDiffInfoForPackage] Starting for package ${appPackage.label}, hash: ${appPackage.packageHash}`);
+    console.log(`[addDiffInfoForPackage] diffPackageMap exists: ${!!diffPackageMap}, entries: ${diffPackageMap ? Object.keys(diffPackageMap).length : 0}`);
+
+    // Add diff info the package
+    if (diffPackageMap) {
+      return storage
+        .getPackageHistory(accountId, appId, deploymentId)
+        .then((history: storageTypes.Package[]) => {
+          console.log(`[addDiffInfoForPackage] Got history with ${history.length} packages`);
+
+          // Update the diff info for the current package we deployed
+          for (const updatedPackage of history) {
+            if (updatedPackage.packageHash === appPackage.packageHash) {
+              updatedPackage.diffPackageMap = diffPackageMap;
+              console.log(`[addDiffInfoForPackage] Updated diff package map for package hash: ${updatedPackage.packageHash}`);
               break;
             }
           }
-
-          if (updateHistory) {
-            return storage.updatePackageHistory(accountId, appId, deploymentId, history);
-          }
-        }
-      })
-      .then(() => {
-        if (updateHistory) {
+
+          console.log(`[addDiffInfoForPackage] Saving updated package history (${history.length} packages)`);
+          return storage.updatePackageHistory(accountId, appId, deploymentId, history);
+        })
+        .then(() => {
+          console.log(`[addDiffInfoForPackage] Successfully updated package history`);
           return storage.getDeployment(accountId, appId, deploymentId).then((deployment: storageTypes.Deployment) => {
+            console.log(`[addDiffInfoForPackage] Invalidating cache for key ${deployment.key}`);
             return invalidateCachedPackage(deployment.key);
           });
-        }
-      })
-      .catch(diffErrorUtils.diffErrorHandler);
+        });
+    } else {
+      console.log(`[addDiffInfoForPackage] No diff package map available, skipping update`);
+      return q(null);
+    }
   }
 
   function processDiff(accountId: string, appId: string, deploymentId: string, appPackage: storageTypes.Package): q.Promise<void> {
-    if (!appPackage.manifestBlobUrl || process.env.ENABLE_PACKAGE_DIFFING) {
+    console.log(`[processDiff] Starting for package ${appPackage.label}, manifestBlobUrl exists: ${!!appPackage.manifestBlobUrl}`);
+    console.log(`[processDiff] ENABLE_PACKAGE_DIFFING env var: ${process.env.ENABLE_PACKAGE_DIFFING}`);
+
+    if (!appPackage.manifestBlobUrl || !process.env.ENABLE_PACKAGE_DIFFING) {
       // No need to process diff because either:
       //   1. The release just contains a single file.
       //   2. Diffing disabled.
+      console.log(`[processDiff] Skipping diff generation: ${!appPackage.manifestBlobUrl ? 'No manifest URL' : 'Diffing disabled by env var'}`);
       return q(<void>null);
     }
 
-    console.log(`Processing package: ${appPackage.label}`);
+    console.log(`[processDiff] Processing package: ${appPackage.label}, hash: ${appPackage.packageHash}`);
 
     return packageDiffing
       .generateDiffPackageMap(accountId, appId, deploymentId, appPackage)
       .then((diffPackageMap: storageTypes.PackageHashToBlobInfoMap) => {
-        console.log(`Package processed, adding diff info`);
-        addDiffInfoForPackage(accountId, appId, deploymentId, appPackage, diffPackageMap);
+        console.log(`[processDiff] Package processed, diffPackageMap exists: ${!!diffPackageMap}`);
+        if (diffPackageMap) {
+          console.log(`[processDiff] diffPackageMap has ${Object.keys(diffPackageMap).length} entries`);
+          Object.keys(diffPackageMap).forEach(key => {
+            console.log(`[processDiff] - Entry: ${key}, size: ${diffPackageMap[key].size} bytes`);
+          });
+        }
+
+        console.log(`[processDiff] Adding diff info to package`);
+        return addDiffInfoForPackage(accountId, appId, deploymentId, appPackage, diffPackageMap);
+      })
+      .then(() => {
+        console.log(`[processDiff] Successfully completed diff processing`);
+      })
+      .catch((error) => {
+        console.error(`[processDiff] Error during diff processing: ${error.message || error}`);
+        return diffErrorUtils.diffErrorHandler(error);
       });
   }
 

api/script/utils/acquisition.ts

@@ -5,6 +5,27 @@ import * as semver from "semver";
 import { UpdateCheckCacheResponse, UpdateCheckRequest, UpdateCheckResponse } from "../types/rest-definitions";
 import { Package } from "../storage/storage";
 import { isUnfinishedRollout } from "./rollout-selector";
+import * as env from "../environment";
+import * as URL from "url";
+
+function proxyBlobUrl(azureUrl: string): string {
+  try {
+    const proxyUrl = env.getUpdateCheckProxyUrl();
+    if (!proxyUrl) {
+      return azureUrl;
+    }
+
+    const parsedUrl = new URL.URL(azureUrl);
+    const newUrl = new URL.URL(proxyUrl);
+    newUrl.pathname = parsedUrl.pathname;
+    newUrl.search = parsedUrl.search;
+
+    return newUrl.toString();
+  } catch (error) {
+    console.warn('Failed to proxy blob URL:', error);
+    return azureUrl;
+  }
+}
 
 interface UpdatePackage {
   response: UpdateCheckResponse;
@@ -114,10 +135,10 @@ function getUpdatePackage(packageHistory: Package[], request: UpdateCheckRequest
     latestSatisfyingEnabledPackage.diffPackageMap &&
     latestSatisfyingEnabledPackage.diffPackageMap[request.packageHash]
   ) {
-    updateDetails.downloadURL = latestSatisfyingEnabledPackage.diffPackageMap[request.packageHash].url;
+    updateDetails.downloadURL = proxyBlobUrl(latestSatisfyingEnabledPackage.diffPackageMap[request.packageHash].url);
     updateDetails.packageSize = latestSatisfyingEnabledPackage.diffPackageMap[request.packageHash].size;
   } else {
-    updateDetails.downloadURL = latestSatisfyingEnabledPackage.blobUrl;
+    updateDetails.downloadURL = proxyBlobUrl(latestSatisfyingEnabledPackage.blobUrl);
     updateDetails.packageSize = latestSatisfyingEnabledPackage.size;
   }