From 3e5a5734dc424ac845c3fa648bffda4bfb215999 Mon Sep 17 00:00:00 2001
From: Perry Zhu <perryzhu2004@outlook.com>
Date: Mon, 13 Oct 2025 12:10:32 -0700
Subject: [PATCH] fix: lambda wait until network idle

---
 scripts/package.json          | 4 ++--
 scripts/src/lambda-handler.ts | 3 +++
 scripts/src/main.ts           | 8 ++++++--
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/scripts/package.json b/scripts/package.json
index b110370..dcf626e 100644
--- a/scripts/package.json
+++ b/scripts/package.json
@@ -9,14 +9,14 @@
         "dev": "ts-node src/main.ts"
     },
     "dependencies": {
+        "@sparticuz/chromium": "^141.0.0",
         "playwright": "^1.40.0"
     },
     "devDependencies": {
         "@types/aws-lambda": "^8.10.155",
         "@types/node": "^20.0.0",
         "ts-node": "^10.9.0",
-        "typescript": "^5.0.0",
-        "@sparticuz/chromium": "^141.0.0"
+        "typescript": "^5.0.0"
     },
     "packageManager": "yarn@1.22.0",
     "engines": {
diff --git a/scripts/src/lambda-handler.ts b/scripts/src/lambda-handler.ts
index 1539108..679df5b 100644
--- a/scripts/src/lambda-handler.ts
+++ b/scripts/src/lambda-handler.ts
@@ -20,6 +20,7 @@ export async function lambdaHandler(
     const imageLinks = await getImageLinksPlaywright(pexelsUrl);
 
     if (imageLinks && imageLinks.length > 0) {
+      console.log(`Total image links found: ${imageLinks.length}`);
       return {
         statusCode: 200,
         body: JSON.stringify({
@@ -29,6 +30,7 @@ export async function lambdaHandler(
         }),
       };
     } else {
+      console.log("No image links found");
       return {
         statusCode: 200,
         body: JSON.stringify({
@@ -40,6 +42,7 @@ export async function lambdaHandler(
       };
     }
   } catch (error) {
+    console.log(`Error in lambdaHandler: ${error}`);
     return {
       statusCode: 500,
       body: JSON.stringify({
diff --git a/scripts/src/main.ts b/scripts/src/main.ts
index 965436f..d56c196 100644
--- a/scripts/src/main.ts
+++ b/scripts/src/main.ts
@@ -76,7 +76,9 @@ async function getImageLinksPlaywright(url: string): Promise<string[]> {
       timeout: 60000,
     });
 
-    await page.waitForTimeout(5000);
+    console.log("Waiting for network idle...");
+    await page.waitForLoadState("networkidle");
+    console.log("Page loaded.");
 
     const maxClicks = 5;
     let clickCount = 0;
@@ -94,7 +96,8 @@ async function getImageLinksPlaywright(url: string): Promise<string[]> {
           await loadMoreButton.click();
 
           // Wait for new content to load
-          await page.waitForTimeout(2000);
+          await page.waitForLoadState("networkidle");
+          console.log("New content loaded after clicking Load More.");
         } catch (error) {
           console.error(`Error clicking button: ${error}`);
           break;
@@ -112,6 +115,7 @@ async function getImageLinksPlaywright(url: string): Promise<string[]> {
     // Extract image links
     const imgLinks = await page.evaluate((): string[] => {
       const imgs = Array.from(document.querySelectorAll("img"));
+      console.log(`Total images found on page: ${imgs.length}`);
       const links: string[] = [];
 
       imgs.forEach((img) => {