Merge main to mainnet

.github/workflows/build-engine.yml

@@ -10,6 +10,7 @@ on:
           - text-generation-webui-gpu
           - text-generation-webui-cpu
           - comfyui
+          - n8n
       target:
         description: 'Target'
         required: true
@@ -30,6 +31,7 @@ env:
   SUBMODULE_PATH: ''
   TARGET: ${{ inputs.target }}
   ALLOW_MULTIPLE_UPDATE: false
+  ARGO_COMMIT_TYPE: ''
 
 jobs:
   build-engine:
@@ -61,6 +63,11 @@ jobs:
             DOCKERFILE_CONTEXT="./ComfyUI/"
             SUBMODULE_PATH="ComfyUI/ComfyUI"
             ALLOW_MULTIPLE_UPDATE="true"
+
+          elif [ "$SOLUTION" == "n8n" ]; then
+            DOCKERFILE="./n8n/Dockerfile"
+            DOCKERFILE_CONTEXT="./n8n/"
+
           else
             echo "Invalid solution name"
             exit 1
@@ -73,6 +80,7 @@ jobs:
           echo "ALLOW_MULTIPLE_UPDATE=$ALLOW_MULTIPLE_UPDATE" >> $GITHUB_ENV
 
       - name: Checkout git submodules
+        if: env.SUBMODULE_PATH != ''
         run: |
           git submodule init && git submodule update "$SUBMODULE_PATH"
 
@@ -151,6 +159,14 @@ jobs:
         with:
           file_to_upload: ${{ env.IMAGE_FILE }}
 
+      - name: Choose commit type
+        run: |
+          if [[ "$TARGET" == "testnet" || "$TARGET" == "mainnet" ]]; then
+            echo "ARGO_COMMIT_TYPE=pull-request" >> $GITHUB_ENV
+          else
+            echo "ARGO_COMMIT_TYPE=direct-commit" >> $GITHUB_ENV
+          fi
+
       - name: Update argocd
         uses: Super-Protocol/sp-build-tools/actions/update-argocd@v1
         with:
@@ -159,5 +175,5 @@ jobs:
           resource_type: solution
           resource_file: ${{ steps.upload_image.outputs.resource }}
           solution_name: ${{ env.SOLUTION }}
-          commit_type: pull-request
+          commit_type: ${{ env.ARGO_COMMIT_TYPE }}
           allow_multiple_update: ${{ env.ALLOW_MULTIPLE_UPDATE }}

.github/workflows/update-engine-configuration.yml

@@ -0,0 +1,201 @@
+name: Update Engine Configuration
+
+on:
+  workflow_dispatch:
+    inputs:
+      solution:
+        description: 'Choose the solution whose offer you want to update.'
+        required: true
+        type: choice
+        options:
+          - 'ComfyUI (CPU only)'
+          - 'ComfyUI (GPU only)'
+          - 'Text Generation Web UI (CPU only)'
+          - 'Text Generation Web UI (GPU only)'
+      target:
+        description: 'Target environment where the offer exists.'
+        required: true
+        type: choice
+        options:
+          - develop
+          - stage
+          - testnet
+          - mainnet
+
+jobs:
+  update-offer-job:
+    environment: ${{ inputs.target }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout main repository
+        uses: actions/checkout@v4
+
+      - name: Prepare Variables and Get Offer ID
+        id: prepare
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const allVars = ${{ toJSON(vars) }};
+            const configMap = {
+              'ComfyUI (CPU only)': { configFile: './ComfyUI/configuration-cpu.json', varName: 'COMFYUI_CPU_OFFER_ID' },
+              'ComfyUI (GPU only)': { configFile: './ComfyUI/configuration-gpu.json', varName: 'COMFYUI_GPU_OFFER_ID' },
+              'Text Generation Web UI (CPU only)': { configFile: './Text Generation WebUI/configuration.json', varName: 'TEXT_GENERATION_WEB_UI_CPU_OFFER_ID' },
+              'Text Generation Web UI (GPU only)': { configFile: './Text Generation WebUI/configuration.json', varName: 'TEXT_GENERATION_WEB_UI_GPU_OFFER_ID' }
+            };
+            const solution = context.payload.inputs.solution;
+            const targetEnv = context.payload.inputs.target;
+            core.info(`Selected solution: "${solution}" for environment "${targetEnv}"`);
+            const config = configMap[solution];
+            if (!config) {
+              core.setFailed(`Internal error: No configuration mapping found for solution "${solution}".`);
+              return;
+            }
+            const offerId = allVars[config.varName];
+            if (!offerId) {
+              core.setFailed(`GitHub Variable '${config.varName}' is not set for the '${targetEnv}' environment.`);
+              return;
+            }
+            core.info(`Found Offer ID: ${offerId}`);
+            core.info(`Using configuration file: ${config.configFile}`);
+            core.setOutput('offer_id', offerId);
+            core.setOutput('config_file_path', config.configFile);
+
+      - name: Process Configuration and Substitute Variables
+        id: process_config
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const allVars = ${{ toJSON(vars) }};
+            const configFilePath = "${{ steps.prepare.outputs.config_file_path }}";
+
+            core.info(`Processing config file: ${configFilePath}`);
+            let content = fs.readFileSync(configFilePath, 'utf8');
+
+            const placeholderRegex = /\$VAR_([A-Z0-9_]+)/g;
+            const placeholders = [...content.matchAll(placeholderRegex)];
+
+            if (placeholders.length === 0) {
+              core.info('No variables to substitute.');
+              core.setOutput('processed_config_path', configFilePath);
+              core.setOutput('substituted_variables', '[]');
+              return;
+            }
+
+            let allVarsFound = true;
+            const uniqueVarNames = [...new Set(placeholders.map(p => p[1]))];
+
+            for (const varName of uniqueVarNames) {
+              if (!Object.prototype.hasOwnProperty.call(allVars, varName)) {
+                core.setFailed(`Required GitHub Variable '${varName}' is not set for the '${context.payload.inputs.target}' environment.`);
+                allVarsFound = false;
+              }
+            }
+
+            if (!allVarsFound) {
+              return;
+            }
+
+            core.info(`Substituting variables: ${uniqueVarNames.join(', ')}`);
+
+            const substitutedData = uniqueVarNames.map(varName => ({
+              name: `$VAR_${varName}`,
+              value: allVars[varName]
+            }));
+            core.setOutput('substituted_variables', JSON.stringify(substitutedData));
+            content = content.replace(placeholderRegex, (match, varName) => {
+              return allVars[varName];
+            });
+
+            const newPath = './processed-config.json';
+            fs.writeFileSync(newPath, content);
+            core.info(`Processed configuration saved to ${newPath}`);
+            core.setOutput('processed_config_path', newPath);
+
+      - name: Set SPCTL repository type
+        id: set-spctl-repo
+        run: |
+          TARGET="${{ inputs.target }}"
+          if [ "$TARGET" == "develop" ] || [ "$TARGET" == "stage" ]; then
+            echo "type=private" >> $GITHUB_OUTPUT
+          else
+            echo "type=public" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Download and install SPCTL
+        uses: Super-Protocol/sp-build-tools/actions/download-spctl@v1
+        with:
+          version: latest
+          repository: ${{ steps.set-spctl-repo.outputs.type }}
+          gh_token: ${{ secrets.GHFG_TOKEN_SPCTL_RELEASES_DOWNLOAD }}
+
+      - name: Prepare SPCTL config
+        uses: actions/github-script@v7
+        env:
+          SECRET_DEVELOP: ${{ secrets.DEVELOP_SPCTL_CONFIG_BASE64 }}
+          SECRET_STAGE: ${{ secrets.STAGING_SPCTL_CONFIG_BASE64 }}
+          SECRET_TESTNET: ${{ secrets.TESTNET_SPCTL_CONFIG_BASE64 }}
+          SECRET_MAINNET: ${{ secrets.MAINNET_SPCTL_CONFIG_BASE64 }}
+        with:
+          script: |
+            const fs = require('fs');
+            const secretEnvMap = {
+              develop: process.env.SECRET_DEVELOP,
+              stage: process.env.SECRET_STAGE,
+              testnet: process.env.SECRET_TESTNET,
+              mainnet: process.env.SECRET_MAINNET
+            };
+            const targetEnv = context.payload.inputs.target;
+            const encodedConfig = secretEnvMap[targetEnv];
+            if (!encodedConfig) {
+              core.setFailed(`Secret for target environment '${targetEnv}' is not set or is empty. Check Actions Secrets.`);
+              return;
+            }
+            const decodedConfig = Buffer.from(encodedConfig, 'base64').toString('utf8');
+            fs.writeFileSync('./config.json', decodedConfig);
+            core.info('SPCTL config file created successfully.');
+
+      - name: Verify SPCTL installation
+        run: ./spctl --version
+
+      - name: Update offer configuration
+        shell: bash
+        run: ./spctl offers update value ${{ steps.prepare.outputs.offer_id }} --configuration ${{ steps.process_config.outputs.processed_config_path  }}
+
+      - name: Generate Job Summary
+        if: always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const { solution, target } = context.payload.inputs;
+            const offerId = '${{ steps.prepare.outputs.offer_id }}';
+            const configFile = '${{ steps.prepare.outputs.config_file_path }}';
+            const substitutedJSON = '${{ steps.process_config.outputs.substituted_variables || '[]' }}';
+            const substitutedVars = JSON.parse(substitutedJSON);
+
+            await core.summary
+              .addHeading('Engine Configuration Update Summary', 2)
+              .addRaw(`The workflow has processed a configuration update for **${solution}**.`)
+              .addTable([
+                [ {data: 'Parameter', header: true}, {data: 'Value', header: true} ],
+                [ 'Environment (Target)', `\`${target}\`` ],
+                [ 'Offer ID', `\`${offerId}\`` ],
+                [ 'Source Configuration', `\`${configFile}\`` ]
+              ])
+              .addHeading('Variable Substitutions', 3)
+              .write();
+
+            if (substitutedVars.length > 0) {
+              const tableData = [
+                [ {data: 'Variable Name', header: true}, {data: 'Substituted Value', header: true} ]
+              ];
+              for (const v of substitutedVars) {
+                tableData.push([`\`${v.name}\``, `\`${v.value}\``]);
+              }
+              await core.summary.addTable(tableData).write();
+            } else {
+              await core.summary.addRaw('No variables were substituted in the configuration file.').write();
+            }

.gitignore

@@ -105,3 +105,8 @@ tools
 .idea
 
 text-generation-webui/
+
+sp-secrets
+sp-inputs
+sp-configurations
+sp-certs

ComfyUI/Dockerfile.local-cpu

@@ -7,7 +7,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 ENV PIP_PREFER_BINARY=1
 ENV PIP_ROOT_USER_ACTION=ignore
 # Ensures output from python is printed immediately to the terminal without buffering
-ENV PYTHONUNBUFFERED=1 
+ENV PYTHONUNBUFFERED=1
 # Speed up some cmake builds
 ENV CMAKE_BUILD_PARALLEL_LEVEL=8
 
@@ -26,8 +26,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
 
 # Set environment variables
-ARG COMFYUI_VERSION=0.3.10
-ARG COMFYUI_MANAGER_VERSION=3.6.5
+ARG COMFYUI_VERSION=0.3.29
+ARG COMFYUI_MANAGER_VERSION=3.31.12
 ARG COMFYUI_PATH=/opt/ComfyUI
 
 RUN git config --global user.email "container@super-protocol.com" \
@@ -74,7 +74,11 @@ RUN <<EOF cat >> user/default/ComfyUI-Manager/config.ini
 file_logging = False
 security_level = weak
 EOF
-
+RUN <<EOF cat >> ${COMFYUI_PATH}/user/default/comfy.settings.json
+{
+    "Comfy.TutorialCompleted": true
+}
+EOF
 # create changeset saving script
 RUN mkdir changeset
 RUN <<EOF cat >> save-changeset.sh
@@ -87,7 +91,7 @@ else
     ls -la ${COMFYUI_PATH}/user/default/workflows/
 fi
 comfy node fix all
-comfy node save-snapshot 
+comfy node save-snapshot
 find . -mindepth 2 -type d -name ".git" -exec rm -rf {} +
 find . -mindepth 2 -type f -name ".gitignore" -exec rm -f {} +
 git add --all

ComfyUI/Dockerfile.prod

@@ -7,7 +7,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 ENV PIP_PREFER_BINARY=1
 ENV PIP_ROOT_USER_ACTION=ignore
 # Ensures output from python is printed immediately to the terminal without buffering
-ENV PYTHONUNBUFFERED=1 
+ENV PYTHONUNBUFFERED=1
 # Speed up some cmake builds
 ENV CMAKE_BUILD_PARALLEL_LEVEL=8
 
@@ -62,8 +62,8 @@ RUN --mount=type=cache,target=/root/.npm \
   npm ci --omit=dev
 
 # Set environment variables
-ARG COMFYUI_VERSION=0.3.10
-ARG COMFYUI_MANAGER_VERSION=3.6.5
+ARG COMFYUI_VERSION=0.3.29
+ARG COMFYUI_MANAGER_VERSION=3.31.12
 ENV COMFYUI_PATH=/opt/ComfyUI
 
 RUN git config --global user.email "container@super-protocol.com" \
@@ -90,12 +90,18 @@ RUN <<EOF cat >> ${COMFYUI_PATH}/user/default/ComfyUI-Manager/config.ini
 [default]
 security_level = strong
 EOF
-# FIXME: following command is failing for some reason - 
+# FIXME: following command is failing for some reason -
 RUN comfy manager disable-gui
 # simulating the same command
 # see https://github.com/jiangyangfan/ComfyUI-Manager/blob/5bf9914e17244560ef69ad4de218150fa755e88d/cm-cli.py#L748-L764
 RUN echo "yes" > ${COMFYUI_PATH}/custom_nodes/ComfyUI-Manager/.enable-cli-only-mode
 
+RUN <<EOF cat >> ${COMFYUI_PATH}/user/default/comfy.settings.json
+{
+    "Comfy.TutorialCompleted": true
+}
+EOF
+
 # Change into ComfyUI directory
 WORKDIR ${COMFYUI_PATH}