Release 5.9

.github/.trufflehog3.yml

@@ -10,19 +10,21 @@ exclude: # exclude matching issues
       - coverage/**
       - tests/**
       - test/**
+      - "*.test.tsx"
+      - "*.test.ts"
       - .scannerwork/**
       - .sonarqube/**
       - zap-report.html
       - zap-report.xml
   - message: Ignore mocks
     paths:
-      - mockServiceWorker.js
+      - "mockServiceWorker.js"
   - message: Ignore auto-generated scan reports
     paths:
       - trufflehog_report.*
   - message: Ignore lockfiles
     paths:
-      - package-lock.json
+      - "package-lock.json"
   - message: Ignore local .env files
     paths:
       - "*.env"
@@ -45,8 +47,8 @@ exclude: # exclude matching issues
     # NOTE pattern is re.Pattern, thus escaping of special characters might be required
     pattern: sha256-.*
     paths:
-      - index.html
-      - nginx.conf.template
+      - "index.html"
+      - "nginx.conf.template"
   - message: Ignore hashes on badge urls
     pattern: https://img\.shields\.io/.*
   - message: Ignore hashes on github gists
@@ -56,7 +58,7 @@ exclude: # exclude matching issues
   - message: Ignore local keycloak configuration (for local development only)
     id: high-entropy
     paths:
-      - realm-export.json
+      - "realm-export.json"
   - message: Ignore SSIS .dtsx files (binary content and verifiers)
     paths:
       - "*.dtsx"
@@ -67,6 +69,9 @@ exclude: # exclude matching issues
   - message: Ignore integrity hashes on Openshift config files
     paths:
       - "Test Data/**"
+  - message: Skip ESRI itemIds everywhere (used mainly in basemaps.json)
+    pattern: ("?)itemId("?):.*
+
 # NOTE you can also specify IDs or patterns together or without paths, e.g.
 #
 #   - message: Skip ID everywhere

.github/workflows/api-dotnetcore.yml

@@ -136,23 +136,6 @@ jobs:
           name: pr-codecov
           path: pr/
 
-      - name: SonarScanner for .NET 8 with pull request decoration support
-        id: scan
-        uses: highbyte/sonarscan-dotnet@v2.3.2
-        if: ${{ github.event_name == 'push' && env.solution-name == 'Pims.sln' }}
-        env:
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-        with:
-          dotnetBuildArguments: ${{env.working-directory}}/${{ env.solution-name }}
-          dotnetTestArguments: ${{env.working-directory}}/${{ env.solution-name }} --logger trx --collect:"XPlat Code Coverage" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=opencover
-          # Optional extra command arguments the the SonarScanner 'begin' command
-          sonarBeginArguments: /d:sonar.cs.opencover.reportsPaths="**/TestResults/**/coverage.opencover.xml" -d:sonar.cs.vstest.reportsPaths="**/TestResults/*.trx" -d:sonar.qualitygate.wait=true
-          # The key of the SonarQube project
-          sonarProjectKey: ${{secrets.PROJECT_KEY_API}}
-          # The name of the SonarQube project
-          sonarProjectName: PIMS-API
-          # The SonarQube server URL. For SonarCloud, skip this setting.
-          sonarHostname: ${{secrets.SONAR_URL}}
   post-build:
     needs: build-backend
     runs-on: ubuntu-22.04
@@ -165,55 +148,11 @@ jobs:
       - name: Failure notification to Teams Channel
         env:
           MS_TEAMS_NOTIFY_URL: ${{ secrets.MS_TEAMS_NOTIFY_URL }}
-        if: env.MS_TEAMS_NOTIFY_URL != '' && failure() && steps.scan.outcome == 'failure'
+        if: env.MS_TEAMS_NOTIFY_URL != '' && failure()
         uses: dragos-cojocari/ms-teams-notification@v1.0.2
         with:
           github-token: ${{ github.token }}
           ms-teams-webhook-uri: ${{ env.MS_TEAMS_NOTIFY_URL }}
-          notification-summary: PIMS API Sonar Scan FAILED in ${{env.GIT_BRANCH}} environment
+          notification-summary: PIMS API FAILED in ${{env.GIT_BRANCH}} environment
           notification-color: ff0000
           timezone: America/Los_Angeles
-
-      - name: Find Comment
-        if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
-        uses: peter-evans/find-comment@v1
-        id: fc
-        with:
-          issue-number: ${{ github.event.pull_request.number }}
-          comment-author: "github-actions[bot]"
-          body-includes: QUALITY GATE STATUS FOR .NET 5
-
-      - name: Check Quality Gate and Create Comment
-        if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && steps.fc.outputs.comment-id == ''
-        uses: peter-evans/create-or-update-comment@v1
-        env:
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-          SONAR_HOST_URL: ${{ secrets.SONAR_URL }}
-          PROJECT_KEY: ${{secrets.PROJECT_KEY_API}}
-        with:
-          issue-number: ${{ github.event.pull_request.number }}
-          body: |
-            QUALITY GATE STATUS FOR .NET 5: FAILED.
-
-            [View and resolve details on][1]
-
-            [1]: ${{env.SONAR_HOST_URL}}/dashboard?id=${{env.PROJECT_KEY}}
-          reactions: confused
-      - name: Check Quality Gate and Update Comment
-        if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && steps.fc.outputs.comment-id != '' && github.event.pull_request.head.repo.full_name == github.repository
-        uses: peter-evans/create-or-update-comment@v1
-        env:
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-          SONAR_HOST_URL: ${{ secrets.SONAR_URL }}
-          PROJECT_KEY: ${{secrets.PROJECT_KEY_APP}}
-        with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
-          issue-number: ${{ github.event.pull_request.number }}
-          body: |
-            QUALITY GATE STATUS FOR .NET 5: FAILED.
-
-            [View and resolve details on][1]
-
-            [1]: ${{env.SONAR_HOST_URL}}/dashboard?id=${{env.PROJECT_KEY}}
-          edit-mode: replace
-          reactions: eyes

.github/workflows/app-react.yml

@@ -68,75 +68,16 @@ jobs:
           name: pr-codecov
           path: pr/
 
-      - name: SonarQube Scan
-        id: scan
-        uses: sonarsource/sonarqube-scan-action@master
-        env:
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-          SONAR_HOST_URL: https://sonarqube-3cd915-tools.apps.silver.devops.gov.bc.ca/
-          PROJECT_KEY: 21faa627-fe70-4a67-89e3-1262a32d2e2c
-          PROJECT_NAME: PIMS-APP
-        with:
-          projectBaseDir: ${{env.working-directory}}
-          args: >
-            -Dsonar.projectKey=21faa627-fe70-4a67-89e3-1262a32d2e2c
-            -Dsonar.projectName=PIMS-APP
-            -Dsonar.qualitygate.wait=true
-
       # Send notifications only if MS_TEAMS_NOTIFY_URL secret has been set
       - name: Failure notification to Teams Channel
         env:
           MS_TEAMS_NOTIFY_URL: ${{ secrets.MS_TEAMS_NOTIFY_URL }}
-        if: env.MS_TEAMS_NOTIFY_URL != '' && failure() && steps.scan.outcome == 'failure'
+        if: env.MS_TEAMS_NOTIFY_URL != '' && failure()
         uses: dragos-cojocari/ms-teams-notification@v1.0.2
         with:
           github-token: ${{ github.token }}
           ms-teams-webhook-uri: ${{ env.MS_TEAMS_NOTIFY_URL }}
-          notification-summary: PIMS APP Sonar Scan FAILED in ${{env.GIT_BRANCH}} environment
+          notification-summary: PIMS APP FAILED in ${{env.GIT_BRANCH}} environment
           notification-color: ff0000
           timezone: America/Los_Angeles
 
-      - name: Find Comment
-        if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
-        uses: peter-evans/find-comment@v1
-        id: fc
-        with:
-          token: ${{secrets.GITHUB_TOKEN}}
-          issue-number: ${{ github.event.pull_request.number }}
-          comment-author: "github-actions[bot]"
-          body-includes: QUALITY GATE STATUS FOR APP-REACT
-
-      - name: Check Quality Gate and Create Comment
-        uses: peter-evans/create-or-update-comment@v1
-        if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && steps.fc.outputs.comment-id == '' && github.event.pull_request.head.repo.full_name == github.repository
-        env:
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-          SONAR_HOST_URL: ${{ secrets.SONAR_URL }}
-          PROJECT_KEY: ${{secrets.PROJECT_KEY_APP}}
-        with:
-          issue-number: ${{ github.event.pull_request.number }}
-          body: |
-            QUALITY GATE STATUS FOR APP-REACT: FAILED.
-
-            [View and resolve details on][1]
-
-            [1]: ${{env.SONAR_HOST_URL}}/dashboard?id=${{env.PROJECT_KEY}}
-          reactions: confused
-      - name: Check Quality Gate and Update Comment
-        if: failure() && steps.scan.outcome == 'failure' && github.event_name == 'pull_request' && steps.fc.outputs.comment-id != '' && github.event.pull_request.head.repo.full_name == github.repository
-        uses: peter-evans/create-or-update-comment@v1
-        env:
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-          SONAR_HOST_URL: ${{ secrets.SONAR_URL }}
-          PROJECT_KEY: ${{secrets.PROJECT_KEY_APP}}
-        with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
-          issue-number: ${{ github.event.pull_request.number }}
-          body: |
-            QUALITY GATE STATUS FOR APP-REACT: FAILED.
-
-            [View and resolve details on][1]
-
-            [1]: ${{env.SONAR_HOST_URL}}/dashboard?id=${{env.PROJECT_KEY}}
-          edit-mode: replace
-          reactions: eyes

.github/workflows/deploy-prod-end.yml

@@ -27,7 +27,7 @@ jobs:
           namespace: 3cd915-tools
       - name: Hide the maintenance page and display the app
         run: |
-          ./tools/cicd/maintenance/maintenance.sh prod off
+          APPLICATION_NAME=pims-app-prod ./tools/cicd/maintenance/maintenance.sh prod off
 
   ci-cd-end-notification:
     name: CI-CD End Notification to Teams Channel

.github/workflows/deploy-prod-start.yml

@@ -143,4 +143,4 @@ jobs:
       - name: call scripts to sync keycloak
         shell: bash
         run: |
-          oc process -f ./openshift/4.0/templates/jobs/keycloak-sync-pipeline-run.yaml -p ASPNETCORE_ENVIRONMENT=$ASPNETCORE_ENVIRONMENT -p NAMESPACE=3cd915-prod -p BRANCH=GIT_BRANCH -p API_URL=http://pims-api:8080/api  | oc create -f - | grep -oP "(?<=\/)[^\s]*" | (read PIPELINE_NAME; oc wait --for=condition=succeeded pipelineruns/$PIPELINE_NAME --timeout=500s)
+          oc process -f ./openshift/4.0/templates/jobs/keycloak-sync-pipeline-run.yaml -p ASPNETCORE_ENVIRONMENT=$ASPNETCORE_ENVIRONMENT -p NAMESPACE=3cd915-prod -p BRANCH=GIT_BRANCH -p API_URL=http://pims-api:8080/api  | oc create -f - | grep -oP "(?<=\/)[^\s]*" | (read PIPELINE_NAME; oc wait --for=condition=succeeded pipelineruns/$PIPELINE_NAME --timeout=750s)

.github/workflows/zap-scan.yml

@@ -18,17 +18,3 @@ jobs:
           target: "https://dev-pims.th.gov.bc.ca/"
           cmd_options: "-r ${{ env.HTML_ZAP_REPORT }} -x ${{ env.ZAP_REPORT }}"
           fail_action: false
-      - name: SonarQube Scan
-        uses: sonarsource/sonarqube-scan-action@master
-        if: ${{ github.event_name == 'push' }}
-        env:
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-          SONAR_HOST_URL: ${{ secrets.SONAR_URL }}
-          PROJECT_KEY: PIMS-ZAP
-          PROJECT_NAME: PIMS-ZAP
-        with:
-          args: >
-            -Dsonar.projectKey=${{ env.PROJECT_KEY }}
-            -Dsonar.projectName=${{ env.PROJECT_NAME }}
-            -Dsonar.zaproxy.reportPath=${{ env.ZAP_REPORT }}
-            -Dsonar.zaproxy.htmlReportPath=${{ env.HTML_ZAP_REPORT }}

Makefile

@@ -180,7 +180,11 @@ refresh: | down build up ## Recreates local docker environment
 .PHONY: infra
 infra: ## Starts infrastructure containers (e.g. database, geoserver). Useful for local debugging
 	@echo "$(P) Starting up infrastructure containers..."
-	@"$(MAKE)" start n="database geoserver grafana prometheus proxy"
+	@"$(MAKE)" start n="database geoserver proxy"
+
+monitoring-up: ## Calls the docker compose up for the monitoring/telemetry images
+	@echo "$(P) Create or start local monitoring containers..."
+	@docker-compose --profile monitoring up -d
 
 start: ## Starts the local containers (n=service name)
 	@echo "$(P) Starting client and server containers..."
@@ -279,7 +283,7 @@ db-upgrade: ## Upgrade an existing database to the TARGET_VERSION (if passed) or
 
 db-scaffold: ## Requires local install of sqlcmd
 	@echo "$(P) regenerate ef core entities from database"
-	@cd source/backend/entities; eval $(grep -v '^#' .env | xargs) dotnet ef dbcontext scaffold Name=PIMS Microsoft.EntityFrameworkCore.SqlServer -o ../entities/ef --schema dbo --context PimsBaseContext --context-namespace Pims.Dal --context-dir . --no-onconfiguring --namespace Pims.Dal.Entities --data-annotations -v -f --startup-project ../api
+	@cd source/backend/entities; eval $(grep -v '^#' .env | xargs) dotnet ef dbcontext scaffold Name=PIMS Microsoft.EntityFrameworkCore.SqlServer -o ../entities/ef --schema "dbo" --schema "pmbc" --context PimsBaseContext --context-namespace Pims.Dal --context-dir . --no-onconfiguring --namespace Pims.Dal.Entities --data-annotations -v -f --startup-project ../api
 
 db-generate-master:
 	@echo "$(P) Generates a master script that for the given sprint."

docker-compose.yml

@@ -187,18 +187,19 @@ services:
     restart: on-failure
     container_name: psp-clamav
     healthcheck:
-      test: [ "CMD", "./check.sh" ]
+      test: ["CMD", "./check.sh"]
       interval: 60s
       retries: 3
       start_period: 120s
     networks:
       - psp
 
+  ####################### Prometheus #######################
   prometheus:
     profiles:
       - all
       - monitoring
-    image: ubuntu/prometheus
+    image: quay.io/prometheus/prometheus:v3.2.0
     container_name: psp-prometheus
     ports:
       - 9000:9090
@@ -210,7 +211,10 @@ services:
     command:
       - "--config.file=/etc/prometheus/prometheus.yml"
       - "--web.enable-lifecycle"
+    networks:
+      - psp
 
+  ####################### Grafana #######################
   grafana:
     profiles:
       - all
@@ -220,8 +224,68 @@ services:
     restart: on-failure
     ports:
       - 4000:3000
-    depends_on:
-      - prometheus
+    networks:
+      - psp
+
+  ####################### OpenTelemetry Collector #######################
+  otel-collector:
+    profiles:
+      - all
+      - monitoring
+    image: otel/opentelemetry-collector-contrib:0.120.0
+    container_name: otel-collector
+    restart: on-failure
+    deploy:
+      resources:
+        limits:
+          memory: 500M
+    command: ["--config=/etc/otel-config.yaml"]
+    volumes:
+      - ./tools/otel-collector/otel-config.yaml:/etc/otel-config.yaml
+    ports:
+      - 4318:4318 # http receiver
+      - 55679:55679 # zPages UI
+      - 9464:9464 # prometheus metrics endpoint
+      - 13133:13133 # health check
+    environment:
+      - OTEL_COLLECTOR_HOST=0.0.0.0
+      - JAEGER_HOST=jaeger
+    healthcheck:
+      test: curl --fail -s http://localhost:13133/ || exit 1
+      interval: 1m30s
+      timeout: 10s
+      retries: 3
+    networks:
+      - psp
+  ####################### Jaeger (browser traces + monitoring dashboards) #######################
+  jaeger:
+    profiles:
+      - all
+      - monitoring
+    image: jaegertracing/jaeger:2.4.0
+    container_name: jaeger
+    command: ["--config=/jaeger/jaeger-config.yaml"]
+    volumes:
+      - ./tools/jaeger/jaeger-config.yaml:/jaeger/jaeger-config.yaml
+    ports:
+      - 16686:16686 # Web UI
+      - 4317:4317 # otlp grpc receiver
+      - 5318:4318 # otlp http receiver
+      - 13123:13133 # health check
+      - 8889:8889 # prometheus metrics endpoint
+      - 8888:8888 # internal container metrics
+    environment:
+      - LOG_LEVEL=debug
+      - JAEGER_LISTEN_HOST=0.0.0.0
+      - PROMETHEUS_HOST=prometheus
+      - PROMETHEUS_PORT=9090
+    healthcheck:
+      test: curl --fail -s http://localhost:13123/status || exit 1
+      interval: 1m30s
+      timeout: 10s
+      retries: 3
+    networks:
+      - psp
 
 ####################### Networks Definition #######################
 networks:

etl/PAT_PIMS_DOCUMENT_IMPORT/PAT_PIMS_DOCUMENT_IMPORT/PAT_PIMS_DOCUMENT_IMPORT.dtproj

@@ -1,12 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
   <DeploymentModel>Project</DeploymentModel>
-  <ProductVersion>16.0.948.0</ProductVersion>
+  <ProductVersion>16.0.5270.0</ProductVersion>
   <SchemaVersion>9.0.1.0</SchemaVersion>
   <State>$base64$PFNvdXJjZUNvbnRyb2xJbmZvIHhtbG5zOnhzZD0iaHR0cDovL3d3dy53My5vcmcvMjAwMS9YTUxTY2hlbWEiIHhtbG5zOnhzaT0iaHR0cDovL3d3dy53My5vcmcvMjAwMS9YTUxTY2hlbWEtaW5zdGFuY2UiIHhtbG5zOmRkbDI9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vYW5hbHlzaXNzZXJ2aWNlcy8yMDAzL2VuZ2luZS8yIiB4bWxuczpkZGwyXzI9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vYW5hbHlzaXNzZXJ2aWNlcy8yMDAzL2VuZ2luZS8yLzIiIHhtbG5zOmRkbDEwMF8xMDA9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vYW5hbHlzaXNzZXJ2aWNlcy8yMDA4L2VuZ2luZS8xMDAvMTAwIiB4bWxuczpkZGwyMDA9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vYW5hbHlzaXNzZXJ2aWNlcy8yMDEwL2VuZ2luZS8yMDAiIHhtbG5zOmRkbDIwMF8yMDA9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vYW5hbHlzaXNzZXJ2aWNlcy8yMDEwL2VuZ2luZS8yMDAvMjAwIiB4bWxuczpkZGwzMDA9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vYW5hbHlzaXNzZXJ2aWNlcy8yMDExL2VuZ2luZS8zMDAiIHhtbG5zOmRkbDMwMF8zMDA9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vYW5hbHlzaXNzZXJ2aWNlcy8yMDExL2VuZ2luZS8zMDAvMzAwIiB4bWxuczpkZGw0MDA9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vYW5hbHlzaXNzZXJ2aWNlcy8yMDEyL2VuZ2luZS80MDAiIHhtbG5zOmRkbDQwMF80MDA9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vYW5hbHlzaXNzZXJ2aWNlcy8yMDEyL2VuZ2luZS80MDAvNDAwIiB4bWxuczpkZGw1MDA9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vYW5hbHlzaXNzZXJ2aWNlcy8yMDEzL2VuZ2luZS81MDAiIHhtbG5zOmRkbDUwMF81MDA9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vYW5hbHlzaXNzZXJ2aWNlcy8yMDEzL2VuZ2luZS81MDAvNTAwIiB4bWxuczpkd2Q9Imh0dHA6Ly9zY2hlbWFzLm1pY3Jvc29mdC5jb20vRGF0YVdhcmVob3VzZS9EZXNpZ25lci8xLjAiPg0KICA8RW5hYmxlZD5mYWxzZTwvRW5hYmxlZD4NCiAgPFByb2plY3ROYW1lPjwvUHJvamVjdE5hbWU+DQogIDxBdXhQYXRoPjwvQXV4UGF0aD4NCiAgPExvY2FsUGF0aD48L0xvY2FsUGF0aD4NCiAgPFByb3ZpZGVyPjwvUHJvdmlkZXI+DQo8L1NvdXJjZUNvbnRyb2xJbmZvPg==</State>
   <Database>
-    <Name>PAT_PIMS_DOCUMENT_IMPORT.database</Name>
-    <FullPath>PAT_PIMS_DOCUMENT_IMPORT.database</FullPath>
+    <Name>PAT_PIMS_DOCUMENT_IMPORT 1.database</Name>
+    <FullPath>PAT_PIMS_DOCUMENT_IMPORT 1.database</FullPath>
   </Database>
   <DataSources />
   <DataSourceViews />