[DO NOT MERGE] rabbit performance test driver

Makefile

@@ -18,7 +18,8 @@ fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice		\
 calling-test demo-smtp elasticsearch-curator elasticsearch-external				\
 elasticsearch-ephemeral minio-external cassandra-external						\
 ingress-nginx-controller nginx-ingress-services reaper restund \
-k8ssandra-test-cluster ldap-scim-bridge wire-server-enterprise
+k8ssandra-test-cluster ldap-scim-bridge wire-server-enterprise \
+integration
 KIND_CLUSTER_NAME     := wire-server
 HELM_PARALLELISM      ?= 1 # 1 for sequential tests; 6 for all-parallel tests
 # (run `psql -h localhost -p 5432 -d backendA -U wire-server -w` for the list of options for PSQL_DB)

charts/integration/templates/configmap.yaml

@@ -77,14 +77,16 @@ data:
       apiPort: 5380
       dohPort: 5381
 
-    originDomain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
+    # originDomain: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
+    # sven-test.wire.link
+    originDomain: dev.zinfra.io
 
     rabbitmq:
       host: rabbitmq
       port: 5671
       adminHost: rabbitmq
-      adminPort: 15671
-      enableTls: true
+      adminPort: 15672
+      enableTls: false
       insecureSkipVerifyTls: true
       vHost: /
 
@@ -310,3 +312,6 @@ data:
 
     integrationTestHostName: integration-headless.{{ .Release.Namespace }}.svc.cluster.local
     cellsEventQueue: cells_events
+    shardingGroupCount: {{ .Values.config.shardingGroupCount }}
+    maxUserNo: {{ .Values.config.maxUserNo }}
+    maxDeliveryDelay: {{ .Values.config.maxDeliveryDelay }}

charts/integration/templates/integration-integration.yaml

@@ -29,10 +29,6 @@
       secret:
         secretName: "brig"
 
-    - name: "turn-servers"
-      configMap:
-        name: "turn"
-
     - name: "cannon-config"
       configMap:
         name: "cannon"
@@ -65,22 +61,6 @@
       configMap:
         name: "background-worker"
 
-    - name: "background-worker-secrets"
-      secret:
-        secretName: "background-worker"
-
-    - name: "stern-config"
-      configMap:
-        name: "backoffice"
-
-    - name: "proxy-config"
-      configMap:
-        name: "proxy"
-
-    - name: "proxy-secrets"
-      secret:
-        secretName: "proxy"
-
     - name: "nginz-config"
       configMap:
         name: "nginz"
@@ -89,28 +69,12 @@
       secret:
         secretName: "nginz"
 
-    - name: elasticsearch-ca
-      secret:
-        secretName: {{ .Values.config.elasticsearch.tlsCaSecretRef.name }}
-
-    - name: redis-ca
-      secret:
-        secretName: {{ .Values.config.redis.tlsCaSecretRef.name }}
-
-    - name: rabbitmq-ca
-      secret:
-        secretName: {{ .Values.config.rabbitmq.tlsCaSecretRef.name }}
-
     {{- if eq (include "useCassandraTLS" .Values.config) "true" }}
     - name: integration-cassandra
       secret:
         secretName: {{ include "cassandraTlsSecretName" .Values.config }}
     {{- end }}
 
-    - name: wire-server-enterprise-config
-      configMap:
-        name: wire-server-enterprise
-
   restartPolicy: Never
 
   initContainers:
@@ -121,14 +85,10 @@
       {{- toYaml .Values.podSecurityContext | nindent 6 }}
         {{- end }}
     volumeMounts:
-      - name: elasticsearch-ca
-        mountPath: "/certs/elasticsearch"
       {{- if eq (include "useCassandraTLS" .Values.config) "true" }}
       - name: "integration-cassandra"
         mountPath: "/certs/cassandra"
       {{- end }}
-      - name: rabbitmq-ca
-        mountPath: /certs/rabbitmq-ca
     env:
     - name: INTEGRATION_DYNAMIC_BACKENDS_POOLSIZE
       value: "{{ .Values.config.dynamicBackendsPoolsize }}"
@@ -141,60 +101,48 @@
     - name: RABBITMQ_USERNAME
       valueFrom:
         secretKeyRef:
-          name: brig
+          name: gundeck
           key: rabbitmqUsername
     - name: RABBITMQ_PASSWORD
       valueFrom:
         secretKeyRef:
-          name: brig
+          name: gundeck
           key: rabbitmqPassword
     command:
       - /bin/sh
       - -c
       - |
         set -euo pipefail
-        # FUTUREWORK: Do all of this in the integration test binary
-        integration-dynamic-backends-db-schemas.sh \
-          --host {{ .Values.config.cassandra.host }} \
-          --port {{ .Values.config.cassandra.port }} \
-          --replication-factor {{ .Values.config.cassandra.replicationFactor }} \
-          {{- if eq (include "useCassandraTLS" .Values.config) "true" }}
-          --tls-ca-certificate-file /certs/cassandra/{{- include "cassandraTlsSecretKey" .Values.config }}
-          {{ end }}
-
-        integration-dynamic-backends-brig-index.sh \
-          --elasticsearch-server https://elastic:changeme@{{ .Values.config.elasticsearch.host }}:9200 \
-          --elasticsearch-ca-cert /certs/elasticsearch/{{ .Values.config.elasticsearch.tlsCaSecretRef.key }}
-        integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }}
-        integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }}
+
         {{- range $name, $dynamicBackend := .Values.config.dynamicBackends }}
         integration-dynamic-backends-vhosts.sh {{ $.Values.config.rabbitmqPutVHostUrl }} {{ $dynamicBackend.federatorExternalHostPrefix}}.{{ $.Release.Namespace }}.svc.cluster.local
         {{- end }}
     resources:
       requests:
         memory: "512Mi"
         cpu: "2"
 
   containers:
-  - name: integration
-    image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        {{- if eq (include "includeSecurityContext" .) "true" }}
+  {{- range $i := until (int .Values.config.shardingGroupCount) }}
+  - name: "integration-{{ int $i}}"
+    image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}"
+        {{- if eq (include "includeSecurityContext" $) "true" }}
     securityContext:
-      {{- toYaml .Values.podSecurityContext | nindent 6 }}
+      {{- toYaml $.Values.podSecurityContext | nindent 6 }}
         {{- end }}
     command:
     - /bin/bash
     - -c
     - |
       set -euo pipefail
 
-      if integration --config /etc/wire/integration/integration.yaml; then
+      if TEST_INCLUDE=testBench integration --config /etc/wire/integration/integration.yaml --sharding-group {{ $i }}; then
         exit_code=$?
       else
         exit_code=$?
       fi
 
-      {{- if .Values.config.uploadXml }}
+      {{- if $.Values.config.uploadXml }}
       # In case a different S3 compliant storage is used to upload test result.
       if ! [[ -z "${UPLOAD_XML_AWS_ACCESS_KEY_ID+x}" ]]; then
         export AWS_ACCESS_KEY_ID="$UPLOAD_XML_AWS_ACCESS_KEY_ID"
@@ -210,7 +158,7 @@
     resources:
       requests:
         memory: "512Mi"
-        cpu: "2"
+        cpu: "0.5"
     volumeMounts:
 
     - name: integration-config
@@ -228,9 +176,6 @@
     - name: brig-secrets
       mountPath: /etc/wire/brig/secrets
 
-    - name: turn-servers
-      mountPath: /etc/wire/brig/turn
-
     - name: cannon-config
       mountPath: /etc/wire/cannon/conf
 
@@ -243,58 +188,13 @@
     - name: spar-config
       mountPath: /etc/wire/spar/conf
 
-    - name: federator-config
-      mountPath: /etc/wire/federator/conf
-
-    - name: federator-secrets
-      mountPath: /etc/wire/federator/secrets
-
-    - name: federator-ca
-      mountPath: /etc/wire/federator/ca
-
-    - name: background-worker-config
-      mountPath: /etc/wire/background-worker/conf
-
-    - name: background-worker-secrets
-      mountPath: /etc/wire/background-worker/secrets
-
-    - name: stern-config
-      mountPath: /etc/wire/stern/conf
-
-    - name: proxy-config
-      mountPath: /etc/wire/proxy/conf
-
-    - name: proxy-secrets
-      mountPath: /etc/wire/proxy/secrets
-
     - name: nginz-config
       mountPath: /etc/wire/nginz/conf
 
     - name: nginz-secrets
       mountPath: /etc/wire/nginz/secrets
 
-    - name: elasticsearch-ca
-      mountPath: /etc/wire/brig/elasticsearch-ca
-
-    - name: redis-ca
-      mountPath: /etc/wire/gundeck/redis-ca
-
-    - name: rabbitmq-ca
-      mountPath: /etc/wire/brig/rabbitmq-ca
-
-    - name: rabbitmq-ca
-      mountPath: /etc/wire/galley/rabbitmq-ca
-
-    - name: rabbitmq-ca
-      mountPath: /etc/wire/background-worker/rabbitmq-ca
-
-    - name: rabbitmq-ca
-      mountPath: /etc/wire/gundeck/rabbitmq-ca
-
-    - name: rabbitmq-ca
-      mountPath: /etc/wire/cannon/rabbitmq-ca
-
-    {{- if eq (include "useCassandraTLS" .Values.config) "true" }}
+    {{- if eq (include "useCassandraTLS" $.Values.config) "true" }}
     - name: "integration-cassandra"
       mountPath: "/certs"
 
@@ -311,9 +211,6 @@
       mountPath: "/etc/wire/spar/cassandra"
     {{- end }}
 
-    - name: wire-server-enterprise-config
-      mountPath: /etc/wire/wire-server-enterprise/conf
-
     env:
     # these dummy values are necessary for Amazonka's "Discover"
     - name: AWS_ACCESS_KEY_ID
@@ -325,35 +222,21 @@
     - name: RABBITMQ_USERNAME
       valueFrom:
         secretKeyRef:
-          name: brig
+          name: gundeck
           key: rabbitmqUsername
     - name: RABBITMQ_PASSWORD
       valueFrom:
         secretKeyRef:
-          name: brig
+          name: gundeck
           key: rabbitmqPassword
-    - name: RABBITMQ_USERNAME_V0
-      value: "wire-server"
-    - name: RABBITMQ_PASSWORD_V0
-      valueFrom:
-        secretKeyRef:
-          name: rabbitmq-v0
-          key: rabbitmq-password
-    - name: RABBITMQ_USERNAME_V1
-      value: "wire-server"
-    - name: RABBITMQ_PASSWORD_V1
-      valueFrom:
-        secretKeyRef:
-          name: rabbitmq-v1
-          key: rabbitmq-password
-    {{- if hasKey .Values.secrets "redisUsername" }}
+    {{- if hasKey $.Values.secrets "redisUsername" }}
     - name: REDIS_USERNAME
       valueFrom:
         secretKeyRef:
           name: integration
           key: redisUsername
     {{- end }}
-    {{- if hasKey .Values.secrets "redisPassword" }}
+    {{- if hasKey $.Values.secrets "redisPassword" }}
     - name: REDIS_PASSWORD
       valueFrom:
         secretKeyRef:
@@ -362,10 +245,10 @@
     {{- end }}
     - name: TEST_XML
       value: /tmp/result.xml
-    {{- if .Values.config.uploadXml }}
+    {{- if $.Values.config.uploadXml }}
     - name: UPLOAD_XML_S3_BASE_URL
-      value: {{ .Values.config.uploadXml.baseUrl }}
-    {{- if .Values.secrets.uploadXmlAwsAccessKeyId }}
+      value: {{ $.Values.config.uploadXml.baseUrl }}
+    {{- if $.Values.secrets.uploadXmlAwsAccessKeyId }}
     - name: UPLOAD_XML_AWS_ACCESS_KEY_ID
       valueFrom:
         secretKeyRef:
@@ -379,8 +262,9 @@
     {{- end }}
     {{- end }}
     - name: ENABLE_FEDERATION_V0
-      value: "1"
+      value: "0"
     - name: ENABLE_FEDERATION_V1
-      value: "1"
+      value: "0"
     - name: ENABLE_FEDERATION_V2
-      value: "1"
+      value: "0"
+  {{- end }}

charts/integration/values.yaml

@@ -12,7 +12,7 @@ podSecurityContext:
     type: RuntimeDefault
 
 config:
-  dynamicBackendsPoolsize: 3
+  dynamicBackendsPoolsize: 0
   dynamicBackends:
     dynamic-backend-1:
       federatorExternalHostPrefix: dynamic-backend-1
@@ -118,8 +118,11 @@ config:
   sqsEndpointUrl: http://fake-aws-sqs:4568
   sesEndpointUrl: http://fake-aws-ses:4569
   s3EndpointUrl: http://fake-aws-s3:9000
-  rabbitmqPutVHostUrl: http://rabbitmq:15672/api/vhosts
-
+  rabbitmqPutVHostUrl: http://rabbitmq-b.databases.svc.cluster.local:15672/api/vhosts
+  shardingGroupCount: 3
+  # shardingGroup: 0
+  maxUserNo: 1000
+  maxDeliveryDelay: 120
 tls:
   verify_depth: 1
   # Namespace from which to obtain the secret containing the CA trusted by

integration/default.nix

@@ -74,6 +74,7 @@
 , split
 , stm
 , streaming-commons
+, streamly
 , string-conversions
 , system-linux-proc
 , tagged
@@ -179,6 +180,7 @@ mkDerivation {
     split
     stm
     streaming-commons
+    streamly
     string-conversions
     system-linux-proc
     tagged

integration/integration.cabal

@@ -184,6 +184,7 @@ library
     Test.MLS.Unreachable
     Test.NginxZAuthModule
     Test.Notifications
+    Test.NotificationsBenchmark
     Test.OAuth
     Test.PasswordReset
     Test.Presence
@@ -299,6 +300,7 @@ library
     , split
     , stm
     , streaming-commons
+    , streamly
     , string-conversions
     , system-linux-proc
     , tagged