diff --git a/api/v1/gateway_types.go b/api/v1/gateway_types.go index 41fcad93..646d9929 100644 --- a/api/v1/gateway_types.go +++ b/api/v1/gateway_types.go @@ -360,17 +360,32 @@ type Otk struct { // This configures a relationship between DMZ and Internal Gateways. InternalOtkGatewayReference string `json:"internalGatewayReference,omitempty"` // InternalGatewayPort defaults to 9443 or graphmanDynamicSync port + // This port is used when the Internal gateway is external (not managed by operator) InternalGatewayPort int `json:"internalGatewayPort,omitempty"` // OTKPort is used in Single mode - sets the otk.port cluster-wide property and in Dual-Mode // sets host_oauth2_auth_server port in #OTK Client Context Variables // TODO: Make this an array for many dmz deployments to one internal DmzOtkGatewayReference string `json:"dmzGatewayReference,omitempty"` + // DmzGatewayPort defaults to 9443 or graphmanDynamicSync port + // This port is used when the DMZ gateway is external (not managed by operator) + DmzGatewayPort int `json:"dmzGatewayPort,omitempty"` // OTKPort defaults to 8443 OTKPort int `json:"port,omitempty"` // MaintenanceTasks for the OTK database are disabled by default MaintenanceTasks OtkMaintenanceTasks `json:"maintenanceTasks,omitempty"` // RuntimeSyncIntervalSeconds how often OTK Gateways should be updated in internal/dmz mode RuntimeSyncIntervalSeconds int `json:"runtimeSyncIntervalSeconds,omitempty"` + // SyncIntervalSeconds determines how often DMZ and Internal gateways should update certificates + // Defaults to RuntimeSyncIntervalSeconds if not specified, or 10 seconds if neither is set + SyncIntervalSeconds int `json:"syncIntervalSeconds,omitempty"` + // DmzKeySecret is a reference to a kubernetes.io/tls Secret containing the DMZ private key and certificate + DmzKeySecret string `json:"dmzKeySecret,omitempty"` + // InternalKeySecret is a reference to a kubernetes.io/tls Secret containing the Internal private key and certificate + InternalKeySecret string `json:"internalKeySecret,omitempty"` + // DmzAuthSecret is a reference to a Secret containing username and password for DMZ authentication + DmzAuthSecret string `json:"dmzAuthSecret,omitempty"` + // InternalAuthSecret is a reference to a Secret containing username and password for Internal authentication + InternalAuthSecret string `json:"internalAuthSecret,omitempty"` } // OtkMaintenanceTasks are included in the install bundle as disabled scheduled tasks @@ -893,6 +908,8 @@ type ExternalKey struct { // only one key usage type is allowed // SSL | CA | AUDIT_SIGNING | AUDIT_VIEWER KeyUsageType KeyUsageType `json:"keyUsageType,omitempty"` + // Otk indicates that this key usage was specific for OTK + Otk bool `json:"otk,omitempty"` } type KeyUsageType string diff --git a/config/crd/bases/security.brcmlabs.com_gateways.yaml b/config/crd/bases/security.brcmlabs.com_gateways.yaml index a473a499..0ebb6746 100644 --- a/config/crd/bases/security.brcmlabs.com_gateways.yaml +++ b/config/crd/bases/security.brcmlabs.com_gateways.yaml @@ -1589,6 +1589,10 @@ spec: description: Name of the kubernetes.io/tls Secret which already exists in Kubernetes type: string + otk: + description: Otk indicates that this key usage was specific + for OTK + type: boolean type: object type: array externalSecrets: @@ -4003,9 +4007,21 @@ spec: description: Type of OTK Database type: string type: object + dmzAuthSecret: + description: DmzAuthSecret is a reference to a Secret containing + username and password... + type: string + dmzGatewayPort: + description: |- + DmzGatewayPort defaults to 9443 or graphmanDynamicSync port + This port is... + type: integer dmzGatewayReference: description: OTKPort is used in Single mode - sets the otk. type: string + dmzKeySecret: + description: DmzKeySecret is a reference to a kubernetes. + type: string enabled: description: Enable or disable the OTK initContainer type: boolean @@ -4142,14 +4158,22 @@ spec: type: string type: object type: object + internalAuthSecret: + description: InternalAuthSecret is a reference to a Secret + containing username and... + type: string internalGatewayPort: - description: InternalGatewayPort defaults to 9443 or graphmanDynamicSync - port + description: |- + InternalGatewayPort defaults to 9443 or graphmanDynamicSync port + This port... type: integer internalGatewayReference: description: InternalOtkGatewayReference to an Operator managed Gateway deployment that... type: string + internalKeySecret: + description: InternalKeySecret is a reference to a kubernetes. + type: string maintenanceTasks: description: MaintenanceTasks for the OTK database are disabled by default @@ -4202,6 +4226,10 @@ spec: items: type: string type: array + syncIntervalSeconds: + description: SyncIntervalSeconds determines how often DMZ + and Internal gateways should... + type: integer type: description: Type of OTK installation single, internal or dmz diff --git a/example/gateway/otk/README-DUAL-GATEWAY-OTK.md b/example/gateway/otk/README-DUAL-GATEWAY-OTK.md new file mode 100644 index 00000000..7e42ea61 --- /dev/null +++ b/example/gateway/otk/README-DUAL-GATEWAY-OTK.md @@ -0,0 +1,498 @@ +# Dual Gateway OTK Configuration Guide + +This guide describes how to configure a Dual Gateway OAuth Toolkit (OTK) deployment using the Layer7 Gateway Operator. In a dual gateway setup, one gateway acts as the DMZ (Demilitarized Zone) gateway and another acts as the Internal gateway, providing enhanced security and separation of concerns. + +## Table of Contents + +- [Overview](#overview) +- [Architecture](#architecture) +- [Prerequisites](#prerequisites) +- [Configuration Overview](#configuration-overview) +- [Step 1: Create Required Secrets](#step-1-create-required-secrets) +- [Step 2: Configure DMZ Gateway](#step-2-configure-dmz-gateway) +- [Step 3: Configure Internal Gateway](#step-3-configure-internal-gateway) +- [Key Configuration Fields](#key-configuration-fields) +- [Deployment](#deployment) +- [Certificate Synchronization](#certificate-synchronization) +- [External Gateway Support](#external-gateway-support) +- [Troubleshooting](#troubleshooting) + +## Overview + +The Dual Gateway OTK deployment consists of: + +- **DMZ Gateway**: Handles external client requests and acts as the OAuth authorization server +- **Internal Gateway**: Handles token validation and resource server operations + +The operator automatically synchronizes certificates and keys between the two gateways, ensuring secure communication and proper OAuth flow. + +## Configuration Overview + +The dual gateway setup requires: + +1. **TLS Secrets**: For DMZ and Internal gateway keys/certificates +2. **Auth Secrets**: For gateway authentication credentials +3. **DMZ Gateway Configuration**: With `type: dmz` +4. **Internal Gateway Configuration**: With `type: internal` + +## Step 1: Create Required Secrets + +### Create TLS Secrets + +You need to create TLS secrets for both DMZ and Internal gateways. These secrets must be of type `kubernetes.io/tls` and contain: +- `tls.crt`: The certificate +- `tls.key`: The private key + +#### Option 1: Using the Provided Script + +A helper script is available to generate self-signed certificates and create all required secrets: + +```bash +cd example/gateway/otk/secrets +./create-secrets.sh +``` + +This script creates: +- `otk-dmz-tls-secret`: TLS secret for DMZ gateway +- `otk-internal-tls-secret`: TLS secret for Internal gateway +- `otk-dmz-auth-secret`: Authentication secret for DMZ gateway (username: `admin`, password: `7layer`) +- `otk-internal-auth-secret`: Authentication secret for Internal gateway (username: `admin`, password: `7layer`) + +#### Option 2: Manual Secret Creation + +**DMZ TLS Secret:** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: otk-dmz-tls-secret + namespace: default +type: kubernetes.io/tls +data: + tls.crt: + tls.key: +``` + +**Internal TLS Secret:** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: otk-internal-tls-secret + namespace: default +type: kubernetes.io/tls +data: + tls.crt: + tls.key: +``` + +**DMZ Auth Secret:** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: otk-dmz-auth-secret + namespace: default +type: Opaque +stringData: + SSG_ADMIN_USERNAME: admin + SSG_ADMIN_PASSWORD: 7layer +``` + +**Internal Auth Secret:** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: otk-internal-auth-secret + namespace: default +type: Opaque +stringData: + SSG_ADMIN_USERNAME: admin + SSG_ADMIN_PASSWORD: 7layer +``` + +## Step 2: Configure DMZ Gateway + +The DMZ gateway configuration should include: + +- `otk.type: dmz` +- Reference to Internal gateway +- DMZ key secret reference +- Internal auth secret for communication with Internal gateway +- Database configuration + +### Sample DMZ Gateway Configuration + +```yaml +apiVersion: security.brcmlabs.com/v1 +kind: Gateway +metadata: + name: otk-ssg-dmz +spec: + version: "11.1.3" + license: + accept: true + secretName: gateway-license + app: + replicas: 1 + image: docker.io/caapim/gateway:11.1.3 + imagePullPolicy: IfNotPresent + resources: + requests: + memory: 8Gi + cpu: 3 + limits: + memory: 8Gi + cpu: 3 + # ExternalKeys with otk flag set to true for OTK-specific key usage + externalKeys: + - name: otk-dmz-tls-secret + enabled: true + alias: otk-dmz-key + keyUsageType: SSL + otk: true + otk: + enabled: true + initContainerImage: docker.io/caapim/otk-install:4.6.4 + type: dmz + # Reference to Internal gateway (can be Gateway name or external hostname) + internalGatewayReference: otk-ssg-internal + # InternalGatewayPort is used when the Internal gateway is external + # If not specified, defaults to 9443 or the gateway's graphmanDynamicSync port + internalGatewayPort: 9443 + # SyncIntervalSeconds determines how often certificates are synchronized + # Defaults to RuntimeSyncIntervalSeconds if not specified, or 10 seconds if neither is set + syncIntervalSeconds: 30 + # Reference to the TLS secret for DMZ key + dmzKeySecret: otk-dmz-tls-secret + # Auth secret for Internal gateway communication + internalAuthSecret: otk-internal-auth-secret + database: + type: mysql + create: true + connectionName: OAuth + auth: + gateway: + username: otk_user + password: otkUserPass + readOnly: + username: readonly_user + password: readonly_userPass + admin: + username: admin + password: adminPass + properties: + minimumPoolSize: 3 + maximumPoolSize: 15 + sql: + databaseName: otk_db + jdbcUrl: jdbc:mysql://mysql.brcmlabs.com:3306/otk_db_init + jdbcDriverClass: com.mysql.cj.jdbc.Driver + connectionProperties: + c3p0.maxConnectionAge: "100" + c3p0.maxIdleTime: "1000" + manageSchema: true + databaseWaitTimeout: 60 + management: + secretName: gateway-secret + graphman: + enabled: true + initContainerImage: docker.io/caapim/graphman-static-init:1.0.4 + dynamicSyncPort: 9443 + cluster: + hostname: gateway.brcmlabs.com + service: + type: ClusterIP + ports: + - name: https + port: 8443 + targetPort: 8443 + protocol: TCP +``` + +## Step 3: Configure Internal Gateway + +The Internal gateway configuration should include: + +- `otk.type: internal` +- Reference to DMZ gateway +- Internal key secret reference +- DMZ auth secret for communication with DMZ gateway +- Database configuration (shared with DMZ) + +### Sample Internal Gateway Configuration + +```yaml +apiVersion: security.brcmlabs.com/v1 +kind: Gateway +metadata: + name: otk-ssg-internal +spec: + version: "11.1.3" + license: + accept: true + secretName: gateway-license + app: + replicas: 1 + image: docker.io/caapim/gateway:11.1.3 + imagePullPolicy: IfNotPresent + resources: + requests: + memory: 8Gi + cpu: 3 + limits: + memory: 8Gi + cpu: 3 + # ExternalKeys with otk flag set to true for OTK-specific key usage + externalKeys: + - name: otk-internal-tls-secret + enabled: true + alias: otk-internal-key + keyUsageType: SSL + otk: true + otk: + enabled: true + initContainerImage: docker.io/caapim/otk-install:4.6.4 + type: internal + # Reference to DMZ gateway (can be Gateway name or external hostname) + dmzGatewayReference: otk-ssg-dmz + # DmzGatewayPort is used when the DMZ gateway is external + # If not specified, defaults to 9443 or the gateway's graphmanDynamicSync port + dmzGatewayPort: 9443 + # SyncIntervalSeconds determines how often certificates are synchronized + # Defaults to RuntimeSyncIntervalSeconds if not specified, or 10 seconds if neither is set + syncIntervalSeconds: 30 + # Reference to the TLS secret for Internal key + internalKeySecret: otk-internal-tls-secret + # Auth secret for DMZ gateway communication + dmzAuthSecret: otk-dmz-auth-secret + database: + type: mysql + create: true + connectionName: OAuth + auth: + gateway: + username: otk_user + password: otkUserPass + readOnly: + username: readonly_user + password: readonly_userPass + admin: + username: admin + password: adminPass + properties: + minimumPoolSize: 3 + maximumPoolSize: 15 + sql: + databaseName: otk_db + jdbcUrl: jdbc:mysql://mysql.brcmlabs.com:3306/otk_db_init + jdbcDriverClass: com.mysql.cj.jdbc.Driver + connectionProperties: + c3p0.maxConnectionAge: "100" + c3p0.maxIdleTime: "1000" + manageSchema: true + databaseWaitTimeout: 60 + management: + secretName: gateway-secret + graphman: + enabled: true + initContainerImage: docker.io/caapim/graphman-static-init:1.0.4 + cluster: + hostname: gateway.brcmlabs.com + service: + type: ClusterIP + ports: + - name: https + port: 8443 + targetPort: 8443 + protocol: TCP + - name: management + port: 9443 + targetPort: 9443 + protocol: TCP +``` + +## Key Configuration Fields + +### OTK-Specific Fields + +| Field | Description | Required | Default | +|-------|-------------|----------|---------| +| `otk.enabled` | Enable OTK installation | Yes | `false` | +| `otk.type` | OTK type: `dmz`, `internal`, or `single` | Yes | - | +| `otk.initContainerImage` | OTK init container image | Yes | - | +| `otk.dmzKeySecret` | Reference to TLS secret containing DMZ key/cert | Yes (DMZ) | - | +| `otk.internalKeySecret` | Reference to TLS secret containing Internal key/cert | Yes (Internal) | - | +| `otk.dmzAuthSecret` | Reference to secret with DMZ gateway credentials | Yes (Internal) | - | +| `otk.internalAuthSecret` | Reference to secret with Internal gateway credentials | Yes (DMZ) | - | +| `otk.dmzGatewayReference` | Reference to DMZ gateway (name or hostname) | Yes (Internal) | - | +| `otk.internalGatewayReference` | Reference to Internal gateway (name or hostname) | Yes (DMZ) | - | +| `otk.dmzGatewayPort` | Port for DMZ gateway (when external) | No | `9443` or `graphmanDynamicSync` port | +| `otk.internalGatewayPort` | Port for Internal gateway (when external) | No | `9443` or `graphmanDynamicSync` port | +| `otk.syncIntervalSeconds` | Certificate sync interval in seconds | No | `RuntimeSyncIntervalSeconds` or `10` | +| `otk.port` | OTK port (defaults to 8443) | No | `8443` | + +### External Keys Configuration + +Both gateways must have `externalKeys` configured with the `otk: true` flag: + +```yaml +externalKeys: +- name: otk-dmz-tls-secret # or otk-internal-tls-secret + enabled: true + alias: otk-dmz-key # or otk-internal-key + keyUsageType: SSL + otk: true # Required for OTK key handling +``` + +## Deployment + +### 1. Create Secrets + +```bash +cd example/gateway/otk/secrets +./create-secrets.sh default +``` + +### 2. Deploy DMZ Gateway + +```bash +kubectl apply -f example/gateway/otk/otk-ssg-dmz.yaml +``` + +### 3. Deploy Internal Gateway + +```bash +kubectl apply -f example/gateway/otk/otk-ssg-internal.yaml +``` + +### 4. Verify Deployment + +```bash +# Check gateway pods +kubectl get pods -l app=gateway + +# Check gateway status +kubectl get gateway otk-ssg-dmz +kubectl get gateway otk-ssg-internal + +# Check logs +kubectl logs -l app=gateway,gateway-name=otk-ssg-dmz +kubectl logs -l app=gateway,gateway-name=otk-ssg-internal +``` + +## Certificate Synchronization + +The operator automatically synchronizes certificates between DMZ and Internal gateways: + +1. **DMZ Certificate → Internal Gateway**: When the DMZ certificate is updated, it's automatically published to the Internal gateway as a trusted certificate and used for FIP (Federated Identity Provider) user creation. + +2. **Internal Certificate → DMZ Gateway**: When the Internal certificate is updated, it's automatically published to the DMZ gateway as a trusted certificate. + +3. **Sync Interval**: Controlled by `syncIntervalSeconds` (default: 10 seconds or `RuntimeSyncIntervalSeconds`). + +4. **Key Updates**: When DMZ or Internal keys are updated: + - The key is synchronized to the respective gateway + - The DMZ private key name is updated in the cluster-wide property `otk.dmz.private_key.name` (DMZ gateway only) + - Old certificates are removed before new ones are published + +## External Gateway Support + +The operator supports scenarios where one or both gateways are external (not managed by the operator): + +### External DMZ Gateway + +If the DMZ gateway is external, configure the Internal gateway with: + +```yaml +otk: + type: internal + dmzGatewayReference: external-dmz-gateway.example.com + dmzGatewayPort: 9443 # Port for Graphman API + dmzAuthSecret: otk-dmz-auth-secret +``` + +### External Internal Gateway + +If the Internal gateway is external, configure the DMZ gateway with: + +```yaml +otk: + type: dmz + internalGatewayReference: external-internal-gateway.example.com + internalGatewayPort: 9443 # Port for Graphman API + internalAuthSecret: otk-internal-auth-secret +``` + +### External Gateway Requirements + +- Graphman API must be enabled and accessible +- Authentication credentials must be provided via auth secrets +- The correct port must be specified if different from default (9443) +- The gateway must be reachable from the operator's network + +## Troubleshooting + +### Common Issues + +1. **Certificates not synchronizing** + - Verify `syncIntervalSeconds` is set appropriately + - Check that Graphman is enabled on both gateways + - Verify auth secrets are correctly configured + - Check operator logs for errors + +2. **Gateway communication failures** + - Verify gateway references are correct (name or hostname) + - Check network connectivity between gateways + - Verify ports are correctly configured + - Ensure auth secrets contain valid credentials + +3. **Key update failures** + - Verify TLS secrets are of type `kubernetes.io/tls` + - Check that secrets contain both `tls.crt` and `tls.key` + - Ensure `externalKeys` have `otk: true` flag + - Verify alias matches the expected value + +4. **Database connection issues** + - Verify database credentials in `otk.database.auth` + - Check JDBC URL is correct and accessible + - Ensure database exists or `create: true` is set + - Verify database wait timeout is sufficient + +### Checking Certificate Sync Status + +```bash +# Check annotations for certificate thumbprints +kubectl get gateway otk-ssg-dmz -o jsonpath='{.metadata.annotations}' +kubectl get gateway otk-ssg-internal -o jsonpath='{.metadata.annotations}' + +# Check cluster-wide properties +kubectl exec -it -- /opt/SecureSpan/Gateway/node/default/bin/ssgconfig \ + get cluster-wide-properties | grep otk.dmz.private_key.name +``` + +### Operator Logs + +```bash +# View operator logs +kubectl logs -n -l control-plane=controller-manager + +# Filter for OTK-related logs +kubectl logs -n -l control-plane=controller-manager | grep -i otk +``` + +## Additional Resources + +- [Layer7 Gateway Operator Documentation](https://github.com/broadcom/layer7-operator) +- [OAuth Toolkit Documentation](https://techdocs.broadcom.com/us/en/ca-enterprise-software/layer7-api-management/api-gateway/11-1.html) +- Example configurations: `example/gateway/otk/` + +--- + +**Note**: This configuration guide assumes you have a working Kubernetes cluster with the Layer7 Gateway Operator installed. Adjust namespaces, hostnames, and other values according to your environment. + diff --git a/example/gateway/otk/otk-ssg-dmz.yaml b/example/gateway/otk/otk-ssg-dmz.yaml new file mode 100644 index 00000000..967b2eed --- /dev/null +++ b/example/gateway/otk/otk-ssg-dmz.yaml @@ -0,0 +1,236 @@ +apiVersion: security.brcmlabs.com/v1 +kind: Gateway +metadata: + name: otk-ssg-dmz +spec: + version: "11.1.3" + license: + accept: true + secretName: gateway-license + app: + replicas: 1 + image: docker.io/caapim/gateway:11.1.3 + imagePullPolicy: IfNotPresent + updateStrategy: + type: rollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 2 + resources: + requests: + memory: 8Gi + cpu: 3 + limits: + memory: 8Gi + cpu: 3 + externalSecrets: + - name: gateway-secret + enabled: true + variableReferencable: true + description: Gateway Secret + # ExternalKeys with otk flag set to true for OTK-specific key usage + externalKeys: + - name: otk-dmz-tls-secret + enabled: true + alias: otk-dmz-key + keyUsageType: SSL + otk: true + otk: + enabled: true + initContainerImage: docker.io/caapim/otk-install:4.6.4 + type: dmz + internalAuthSecret: otk-internal-auth-secret + internalGatewayReference: as673366-gw-upgrade-0.apim.labs.broadcom.net + # InternalGatewayPort is used when the Internal gateway is external (not managed by operator) + # If not specified, defaults to 9443 or the gateway's graphmanDynamicSync port + internalGatewayPort: 8443 + # SyncIntervalSeconds determines how often DMZ and Internal gateways should update certificates + # Defaults to RuntimeSyncIntervalSeconds if not specified, or 10 seconds if neither is set + syncIntervalSeconds: 30 + # Reference to the TLS secret for DMZ key (used by OTK reconciliation) + dmzKeySecret: otk-dmz-tls-secret + database: + type: mysql + create: true + connectionName: OAuth + auth: + # A single secret containing all of the values defined here will be created + # if existingSecret is set the corresponding gateway, readOnly or admin will be omitted from the secret + # if no values are set, a secret will not be created or referenced and the deployment will be invalidated. + # existingSecret: otk-db-secret + gateway: + username: otk_user + password: otkUserPass + readOnly: + # username: readonly_user + username: readonly_user + password: readonly_userPass + admin: + # username: admin + username: admin + password: adminPass + properties: + minimumPoolSize: 3 + maximumPoolSize: 15 + sql: + databaseName: otk_db + #jdbcUrl: jdbc:mysql://:/ + jdbcUrl: jdbc:mysql://mysql.brcmlabs.com:3306/otk_db_init + jdbcDriverClass: com.mysql.cj.jdbc.Driver + connectionProperties: + c3p0.maxConnectionAge: "100" + c3p0.maxIdleTime: "1000" + manageSchema: true + databaseWaitTimeout: 60 + autoscaling: + enabled: false + bundle: + - type: restman + source: secret + name: restman-bootstrap-bundle + - type: graphman + source: secret + name: graphman-bootstrap-bundle + repositoryReferences: + - name: l7-gw-myframework + enabled: true + type: static + encryption: + existingSecret: graphman-encryption-secret + key: FRAMEWORK_ENCRYPTION_PASSPHRASE + - name: l7-gw-myapis + enabled: true + type: dynamic + encryption: + existingSecret: graphman-encryption-secret + key: APIS_ENCRYPTION_PASSPHRASE + - name: l7-gw-mysubscriptions + enabled: true + type: dynamic + encryption: + existingSecret: graphman-encryption-secret + key: SUBSCRIPTIONS_ENCRYPTION_PASSPHRASE + - name: local-reference-repository + enabled: true + type: dynamic + encryption: { } + - name: otk-customizations-dmz + enabled: true + type: dynamic + encryption: { } + bootstrap: + script: + enabled: true + initContainers: [] + hazelcast: + external: false + endpoint: hazelcast.example.com:5701 + management: + secretName: gateway-secret + #username: admin + #password: 7layer + # Management port requires a separate service... + service: + enabled: true + #annotations: + # cloud.google.com/load-balancer-type: "Internal" + type: LoadBalancer + ports: + - name: management + port: 9443 + targetPort: 9443 + protocol: TCP + restman: + enabled: false + graphman: + enabled: true + initContainerImage: docker.io/caapim/graphman-static-init:1.0.4 + dynamicSyncPort: 9443 + cluster: + #password: 7layer + hostname: gateway.brcmlabs.com + database: + enabled: false # this runs the gateway in dbbacked/ephemeral mode + # jdbcUrl: "jdbc:mysql://cluster1-haproxy.pxc.svc.cluster.local:3306/ssg" + # username: "gateway" + # password: "ACm8BDr3Rfk2Flx9V" + java: + jvmHeap: + calculate: true + percentage: 50 + default: 4g + extraArgs: + - -Dcom.l7tech.server.audit.message.saveToInternal=false + - -Dcom.l7tech.server.audit.admin.saveToInternal=false + - -Dcom.l7tech.server.audit.system.saveToInternal=false + - -Dcom.l7tech.server.audit.log.format=json + - -Djava.util.logging.config.file=/opt/SecureSpan/Gateway/node/default/etc/conf/log-override.properties + - -Dcom.l7tech.security.ssl.hostAllowWildcard=true + - -Dcom.l7tech.server.pkix.useDefaultTrustAnchors=true + #- -Dcom.l7tech.bootstrap.autoTrustSslKey=trustAnchor,TrustedFor.SSL,TrustedFor.SAML_ISSUER + listenPorts: + harden: true + custom: + enabled: false + ports: [] + cwp: + enabled: true + properties: + - name: io.httpsHostAllowWildcard + value: "true" + - name: log.levels + value: | + com.l7tech.level = CONFIG + com.l7tech.server.policy.variable.ServerVariables.level = SEVERE + com.l7tech.external.assertions.odata.server.producer.jdbc.GenerateSqlQuery.level = SEVERE + com.l7tech.server.policy.assertion.ServerSetVariableAssertion.level = SEVERE + com.l7tech.external.assertions.comparison.server.ServerComparisonAssertion.level = SEVERE + - name: audit.setDetailLevel.FINE + value: 152 7101 7103 9648 9645 7026 7027 4155 150 4716 4114 6306 4100 9655 150 151 11000 4104 + system: + properties: |- + # Default Gateway system properties + # Configuration properties for shared state extensions. + com.l7tech.server.extension.sharedKeyValueStoreProvider=embeddedhazelcast + com.l7tech.server.extension.sharedCounterProvider=ssgdb + com.l7tech.server.extension.sharedClusterInfoProvider=ssgdb + # By default, FIPS module will block an RSA modulus from being used for encryption if it has been used for + # signing, or visa-versa. Set true to disable this default behaviour and remain backwards compatible. + com.l7tech.org.bouncycastle.rsa.allow_multi_use=true + # Specifies the type of Trust Store (JKS/PKCS12) provided by AdoptOpenJDK that is used by Gateway. + # Must be set correctly when Gateway is running in FIPS mode. If not specified it will default to PKCS12. + javax.net.ssl.trustStoreType=jks + com.l7tech.server.clusterStaleNodeCleanupTimeoutSeconds=86400 + # Additional properties go here + service: + # annotations: + type: ClusterIP + ports: + - name: https + port: 8443 + targetPort: 8443 + protocol: TCP + ingress: + enabled: true + ingressClassName: nginx + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + # nginx.ingress.kubernetes.io/ssl-passthrough: "true" + tls: + - hosts: + - otk-ssg-dmz.brcmlabs.com + secretName: brcmlabs + rules: + - host: otk-ssg-dmz.brcmlabs.com + # containerSecurityContext: + # runAsNonRoot: true + # runAsUser: 1000669998 + # capabilities: + # drop: + # - ALL + # allowPrivilegeEscalation: false + # podSecurityContext: + # runAsUser: 1000669998 + # runAsGroup: 1000669998 + + diff --git a/example/gateway/otk/otk-ssg-internal.yaml b/example/gateway/otk/otk-ssg-internal.yaml new file mode 100644 index 00000000..c0cd32dd --- /dev/null +++ b/example/gateway/otk/otk-ssg-internal.yaml @@ -0,0 +1,233 @@ +apiVersion: security.brcmlabs.com/v1 +kind: Gateway +metadata: + name: otk-ssg-internal +spec: + version: "11.1.3" + license: + accept: true + secretName: gateway-license + app: + replicas: 1 + image: docker.io/caapim/gateway:11.1.3 + imagePullPolicy: IfNotPresent + updateStrategy: + type: rollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 2 + resources: + requests: + memory: 8Gi + cpu: 3 + limits: + memory: 8Gi + cpu: 3 + # ExternalKeys with otk flag set to true for OTK-specific key usage + externalKeys: + - name: otk-internal-tls-secret + enabled: true + alias: otk-internal-key + keyUsageType: SSL + otk: true + otk: + enabled: true + initContainerImage: docker.io/caapim/otk-install:4.6.4 + type: internal + dmzGatewayReference: otk-ssg-dmz + # DmzGatewayPort is used when the DMZ gateway is external (not managed by operator) + # If not specified, defaults to 9443 or the gateway's graphmanDynamicSync port + dmzGatewayPort: 9443 + # SyncIntervalSeconds determines how often DMZ and Internal gateways should update certificates + # Defaults to RuntimeSyncIntervalSeconds if not specified, or 10 seconds if neither is set + syncIntervalSeconds: 30 + # Reference to the TLS secret for Internal key (used by OTK reconciliation) + internalKeySecret: otk-internal-tls-secret + database: + type: mysql + create: true + connectionName: OAuth + auth: + # A single secret containing all of the values defined here will be created + # if existingSecret is set the corresponding gateway, readOnly or admin will be omitted from the secret + # if no values are set, a secret will not be created or referenced and the deployment will be invalidated. + # existingSecret: otk-db-secret + gateway: + username: otk_user + password: otkUserPass + readOnly: + # username: readonly_user + username: readonly_user + password: readonly_userPass + admin: + # username: admin + username: admin + password: adminPass + properties: + minimumPoolSize: 3 + maximumPoolSize: 15 + sql: + databaseName: otk_db + #jdbcUrl: jdbc:mysql://:/ + jdbcUrl: jdbc:mysql://mysql.brcmlabs.com:3306/otk_db_init + jdbcDriverClass: com.mysql.cj.jdbc.Driver + connectionProperties: + c3p0.maxConnectionAge: "100" + c3p0.maxIdleTime: "1000" + manageSchema: true + databaseWaitTimeout: 60 + autoscaling: + enabled: false + bundle: + - type: restman + source: secret + name: restman-bootstrap-bundle + - type: graphman + source: secret + name: graphman-bootstrap-bundle + repositoryReferences: + - name: l7-gw-myframework + enabled: true + type: static + encryption: + existingSecret: graphman-encryption-secret + key: FRAMEWORK_ENCRYPTION_PASSPHRASE + - name: l7-gw-myapis + enabled: true + type: dynamic + encryption: + existingSecret: graphman-encryption-secret + key: APIS_ENCRYPTION_PASSPHRASE + - name: l7-gw-mysubscriptions + enabled: true + type: dynamic + encryption: + existingSecret: graphman-encryption-secret + key: SUBSCRIPTIONS_ENCRYPTION_PASSPHRASE + - name: local-reference-repository + enabled: true + type: dynamic + encryption: { } + - name: otk-customizations-internal + enabled: true + type: dynamic + encryption: { } + bootstrap: + script: + enabled: true + initContainers: [] + hazelcast: + external: false + endpoint: hazelcast.example.com:5701 + management: + secretName: gateway-secret + #username: admin + #password: 7layer + # Management port requires a separate service... + service: + enabled: false + #annotations: + # cloud.google.com/load-balancer-type: "Internal" + type: LoadBalancer + ports: + - name: management + port: 9443 + targetPort: 9443 + protocol: TCP + restman: + enabled: false + graphman: + enabled: true + initContainerImage: docker.io/caapim/graphman-static-init:1.0.4 + cluster: + #password: 7layer + hostname: gateway.brcmlabs.com + database: + enabled: false # this runs the gateway in dbbacked/ephemeral mode + # jdbcUrl: "jdbc:mysql://cluster1-haproxy.pxc.svc.cluster.local:3306/ssg" + # username: "gateway" + # password: "ACm8BDr3Rfk2Flx9V" + java: + jvmHeap: + calculate: true + percentage: 50 + default: 4g + extraArgs: + - -Dcom.l7tech.server.audit.message.saveToInternal=false + - -Dcom.l7tech.server.audit.admin.saveToInternal=false + - -Dcom.l7tech.server.audit.system.saveToInternal=false + - -Dcom.l7tech.server.audit.log.format=json + - -Djava.util.logging.config.file=/opt/SecureSpan/Gateway/node/default/etc/conf/log-override.properties + - -Dcom.l7tech.security.ssl.hostAllowWildcard=true + - -Dcom.l7tech.server.pkix.useDefaultTrustAnchors=true + #- -Dcom.l7tech.bootstrap.autoTrustSslKey=trustAnchor,TrustedFor.SSL,TrustedFor.SAML_ISSUER + listenPorts: + harden: true + custom: + enabled: false + ports: [] + cwp: + enabled: true + properties: + - name: io.httpsHostAllowWildcard + value: "true" + - name: log.levels + value: | + com.l7tech.level = CONFIG + com.l7tech.server.policy.variable.ServerVariables.level = SEVERE + com.l7tech.external.assertions.odata.server.producer.jdbc.GenerateSqlQuery.level = SEVERE + com.l7tech.server.policy.assertion.ServerSetVariableAssertion.level = SEVERE + com.l7tech.external.assertions.comparison.server.ServerComparisonAssertion.level = SEVERE + - name: audit.setDetailLevel.FINE + value: 152 7101 7103 9648 9645 7026 7027 4155 150 4716 4114 6306 4100 9655 150 151 11000 4104 + system: + properties: |- + # Default Gateway system properties + # Configuration properties for shared state extensions. + com.l7tech.server.extension.sharedKeyValueStoreProvider=embeddedhazelcast + com.l7tech.server.extension.sharedCounterProvider=ssgdb + com.l7tech.server.extension.sharedClusterInfoProvider=ssgdb + # By default, FIPS module will block an RSA modulus from being used for encryption if it has been used for + # signing, or visa-versa. Set true to disable this default behaviour and remain backwards compatible. + com.l7tech.org.bouncycastle.rsa.allow_multi_use=true + # Specifies the type of Trust Store (JKS/PKCS12) provided by AdoptOpenJDK that is used by Gateway. + # Must be set correctly when Gateway is running in FIPS mode. If not specified it will default to PKCS12. + javax.net.ssl.trustStoreType=jks + com.l7tech.server.clusterStaleNodeCleanupTimeoutSeconds=86400 + # Additional properties go here + service: + # annotations: + type: ClusterIP + ports: + - name: https + port: 8443 + targetPort: 8443 + protocol: TCP + - name: management + port: 9443 + targetPort: 9443 + protocol: TCP + ingress: + enabled: true + ingressClassName: nginx + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + # nginx.ingress.kubernetes.io/ssl-passthrough: "true" + tls: + - hosts: + - otk-ssg-internal.brcmlabs.com + secretName: brcmlabs + rules: + - host: otk-ssg-internal.brcmlabs.com + # containerSecurityContext: + # runAsNonRoot: true + # runAsUser: 1000669998 + # capabilities: + # drop: + # - ALL + # allowPrivilegeEscalation: false + # podSecurityContext: + # runAsUser: 1000669998 + # runAsGroup: 1000669998 + + diff --git a/example/repositories/kustomization.yaml b/example/repositories/kustomization.yaml index 46194a77..834d6c23 100644 --- a/example/repositories/kustomization.yaml +++ b/example/repositories/kustomization.yaml @@ -6,4 +6,6 @@ resources: - ./apis-repository.yaml - ./local-reference-repository.yaml - ./otk-customizations-single.yaml + - ./otk-customizations-dmz.yaml + - ./otk-customizations-internal.yaml - ../base diff --git a/internal/controller/gateway/controller.go b/internal/controller/gateway/controller.go index 1ee34b0b..af83bb95 100644 --- a/internal/controller/gateway/controller.go +++ b/internal/controller/gateway/controller.go @@ -236,6 +236,11 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { req = append(req, creconcile.Request{NamespacedName: types.NamespacedName{Namespace: gateway.Namespace, Name: gateway.Name}}) } } + if gateway.Spec.App.Otk.Enabled { + if gateway.Spec.App.Otk.DmzKeySecret == a.GetName() || gateway.Spec.App.Otk.InternalKeySecret == a.GetName() { + req = append(req, creconcile.Request{NamespacedName: types.NamespacedName{Namespace: gateway.Namespace, Name: gateway.Name}}) + } + } } return req }), diff --git a/pkg/gateway/reconcile/cron.go b/pkg/gateway/reconcile/cron.go index 979c54f7..2de5da1c 100644 --- a/pkg/gateway/reconcile/cron.go +++ b/pkg/gateway/reconcile/cron.go @@ -72,16 +72,37 @@ func registerJobs(ctx context.Context, params Params) { if err != nil { params.Log.V(2).Info("otk policy sync job already registered", "name", params.Instance.Name, "namespace", params.Instance.Namespace) } - if params.Instance.Spec.App.Otk.Type == securityv1.OtkTypeDMZ || params.Instance.Spec.App.Otk.Type == securityv1.OtkTypeInternal { - _, err = s.Every(otkSyncInterval).Seconds().Tag(params.Instance.Name+"-"+params.Instance.Namespace+"-sync-otk-certificates").Do(syncOtkCertificates, ctx, params) - if err != nil { - params.Log.V(2).Info("otk certificate sync job already registered", "name", params.Instance.Name, "namespace", params.Instance.Namespace) - } - _, err = s.Every(otkSyncInterval).Seconds().Tag(params.Instance.Name+"-"+params.Instance.Namespace+"-sync-otk-certificate-secret").Do(manageCertificateSecrets, ctx, params) - if err != nil { - params.Log.V(2).Info("otk certificate secret sync job already registered", "name", params.Instance.Name, "namespace", params.Instance.Namespace) - } + + // Register certificate sync job for DMZ and Internal gateways + // Use SyncIntervalSeconds if specified, otherwise fall back to RuntimeSyncIntervalSeconds or default + certSyncInterval := otkSyncInterval + if params.Instance.Spec.App.Otk.SyncIntervalSeconds != 0 { + certSyncInterval = params.Instance.Spec.App.Otk.SyncIntervalSeconds + } + + _, err = s.Every(certSyncInterval).Seconds().Tag(params.Instance.Name+"-sync-otk-certificates").Do(syncOtkCertificates, ctx, params) + + if err != nil { + params.Log.V(2).Info("otk certificate sync job already registered", "name", params.Instance.Name, "namespace", params.Instance.Namespace) } + + // Register external keys sync job for certificate publishing between DMZ and Internal + _, err = s.Every(certSyncInterval).Seconds().Tag(params.Instance.Name+"-sync-otk-external-keys").Do(syncOtkExternalKeys, ctx, params) + + if err != nil { + params.Log.V(2).Info("otk external keys sync job already registered", "name", params.Instance.Name, "namespace", params.Instance.Namespace) + } + } +} + +func syncOtkExternalKeys(ctx context.Context, params Params) { + // Sync certificates between DMZ and Internal gateways via ExternalKeys + // This handles OTK certificate publishing (publishDmzCertToInternal, publishInternalCertToDmz) + err := ExternalKeys(ctx, params) + if err != nil { + params.Log.Error(err, "failed to sync OTK external keys certificates", "name", params.Instance.Name, "namespace", params.Instance.Namespace) + } else { + params.Log.V(2).Info("OTK external keys certificates synced", "name", params.Instance.Name, "namespace", params.Instance.Namespace, "interval", params.Instance.Spec.App.Otk.SyncIntervalSeconds) } } diff --git a/pkg/gateway/reconcile/externalkeys.go b/pkg/gateway/reconcile/externalkeys.go index de07af50..a19e3300 100644 --- a/pkg/gateway/reconcile/externalkeys.go +++ b/pkg/gateway/reconcile/externalkeys.go @@ -28,10 +28,35 @@ package reconcile import ( "context" + "crypto/sha1" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "strings" + + securityv1 "github.com/caapim/layer7-operator/api/v1" + "github.com/caapim/layer7-operator/internal/graphman" + "github.com/caapim/layer7-operator/pkg/util" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) func ExternalKeys(ctx context.Context, params Params) error { gateway := params.Instance + + // Handle OTK keys if OTK is enabled + if gateway.Spec.App.Otk.Enabled { + err := handleOtkKeys(ctx, params, gateway) + if err != nil { + params.Log.Error(err, "failed to handle OTK keys", "name", gateway.Name, "namespace", gateway.Namespace) + return err + } + } + + // Handle regular external keys if len(gateway.Spec.App.ExternalKeys) == 0 && len(gateway.Status.LastAppliedExternalKeys) == 0 { return nil } @@ -66,3 +91,512 @@ func ExternalKeys(ctx context.Context, params Params) error { return nil } + +func handleOtkKeys(ctx context.Context, params Params, gateway *securityv1.Gateway) error { + // Handle DMZ key updates only if there's an externalKey with otk: true referencing the same secret + if gateway.Spec.App.Otk.DmzKeySecret != "" && gateway.Spec.App.Otk.Type == securityv1.OtkTypeDMZ { + // Check if there's an externalKey with otk: true that references this secret + hasOtkExternalKey := false + for _, ek := range gateway.Spec.App.ExternalKeys { + if ek.Enabled && ek.Otk && ek.Name == gateway.Spec.App.Otk.DmzKeySecret { + hasOtkExternalKey = true + break + } + } + + // Only process if there's an externalKey with otk: true + if hasOtkExternalKey { + err := handleDmzKeyUpdate(ctx, params, gateway) + if err != nil { + params.Log.Error(err, "failed to handle DMZ key update", "name", gateway.Name, "namespace", gateway.Namespace) + return err + } + } else { + params.Log.V(2).Info("Skipping DMZ key update - no externalKey with otk: true found", "secret", gateway.Spec.App.Otk.DmzKeySecret) + } + } + + // Handle Internal key updates only if there's an externalKey with otk: true referencing the same secret + if gateway.Spec.App.Otk.InternalKeySecret != "" && gateway.Spec.App.Otk.Type == securityv1.OtkTypeInternal { + // Check if there's an externalKey with otk: true that references this secret + hasOtkExternalKey := false + for _, ek := range gateway.Spec.App.ExternalKeys { + if ek.Enabled && ek.Otk && ek.Name == gateway.Spec.App.Otk.InternalKeySecret { + hasOtkExternalKey = true + break + } + } + + // Only process if there's an externalKey with otk: true + if hasOtkExternalKey { + err := handleInternalKeyUpdate(ctx, params, gateway) + if err != nil { + params.Log.Error(err, "failed to handle Internal key update", "name", gateway.Name, "namespace", gateway.Namespace) + return err + } + } else { + params.Log.V(2).Info("Skipping Internal key update - no externalKey with otk: true found", "secret", gateway.Spec.App.Otk.InternalKeySecret) + } + } + + return nil +} + +func handleDmzKeyUpdate(ctx context.Context, params Params, gateway *securityv1.Gateway) error { + // Get DMZ key secret + dmzKeySecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.DmzKeySecret) + if err != nil { + if k8serrors.IsNotFound(err) { + params.Log.V(2).Info("DMZ key secret not found, skipping", "secret", gateway.Spec.App.Otk.DmzKeySecret) + return nil + } + return err + } + + // Check if operator managed (ephemeral mode) + isOperatorManaged := !gateway.Spec.App.Management.Database.Enabled + + if isOperatorManaged { + // Update DMZ with the new key (key sync only, cert publishing handled by syncOtkCertificates) + err = updateDmzWithKey(ctx, params, gateway, dmzKeySecret) + if err != nil { + return fmt.Errorf("failed to update DMZ with key: %w", err) + } + } + + return nil +} + +func handleInternalKeyUpdate(ctx context.Context, params Params, gateway *securityv1.Gateway) error { + // Get Internal key secret + internalKeySecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.InternalKeySecret) + if err != nil { + if k8serrors.IsNotFound(err) { + params.Log.V(2).Info("Internal key secret not found, skipping", "secret", gateway.Spec.App.Otk.InternalKeySecret) + return nil + } + return err + } + + // Check if operator managed (ephemeral mode) + isOperatorManaged := !gateway.Spec.App.Management.Database.Enabled + + if isOperatorManaged { + // Update Internal with the new key (key sync only, cert publishing handled by syncOtkCertificates) + err = updateInternalWithKey(ctx, params, gateway, internalKeySecret) + if err != nil { + return fmt.Errorf("failed to update Internal with key: %w", err) + } + } + + return nil +} + +func updateDmzWithKey(ctx context.Context, params Params, gateway *securityv1.Gateway, keySecret *corev1.Secret) error { + if keySecret.Type != corev1.SecretTypeTLS { + return fmt.Errorf("DMZ key secret must be of type kubernetes.io/tls") + } + + certData := keySecret.Data["tls.crt"] + keyData := keySecret.Data["tls.key"] + + if len(certData) == 0 || len(keyData) == 0 { + return fmt.Errorf("DMZ key secret must contain tls.crt and tls.key") + } + + // Extract certificate from chain + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + if len(crtStrings) == 0 { + return fmt.Errorf("invalid certificate format in DMZ key secret") + } + + // Use first certificate in chain + firstCert := crtStrings[0] + b, _ := pem.Decode([]byte(firstCert)) + if b == nil { + return fmt.Errorf("failed to decode certificate") + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + return fmt.Errorf("failed to parse certificate: %w", err) + } + + // Create Graphman key bundle + keySecretMap := []util.GraphmanKey{ + { + Name: crtX509.Subject.CommonName, + Crt: string(certData), + Key: string(keyData), + Alias: "otk-dmz-key", + UsageType: "", + }, + } + + bundleBytes, err := util.ConvertX509ToGraphmanBundle(keySecretMap, []string{}) + if err != nil { + return fmt.Errorf("failed to convert key to bundle: %w", err) + } + + // Calculate checksum + dataBytes, _ := json.Marshal(&keySecretMap) + h := sha1.New() + h.Write(dataBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Get gateway secret for authentication + name := gateway.Name + if gateway.Spec.App.Management.SecretName != "" { + name = gateway.Spec.App.Management.SecretName + } + gwSecret, err := getGatewaySecret(ctx, params, name) + if err != nil { + return err + } + + annotation := "security.brcmlabs.com/otk-dmz-key" + + // Check if DMZ key was already applied (to determine if update is needed) + keyWasUpdated := false + currentSha1Sum := "" + if !gateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, params) + if err != nil { + return err + } + // Check current annotation value before update + for _, pod := range podList.Items { + if val, ok := pod.ObjectMeta.Annotations[annotation]; ok { + currentSha1Sum = val + break + } + } + err = ReconcileEphemeralGateway(ctx, params, "otk dmz key", *podList, gateway, gwSecret, "", annotation, sha1Sum, false, "otk dmz key", bundleBytes) + if err != nil { + return err + } + // Key was updated if sha1Sum changed + keyWasUpdated = currentSha1Sum != sha1Sum + } else { + gatewayDeployment, err := getGatewayDeployment(ctx, params) + if err != nil { + return err + } + // Check current annotation value before update + currentSha1Sum = gatewayDeployment.ObjectMeta.Annotations[annotation] + err = ReconcileDBGateway(ctx, params, "otk dmz key", *gatewayDeployment, gateway, gwSecret, "", annotation, sha1Sum, false, "otk dmz key", bundleBytes) + if err != nil { + return err + } + // Key was updated if sha1Sum changed (ReconcileDBGateway returns early if already applied) + keyWasUpdated = currentSha1Sum != sha1Sum + } + + // Update cluster property if DMZ key was updated OR if this is the first reconciliation (currentSha1Sum is empty) + // This ensures the CWP is set on first reconciliation, not just on key updates + if keyWasUpdated || currentSha1Sum == "" { + if err := updateDmzPrivateKeyClusterProperty(ctx, params, gateway, "otk-dmz-key"); err != nil { + params.Log.V(2).Info("Failed to update DMZ private key cluster property", "error", err, "gateway", gateway.Name) + // Don't fail the entire operation if cluster property update fails + } + } else { + params.Log.V(2).Info("DMZ key was not updated, skipping cluster property update", "gateway", gateway.Name) + } + + return nil +} + +func updateInternalWithKey(ctx context.Context, params Params, gateway *securityv1.Gateway, keySecret *corev1.Secret) error { + if keySecret.Type != corev1.SecretTypeTLS { + return fmt.Errorf("Internal key secret must be of type kubernetes.io/tls") + } + + certData := keySecret.Data["tls.crt"] + keyData := keySecret.Data["tls.key"] + + if len(certData) == 0 || len(keyData) == 0 { + return fmt.Errorf("Internal key secret must contain tls.crt and tls.key") + } + + // Extract certificate from chain + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + if len(crtStrings) == 0 { + return fmt.Errorf("invalid certificate format in Internal key secret") + } + + // Use first certificate in chain + firstCert := crtStrings[0] + b, _ := pem.Decode([]byte(firstCert)) + if b == nil { + return fmt.Errorf("failed to decode certificate") + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + return fmt.Errorf("failed to parse certificate: %w", err) + } + + // Create Graphman key bundle + keySecretMap := []util.GraphmanKey{ + { + Name: crtX509.Subject.CommonName, + Crt: string(certData), + Key: string(keyData), + Alias: "otk-internal-key", + UsageType: "", + }, + } + + bundleBytes, err := util.ConvertX509ToGraphmanBundle(keySecretMap, []string{}) + if err != nil { + return fmt.Errorf("failed to convert key to bundle: %w", err) + } + + // Calculate checksum + dataBytes, _ := json.Marshal(&keySecretMap) + h := sha1.New() + h.Write(dataBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Get gateway secret for authentication + name := gateway.Name + if gateway.Spec.App.Management.SecretName != "" { + name = gateway.Spec.App.Management.SecretName + } + gwSecret, err := getGatewaySecret(ctx, params, name) + if err != nil { + return err + } + + annotation := "security.brcmlabs.com/otk-internal-key" + + if !gateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, params) + if err != nil { + return err + } + err = ReconcileEphemeralGateway(ctx, params, "otk internal key", *podList, gateway, gwSecret, "", annotation, sha1Sum, false, "otk internal key", bundleBytes) + if err != nil { + return err + } + } else { + gatewayDeployment, err := getGatewayDeployment(ctx, params) + if err != nil { + return err + } + err = ReconcileDBGateway(ctx, params, "otk internal key", *gatewayDeployment, gateway, gwSecret, "", annotation, sha1Sum, false, "otk internal key", bundleBytes) + if err != nil { + return err + } + } + + return nil +} + +// checkClusterPropertyExists checks if the cluster property exists in the ConfigMap +func checkClusterPropertyExists(ctx context.Context, params Params, gateway *securityv1.Gateway, propertyName string) bool { + // Only check for DMZ gateway type + if gateway.Spec.App.Otk.Type != securityv1.OtkTypeDMZ { + return false + } + + // Get the cluster properties ConfigMap + cmName := gateway.Name + "-cwp-bundle" + cm, err := getGatewayConfigMap(ctx, params, cmName) + if err != nil { + return false + } + + // Parse existing bundle + bundleJSON := cm.Data["cwp.json"] + if bundleJSON == "" { + return false + } + + bundle := graphman.Bundle{} + err = json.Unmarshal([]byte(bundleJSON), &bundle) + if err != nil { + return false + } + + // Check if property exists + for _, cwp := range bundle.ClusterProperties { + if cwp.Name == propertyName { + return true + } + } + + return false +} + +// updateDmzPrivateKeyClusterProperty updates the cluster property otk.dmz.private_key.name +// with the DMZ private key name. This function only updates the property if it exists. +// It does not create the property if it doesn't exist. +func updateDmzPrivateKeyClusterProperty(ctx context.Context, params Params, gateway *securityv1.Gateway, keyName string) error { + // Only update cluster property for DMZ gateway type + if gateway.Spec.App.Otk.Type != securityv1.OtkTypeDMZ { + return nil + } + + // Get the cluster properties ConfigMap + cmName := gateway.Name + "-cwp-bundle" + cm, err := getGatewayConfigMap(ctx, params, cmName) + if err != nil { + if !k8serrors.IsNotFound(err) { + return fmt.Errorf("failed to get cluster properties ConfigMap: %w", err) + } + // ConfigMap doesn't exist, property doesn't exist - skip update + return fmt.Errorf("cluster property ConfigMap does not exist") + } + + // Parse existing bundle + bundle := graphman.Bundle{} + bundleJSON := cm.Data["cwp.json"] + if bundleJSON == "" { + // Empty bundle, property doesn't exist - skip update + return fmt.Errorf("cluster property bundle is empty") + } + + err = json.Unmarshal([]byte(bundleJSON), &bundle) + if err != nil { + return fmt.Errorf("failed to parse cluster properties bundle: %w", err) + } + + // Initialize bundle properties if nil + if bundle.Properties == nil { + bundle.Properties = &graphman.BundleProperties{ + Mappings: graphman.BundleMappings{}, + } + } + + // Check if property exists and update it + propertyName := "otk.dmz.private_key.name" + found := false + for _, cwp := range bundle.ClusterProperties { + if cwp.Name == propertyName { + cwp.Value = keyName + found = true + break + } + } + + if !found { + // Property doesn't exist - skip update (only update, don't create) + return fmt.Errorf("cluster property %s does not exist", propertyName) + } + + // Marshal bundle back to JSON + bundleBytes, err := json.Marshal(bundle) + if err != nil { + return fmt.Errorf("failed to marshal cluster properties bundle: %w", err) + } + + // Calculate checksum + h := sha1.New() + h.Write(bundleBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Update ConfigMap + cm.Data["cwp.json"] = string(bundleBytes) + if cm.ObjectMeta.Annotations == nil { + cm.ObjectMeta.Annotations = make(map[string]string) + } + cm.ObjectMeta.Annotations["checksum/data"] = sha1Sum + + err = params.Client.Update(ctx, cm) + if err != nil { + return fmt.Errorf("failed to update cluster properties ConfigMap: %w", err) + } + + params.Log.V(2).Info("Updated cluster property ConfigMap", "property", propertyName, "value", keyName, "gateway", gateway.Name) + + // Apply the cluster property using the existing mechanism + gwUpdReq, err := NewGwUpdateRequest( + ctx, + gateway, + params, + WithBundleType(BundleTypeClusterProp), + ) + if err != nil { + return fmt.Errorf("failed to create gateway update request: %w", err) + } + + err = SyncGateway(ctx, params, *gwUpdReq) + if err != nil { + return fmt.Errorf("failed to sync cluster property: %w", err) + } + + params.Log.V(2).Info("Applied cluster property", "property", propertyName, "value", keyName, "gateway", gateway.Name) + + return nil +} + +// createDmzPrivateKeyClusterProperty creates a new cluster properties ConfigMap with the DMZ private key property +func createDmzPrivateKeyClusterProperty(ctx context.Context, params Params, gateway *securityv1.Gateway, keyName string, cmName string) error { + // Create new bundle with the property + bundle := graphman.Bundle{ + ClusterProperties: []*graphman.ClusterPropertyInput{ + { + Name: "otk.dmz.private_key.name", + Value: keyName, + }, + }, + Properties: &graphman.BundleProperties{ + Mappings: graphman.BundleMappings{}, + }, + } + + bundleBytes, err := json.Marshal(bundle) + if err != nil { + return fmt.Errorf("failed to marshal cluster properties bundle: %w", err) + } + + // Calculate checksum + h := sha1.New() + h.Write(bundleBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Create ConfigMap + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: gateway.Namespace, + Annotations: map[string]string{ + "checksum/data": sha1Sum, + }, + }, + Data: map[string]string{ + "cwp.json": string(bundleBytes), + }, + } + + // Set controller reference + if err := controllerutil.SetControllerReference(gateway, cm, params.Scheme); err != nil { + return fmt.Errorf("failed to set controller reference: %w", err) + } + + err = params.Client.Create(ctx, cm) + if err != nil { + return fmt.Errorf("failed to create cluster properties ConfigMap: %w", err) + } + + params.Log.V(2).Info("Created cluster property ConfigMap", "property", "otk.dmz.private_key.name", "value", keyName, "gateway", gateway.Name) + + // Apply the cluster property using the existing mechanism + gwUpdReq, err := NewGwUpdateRequest( + ctx, + gateway, + params, + WithBundleType(BundleTypeClusterProp), + ) + if err != nil { + return fmt.Errorf("failed to create gateway update request: %w", err) + } + + err = SyncGateway(ctx, params, *gwUpdReq) + if err != nil { + return fmt.Errorf("failed to sync cluster property: %w", err) + } + + params.Log.V(2).Info("Applied cluster property", "property", "otk.dmz.private_key.name", "value", keyName, "gateway", gateway.Name) + + return nil +} diff --git a/pkg/gateway/reconcile/gateway.go b/pkg/gateway/reconcile/gateway.go index f01acbc3..f2ca4434 100644 --- a/pkg/gateway/reconcile/gateway.go +++ b/pkg/gateway/reconcile/gateway.go @@ -499,7 +499,8 @@ func NewGwUpdateRequest(ctx context.Context, gateway *securityv1.Gateway, params for _, k := range gateway.Status.LastAppliedExternalKeys { found := false for _, ek := range gateway.Spec.App.ExternalKeys { - if k == ek.Alias && ek.Enabled { + if k == ek.Alias && ek.Enabled && !ek.Otk { + // Only process non-OTK keys in regular external keys flow found = true } } @@ -511,7 +512,8 @@ func NewGwUpdateRequest(ctx context.Context, gateway *securityv1.Gateway, params var sha1Sum string for _, externalKey := range gateway.Spec.App.ExternalKeys { - if externalKey.Enabled { + if externalKey.Enabled && !externalKey.Otk { + // Skip keys with otk: true - they are handled separately by OTK reconciliation secret, err := getGatewaySecret(ctx, params, externalKey.Name) if err != nil { return nil, err diff --git a/pkg/gateway/reconcile/l7otkcertificates.go b/pkg/gateway/reconcile/l7otkcertificates.go index c290f3f5..04de4470 100644 --- a/pkg/gateway/reconcile/l7otkcertificates.go +++ b/pkg/gateway/reconcile/l7otkcertificates.go @@ -26,15 +26,20 @@ package reconcile import ( + "bytes" "context" - "crypto/tls" + "crypto/sha1" + "crypto/x509" "encoding/base64" + "encoding/hex" "encoding/json" - "strconv" + "encoding/pem" + "fmt" + "strings" securityv1 "github.com/caapim/layer7-operator/api/v1" "github.com/caapim/layer7-operator/internal/graphman" - "github.com/caapim/layer7-operator/pkg/gateway" + "github.com/caapim/layer7-operator/pkg/util" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -54,88 +59,423 @@ func syncOtkCertificates(ctx context.Context, params Params) { return } - err = applyOtkCertificates(ctx, params, gateway) - if err != nil { - params.Log.Info("failed to reconcile otk certificates", "name", gateway.Name, "namespace", gateway.Namespace, "error", err.Error()) + // Publish DMZ certs to Internal Gateway when DMZ key is updated + if gateway.Spec.App.Otk.Type == securityv1.OtkTypeDMZ && gateway.Spec.App.Otk.DmzKeySecret != "" { + err = publishDmzCertificatesToInternal(ctx, params, gateway) + if err != nil { + params.Log.V(2).Info("failed to publish DMZ certificates to Internal", "name", gateway.Name, "namespace", gateway.Namespace, "error", err.Error()) + } + } + + // Publish Internal certs to DMZ Gateway when Internal key is updated + if gateway.Spec.App.Otk.Type == securityv1.OtkTypeInternal && gateway.Spec.App.Otk.InternalKeySecret != "" { + err = publishInternalCertificatesToDmz(ctx, params, gateway) + if err != nil { + params.Log.V(2).Info("failed to publish Internal certificates to DMZ", "name", gateway.Name, "namespace", gateway.Namespace, "error", err.Error()) + } } } -func applyOtkCertificates(ctx context.Context, params Params, gateway *securityv1.Gateway) error { +// publishDmzCertificatesToInternal publishes DMZ certificates to Internal gateway when DMZ key is updated +// Handles ephemeral, DB-backed, and external gateways +func publishDmzCertificatesToInternal(ctx context.Context, params Params, gateway *securityv1.Gateway) error { + // Check if Internal gateway reference is specified + if gateway.Spec.App.Otk.InternalOtkGatewayReference == "" { + return nil + } - bundle := graphman.Bundle{} - annotation := "" - sha1Sum := "" + // Get DMZ key secret + dmzKeySecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.DmzKeySecret) + if err != nil { + if k8serrors.IsNotFound(err) { + params.Log.V(2).Info("DMZ key secret not found, skipping cert publish", "secret", gateway.Spec.App.Otk.DmzKeySecret) + return nil + } + return err + } - switch gateway.Spec.App.Otk.Type { - case securityv1.OtkTypeDMZ: - internalSecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.InternalOtkGatewayReference+"-otk-internal-certificates") - sha1Sum = internalSecret.ObjectMeta.Annotations["checksum/data"] + // Check if key was updated by comparing annotation + annotation := "security.brcmlabs.com/otk-dmz-key" + currentSha1Sum := "" + if !gateway.Spec.App.Management.Database.Enabled { + // Ephemeral gateway - check pod annotations + podList, err := getGatewayPods(ctx, params) if err != nil { return err } - annotation = "security.brcmlabs.com/" + gateway.Name + "-" + string(gateway.Spec.App.Otk.Type) + "-certificates" - for k, v := range internalSecret.Data { - bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ - Name: k, - CertBase64: base64.StdEncoding.EncodeToString(v), - TrustAnchor: true, - VerifyHostname: false, - RevocationCheckPolicyType: "USE_DEFAULT", - TrustedFor: []graphman.TrustedForType{ - "SSL", - "SIGNING_SERVER_CERTS", - }, - }) + for _, pod := range podList.Items { + if val, ok := pod.ObjectMeta.Annotations[annotation]; ok { + currentSha1Sum = val + break + } + } + } else { + // DB-backed gateway - check deployment annotations + gatewayDeployment, err := getGatewayDeployment(ctx, params) + if err != nil { + return err } + currentSha1Sum = gatewayDeployment.ObjectMeta.Annotations[annotation] + } + + // Calculate current key checksum + certData := dmzKeySecret.Data["tls.crt"] + keyData := dmzKeySecret.Data["tls.key"] + if len(certData) == 0 || len(keyData) == 0 { + return fmt.Errorf("DMZ key secret must contain tls.crt and tls.key") + } + + keySecretMap := []struct { + Name string + Crt string + Key string + Alias string + UsageType string + }{ + { + Name: "dmz-key", + Crt: string(certData), + Key: string(keyData), + Alias: "otk-dmz-key", + UsageType: "", + }, + } - case securityv1.OtkTypeInternal: - dmzSecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.DmzOtkGatewayReference+"-otk-dmz-certificates") - sha1Sum = dmzSecret.ObjectMeta.Annotations["checksum/data"] + dataBytes, _ := json.Marshal(&keySecretMap) + h := sha1.New() + h.Write(dataBytes) + newSha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Only publish if key was updated + if currentSha1Sum == newSha1Sum { + params.Log.V(2).Info("DMZ key not updated, skipping cert publish", "gateway", gateway.Name) + return nil + } + + // Publish DMZ cert to Internal (handles ephemeral, DB-backed, and external gateways) + return publishDmzCertToInternal(ctx, params, gateway, dmzKeySecret) +} + +// publishInternalCertificatesToDmz publishes Internal certificates to DMZ gateway when Internal key is updated +// Handles ephemeral, DB-backed, and external gateways +func publishInternalCertificatesToDmz(ctx context.Context, params Params, gateway *securityv1.Gateway) error { + // Check if DMZ gateway reference is specified + if gateway.Spec.App.Otk.DmzOtkGatewayReference == "" { + return nil + } + + // Get Internal key secret + internalKeySecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.InternalKeySecret) + if err != nil { + if k8serrors.IsNotFound(err) { + params.Log.V(2).Info("Internal key secret not found, skipping cert publish", "secret", gateway.Spec.App.Otk.InternalKeySecret) + return nil + } + return err + } + + // Check if key was updated by comparing annotation + annotation := "security.brcmlabs.com/otk-internal-key" + currentSha1Sum := "" + + if !gateway.Spec.App.Management.Database.Enabled { + // Ephemeral gateway - check pod annotations + podList, err := getGatewayPods(ctx, params) + if err != nil { + return err + } + for _, pod := range podList.Items { + if val, ok := pod.ObjectMeta.Annotations[annotation]; ok { + currentSha1Sum = val + break + } + } + } else { + // DB-backed gateway - check deployment annotations + gatewayDeployment, err := getGatewayDeployment(ctx, params) if err != nil { return err } - annotation = "security.brcmlabs.com/" + gateway.Name + "-" + string(gateway.Spec.App.Otk.Type) + "-fips-users" - for k, v := range dmzSecret.Data { - bundle.FipUsers = append(bundle.FipUsers, &graphman.FipUserInput{ - Name: k, - ProviderName: "otk-fips-provider", - SubjectDn: "cn=" + k, - CertBase64: base64.RawStdEncoding.EncodeToString(v), + currentSha1Sum = gatewayDeployment.ObjectMeta.Annotations[annotation] + } + + // Calculate current key checksum + certData := internalKeySecret.Data["tls.crt"] + keyData := internalKeySecret.Data["tls.key"] + if len(certData) == 0 || len(keyData) == 0 { + return fmt.Errorf("Internal key secret must contain tls.crt and tls.key") + } + + keySecretMap := []struct { + Name string + Crt string + Key string + Alias string + UsageType string + }{ + { + Name: "internal-key", + Crt: string(certData), + Key: string(keyData), + Alias: "otk-internal-key", + UsageType: "", + }, + } + + dataBytes, _ := json.Marshal(&keySecretMap) + h := sha1.New() + h.Write(dataBytes) + newSha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // Only publish if key was updated + if currentSha1Sum == newSha1Sum { + params.Log.V(2).Info("Internal key not updated, skipping cert publish", "gateway", gateway.Name) + return nil + } + + // Publish Internal cert to DMZ (handles ephemeral, DB-backed, and external gateways) + return publishInternalCertToDmz(ctx, params, gateway, internalKeySecret) +} + +func publishDmzCertToInternal(ctx context.Context, params Params, gateway *securityv1.Gateway, dmzKeySecret *corev1.Secret) error { + // Get Internal gateway + internalGateway := &securityv1.Gateway{} + err := params.Client.Get(ctx, types.NamespacedName{ + Name: gateway.Spec.App.Otk.InternalOtkGatewayReference, + Namespace: gateway.Namespace, + }, internalGateway) + + isExternalGateway := false + if err != nil { + if k8serrors.IsNotFound(err) { + // Gateway not found - check if it's external (port specified) + if gateway.Spec.App.Otk.InternalGatewayPort != 0 { + params.Log.V(2).Info("Internal gateway not found but port specified, treating as external", + "gateway", gateway.Spec.App.Otk.InternalOtkGatewayReference, + "port", gateway.Spec.App.Otk.InternalGatewayPort) + isExternalGateway = true + } else { + params.Log.V(2).Info("Internal gateway not found and no port specified, skipping cert publish", + "gateway", gateway.Spec.App.Otk.InternalOtkGatewayReference) + return nil + } + } else { + return err + } + } + + certData := dmzKeySecret.Data["tls.crt"] + if len(certData) == 0 { + return fmt.Errorf("DMZ key secret must contain tls.crt") + } + + // Parse certificate + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + if len(crtStrings) == 0 { + return fmt.Errorf("invalid certificate format") + } + + // Before adding new certs, remove existing ones if they were previously applied + // Check if certs were previously applied by checking the annotation + annotation := "security.brcmlabs.com/" + gateway.Name + "-dmz-certificates" + thumbprintAnnotation := "security.brcmlabs.com/" + gateway.Name + "-dmz-certificates-thumbprints" + previousCertChecksum := "" + var oldThumbprints []string + if !isExternalGateway { + if !internalGateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, params) + if err == nil { + for _, pod := range podList.Items { + if val, ok := pod.ObjectMeta.Annotations[annotation]; ok { + previousCertChecksum = val + } + if val, ok := pod.ObjectMeta.Annotations[thumbprintAnnotation]; ok && val != "" { + // Parse comma-separated thumbprints + oldThumbprints = strings.Split(val, ",") + } + if previousCertChecksum != "" { + break + } + } + } + } else { + gatewayDeployment, err := getGatewayDeployment(ctx, params) + if err == nil { + previousCertChecksum = gatewayDeployment.ObjectMeta.Annotations[annotation] + if val, ok := gatewayDeployment.ObjectMeta.Annotations[thumbprintAnnotation]; ok && val != "" { + oldThumbprints = strings.Split(val, ",") + } + } + } + } + + bundle := graphman.Bundle{} + + // If we have old thumbprints, add deletion mappings before adding new certs + if len(oldThumbprints) > 0 && previousCertChecksum != "" { + if bundle.Properties == nil { + bundle.Properties = &graphman.BundleProperties{ + Mappings: graphman.BundleMappings{}, + } + } + for _, thumbprint := range oldThumbprints { + thumbprint = strings.TrimSpace(thumbprint) + if thumbprint != "" { + bundle.Properties.Mappings.TrustedCerts = append(bundle.Properties.Mappings.TrustedCerts, &graphman.MappingInstructionInput{ + Action: graphman.MappingActionDelete, + Source: graphman.MappingSource{ThumbprintSha1: thumbprint}, + }) + } + } + // Also remove old FIP users with the same names + // We'll identify them by the cert CommonName pattern + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue + } + // Remove FIP user by name (CommonName) + bundle.Properties.Mappings.FipUsers = append(bundle.Properties.Mappings.FipUsers, &graphman.MappingInstructionInput{ + Action: graphman.MappingActionDelete, + Source: graphman.MappingSource{Name: crtX509.Subject.CommonName}, }) } } + // Calculate thumbprints for new certs and add to TrustedCerts + var newThumbprints []string + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue + } + + // Calculate thumbprint for this cert + thumbprint, err := calculateCertThumbprint(crtX509.Raw) + if err != nil { + params.Log.V(2).Info("Failed to calculate cert thumbprint", "error", err, "cert", crtX509.Subject.CommonName) + thumbprint = "" // Continue without thumbprint + } else { + newThumbprints = append(newThumbprints, thumbprint) + } + + bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ + Name: crtX509.Subject.CommonName, + CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), + ThumbprintSha1: thumbprint, + TrustAnchor: true, + VerifyHostname: false, + RevocationCheckPolicyType: "USE_DEFAULT", + TrustedFor: []graphman.TrustedForType{ + "SSL", + "SIGNING_SERVER_CERTS", + }, + }) + + // Add to FIP Users + bundle.FipUsers = append(bundle.FipUsers, &graphman.FipUserInput{ + Name: crtX509.Subject.CommonName, + ProviderName: "otk-fips-provider", + SubjectDn: "cn=" + crtX509.Subject.CommonName, + CertBase64: base64.RawStdEncoding.EncodeToString(crtX509.Raw), + }) + } + bundleBytes, err := json.Marshal(bundle) if err != nil { return err } - name := gateway.Name - if gateway.Spec.App.Management.SecretName != "" { - name = gateway.Spec.App.Management.SecretName + // Calculate checksum + h := sha1.New() + h.Write(bundleBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // If gateway is external (not managed by operator), use specified port and auth secret + if isExternalGateway { + // Parse certificates to extract information for FIP user creation + var certInfo []struct { + commonName string + subjectDn string + certRaw []byte + } + + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue + } + + // Extract full Subject DN from certificate + subjectDn := extractSubjectDN(crtX509) + + certInfo = append(certInfo, struct { + commonName string + subjectDn string + certRaw []byte + }{ + commonName: crtX509.Subject.CommonName, + subjectDn: subjectDn, + certRaw: crtX509.Raw, + }) + } + + return syncDmzCertToExternalInternalGateway(ctx, params, gateway, dmzKeySecret, certInfo) } - gwSecret, err := getGatewaySecret(ctx, params, name) + // Get Internal gateway secret + name := internalGateway.Name + if internalGateway.Spec.App.Management.SecretName != "" { + name = internalGateway.Spec.App.Management.SecretName + } + gwSecret, err := getGatewaySecret(ctx, params, name) if err != nil { return err } - if !gateway.Spec.App.Management.Database.Enabled { - podList, err := getGatewayPods(ctx, params) + internalParams := params + internalParams.Instance = internalGateway + + // Note: InternalGatewayPort is used when the gateway is external (not found in cluster) + // For operator-managed gateways, the gateway's own graphman port configuration is used + + if !internalGateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, internalParams) if err != nil { return err } - err = ReconcileEphemeralGateway(ctx, params, "otk certificates", *podList, gateway, gwSecret, "", annotation, sha1Sum, true, "otk certificates", bundleBytes) + err = ReconcileEphemeralGateway(ctx, internalParams, "otk certificates", *podList, internalGateway, gwSecret, "", annotation, sha1Sum, true, "otk certificates", bundleBytes) if err != nil { return err } } else { - gatewayDeployment, err := getGatewayDeployment(ctx, params) + gatewayDeployment, err := getGatewayDeployment(ctx, internalParams) if err != nil { return err } - err = ReconcileDBGateway(ctx, params, "otk certificates", *gatewayDeployment, gateway, gwSecret, "", annotation, sha1Sum, false, "otk certificates", bundleBytes) + err = ReconcileDBGateway(ctx, internalParams, "otk certificates", *gatewayDeployment, internalGateway, gwSecret, "", annotation, sha1Sum, false, "otk certificates", bundleBytes) if err != nil { return err } @@ -144,114 +484,467 @@ func applyOtkCertificates(ctx context.Context, params Params, gateway *securityv return nil } -func manageCertificateSecrets(ctx context.Context, params Params) { - gw := &securityv1.Gateway{} - err := params.Client.Get(ctx, types.NamespacedName{Name: params.Instance.Name, Namespace: params.Instance.Namespace}, gw) - if err != nil && k8serrors.IsNotFound(err) { - params.Log.Error(err, "gateway not found", "name", params.Instance.Name, "namespace", params.Instance.Namespace) - _ = removeJob(params.Instance.Name + "-sync-otk-certificate-secret") - return +func publishInternalCertToDmz(ctx context.Context, params Params, gateway *securityv1.Gateway, internalKeySecret *corev1.Secret) error { + // Get DMZ gateway + dmzGateway := &securityv1.Gateway{} + err := params.Client.Get(ctx, types.NamespacedName{ + Name: gateway.Spec.App.Otk.DmzOtkGatewayReference, + Namespace: gateway.Namespace, + }, dmzGateway) + + isExternalGateway := false + if err != nil { + if k8serrors.IsNotFound(err) { + // Gateway not found - check if it's external (port specified) + if gateway.Spec.App.Otk.DmzGatewayPort != 0 { + params.Log.V(2).Info("DMZ gateway not found but port specified, treating as external", + "gateway", gateway.Spec.App.Otk.DmzOtkGatewayReference, + "port", gateway.Spec.App.Otk.DmzGatewayPort) + isExternalGateway = true + } else { + params.Log.V(2).Info("DMZ gateway not found and no port specified, skipping cert publish", + "gateway", gateway.Spec.App.Otk.DmzOtkGatewayReference) + return nil + } + } else { + return err + } } - if !gw.Spec.App.Otk.Enabled { - _ = removeJob(params.Instance.Name + "-sync-otk-certificate-secret") - return + certData := internalKeySecret.Data["tls.crt"] + if len(certData) == 0 { + return fmt.Errorf("Internal key secret must contain tls.crt") } - params.Instance = gw - internalGatewayPort := 9443 - defaultOtkPort := 8443 - rawInternalCertList := map[string][]byte{} - rawDMZCertList := map[string][]byte{} - desiredSecrets := []*corev1.Secret{} - if gw.Spec.App.Management.Graphman.DynamicSyncPort != 0 { - internalGatewayPort = gw.Spec.App.Management.Graphman.DynamicSyncPort + // Parse certificate + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + if len(crtStrings) == 0 { + return fmt.Errorf("invalid certificate format") } - if gw.Spec.App.Otk.InternalGatewayPort != 0 { - internalGatewayPort = gw.Spec.App.Otk.InternalGatewayPort + // Before adding new certs, remove existing ones if they were previously applied + // Check if certs were previously applied by checking the annotation + annotation := "security.brcmlabs.com/" + gateway.Name + "-internal-certificates" + thumbprintAnnotation := "security.brcmlabs.com/" + gateway.Name + "-internal-certificates-thumbprints" + previousCertChecksum := "" + var oldThumbprints []string + if !isExternalGateway { + if !dmzGateway.Spec.App.Management.Database.Enabled { + dmzParams := params + dmzParams.Instance = dmzGateway + podList, err := getGatewayPods(ctx, dmzParams) + if err == nil { + for _, pod := range podList.Items { + if val, ok := pod.ObjectMeta.Annotations[annotation]; ok { + previousCertChecksum = val + } + if val, ok := pod.ObjectMeta.Annotations[thumbprintAnnotation]; ok && val != "" { + // Parse comma-separated thumbprints + oldThumbprints = strings.Split(val, ",") + } + if previousCertChecksum != "" { + break + } + } + } + } else { + dmzParams := params + dmzParams.Instance = dmzGateway + gatewayDeployment, err := getGatewayDeployment(ctx, dmzParams) + if err == nil { + previousCertChecksum = gatewayDeployment.ObjectMeta.Annotations[annotation] + if val, ok := gatewayDeployment.ObjectMeta.Annotations[thumbprintAnnotation]; ok && val != "" { + oldThumbprints = strings.Split(val, ",") + } + } + } } - if gw.Spec.App.Otk.OTKPort != 0 { - defaultOtkPort = gw.Spec.App.Otk.OTKPort + bundle := graphman.Bundle{} + + // If we have old thumbprints, add deletion mappings before adding new certs + if len(oldThumbprints) > 0 && previousCertChecksum != "" { + if bundle.Properties == nil { + bundle.Properties = &graphman.BundleProperties{ + Mappings: graphman.BundleMappings{}, + } + } + for _, thumbprint := range oldThumbprints { + thumbprint = strings.TrimSpace(thumbprint) + if thumbprint != "" { + bundle.Properties.Mappings.TrustedCerts = append(bundle.Properties.Mappings.TrustedCerts, &graphman.MappingInstructionInput{ + Action: graphman.MappingActionDelete, + Source: graphman.MappingSource{ThumbprintSha1: thumbprint}, + }) + } + } + } + + // Calculate thumbprints for new certs and add to TrustedCerts + var newThumbprints []string + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue + } + + // Calculate thumbprint for this cert + thumbprint, err := calculateCertThumbprint(crtX509.Raw) + if err != nil { + params.Log.V(2).Info("Failed to calculate cert thumbprint", "error", err, "cert", crtX509.Subject.CommonName) + thumbprint = "" // Continue without thumbprint + } else { + newThumbprints = append(newThumbprints, thumbprint) + } + + bundle.TrustedCerts = append(bundle.TrustedCerts, &graphman.TrustedCertInput{ + Name: crtX509.Subject.CommonName, + CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), + ThumbprintSha1: thumbprint, + TrustAnchor: true, + VerifyHostname: false, + RevocationCheckPolicyType: "USE_DEFAULT", + TrustedFor: []graphman.TrustedForType{ + "SSL", + "SIGNING_SERVER_CERTS", + }, + }) } - podList, err := getGatewayPods(ctx, params) + + bundleBytes, err := json.Marshal(bundle) if err != nil { - params.Log.Error(err, "failed to retrieve gateway pods", "name", params.Instance.Name, "namespace", params.Instance.Namespace) - return + return err } - for _, pod := range podList.Items { - for _, containerStatus := range pod.Status.ContainerStatuses { - if containerStatus.Name == "gateway" { - if !containerStatus.Ready { - params.Log.V(2).Info("pod not ready", "pod", pod.Name, "name", params.Instance.Name, "namespace", params.Instance.Namespace) - return - } - } + // Calculate checksum + h := sha1.New() + h.Write(bundleBytes) + sha1Sum := fmt.Sprintf("%x", h.Sum(nil)) + + // If gateway is external (not managed by operator), use specified port and auth secret + if isExternalGateway { + return syncInternalCertToExternalDmzGateway(ctx, params, gateway, bundleBytes, sha1Sum) + } + + // Get DMZ gateway secret + name := dmzGateway.Name + if dmzGateway.Spec.App.Management.SecretName != "" { + name = dmzGateway.Spec.App.Management.SecretName + } + gwSecret, err := getGatewaySecret(ctx, params, name) + if err != nil { + return err + } + + // annotation is already declared above, reuse it + // annotation := "security.brcmlabs.com/" + gateway.Name + "-internal-certificates" + + dmzParams := params + dmzParams.Instance = dmzGateway + + // Note: DmzGatewayPort is used when the gateway is external (not found in cluster) + // For operator-managed gateways, the gateway's own graphman port configuration is used + + if !dmzGateway.Spec.App.Management.Database.Enabled { + podList, err := getGatewayPods(ctx, dmzParams) + if err != nil { + return err } + err = ReconcileEphemeralGateway(ctx, dmzParams, "otk certificates", *podList, dmzGateway, gwSecret, "", annotation, sha1Sum, true, "otk certificates", bundleBytes) + if err != nil { + return err + } + } else { + gatewayDeployment, err := getGatewayDeployment(ctx, dmzParams) + if err != nil { + return err + } + err = ReconcileDBGateway(ctx, dmzParams, "otk certificates", *gatewayDeployment, dmzGateway, gwSecret, "", annotation, sha1Sum, false, "otk certificates", bundleBytes) + if err != nil { + return err + } + } - switch gw.Spec.App.Otk.Type { - case securityv1.OtkTypeDMZ: - rawCert, err := retrieveCertificate(pod.Status.PodIP, strconv.Itoa(defaultOtkPort)) - if err != nil { - params.Log.Error(err, "failed to retrieve certificate", "pod", pod.Name, "name", params.Instance.Name, "namespace", params.Instance.Namespace) - return - } - if len(rawDMZCertList) > 0 { - for _, cert := range rawDMZCertList { - if string(rawCert) != string(cert) { - rawDMZCertList[pod.Name] = rawCert - } - } - } else { - rawDMZCertList[pod.Name] = rawCert - } - case securityv1.OtkTypeInternal: - rawCert, err := retrieveCertificate(pod.Status.PodIP, strconv.Itoa(internalGatewayPort)) - if err != nil { - params.Log.Error(err, "failed to retrieve certificate", "pod", pod.Name, "name", params.Instance.Name, "namespace", params.Instance.Namespace) - return - } - if len(rawInternalCertList) > 0 { - for _, cert := range rawInternalCertList { - if string(rawCert) != string(cert) { - rawInternalCertList[pod.Name] = rawCert - } - } - } else { - rawInternalCertList[pod.Name] = rawCert - } + return nil +} + +// syncDmzCertToExternalInternalGateway syncs DMZ certificate to an external Internal gateway +// using graphman. First it adds the certificate as a trusted cert, then creates a FIP user. +func syncDmzCertToExternalInternalGateway(ctx context.Context, params Params, gateway *securityv1.Gateway, dmzKeySecret *corev1.Secret, certInfo []struct { + commonName string + subjectDn string + certRaw []byte +}) error { + // Get auth secret for external Internal gateway + if gateway.Spec.App.Otk.InternalAuthSecret == "" { + return fmt.Errorf("internalAuthSecret is required for external Internal gateway") + } + authSecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.InternalAuthSecret) + if err != nil { + return fmt.Errorf("failed to get auth secret for external Internal gateway: %w", err) + } + + // Parse username and password from auth secret + username, password := parseGatewaySecret(authSecret) + if username == "" || password == "" { + return fmt.Errorf("could not retrieve gateway credentials from auth secret: %s", gateway.Spec.App.Otk.InternalAuthSecret) + } + + // Build endpoint URL for external gateway + // Format: :/graphman + // ApplyGraphmanBundle expects format: host:port/path (without https://) + gatewayReference := gateway.Spec.App.Otk.InternalOtkGatewayReference + port := gateway.Spec.App.Otk.InternalGatewayPort + if port == 0 { + port = 9443 // Default graphman port + } + + // For external gateways, the reference might be a hostname or IP + // If it's just a name without domain, we might need to construct a full hostname + // For now, use the reference as-is (could be FQDN, hostname, or IP) + endpoint := fmt.Sprintf("%s:%d/graphman", gatewayReference, port) + + // Step 1: Sync DMZ certificate as TrustedCert first + certData := dmzKeySecret.Data["tls.crt"] + if len(certData) == 0 { + return fmt.Errorf("DMZ key secret must contain tls.crt") + } + + crtStrings := strings.SplitAfter(string(certData), "-----END CERTIFICATE-----") + trustedCertBundle := graphman.Bundle{} + + for _, certStr := range crtStrings { + if certStr == "" { + continue + } + b, _ := pem.Decode([]byte(certStr)) + if b == nil { + continue + } + crtX509, err := x509.ParseCertificate(b.Bytes) + if err != nil { + continue } + + trustedCertBundle.TrustedCerts = append(trustedCertBundle.TrustedCerts, &graphman.TrustedCertInput{ + Name: crtX509.Subject.CommonName, + CertBase64: base64.StdEncoding.EncodeToString([]byte(certStr)), + TrustAnchor: true, + VerifyHostname: false, + RevocationCheckPolicyType: "USE_DEFAULT", + TrustedFor: []graphman.TrustedForType{ + "SSL", + "SIGNING_SERVER_CERTS", + }, + }) + } + + trustedCertBundleBytes, err := json.Marshal(trustedCertBundle) + if err != nil { + return fmt.Errorf("failed to marshal trusted cert bundle: %w", err) } - if gw.Spec.App.Otk.Type == securityv1.OtkTypeDMZ && len(rawDMZCertList) > 0 { - desiredSecrets = append(desiredSecrets, gateway.NewOtkCertificateSecret(gw, gw.Name+"-otk-dmz-certificates", rawDMZCertList)) + params.Log.V(2).Info("Syncing DMZ certificate as TrustedCert to external Internal gateway", + "gateway", gatewayReference, + "endpoint", endpoint) + + // Apply trusted cert bundle first + err = util.ApplyGraphmanBundle(username, password, endpoint, "", trustedCertBundleBytes) + if err != nil { + return fmt.Errorf("failed to sync DMZ certificate as TrustedCert to external Internal gateway: %w", err) } - if gw.Spec.App.Otk.Type == securityv1.OtkTypeInternal && len(rawInternalCertList) > 0 { - desiredSecrets = append(desiredSecrets, gateway.NewOtkCertificateSecret(gw, gw.Name+"-otk-internal-certificates", rawInternalCertList)) + params.Log.Info("Successfully synced DMZ certificate as TrustedCert to external Internal gateway", + "gateway", gatewayReference, + "endpoint", endpoint) + + // Step 2: Create FIP user with DMZ certificate + if len(certInfo) == 0 { + params.Log.V(2).Info("No certificate info available, skipping FIP user creation") + return nil } - err = reconcileSecrets(ctx, params, desiredSecrets) + fipUserBundle := graphman.Bundle{} + + for _, info := range certInfo { + // Use the extracted Subject DN (not just "cn=" + CommonName) + // Since FIP identity provider doesn't have a default subject dn, we must provide it + fipUserBundle.FipUsers = append(fipUserBundle.FipUsers, &graphman.FipUserInput{ + Name: info.commonName, + ProviderName: "otk-fips-provider", + SubjectDn: info.subjectDn, // Full Subject DN from certificate + CertBase64: base64.RawStdEncoding.EncodeToString(info.certRaw), + }) + } + + fipUserBundleBytes, err := json.Marshal(fipUserBundle) if err != nil { - params.Log.Error(err, "failed to reconcile otk certificates", "Name", gw.Name, "namespace", gw.Namespace) - return + return fmt.Errorf("failed to marshal FIP user bundle: %w", err) + } + + params.Log.V(2).Info("Creating FIP user with DMZ certificate in external Internal gateway", + "gateway", gatewayReference, + "endpoint", endpoint) + + // Apply FIP user bundle after certificate is synced + err = util.ApplyGraphmanBundle(username, password, endpoint, "", fipUserBundleBytes) + if err != nil { + return fmt.Errorf("failed to create FIP user with DMZ certificate in external Internal gateway: %w", err) + } + + params.Log.Info("Successfully created FIP user with DMZ certificate in external Internal gateway", + "gateway", gatewayReference, + "endpoint", endpoint) + + return nil +} + +// extractSubjectDN extracts the full Subject DN from an x509 certificate +// Format: CN=name,OU=org unit,O=org,C=country, etc. +func extractSubjectDN(cert *x509.Certificate) string { + var parts []string + + // Add CommonName + if cert.Subject.CommonName != "" { + parts = append(parts, "CN="+cert.Subject.CommonName) + } + + // Add Country + for _, c := range cert.Subject.Country { + if c != "" { + parts = append(parts, "C="+c) + } + } + + // Add Organization + for _, o := range cert.Subject.Organization { + if o != "" { + parts = append(parts, "O="+o) + } + } + + // Add Organizational Unit + for _, ou := range cert.Subject.OrganizationalUnit { + if ou != "" { + parts = append(parts, "OU="+ou) + } + } + + // Add Locality + for _, l := range cert.Subject.Locality { + if l != "" { + parts = append(parts, "L="+l) + } + } + + // Add Province/State + for _, p := range cert.Subject.Province { + if p != "" { + parts = append(parts, "ST="+p) + } + } + + // Add Street Address + for _, s := range cert.Subject.StreetAddress { + if s != "" { + parts = append(parts, "STREET="+s) + } + } + + // Add Postal Code + for _, pc := range cert.Subject.PostalCode { + if pc != "" { + parts = append(parts, "POSTALCODE="+pc) + } + } + + // Add Serial Number + if cert.Subject.SerialNumber != "" { + parts = append(parts, "SERIALNUMBER="+cert.Subject.SerialNumber) + } + + // Join all parts with comma + if len(parts) == 0 { + // Fallback to CN if nothing else is available + if cert.Subject.CommonName != "" { + return "CN=" + cert.Subject.CommonName + } + return "" } + return strings.Join(parts, ",") } -func retrieveCertificate(host string, port string) ([]byte, error) { - conf := &tls.Config{ - InsecureSkipVerify: true, +// syncInternalCertToExternalDmzGateway syncs Internal certificate to an external DMZ gateway +// using graphman. It adds the certificate as a trusted cert. +func syncInternalCertToExternalDmzGateway(ctx context.Context, params Params, gateway *securityv1.Gateway, bundleBytes []byte, sha1Sum string) error { + // Get auth secret for external DMZ gateway + if gateway.Spec.App.Otk.DmzAuthSecret == "" { + return fmt.Errorf("dmzAuthSecret is required for external DMZ gateway") } - conn, err := tls.Dial("tcp", host+":"+port, conf) + authSecret, err := getGatewaySecret(ctx, params, gateway.Spec.App.Otk.DmzAuthSecret) + if err != nil { + return fmt.Errorf("failed to get auth secret for external DMZ gateway: %w", err) + } + + // Parse username and password from auth secret + username, password := parseGatewaySecret(authSecret) + if username == "" || password == "" { + return fmt.Errorf("could not retrieve gateway credentials from auth secret: %s", gateway.Spec.App.Otk.DmzAuthSecret) + } + + // Build endpoint URL for external gateway + // Format: :/graphman + // ApplyGraphmanBundle expects format: host:port/path (without https://) + gatewayReference := gateway.Spec.App.Otk.DmzOtkGatewayReference + port := gateway.Spec.App.Otk.DmzGatewayPort + if port == 0 { + port = 9443 // Default graphman port + } + + // For external gateways, the reference might be a hostname or IP + endpoint := fmt.Sprintf("%s:%d/graphman", gatewayReference, port) + + params.Log.V(2).Info("Syncing Internal certificate to external DMZ gateway", + "gateway", gatewayReference, + "endpoint", endpoint, + "sha1Sum", sha1Sum) + + // Apply bundle to external gateway using graphman + err = util.ApplyGraphmanBundle(username, password, endpoint, "", bundleBytes) + if err != nil { + return fmt.Errorf("failed to sync Internal certificate to external DMZ gateway: %w", err) + } + + params.Log.Info("Successfully synced Internal certificate to external DMZ gateway", + "gateway", gatewayReference, + "endpoint", endpoint, + "sha1Sum", sha1Sum) + + return nil +} + +// calculateCertThumbprint calculates the SHA1 thumbprint of a certificate in the format expected by Graphman +// Format: base64-encoded hex string of SHA1 fingerprint +func calculateCertThumbprint(rawCert []byte) (string, error) { + fingerprint := sha1.Sum(rawCert) + var buf bytes.Buffer + for _, f := range fingerprint { + fmt.Fprintf(&buf, "%02X", f) + } + hexDump, err := hex.DecodeString(buf.String()) if err != nil { - return nil, err + return "", err } - defer conn.Close() - cert := conn.ConnectionState().PeerCertificates[0].Raw - return cert, nil + buf.Reset() + return base64.StdEncoding.EncodeToString(hexDump), nil } diff --git a/pkg/util/graphman.go b/pkg/util/graphman.go index cabd1049..02e0c799 100644 --- a/pkg/util/graphman.go +++ b/pkg/util/graphman.go @@ -595,6 +595,11 @@ func BuildOtkOverrideBundle(mode string, gatewayHost string, otkPort int) ([]byt Soap: false, }) } + bundle.ClusterProperties = append(bundle.ClusterProperties, &graphman.ClusterPropertyInput{ + Name: "otk.port", + Value: strconv.Itoa(otkPort), + Description: "OTK Port", + }) } bundle.FederatedIdps = append(bundle.FederatedIdps, &graphman.FederatedIdpInput{ @@ -629,6 +634,11 @@ func BuildOtkOverrideBundle(mode string, gatewayHost string, otkPort int) ([]byt Soap: false, }) } + bundle.ClusterProperties = append(bundle.ClusterProperties, &graphman.ClusterPropertyInput{ + Name: "otk.port", + Value: strconv.Itoa(otkPort), + Description: "OTK Port", + }) } case "SINGLE": bundle.ClusterProperties = append(bundle.ClusterProperties, &graphman.ClusterPropertyInput{