diff --git a/hazelcast/src/main/java/org/apache/karaf/cellar/hazelcast/factory/HazelcastConfigurationManager.java b/hazelcast/src/main/java/org/apache/karaf/cellar/hazelcast/factory/HazelcastConfigurationManager.java index bc667b1f6..a55d2c207 100644 --- a/hazelcast/src/main/java/org/apache/karaf/cellar/hazelcast/factory/HazelcastConfigurationManager.java +++ b/hazelcast/src/main/java/org/apache/karaf/cellar/hazelcast/factory/HazelcastConfigurationManager.java @@ -85,17 +85,19 @@ public boolean isUpdated(Map properties) { if (!CellarUtils.collectionEquals(discoveredMemberSet, newDiscoveredMemberSet)) { LOGGER.debug("Hazelcast discoveredMemberSet has been changed from {} to {}", discoveredMemberSet, newDiscoveredMemberSet); discoveredMemberSet = newDiscoveredMemberSet; - for (String discoveredMember:discoveredMemberSet) { - if (discoveredMember != null && !String.valueOf(discoveredMember).equals("null") && !tcpIpConfig.getMembers().contains(discoveredMember)) { - tcpIpConfig.getMembers().add(discoveredMember); - } - } - Iterator iterator = tcpIpConfig.getMembers().iterator(); - while (iterator.hasNext()) { - String member = iterator.next(); - if (!discoveredMemberSet.contains(member)) { - iterator.remove(); - } + if (tcpIpConfig != null) { + for (String discoveredMember:discoveredMemberSet) { + if (discoveredMember != null && !String.valueOf(discoveredMember).equals("null") && !tcpIpConfig.getMembers().contains(discoveredMember)) { + tcpIpConfig.getMembers().add(discoveredMember); + } + } + Iterator iterator = tcpIpConfig.getMembers().iterator(); + while (iterator.hasNext()) { + String member = iterator.next(); + if (!discoveredMemberSet.contains(member)) { + iterator.remove(); + } + } } updated = Boolean.TRUE; } diff --git a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/ConfigKey.java b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/ConfigKey.java index c26c5b8e2..d6aa67011 100644 --- a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/ConfigKey.java +++ b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/ConfigKey.java @@ -11,8 +11,10 @@ * GitHub */ public enum ConfigKey { + KUBERNETES_ENABLE_AUTOCONFIGURE("kubernetes.autoConfig"), KUBERNETES_MASTER(Config.KUBERNETES_MASTER_SYSTEM_PROPERTY), KUBERNETES_API_VERSION(Config.KUBERNETES_API_VERSION_SYSTEM_PROPERTY), + KUBERNETES_NAMESPACE(Config.KUBERNETES_NAMESPACE_SYSTEM_PROPERTY), KUBERNETES_TRUST_CERTIFICATES(Config.KUBERNETES_TRUST_CERT_SYSTEM_PROPERTY), KUBERNETES_DISABLE_HOSTNAME_VERIFICATION(Config.KUBERNETES_DISABLE_HOSTNAME_VERIFICATION_SYSTEM_PROPERTY), KUBERNETES_CERTS_CA_FILE(Config.KUBERNETES_CA_CERTIFICATE_FILE_SYSTEM_PROPERTY), diff --git a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryService.java b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryService.java index bed2dcb73..2fd5dbec3 100644 --- a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryService.java +++ b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryService.java @@ -13,6 +13,7 @@ */ package org.apache.karaf.cellar.kubernetes; +import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodList; import io.fabric8.kubernetes.client.Config; @@ -35,8 +36,11 @@ public class KubernetesDiscoveryService implements DiscoveryService { private static final Logger LOGGER = LoggerFactory.getLogger(KubernetesDiscoveryService.class); + private boolean kubernetesAutoconfigure; + private String kubernetesMaster; private String kubernetesApiVersion; + private String kubernetesNamespace; private boolean kubernetesTrustCertificates; private boolean kubernetesDisableHostnameVerification; private String kubernetesCertsCaFile; @@ -68,31 +72,46 @@ public KubernetesDiscoveryService() { } Config createConfig() { - return new ConfigBuilder() - .withMasterUrl(kubernetesMaster) - .withApiVersion(kubernetesApiVersion) - .withTrustCerts(kubernetesTrustCertificates) - .withDisableHostnameVerification(kubernetesDisableHostnameVerification) - .withCaCertFile(kubernetesCertsCaFile) - .withCaCertData(kubernetesCertsCaData) - .withClientCertFile(kubernetesCertsClientFile) - .withClientCertData(kubernetesCertsClientData) - .withClientKeyFile(kubernetesCertsClientKeyFile) - .withClientKeyData(kubernetesCertsClientKeyData) - .withClientKeyAlgo(kubernetesCertsClientKeyAlgo) - .withClientKeyPassphrase(kubernetesCertsClientKeyPassphrase) - .withUsername(kubernetesAuthBasicUsername) - .withPassword(kubernetesAuthBasicPassword) - .withOauthToken(kubernetesOauthToken) - .withWatchReconnectInterval(kubernetesWatchReconnectInterval) - .withWatchReconnectLimit(kubernetesWatchReconnectLimit) - .withUserAgent(kubernetesUserAgent) - .withTlsVersions(TlsVersion.forJavaName(kubernetesTlsVersion)) - .withTrustStoreFile(kubernetesTruststoreFile) - .withTrustStorePassphrase(kubernetesTruststorePassphrase) - .withKeyStoreFile(kubernetesKeystoreFile) - .withKeyStorePassphrase(kubernetesKeystorePassphrase) - .build(); + TlsVersion[] tlsVersions = {}; + if (kubernetesTlsVersion != null) { + tlsVersions = new TlsVersion[1]; + tlsVersions[0] = TlsVersion.forJavaName(kubernetesTlsVersion); + } + + Config config; + if (kubernetesAutoconfigure) { + config = Config.autoConfigure(null); + config.setTlsVersions(tlsVersions); // avoid NPE in io.fabric8.kubernetes.client.utils.URLUtils.join(URLUtils.java:47) + } else { + config = new ConfigBuilder() + .withMasterUrl(kubernetesMaster) + .withApiVersion(kubernetesApiVersion) + .withNamespace(kubernetesNamespace) + .withTrustCerts(kubernetesTrustCertificates) + .withDisableHostnameVerification(kubernetesDisableHostnameVerification) + .withCaCertFile(kubernetesCertsCaFile) + .withCaCertData(kubernetesCertsCaData) + .withClientCertFile(kubernetesCertsClientFile) + .withClientCertData(kubernetesCertsClientData) + .withClientKeyFile(kubernetesCertsClientKeyFile) + .withClientKeyData(kubernetesCertsClientKeyData) + .withClientKeyAlgo(kubernetesCertsClientKeyAlgo) + .withClientKeyPassphrase(kubernetesCertsClientKeyPassphrase) + .withUsername(kubernetesAuthBasicUsername) + .withPassword(kubernetesAuthBasicPassword) + .withOauthToken(kubernetesOauthToken) + .withWatchReconnectInterval(kubernetesWatchReconnectInterval) + .withWatchReconnectLimit(kubernetesWatchReconnectLimit) + .withUserAgent(kubernetesUserAgent) + .withTlsVersions(tlsVersions) + .withTrustStoreFile(kubernetesTruststoreFile) + .withTrustStorePassphrase(kubernetesTruststorePassphrase) + .withKeyStoreFile(kubernetesKeystoreFile) + .withKeyStorePassphrase(kubernetesKeystorePassphrase) + .build(); + } + + return config; } public void init() { @@ -121,8 +140,8 @@ public Set discoverMembers() { try { PodList podList = kubernetesClient.pods().list(); for (Pod pod : podList.getItems()) { - String value = pod.getMetadata().getLabels().get(kubernetesPodLabelKey); - if (value != null && !value.isEmpty() && value.equals(kubernetesPodLabelValue)) { + String value = getKubernetesPodLabelKey(pod); + if (value != null && !value.isEmpty() && value.equals(kubernetesPodLabelValue) && pod.getStatus().getPodIP() != null) { members.add(pod.getStatus().getPodIP()); } } @@ -132,6 +151,16 @@ public Set discoverMembers() { return members; } + private String getKubernetesPodLabelKey(Pod pod) { + ObjectMeta metadata = pod.getMetadata(); + if (metadata == null) + return null; + Map labels = metadata.getLabels(); + if (labels == null) + return null; + return labels.get(kubernetesPodLabelKey); + } + @Override public void signIn() { // nothing to do for Kubernetes @@ -147,6 +176,16 @@ public void signOut() { // nothing to do for Kubernetes } + public boolean isKubernetesAutoconfigure() { + return kubernetesAutoconfigure; + } + + public void setKubernetesAutoconfigure(String kubernetesAutoconfigure) { + if (kubernetesAutoconfigure != null) { + this.kubernetesAutoconfigure = Boolean.parseBoolean(kubernetesAutoconfigure); + } + } + void setKubernetesClient(KubernetesClient kubernetesClient) { this.kubernetesClient = kubernetesClient; } @@ -183,6 +222,14 @@ public void setKubernetesApiVersion(String kubernetesApiVersion) { this.kubernetesApiVersion = kubernetesApiVersion; } + public String getKubernetesNamespace() { + return kubernetesNamespace; + } + + public void setKubernetesNamespace(String kubernetesNamespace) { + this.kubernetesNamespace = kubernetesNamespace; + } + public boolean isKubernetesTrustCertificates() { return kubernetesTrustCertificates; } diff --git a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactory.java b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactory.java index 44b73dc79..628233ef1 100644 --- a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactory.java +++ b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactory.java @@ -13,6 +13,7 @@ */ package org.apache.karaf.cellar.kubernetes; +import io.fabric8.kubernetes.client.Config; import org.apache.karaf.cellar.core.discovery.DiscoveryService; import org.osgi.framework.BundleContext; import org.osgi.framework.ServiceRegistration; @@ -40,9 +41,11 @@ import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_CERTS_CLIENT_KEY_FILE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_CERTS_CLIENT_KEY_PASSPHRASE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_DISABLE_HOSTNAME_VERIFICATION; +import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_ENABLE_AUTOCONFIGURE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_KEYSTORE_FILE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_KEYSTORE_PASSPHRASE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_MASTER; +import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_NAMESPACE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_TLS_VERSIONS; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_TRUSTSTORE_FILE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_TRUSTSTORE_PASSPHRASE; @@ -131,8 +134,10 @@ public void updated(String pid, Dictionary properties) throws ConfigurationExcep kubernetesMaster = "http://" + kubernetesHost + ":" + kubernetesPort; } + kubernetesDiscoveryService.setKubernetesAutoconfigure(KUBERNETES_ENABLE_AUTOCONFIGURE.getValue(properties)); kubernetesDiscoveryService.setKubernetesMaster(kubernetesMaster); kubernetesDiscoveryService.setKubernetesApiVersion(KUBERNETES_API_VERSION.getValue(properties)); + kubernetesDiscoveryService.setKubernetesNamespace(KUBERNETES_NAMESPACE.getValue(properties)); kubernetesDiscoveryService.setKubernetesTrustCertificates(KUBERNETES_TRUST_CERTIFICATES.getValue(properties)); kubernetesDiscoveryService.setKubernetesDisableHostnameVerification(KUBERNETES_DISABLE_HOSTNAME_VERIFICATION.getValue(properties)); kubernetesDiscoveryService.setKubernetesCertsCaFile(KUBERNETES_CERTS_CA_FILE.getValue(properties)); diff --git a/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactoryTest.java b/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactoryTest.java index 01899c304..0d03ec26c 100644 --- a/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactoryTest.java +++ b/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactoryTest.java @@ -150,6 +150,7 @@ public void verifyUpdateKubernetesConfig() throws Exception { properties.put(KUBERNETES_KEYSTORE_PASSPHRASE.propertyName, EXPECTED_KUBERNETES_KEYSTORE_PASSPHRASE); update(); assertEquals(EXPECTED_KUBERNETES_API_VERSION, registeredService.getKubernetesApiVersion()); + assertEquals(EXPECTED_KUBERNETES_API_VERSION, registeredService.getKubernetesApiVersion()); assertEquals(Boolean.parseBoolean(EXPECTED_KUBERNETES_TRUST_CERTIFICATES), registeredService.isKubernetesTrustCertificates()); assertEquals(Boolean.parseBoolean(EXPECTED_KUBERNETES_DISABLE_HOSTNAME_VERIFICATION), registeredService.isKubernetesDisableHostnameVerification()); assertEquals(EXPECTED_KUBERNETES_CERTS_CA_FILE, registeredService.getKubernetesCertsCaFile()); diff --git a/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceTest.java b/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceTest.java index e6e792e8f..d2313f838 100644 --- a/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceTest.java +++ b/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceTest.java @@ -29,6 +29,7 @@ public class KubernetesDiscoveryServiceTest { static final String EXPECTED_KUBERNETES_MASTER = "http://master/"; static final String EXPECTED_KUBERNETES_API_VERSION = "api version"; + static final String EXPECTED_KUBERNETES_NAMESPACE = "default"; static final String EXPECTED_KUBERNETES_TRUST_CERTIFICATES = "true"; static final String EXPECTED_KUBERNETES_DISABLE_HOSTNAME_VERIFICATION = "true"; static final String EXPECTED_KUBERNETES_CERTS_CA_FILE = "certs ca file"; @@ -84,6 +85,7 @@ public void setup() { public void createConfig() { service.setKubernetesMaster(EXPECTED_KUBERNETES_MASTER); service.setKubernetesApiVersion(EXPECTED_KUBERNETES_API_VERSION); + service.setKubernetesNamespace(EXPECTED_KUBERNETES_NAMESPACE); service.setKubernetesTrustCertificates(EXPECTED_KUBERNETES_TRUST_CERTIFICATES); service.setKubernetesDisableHostnameVerification(EXPECTED_KUBERNETES_DISABLE_HOSTNAME_VERIFICATION); service.setKubernetesCertsCaFile(EXPECTED_KUBERNETES_CERTS_CA_FILE); @@ -109,6 +111,7 @@ public void createConfig() { Config config = service.createConfig(); assertEquals(EXPECTED_KUBERNETES_MASTER, config.getMasterUrl()); assertEquals(EXPECTED_KUBERNETES_API_VERSION, config.getApiVersion()); + assertEquals(EXPECTED_KUBERNETES_NAMESPACE, config.getNamespace()); assertTrue(config.isTrustCerts()); assertTrue(config.isDisableHostnameVerification()); assertEquals(EXPECTED_KUBERNETES_CERTS_CA_FILE, config.getCaCertFile()); diff --git a/manual/src/main/asciidoc/user-guide/cloud.adoc b/manual/src/main/asciidoc/user-guide/cloud.adoc index d484b1de4..ab5e7ae01 100644 --- a/manual/src/main/asciidoc/user-guide/cloud.adoc +++ b/manual/src/main/asciidoc/user-guide/cloud.adoc @@ -129,4 +129,89 @@ pod.label.value=cellar ---- In case you change the file, the discovery service will check again for new nodes. If new nodes are found, Hazelcast configuration will be -updated and the instance restarted. \ No newline at end of file +updated and the instance restarted. + +===== Hazelcast configuration for Kubernetes + +Even within Kubernetes, the default `hazelcast.xml` will discover nodes on the network via multicast. If this is not desired, make sure to disable multicast discovery and enable TCP/IP in `hazelcast.xml`. You can also disable the port auto-increment, since each container will have a different IP, and a predictable port is preferable: + +---- + + + 5701 + + + + + .... + + .... + + +---- + +===== Using Kubernetes service accounts + +https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/[Kubernetes service accounts] allow the processes running within pods to access the Kubernetes API server. Make sure the chosen service account has the appropriate authorization to enumerate Karaf Cellar pods in the appropriate Kubernetes namespace(s). Minimally, allow `get`, `watch` and `list` for `pod` resources: + +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role / ClusterRole +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + +---- + +===== Kubernetes client automatic configuration + +The Kubernetes client used by Karaf Cellar supports autoconfiguration as described https://github.com/fabric8io/kubernetes-client#configuring-the-client[here]. This is turned off by default in Karaf Cellar. However, this feature can be turned on by using the setting `kubernetes.autoConfig = true` in the Karaf Cellar Kubernetes discovery service. + +Note: not all the options provided by the Kubernetes client are actually usable within Cellar. + +Here is a sample `etc/org.apache.karaf.cellar.kubernetes-mycellarcluster.cfg`: + +---- +pod.label.key=cellar_cluster +pod.label.value=mycluster +kubernetes.autoConfig=true +---- + +Such a setup is suitable to be dropped in a docker container intended to run in a pod configured as follows: + +---- +apiVersion: v1 +kind: Pod +metadata: + labels: + cellar_cluster: mycellarcluster +... +spec: + automountServiceAccountToken: true +... +---- + +===== Manual Kubernetes service account configuration + +If the automatic configuration does not work, a good starting point to replicate the same function manually would be: + +---- +pod.label.key=cellar_cluster +pod.label.value=mycluster + +kubernetes.master=https://$[env:KUBERNETES_SERVICE_HOST]:$[env:KUBERNETES_SERVICE_PORT_HTTPS] +kubernetes.api.version=v1 + +kubernetes.certs.ca.file=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt +kubernetes.auth.token=$[secret:kube-secrets/kubernetes.io/serviceaccount/token] +kubernetes.namespace=$[secret:kube-secrets/kubernetes.io/serviceaccount/namespace] +---- + +The above relies on Karaf's configuration admin environment and secret interpolation. See https://karaf.apache.org/manual/latest/#_secret_files[Secret files] and https://karaf.apache.org/manual/latest/#_environment_variables[Environment variables] in the Karaf manual. You will have to https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#service-account-admission-controller[configure kubernetes to mount service account secrets] at `${karaf.etc}/kube-secrets`, or, alternatively, provide a symlink towards `/var/run/secrets`, or change the corresponding setting in `config.properties`: