diff --git a/snakebite/config.py b/snakebite/config.py index 3a060d3..a61aeba 100644 --- a/snakebite/config.py +++ b/snakebite/config.py @@ -20,8 +20,16 @@ def get_config_from_env(cls): core_path = os.path.join(os.environ['HADOOP_HOME'], 'conf', 'core-site.xml') core_configs = cls.read_core_config(core_path) + maybe_ha_name = None + if len(core_configs.get('namenodes', [])) == 1: + # We may have gotten an HA config from core-site.xml, try + # to use it to resolve HA names + maybe_ha_name = core_configs.get('namenodes')[0].get('namenode') + hdfs_path = os.path.join(os.environ['HADOOP_HOME'], 'conf', 'hdfs-site.xml') - hdfs_configs = cls.read_hdfs_config(hdfs_path) + # May have interpreted defaultFS as a NN, pass this to + # read_hdfs_config to try to resolve an HA configuration + hdfs_configs = cls.read_hdfs_config(hdfs_path, maybe_ha_name) if (not core_configs) and (not hdfs_configs): raise Exception("No config found in %s nor in %s" % (core_path, hdfs_path)) @@ -73,23 +81,32 @@ def read_core_config(cls, core_site_path): else: configs['use_sasl'] = False - if namenodes: + if namenodes: configs['namenodes'] = namenodes return configs @classmethod - def read_hdfs_config(cls, hdfs_site_path): + def read_hdfs_config(cls, hdfs_site_path, maybe_ha_name=None): configs = {} + ha_configs = cls.read_hdfs_ha_configs(hdfs_site_path) namenodes = [] for property in cls.read_hadoop_config(hdfs_site_path): - if property.findall('name')[0].text.startswith("dfs.namenode.rpc-address"): + if property.findall('name')[0].text.startswith('dfs.namenode.rpc-address'): + prop_name = property.findall('name')[0].text parse_result = urlparse("//" + property.findall('value')[0].text) - log.debug("Got namenode '%s' from %s" % (parse_result.geturl(), hdfs_site_path)) - namenodes.append({"namenode": parse_result.hostname, - "port": parse_result.port if parse_result.port - else Namenode.DEFAULT_PORT}) + + if (prop_name == 'dfs.namenode.rpc-address' or + cls.valid_ha_namenode(maybe_ha_name, + ha_configs, + parse_result.geturl(), + hdfs_site_path, + prop_name)): + log.debug("Got namenode '%s' from %s" % (parse_result.geturl(), hdfs_site_path)) + namenodes.append({"namenode": parse_result.hostname, + "port": parse_result.port if parse_result.port + else Namenode.DEFAULT_PORT}) if property.findall('name')[0].text == 'fs.trash.interval': configs['use_trash'] = True @@ -151,7 +168,11 @@ def get_external_config(cls): hdfs_configs = {} for hdfs_conf_path in cls.hdfs_try_paths: - hdfs_configs = cls.read_hdfs_config(hdfs_conf_path) + if len(core_configs.get('namenodes', [])) == 1: + hdfs_configs = cls.read_hdfs_config(hdfs_conf_path, + core_configs.get('namenodes')[0].get('namenode')) + else: + hdfs_configs = cls.read_hdfs_config(hdfs_conf_path) if hdfs_configs: break @@ -169,3 +190,59 @@ def get_external_config(cls): } return configs + + @classmethod + def valid_ha_namenode(cls, maybe_ha_name, ha_configs, nn_url, hdfs_site_path, name): + name_parts = name.split('.') + + if len(name_parts) != 5: + log.debug("Could not parse cluster name from %s, skipping %s from %s" % + (name, + nn_url, + hdfs_site_path)) + return False + + cluster = name_parts[-2] + if cluster != maybe_ha_name: + log.debug("Skipping %s from %s, becuause it does not belong to our active cluster %s" % + (nn_url, hdfs_site_path, maybe_ha_name)) + return False + + if cluster not in ha_configs.get('clusters', []): + log.debug("Skipping %s from %s, becuause it is no in the configured cluster list: %s" % + (nn_url, hdfs_site_path, ha_configs.get('clusters'))) + return False + + logical_namenode = name_parts[-1] + cluster_logical_namenodes = (ha_configs + .get('logical_namenodes', {}) + .get(cluster, [])) + if logical_namenode not in cluster_logical_namenodes: + log.debug("Could not find logical mapping for %s in cluster %s from %s, skipping" % + (nn_url, + cluster, + hdfs_site_path)) + return False + return True + + @classmethod + def read_hdfs_ha_configs(cls, hdfs_site_path): + ha_configs = {} + for property in cls.read_hadoop_config(hdfs_site_path): + name = property.findall('name')[0].text + value = property.findall('value')[0].text + + if name == 'dfs.nameservices': + ha_configs['clusters'] = value.split(',') + + if name.startswith('dfs.ha.namenodes'): + name_parts = property.findall('name')[0].text.split('.') + if len(name_parts) != 4: + log.debug("Could not parse cluster name from %s, skipping" % + (property.findall('name')[0].text)) + continue + if 'logical_namenodes' not in ha_configs: + ha_configs['logical_namenodes'] = {} + cluster = name_parts[-1] + ha_configs['logical_namenodes'][cluster] = value.split(',') + return ha_configs diff --git a/test/config_test.py b/test/config_test.py index 09c449c..063c494 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -44,7 +44,7 @@ def _verify_hdfs_noport_settings(self, config): def test_read_hdfs_config_ha(self): hdfs_site_path = self.get_config_path('ha-port-hdfs-site.xml') - config = HDFSConfig.read_hdfs_config(hdfs_site_path) + config = HDFSConfig.read_hdfs_config(hdfs_site_path, 'testha') self._verify_hdfs_settings(config) def test_read_core_config_ha(self): @@ -97,7 +97,7 @@ def test_ha_without_ports(self, environ_get): @patch('os.environ.get') def test_ha_config_trash_in_core(self, environ_get): environ_get.return_value = False - HDFSConfig.core_try_paths = (self.get_config_path('core-with-trash.xml'),) + HDFSConfig.core_try_paths = (self.get_config_path('ha-core-with-trash.xml'),) HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-noport-hdfs-site.xml'),) config = HDFSConfig.get_external_config() @@ -143,3 +143,32 @@ def test_use_datanode_hostname_configs(self): conf_path = self.get_config_path('use-datanode-hostname-hdfs-site.xml') config = HDFSConfig.read_hdfs_config(conf_path) self.assertTrue(config['use_datanode_hostname']) + + def test_ha_multi(self): + HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),) + HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-multi-hdfs-site.xml'),) + config = HDFSConfig.get_external_config() + + self._verify_hdfs_settings(config) + + def test_ha_multi_missing_nameservices(self): + HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),) + HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-multi-no-nameservices-hdfs-site.xml'),) + config = HDFSConfig.get_external_config() + + self.assertEquals(config['namenodes'], [{'namenode': 'testha', 'port': 8020}]) + + def test_ha_multi_bad_logical_nn_mapping(self): + HDFSConfig.core_try_paths = (self.get_config_path('ha-core-site.xml'),) + HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-multi-bad-nn-hdfs-site.xml'),) + config = HDFSConfig.get_external_config() + + self.assertEquals(config['namenodes'], [{'namenode': 'testha', 'port': 8020}]) + + def test_ha_multi_missing_default_fs(self): + HDFSConfig.core_try_paths = (self.get_config_path('ha-no-default-fs-core-site.xml'),) + HDFSConfig.hdfs_try_paths = (self.get_config_path('ha-multi-hdfs-site.xml'),) + config = HDFSConfig.get_external_config() + + print config + self.assertEquals(config['namenodes'], []) diff --git a/test/testconfig/conf/ha-core-with-trash.xml b/test/testconfig/conf/ha-core-with-trash.xml new file mode 100644 index 0000000..0c5ad88 --- /dev/null +++ b/test/testconfig/conf/ha-core-with-trash.xml @@ -0,0 +1,19 @@ + + + + + + fs.defaultFS + hdfs://testha + + + + fs.trash.interval + 1 + + + + fs.trash.checkpoint.interval + 1 + + diff --git a/test/testconfig/conf/ha-multi-bad-nn-hdfs-site.xml b/test/testconfig/conf/ha-multi-bad-nn-hdfs-site.xml new file mode 100644 index 0000000..168a414 --- /dev/null +++ b/test/testconfig/conf/ha-multi-bad-nn-hdfs-site.xml @@ -0,0 +1,60 @@ + + + + + + dfs.nameservices + testha,testotherha + + + + dfs.ha.namenodes.testha + nonexistent1,nonexistent2 + + + + dfs.namenode.rpc-address.testha.namenode1-mydomain + namenode1.mydomain:8888 + + + + dfs.namenode.rpc-address.testha.namenode2-mydomain + namenode2.mydomain:8888 + + + + dfs.namenode.http-address.testha.namenode1-mydomain + namenode1.mydomain:50070 + + + + dfs.namenode.http-address.testha.namenode2-mydomain + namenode2.mydomain:50070 + + + + dfs.ha.namenodes.testotherha + namenode-other1-mydomain,namenode-other2-mydomain + + + + dfs.namenode.rpc-address.testotherha.namenode-other1-mydomain + namenode-other1.mydomain:8888 + + + + dfs.namenode.rpc-address.testotherha.namenode-other2-mydomain + namenode-other2.mydomain:8888 + + + + dfs.namenode.http-address.testotherha.namenode-other1-mydomain + namenode-other1.mydomain:50070 + + + + dfs.namenode.http-address.testotherha.namenode-other2-mydomain + namenode-other2.mydomain:50070 + + + diff --git a/test/testconfig/conf/ha-multi-hdfs-site.xml b/test/testconfig/conf/ha-multi-hdfs-site.xml new file mode 100644 index 0000000..e40a253 --- /dev/null +++ b/test/testconfig/conf/ha-multi-hdfs-site.xml @@ -0,0 +1,60 @@ + + + + + + dfs.nameservices + testha,testotherha + + + + dfs.ha.namenodes.testha + namenode1-mydomain,namenode2-mydomain + + + + dfs.namenode.rpc-address.testha.namenode1-mydomain + namenode1.mydomain:8888 + + + + dfs.namenode.rpc-address.testha.namenode2-mydomain + namenode2.mydomain:8888 + + + + dfs.namenode.http-address.testha.namenode1-mydomain + namenode1.mydomain:50070 + + + + dfs.namenode.http-address.testha.namenode2-mydomain + namenode2.mydomain:50070 + + + + dfs.ha.namenodes.testotherha + namenode-other1-mydomain,namenode-other2-mydomain + + + + dfs.namenode.rpc-address.testotherha.namenode-other1-mydomain + namenode-other1.mydomain:8888 + + + + dfs.namenode.rpc-address.testotherha.namenode-other2-mydomain + namenode-other2.mydomain:8888 + + + + dfs.namenode.http-address.testotherha.namenode-other1-mydomain + namenode-other1.mydomain:50070 + + + + dfs.namenode.http-address.testotherha.namenode-other2-mydomain + namenode-other2.mydomain:50070 + + + diff --git a/test/testconfig/conf/ha-multi-no-nameservices-hdfs-site.xml b/test/testconfig/conf/ha-multi-no-nameservices-hdfs-site.xml new file mode 100644 index 0000000..1cdf733 --- /dev/null +++ b/test/testconfig/conf/ha-multi-no-nameservices-hdfs-site.xml @@ -0,0 +1,55 @@ + + + + + + dfs.ha.namenodes.testha + namenode1-mydomain,namenode2-mydomain + + + + dfs.namenode.rpc-address.testha.namenode1-mydomain + namenode1.mydomain:8888 + + + + dfs.namenode.rpc-address.testha.namenode2-mydomain + namenode2.mydomain:8888 + + + + dfs.namenode.http-address.testha.namenode1-mydomain + namenode1.mydomain:50070 + + + + dfs.namenode.http-address.testha.namenode2-mydomain + namenode2.mydomain:50070 + + + + dfs.ha.namenodes.testotherha + namenode-other1-mydomain,namenode-other2-mydomain + + + + dfs.namenode.rpc-address.testotherha.namenode-other1-mydomain + namenode-other1.mydomain:8888 + + + + dfs.namenode.rpc-address.testotherha.namenode-other2-mydomain + namenode-other2.mydomain:8888 + + + + dfs.namenode.http-address.testotherha.namenode-other1-mydomain + namenode-other1.mydomain:50070 + + + + dfs.namenode.http-address.testotherha.namenode-other2-mydomain + namenode-other2.mydomain:50070 + + + diff --git a/test/testconfig/conf/ha-no-default-fs-core-site.xml b/test/testconfig/conf/ha-no-default-fs-core-site.xml new file mode 100644 index 0000000..d57a965 --- /dev/null +++ b/test/testconfig/conf/ha-no-default-fs-core-site.xml @@ -0,0 +1,5 @@ + + + + +