diff --git a/amazon-efs-utils.spec b/amazon-efs-utils.spec index 46fe2de6..d2c20c77 100644 --- a/amazon-efs-utils.spec +++ b/amazon-efs-utils.spec @@ -41,7 +41,7 @@ %{?!include_vendor_tarball:%define include_vendor_tarball true} Name : amazon-efs-utils -Version : 2.4.1 +Version : 2.4.2 Release : 1%{platform} Summary : This package provides utilities for simplifying the use of EFS file systems @@ -196,6 +196,13 @@ fi %clean %changelog +* Tue Dec 23 2025 Samuel Hale - 2.4.2 +- Skip stunnel binary invocation when efs-proxy mode is enabled +- Retry "access denied" only for access point mounting +- Fix issue for missing PATH in env when check stunnel lib +- Fix EFS_FQDN_RE to support ADC DNS suffixes with hyphens +- Fix IPv6-only mount target FQDN resolution in match_device + * Thu Nov 20 2025 Anthony Tse - 2.4.1 - Add cafile override for eusc-de-east-1 in efs-utils.conf diff --git a/build-deb.sh b/build-deb.sh index 93cdee72..a62a3944 100755 --- a/build-deb.sh +++ b/build-deb.sh @@ -11,7 +11,7 @@ set -ex BASE_DIR=$(pwd) BUILD_ROOT=${BASE_DIR}/build/debbuild -VERSION=2.4.1 +VERSION=2.4.2 RELEASE=1 ARCH=$(dpkg --print-architecture) DEB_SYSTEM_RELEASE_PATH=/etc/os-release diff --git a/config.ini b/config.ini index 90025d23..19ddb2e0 100644 --- a/config.ini +++ b/config.ini @@ -7,5 +7,5 @@ # [global] -version=2.4.1 +version=2.4.2 release=1 diff --git a/src/mount_efs/__init__.py b/src/mount_efs/__init__.py index 9b5fe51b..30aa5336 100755 --- a/src/mount_efs/__init__.py +++ b/src/mount_efs/__init__.py @@ -86,7 +86,7 @@ BOTOCORE_PRESENT = False -VERSION = "2.4.1" +VERSION = "2.4.2" SERVICE = "elasticfilesystem" AMAZON_LINUX_2_RELEASE_ID = "Amazon Linux release 2 (Karoo)" @@ -1425,10 +1425,15 @@ def find_command_path(command, install_method): # For more information, see https://brew.sh/2021/02/05/homebrew-3.0.0/ else: env_path = "/opt/homebrew/bin:/usr/local/bin" - os.putenv("PATH", env_path) + + existing_path = os.environ.get("PATH", "") + search_path = env_path + ":" + existing_path if existing_path else env_path + + env = os.environ.copy() + env["PATH"] = search_path try: - path = subprocess.check_output(["which", command]) + path = subprocess.check_output(["which", command], env=env) return path.strip().decode() except subprocess.CalledProcessError as e: fatal_error( @@ -1479,7 +1484,7 @@ def write_stunnel_config_file( hand-serialize it. """ - stunnel_options = get_stunnel_options() + stunnel_options = [] if efs_proxy_enabled else get_stunnel_options() mount_filename = get_mount_specific_filename(fs_id, mountpoint, tls_port) system_release_version = get_system_release_version() @@ -2223,9 +2228,15 @@ def backoff_function(i): out, err = proc.communicate(timeout=retry_nfs_mount_command_timeout_sec) rc = proc.poll() if rc != 0: + is_access_point_mount = "accesspoint" in options continue_retry = any( error_string in str(err) for error_string in RETRYABLE_ERRORS ) + + # Only retry "access denied" for access point mounts, handles race condition that can occur during AP backend provisioning + if not continue_retry and "access denied by server" in str(err): + continue_retry = is_access_point_mount + if continue_retry: logging.error( 'Mounting %s to %s failed, return code=%s, stdout="%s", stderr="%s", mount attempt %d/%d, ' @@ -3223,8 +3234,18 @@ def match_device(config, device, options): return remote, path, None try: - primary, secondaries, _ = socket.gethostbyname_ex(remote) - hostnames = list(filter(lambda e: e is not None, [primary] + secondaries)) + addrinfo = socket.getaddrinfo( + remote, None, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_CANONNAME + ) + hostnames = list( + set( + filter( + lambda e: e is not None and e != "", [info[3] for info in addrinfo] + ) + ) + ) + if not hostnames: + hostnames = [remote] except socket.gaierror: create_default_cloudwatchlog_agent_if_not_exist(config, options) fatal_error( diff --git a/src/proxy/Cargo.lock b/src/proxy/Cargo.lock index 66cf7425..202db8f4 100644 --- a/src/proxy/Cargo.lock +++ b/src/proxy/Cargo.lock @@ -337,7 +337,7 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "efs-proxy" -version = "2.4.1" +version = "2.4.2" dependencies = [ "anyhow", "async-trait", diff --git a/src/proxy/Cargo.toml b/src/proxy/Cargo.toml index cb469919..b5c020f2 100644 --- a/src/proxy/Cargo.toml +++ b/src/proxy/Cargo.toml @@ -3,7 +3,7 @@ name = "efs-proxy" edition = "2021" build = "build.rs" # The version of efs-proxy is tied to efs-utils. -version = "2.4.1" +version = "2.4.2" publish = false license = "MIT" @@ -34,6 +34,7 @@ xdr-codec = { path = "rust-xdr/xdr-codec"} test-case = "*" tokio = { version = "1.29.0", features = ["test-util"] } tempfile = "3.10.1" +regex = "1.10.2" [build-dependencies] xdrgen = { path = "rust-xdr/xdrgen" } diff --git a/src/proxy/build.rs b/src/proxy/build.rs index dbc10216..81bcde27 100644 --- a/src/proxy/build.rs +++ b/src/proxy/build.rs @@ -1,5 +1,3 @@ -use xdrgen; - fn main() { xdrgen::compile("src/efs_prot.x").expect("xdrgen efs_prot.x failed"); } diff --git a/src/proxy/rust-xdr/xdrgen/src/spec/mod.rs b/src/proxy/rust-xdr/xdrgen/src/spec/mod.rs index e259ae08..15b6bb67 100644 --- a/src/proxy/rust-xdr/xdrgen/src/spec/mod.rs +++ b/src/proxy/rust-xdr/xdrgen/src/spec/mod.rs @@ -1073,15 +1073,15 @@ impl Symtab { } } - pub fn constants(&self) -> Iter)> { + pub fn constants(&self) -> Iter<'_, String, (i64, Option)> { self.consts.iter() } - pub fn typespecs(&self) -> Iter { + pub fn typespecs(&self) -> Iter<'_, String, Type> { self.typespecs.iter() } - pub fn typesyns(&self) -> Iter { + pub fn typesyns(&self) -> Iter<'_, String, Type> { self.typesyns.iter() } } diff --git a/src/proxy/src/config_parser.rs b/src/proxy/src/config_parser.rs index 0a49fb14..0a367a3a 100644 --- a/src/proxy/src/config_parser.rs +++ b/src/proxy/src/config_parser.rs @@ -20,6 +20,10 @@ where } } +fn default_log_format() -> Option { + Some("file".to_string()) +} + #[derive(Default, Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct ProxyConfig { #[serde(alias = "fips", deserialize_with = "deserialize_bool")] @@ -33,6 +37,11 @@ pub struct ProxyConfig { #[serde(alias = "output")] pub output: Option, + /// The format to use for logging. Values can be "file", "stdout" + /// Default is "file" if not specified. + #[serde(alias = "log_format", default = "default_log_format")] + pub log_format: Option, + /// The proxy process is responsible for writing it's PID into this file so that the Watchdog /// process can monitor it #[serde(alias = "pid")] @@ -136,6 +145,7 @@ checkHost = fs-12341234.efs.us-east-1.amazonaws.com output: Some(String::from( "/var/log/amazon/efs/fs-12341234.home.ec2-user.efs.21036.efs-proxy.log", )), + log_format: Some(String::from("file")), nested_config: EfsConfig { listen_addr: String::from("127.0.0.1:21036"), mount_target_addr: String::from("fs-12341234.efs.us-east-1.amazonaws.com:2049"), @@ -162,6 +172,7 @@ socket = a:SO_BINDTODEVICE=lo pid = /var/run/efs/fs-12341234.home.ec2-user.efs.21036+/stunnel.pid port = 8081 initial_partition_ip = 127.0.0.1:2049 +log_format = stdout [efs] accept = 127.0.0.1:21036 @@ -187,6 +198,7 @@ checkHost = fs-12341234.efs.us-east-1.amazonaws.com ), debug: DEFAULT_LOG_LEVEL.to_string(), output: None, + log_format: Some(String::from("stdout")), nested_config: EfsConfig { listen_addr: String::from("127.0.0.1:21036"), mount_target_addr: String::from("fs-12341234.efs.us-east-1.amazonaws.com:2049"), diff --git a/src/proxy/src/efs_prot.x b/src/proxy/src/efs_prot.x index d0faeb4f..eac14ae9 100644 --- a/src/proxy/src/efs_prot.x +++ b/src/proxy/src/efs_prot.x @@ -48,10 +48,3 @@ struct BindClientResponse { BindResponse bind_response; ScaleUpConfig scale_up_config; }; - -union OperationResponse switch (OperationType operation_type) { - case OP_BIND_CLIENT_TO_PARTITION: - BindClientResponse response; - default: - void; -}; diff --git a/src/proxy/src/lib.rs b/src/proxy/src/lib.rs index 42111954..ee41164e 100644 --- a/src/proxy/src/lib.rs +++ b/src/proxy/src/lib.rs @@ -10,6 +10,7 @@ pub mod connections; pub mod controller; pub mod efs_rpc; pub mod error; +pub mod log_encoder; pub mod logger; pub mod proxy; pub mod proxy_identifier; diff --git a/src/proxy/src/log_encoder.rs b/src/proxy/src/log_encoder.rs new file mode 100644 index 00000000..e0529bf5 --- /dev/null +++ b/src/proxy/src/log_encoder.rs @@ -0,0 +1,132 @@ +use anyhow::Result; +use chrono::Utc; +use log4rs::encode::{Encode, Write}; +use std::fmt; + +/// Custom encoder that replaces newlines with spaces to keep multi-line logs on a single line +pub struct SingleLineEncoder; + +impl Encode for SingleLineEncoder { + fn encode(&self, w: &mut dyn Write, record: &log::Record<'_>) -> Result<()> { + let timestamp = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ"); + let level = record.level(); + let module = record.module_path().unwrap_or("-"); + let message = format!("{}", record.args()); + let single_line_message = message.replace('\n', " "); + + writeln!( + w, + "{} {} {} {} {}", + timestamp, + std::process::id(), + level, + module, + single_line_message + ) + .map_err(|e| anyhow::anyhow!(e)) + } +} + +impl fmt::Debug for SingleLineEncoder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SingleLineEncoder").finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use log::{Level, Record}; + use regex::Regex; + use std::io; + + struct BufferWriter<'a>(&'a mut Vec); + + impl<'a> io::Write for BufferWriter<'a> { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + impl<'a> Write for BufferWriter<'a> { + // This trait is implemented automatically because BufferWriter implements io::Write + } + + #[test] + fn test_format_log_message() { + let encoder = SingleLineEncoder; + + let record = Record::builder() + .args(format_args!("Test message")) + .level(Level::Info) + .target("test_target") + .module_path(Some("test_module")) + .file(Some("test_file.rs")) + .line(Some(42)) + .build(); + + let mut buffer = Vec::new(); + + let mut writer = BufferWriter(&mut buffer); + encoder.encode(&mut writer, &record).unwrap(); + + let output = String::from_utf8_lossy(&buffer); + + let timestamp_regex = r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z"; + let pid_regex = r"\d+"; + let level_regex = r"INFO"; + let module_regex = r"test_module"; + let message_regex = r"Test message"; + + let pattern = format!( + "^{} {} {} {} {}$", + timestamp_regex, pid_regex, level_regex, module_regex, message_regex + ); + + let regex = Regex::new(&pattern).unwrap(); + assert!( + regex.is_match(output.trim()), + "Output format doesn't match expected pattern. Got: {}", + output + ); + } + + #[test] + fn test_multiline_message() { + let encoder = SingleLineEncoder; + + let record = Record::builder() + .args(format_args!("Test\nmultiline\nmessage")) + .level(Level::Warn) + .target("test_target") + .module_path(Some("test_module")) + .file(Some("test_file.rs")) + .line(Some(42)) + .build(); + + let mut buffer = Vec::new(); + + let mut writer = BufferWriter(&mut buffer); + encoder.encode(&mut writer, &record).unwrap(); + + let output = String::from_utf8_lossy(&buffer); + + assert!( + output.contains("Test multiline message"), + "Multiline message not properly formatted. Got: {}", + output + ); + + let newline_count = output.chars().filter(|&c| c == '\n').count(); + assert_eq!( + newline_count, 1, + "Expected only one newline at the end. Got: {}", + output + ); + } +} diff --git a/src/proxy/src/logger.rs b/src/proxy/src/logger.rs index 2b7a3c70..a82b504a 100644 --- a/src/proxy/src/logger.rs +++ b/src/proxy/src/logger.rs @@ -16,50 +16,187 @@ use log4rs::{ use std::{path::Path, str::FromStr}; use crate::config_parser::ProxyConfig; +use crate::log_encoder::SingleLineEncoder; const LOG_FILE_MAX_BYTES: u64 = 1048576; const LOG_FILE_COUNT: u32 = 10; -pub fn init(config: &ProxyConfig) { - let log_file_path_string = config - .output - .clone() - .expect("config value `output` is not set"); - let log_file_path = Path::new(&log_file_path_string); +pub fn create_config(config: &ProxyConfig) -> Config { let level_filter = LevelFilter::from_str(&config.debug).expect("config value for `debug` is invalid"); - let stderr = ConsoleAppender::builder().target(Target::Stderr).build(); - - let trigger = SizeTrigger::new(LOG_FILE_MAX_BYTES); - let mut pattern = log_file_path_string.clone(); - pattern.push_str(".{}"); - let roller = FixedWindowRoller::builder() - .build(&pattern, LOG_FILE_COUNT) - .expect("Unable to create roller"); - let policy = CompoundPolicy::new(Box::new(trigger), Box::new(roller)); - - let log_file = RollingFileAppender::builder() - .encoder(Box::new(PatternEncoder::new( - "{d(%Y-%m-%dT%H:%M:%S%.3fZ)(utc)} {P} {l} {M} {m}{n}", - ))) - .build(log_file_path, Box::new(policy)) - .expect("Unable to create log file"); - - let config = Config::builder() - .appender(Appender::builder().build("logfile", Box::new(log_file))) - .appender( - Appender::builder() - .filter(Box::new(ThresholdFilter::new(LevelFilter::Error))) - .build("stderr", Box::new(stderr)), - ) - .build( - Root::builder() - .appender("logfile") - .appender("stderr") - .build(level_filter), - ) - .expect("Invalid logger config"); - - let _ = log4rs::init_config(config).expect("Unable to initialize logger"); + let log_format = config.log_format.as_deref().unwrap_or("file"); + + let mut config_builder = Config::builder(); + let mut root_builder = Root::builder(); + + match log_format { + "file" => { + let log_file_path_string = config + .output + .clone() + .expect("config value `output` is not set"); + + let log_file_path = Path::new(&log_file_path_string); + + let stderr = ConsoleAppender::builder().target(Target::Stderr).build(); + + config_builder = config_builder.appender( + Appender::builder() + .filter(Box::new(ThresholdFilter::new(LevelFilter::Error))) + .build("stderr", Box::new(stderr)), + ); + + let trigger = SizeTrigger::new(LOG_FILE_MAX_BYTES); + let mut pattern = log_file_path_string.clone(); + pattern.push_str(".{}"); + let roller = FixedWindowRoller::builder() + .build(&pattern, LOG_FILE_COUNT) + .expect("Unable to create roller"); + let policy = CompoundPolicy::new(Box::new(trigger), Box::new(roller)); + + let log_file = RollingFileAppender::builder() + .encoder(Box::new(PatternEncoder::new( + "{d(%Y-%m-%dT%H:%M:%S%.3fZ)(utc)} {P} {l} {M} {m}{n}", + ))) + .build(log_file_path, Box::new(policy)) + .expect("Unable to create log file"); + + config_builder = + config_builder.appender(Appender::builder().build("logfile", Box::new(log_file))); + + root_builder = root_builder.appender("logfile").appender("stderr"); + } + "stdout" => { + let stderr = ConsoleAppender::builder() + .target(Target::Stderr) + .encoder(Box::new(SingleLineEncoder)) + .build(); + + config_builder = config_builder.appender( + Appender::builder() + .filter(Box::new(ThresholdFilter::new(LevelFilter::Error))) + .build("stderr", Box::new(stderr)), + ); + + let stdout = ConsoleAppender::builder() + .target(Target::Stdout) + .encoder(Box::new(SingleLineEncoder)) + .build(); + + config_builder = + config_builder.appender(Appender::builder().build("stdout", Box::new(stdout))); + + root_builder = root_builder.appender("stderr").appender("stdout"); + } + _ => panic!("Invalid `log_format` value. Must be either 'file' or 'stdout'"), + } + + config_builder + .build(root_builder.build(level_filter)) + .expect("Invalid logger config") +} + +pub fn init(config: &ProxyConfig) { + let log_format = config.log_format.as_deref().unwrap_or("file"); + if log_format == "file" && config.output.is_none() { + return; + } + + let log_config = create_config(config); + let _ = log4rs::init_config(log_config).expect("Unable to initialize logger"); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config_parser::ProxyConfig; + use std::panic; + use tempfile::tempdir; + + #[test] + fn test_logger_init_with_file() { + let temp_dir = tempdir().expect("Failed to create temporary directory"); + let log_path = temp_dir.path().join("test.log"); + let log_path_str = log_path.to_str().expect("Failed to convert path to string"); + + let config = ProxyConfig { + fips: false, + debug: "info".to_string(), + output: Some(log_path_str.to_string()), + log_format: Some("file".to_string()), + pid_file_path: "".to_string(), + nested_config: Default::default(), + }; + + let result = panic::catch_unwind(|| { + init(&config); + }); + + let _ = temp_dir.close(); + + assert!( + result.is_ok(), + "Logger initialization panicked with valid config" + ); + } + + #[test] + fn test_create_config_with_file() { + let temp_dir = tempdir().expect("Failed to create temporary directory"); + let log_path = temp_dir.path().join("test.log"); + let log_path_str = log_path.to_str().expect("Failed to convert path to string"); + + let config = ProxyConfig { + fips: false, + debug: "info".to_string(), + output: Some(log_path_str.to_string()), + log_format: Some("file".to_string()), + pid_file_path: "".to_string(), + nested_config: Default::default(), + }; + + let log_config = create_config(&config); + + assert_eq!(log_config.root().level(), LevelFilter::Info); + + let _ = temp_dir.close(); + } + + #[test] + fn test_create_config_with_stdout() { + let config = ProxyConfig { + fips: false, + debug: "debug".to_string(), + output: None, + log_format: Some("stdout".to_string()), + pid_file_path: "".to_string(), + nested_config: Default::default(), + }; + + let log_config = create_config(&config); + + assert_eq!(log_config.root().level(), LevelFilter::Debug); + } + + #[test] + fn test_init_skips_when_output_none() { + let config = ProxyConfig { + fips: false, + debug: "info".to_string(), + output: None, + log_format: Some("file".to_string()), + pid_file_path: "".to_string(), + nested_config: Default::default(), + }; + + let result = panic::catch_unwind(|| { + init(&config); + }); + + assert!( + result.is_ok(), + "Logger initialization should not panic when output is None" + ); + } } diff --git a/src/proxy/src/main.rs b/src/proxy/src/main.rs index 92d4d1e4..65ecd409 100644 --- a/src/proxy/src/main.rs +++ b/src/proxy/src/main.rs @@ -18,6 +18,7 @@ mod connections; mod controller; mod efs_rpc; mod error; +mod log_encoder; mod logger; mod proxy; mod proxy_identifier; @@ -46,9 +47,7 @@ async fn main() { Err(e) => panic!("Failed to read configuration. {}", e), }; - if let Some(_log_file_path) = &proxy_config.output { - logger::init(&proxy_config) - } + logger::init(&proxy_config); info!("Running with configuration: {:?}", proxy_config); diff --git a/src/proxy/src/proxy_identifier.rs b/src/proxy/src/proxy_identifier.rs index 0e986864..75c71f22 100644 --- a/src/proxy/src/proxy_identifier.rs +++ b/src/proxy/src/proxy_identifier.rs @@ -8,6 +8,12 @@ pub struct ProxyIdentifier { pub incarnation: i64, } +impl Default for ProxyIdentifier { + fn default() -> Self { + Self::new() + } +} + impl ProxyIdentifier { pub fn new() -> Self { ProxyIdentifier { @@ -32,7 +38,7 @@ mod tests { #[test] fn test_increment() { - let mut proxy_id = ProxyIdentifier::new(); + let mut proxy_id = ProxyIdentifier::default(); let proxy_id_original = proxy_id; for i in 0..5 { assert_eq!(i, proxy_id.incarnation); diff --git a/src/watchdog/__init__.py b/src/watchdog/__init__.py index f66e7fb4..3a196979 100755 --- a/src/watchdog/__init__.py +++ b/src/watchdog/__init__.py @@ -56,7 +56,7 @@ AMAZON_LINUX_2_RELEASE_ID, AMAZON_LINUX_2_PRETTY_NAME, ] -VERSION = "2.4.1" +VERSION = "2.4.2" SERVICE = "elasticfilesystem" CONFIG_FILE = "/etc/amazon/efs/efs-utils.conf" @@ -992,10 +992,15 @@ def find_command_path(command, install_method): # For more information, see https://brew.sh/2021/02/05/homebrew-3.0.0/ else: env_path = "/opt/homebrew/bin:/usr/local/bin" - os.putenv("PATH", env_path) + + existing_path = os.environ.get("PATH", "") + search_path = env_path + ":" + existing_path if existing_path else env_path + + env = os.environ.copy() + env["PATH"] = search_path try: - path = subprocess.check_output(["which", command]) + path = subprocess.check_output(["which", command], env=env) return path.strip().decode() except subprocess.CalledProcessError as e: fatal_error( diff --git a/test/common.py b/test/common.py index 474746dd..d0dd9820 100644 --- a/test/common.py +++ b/test/common.py @@ -47,6 +47,14 @@ def _create_mock(self): communicate_return_value=(b"", b"mount.nfs4: Connection reset by peer"), ) DEFAULT_NON_RETRYABLE_FAILURE_POPEN = PopenMock( + return_code=1, + poll_result=1, + communicate_return_value=( + b"", + b"mount.nfs4: Protocol not supported", + ), +) +ACCESS_DENIED_FAILURE_POPEN = PopenMock( return_code=1, poll_result=1, communicate_return_value=( diff --git a/test/mount_efs_test/test_match_device.py b/test/mount_efs_test/test_match_device.py index af06baf4..d74401f1 100644 --- a/test/mount_efs_test/test_match_device.py +++ b/test/mount_efs_test/test_match_device.py @@ -1,123 +1,131 @@ -# Copyright 2017-2018 Amazon.com, Inc. and its affiliates. All Rights Reserved. -# -# Licensed under the MIT License. See the LICENSE accompanying this file -# for the specific language governing permissions and limitations under -# the License. - -import socket - -import pytest - -import mount_efs - -from .. import utils - -try: - import ConfigParser -except ImportError: - from configparser import ConfigParser - -DEFAULT_AZ = "us-east-1a" -CORRECT_DEVICE_DESCRIPTORS_FS_ID = [ - ("fs-deadbeef", ("fs-deadbeef", "/", None)), - ("fs-deadbeef:/", ("fs-deadbeef", "/", None)), - ("fs-deadbeef:/some/subpath", ("fs-deadbeef", "/some/subpath", None)), - ( - "fs-deadbeef:/some/subpath/with:colons", - ("fs-deadbeef", "/some/subpath/with:colons", None), - ), -] -CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS = [ - ("custom-cname.example.com", ("fs-deadbeef", "/", None)), - ("custom-cname.example.com:/", ("fs-deadbeef", "/", None)), - ("custom-cname.example.com:/some/subpath", ("fs-deadbeef", "/some/subpath", None)), - ( - "custom-cname.example.com:/some/subpath/with:colons", - ("fs-deadbeef", "/some/subpath/with:colons", None), - ), -] -CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS_WITH_AZ = [ - ("custom-cname.example.com", ("fs-deadbeef", "/", DEFAULT_AZ)), - ("custom-cname.example.com:/", ("fs-deadbeef", "/", DEFAULT_AZ)), - ( - "custom-cname.example.com:/some/subpath", - ("fs-deadbeef", "/some/subpath", DEFAULT_AZ), - ), - ( - "custom-cname.example.com:/some/subpath/with:colons", - ("fs-deadbeef", "/some/subpath/with:colons", DEFAULT_AZ), - ), -] -DEFAULT_REGION = "us-east-1" -DEFAULT_NFS_OPTIONS = {} -FS_ID = "fs-deadbeef" -OPTIONS_WITH_AZ = {"az": DEFAULT_AZ} -TEST_SOCKET_GET_ADDR_INFO_RETURN = [ - (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80)) -] - - -@pytest.fixture(autouse=True) -def setup(mocker): - mocker.patch("mount_efs.get_target_region", return_value=DEFAULT_REGION) - mocker.patch( - "socket.getaddrinfo", - return_value=TEST_SOCKET_GET_ADDR_INFO_RETURN, - ) - - -def _get_mock_config( - dns_name_format="{az}.{fs_id}.efs.{region}.{dns_name_suffix}", - dns_name_suffix="amazonaws.com", - cloudwatch_enabled="false", - has_fallback_to_mount_target_ip_address_item=True, - fallback_to_mount_target_ip_address=False, -): - try: - config = ConfigParser.SafeConfigParser() - except AttributeError: - config = ConfigParser() - config.add_section(mount_efs.CONFIG_SECTION) - config.add_section(mount_efs.CLOUDWATCH_LOG_SECTION) - config.set(mount_efs.CONFIG_SECTION, "dns_name_format", dns_name_format) - config.set(mount_efs.CONFIG_SECTION, "dns_name_suffix", dns_name_suffix) - config.set(mount_efs.CLOUDWATCH_LOG_SECTION, "enabled", cloudwatch_enabled) - if has_fallback_to_mount_target_ip_address_item: - config.set( - mount_efs.CONFIG_SECTION, - mount_efs.FALLBACK_TO_MOUNT_TARGET_IP_ADDRESS_ITEM, - str(fallback_to_mount_target_ip_address), - ) - - return config - - -def test_match_device_correct_descriptors_fs_id(mocker): - config = _get_mock_config() - for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_FS_ID: - assert (fs_id, path, az) == mount_efs.match_device( - config, device, DEFAULT_NFS_OPTIONS - ) - - -def test_match_device_correct_descriptors_cname_dns_suffix_override_region(mocker): - get_dns_name_mock = mocker.patch( - "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", - return_value=("fs-deadbeef.efs.cn-north-1.amazonaws.com.cn", None), - ) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", - return_value=("fs-deadbeef.efs.cn-north-1.amazonaws.com.cn", [], None), - ) - config = _get_mock_config() - for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: - assert (fs_id, path, az) == mount_efs.match_device( - config, device, DEFAULT_NFS_OPTIONS - ) - utils.assert_called(get_dns_name_mock) - utils.assert_called(gethostbyname_ex_mock) - - +# Copyright 2017-2018 Amazon.com, Inc. and its affiliates. All Rights Reserved. +# +# Licensed under the MIT License. See the LICENSE accompanying this file +# for the specific language governing permissions and limitations under +# the License. + +import socket + +import pytest + +import mount_efs + +from .. import utils + +try: + import ConfigParser +except ImportError: + from configparser import ConfigParser + +DEFAULT_AZ = "us-east-1a" +CORRECT_DEVICE_DESCRIPTORS_FS_ID = [ + ("fs-deadbeef", ("fs-deadbeef", "/", None)), + ("fs-deadbeef:/", ("fs-deadbeef", "/", None)), + ("fs-deadbeef:/some/subpath", ("fs-deadbeef", "/some/subpath", None)), + ( + "fs-deadbeef:/some/subpath/with:colons", + ("fs-deadbeef", "/some/subpath/with:colons", None), + ), +] +CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS = [ + ("custom-cname.example.com", ("fs-deadbeef", "/", None)), + ("custom-cname.example.com:/", ("fs-deadbeef", "/", None)), + ("custom-cname.example.com:/some/subpath", ("fs-deadbeef", "/some/subpath", None)), + ( + "custom-cname.example.com:/some/subpath/with:colons", + ("fs-deadbeef", "/some/subpath/with:colons", None), + ), +] +CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS_WITH_AZ = [ + ("custom-cname.example.com", ("fs-deadbeef", "/", DEFAULT_AZ)), + ("custom-cname.example.com:/", ("fs-deadbeef", "/", DEFAULT_AZ)), + ( + "custom-cname.example.com:/some/subpath", + ("fs-deadbeef", "/some/subpath", DEFAULT_AZ), + ), + ( + "custom-cname.example.com:/some/subpath/with:colons", + ("fs-deadbeef", "/some/subpath/with:colons", DEFAULT_AZ), + ), +] +DEFAULT_REGION = "us-east-1" +DEFAULT_NFS_OPTIONS = {} +FS_ID = "fs-deadbeef" +OPTIONS_WITH_AZ = {"az": DEFAULT_AZ} +TEST_SOCKET_GET_ADDR_INFO_RETURN = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80)) +] + + +@pytest.fixture(autouse=True) +def setup(mocker): + mocker.patch("mount_efs.get_target_region", return_value=DEFAULT_REGION) + mocker.patch( + "socket.getaddrinfo", + return_value=TEST_SOCKET_GET_ADDR_INFO_RETURN, + ) + + +def _get_mock_config( + dns_name_format="{az}.{fs_id}.efs.{region}.{dns_name_suffix}", + dns_name_suffix="amazonaws.com", + cloudwatch_enabled="false", + has_fallback_to_mount_target_ip_address_item=True, + fallback_to_mount_target_ip_address=False, +): + try: + config = ConfigParser.SafeConfigParser() + except AttributeError: + config = ConfigParser() + config.add_section(mount_efs.CONFIG_SECTION) + config.add_section(mount_efs.CLOUDWATCH_LOG_SECTION) + config.set(mount_efs.CONFIG_SECTION, "dns_name_format", dns_name_format) + config.set(mount_efs.CONFIG_SECTION, "dns_name_suffix", dns_name_suffix) + config.set(mount_efs.CLOUDWATCH_LOG_SECTION, "enabled", cloudwatch_enabled) + if has_fallback_to_mount_target_ip_address_item: + config.set( + mount_efs.CONFIG_SECTION, + mount_efs.FALLBACK_TO_MOUNT_TARGET_IP_ADDRESS_ITEM, + str(fallback_to_mount_target_ip_address), + ) + + return config + + +def test_match_device_correct_descriptors_fs_id(mocker): + config = _get_mock_config() + for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_FS_ID: + assert (fs_id, path, az) == mount_efs.match_device( + config, device, DEFAULT_NFS_OPTIONS + ) + + +def _mock_getaddrinfo_return(canonname, family=socket.AF_INET, ip="93.184.216.34"): + """Helper to build a mock getaddrinfo return value with AI_CANONNAME.""" + sockaddr = (ip, 0) if family == socket.AF_INET else (ip, 0, 0, 0) + return [(family, socket.SOCK_STREAM, 6, canonname, sockaddr)] + + +def test_match_device_correct_descriptors_cname_dns_suffix_override_region(mocker): + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=("fs-deadbeef.efs.cn-north-1.amazonaws.com.cn", None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return( + "fs-deadbeef.efs.cn-north-1.amazonaws.com.cn" + ), + ) + config = _get_mock_config() + for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: + assert (fs_id, path, az) == mount_efs.match_device( + config, device, DEFAULT_NFS_OPTIONS + ) + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + def test_match_device_correct_descriptors_cname_dns_adc_suffix(mocker): """ADC regions use DNS suffixes with hyphens (e.g. cloud.adc-e.uk)""" adc_dns_name = "fs-deadbeef.efs.eu-isoe-west-1.cloud.adc-e.uk" @@ -125,9 +133,9 @@ def test_match_device_correct_descriptors_cname_dns_adc_suffix(mocker): "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", return_value=(adc_dns_name, None), ) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", - return_value=(adc_dns_name, [], None), + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return(adc_dns_name), ) config = _get_mock_config(dns_name_suffix="cloud.adc-e.uk") for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: @@ -135,290 +143,367 @@ def test_match_device_correct_descriptors_cname_dns_adc_suffix(mocker): config, device, DEFAULT_NFS_OPTIONS ) utils.assert_called(get_dns_name_mock) - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_correct_descriptors_cname_dns_primary(mocker): - get_dns_name_mock = mocker.patch( - "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", - return_value=("fs-deadbeef.efs.us-east-1.amazonaws.com", None), - ) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", - return_value=("fs-deadbeef.efs.us-east-1.amazonaws.com", [], None), - ) - config = _get_mock_config() - for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: - assert (fs_id, path, az) == mount_efs.match_device( - config, device, DEFAULT_NFS_OPTIONS - ) - utils.assert_called(get_dns_name_mock) - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_correct_descriptors_cname_dns_secondary(mocker): - get_dns_name_mock = mocker.patch( - "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", - return_value=("fs-deadbeef.efs.us-east-1.amazonaws.com", None), - ) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", - return_value=(None, ["fs-deadbeef.efs.us-east-1.amazonaws.com"], None), - ) - config = _get_mock_config() - for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: - assert (fs_id, path, az) == mount_efs.match_device( - config, device, DEFAULT_NFS_OPTIONS - ) - utils.assert_called(get_dns_name_mock) - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_correct_descriptors_cname_dns_tertiary(mocker): - get_dns_name_mock = mocker.patch( - "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", - return_value=("fs-deadbeef.efs.us-east-1.amazonaws.com", None), - ) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", - return_value=(None, [None, "fs-deadbeef.efs.us-east-1.amazonaws.com"], None), - ) - config = _get_mock_config() - for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: - assert (fs_id, path, az) == mount_efs.match_device( - config, device, DEFAULT_NFS_OPTIONS - ) - utils.assert_called(get_dns_name_mock) - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_correct_descriptors_cname_dns_amongst_invalid(mocker): - get_dns_name_mock = mocker.patch( - "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", - return_value=("fs-deadbeef.efs.us-east-1.amazonaws.com", None), - ) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", - return_value=( - "fs-deadbeef.efs.us-west-1.amazonaws.com", - ["fs-deadbeef.efs.us-east-1.amazonaws.com", "invalid-efs-name.example.com"], - None, - ), - ) - config = _get_mock_config() - for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: - assert (fs_id, path, az) == mount_efs.match_device( - config, device, DEFAULT_NFS_OPTIONS - ) - utils.assert_called(get_dns_name_mock) - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_unresolvable_domain(mocker, capsys): - mocker.patch("socket.gethostbyname_ex", side_effect=socket.gaierror) - config = _get_mock_config() - with pytest.raises(SystemExit) as ex: - mount_efs.match_device(config, "custom-cname.example.com", DEFAULT_NFS_OPTIONS) - - assert 0 != ex.value.code - out, err = capsys.readouterr() - assert "Failed to resolve" in err - - -def test_match_device_no_hostnames(mocker, capsys): - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", return_value=(None, [], None) - ) - config = _get_mock_config() - with pytest.raises(SystemExit) as ex: - mount_efs.match_device(config, "custom-cname.example.com", DEFAULT_NFS_OPTIONS) - - assert 0 != ex.value.code - out, err = capsys.readouterr() - assert "did not resolve to an EFS mount target" in err - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_no_hostnames2(mocker, capsys): - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", return_value=(None, [None, None], None) - ) - config = _get_mock_config() - with pytest.raises(SystemExit) as ex: - mount_efs.match_device(config, "custom-cname.example.com", DEFAULT_NFS_OPTIONS) - - assert 0 != ex.value.code - out, err = capsys.readouterr() - assert "did not resolve to an EFS mount target" in err - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_resolve_to_invalid_efs_dns_name(mocker, capsys): - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", - return_value=("invalid-efs-name.example.com", [], None), - ) - config = _get_mock_config() - with pytest.raises(SystemExit) as ex: - mount_efs.match_device(config, "custom-cname.example.com", DEFAULT_NFS_OPTIONS) - - assert 0 != ex.value.code - out, err = capsys.readouterr() - assert "did not resolve to a valid DNS name" in err - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_resolve_to_unexpected_efs_dns_name(mocker, capsys): - get_dns_name_mock = mocker.patch( - "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", - return_value=("fs-deadbeef.efs.us-west-1.amazonaws.com", None), - ) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", - return_value=("fs-deadbeef.efs.us-east-1.amazonaws.com", [], None), - ) - config = _get_mock_config() - with pytest.raises(SystemExit) as ex: - mount_efs.match_device(config, "custom-cname.example.com", DEFAULT_NFS_OPTIONS) - - assert 0 != ex.value.code - out, err = capsys.readouterr() - assert "did not resolve to a valid DNS name" in err - utils.assert_called(get_dns_name_mock) - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_fqdn_same_as_dns_name(mocker, capsys): - dns_name = "%s.efs.us-east-1.amazonaws.com" % FS_ID - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", return_value=(dns_name, [], None) - ) - efs_fqdn_match = mount_efs.EFS_FQDN_RE.match(dns_name) - assert efs_fqdn_match - assert FS_ID == efs_fqdn_match.group("fs_id") - - config = _get_mock_config() - ( - expected_dns_name, - ip_address, - ) = mount_efs.get_dns_name_and_fallback_mount_target_ip_address( - config, FS_ID, DEFAULT_NFS_OPTIONS - ) - assert dns_name == expected_dns_name - assert None == ip_address - - for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: - assert (fs_id, path, az) == mount_efs.match_device( - config, device, DEFAULT_NFS_OPTIONS - ) - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_fqdn_same_as_dns_name_with_az(mocker, capsys): - dns_name = "%s.%s.efs.us-east-1.amazonaws.com" % (DEFAULT_AZ, FS_ID) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", return_value=(dns_name, [], None) - ) - efs_fqdn_match = mount_efs.EFS_FQDN_RE.match(dns_name) - assert efs_fqdn_match - assert FS_ID == efs_fqdn_match.group("fs_id") - - config = _get_mock_config() - ( - expected_dns_name, - ip_address, - ) = mount_efs.get_dns_name_and_fallback_mount_target_ip_address( - config, FS_ID, OPTIONS_WITH_AZ - ) - assert dns_name == expected_dns_name - assert None == ip_address - for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS_WITH_AZ: - assert (fs_id, path, az) == mount_efs.match_device( - config, device, OPTIONS_WITH_AZ - ) - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_with_az_dns_name_mount_az_not_in_option(mocker): - # When dns_name is provided for mounting, if the az is not provided in the mount option, also dns_name contains az - # info, verify that the az info returned is equal to the az info in the dns name - dns_name = "us-east-1a.fs-deadbeef.efs.us-east-1.amazonaws.com" - config = _get_mock_config() - get_dns_name_mock = mocker.patch( - "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", - return_value=(dns_name, None), - ) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", return_value=(dns_name, [], None) - ) - fsid, path, az = mount_efs.match_device(config, dns_name, DEFAULT_NFS_OPTIONS) - - assert az == "us-east-1a" - - utils.assert_called(get_dns_name_mock) - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_with_az_dns_name_mount_az_in_option(mocker): - # When dns_name is provided for mounting, if the az is provided in the mount option, also dns_name contains az - # info, verify that the az info returned is equal to the az info in the dns name - dns_name = "us-east-1a.fs-deadbeef.efs.us-east-1.amazonaws.com" - config = _get_mock_config() - get_dns_name_mock = mocker.patch( - "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", - return_value=(dns_name, None), - ) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", return_value=(dns_name, [], None) - ) - fsid, path, az = mount_efs.match_device(config, dns_name, OPTIONS_WITH_AZ) - - assert az == "us-east-1a" - - utils.assert_called(get_dns_name_mock) - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_with_dns_name_mount_az_in_option(mocker): - # When dns_name is mapping to the az_dns_name, and the az field is provided to the option, verify that the az info returned is - # equal to the az info in the dns name - dns_name = "example.random.com" - az_dns_name = "us-east-1a.fs-deadbeef.efs.us-east-1.amazonaws.com" - config = _get_mock_config() - get_dns_name_mock = mocker.patch( - "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", - return_value=(az_dns_name, None), - ) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", return_value=(az_dns_name, [], None) - ) - fsid, path, az = mount_efs.match_device(config, dns_name, OPTIONS_WITH_AZ) - - assert az == "us-east-1a" - - utils.assert_called(get_dns_name_mock) - utils.assert_called(gethostbyname_ex_mock) - - -def test_match_device_with_dns_name_mount_az_in_option_not_match(mocker, capsys): - # When dns_name is mapping to the az_dns_name, and the az field is provided to the option, while the two az value is not - # the same, verify that exception is thrown - dns_name = "example.random.com" - az_dns_name = "us-east-1b.fs-deadbeef.efs.us-east-1.amazonaws.com" - config = _get_mock_config() - get_dns_name_mock = mocker.patch( - "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", - return_value=(az_dns_name, None), - ) - gethostbyname_ex_mock = mocker.patch( - "socket.gethostbyname_ex", return_value=(az_dns_name, [], None) - ) - - with pytest.raises(SystemExit) as ex: - mount_efs.match_device(config, dns_name, OPTIONS_WITH_AZ) - - assert 0 != ex.value.code - out, err = capsys.readouterr() - assert "does not match the az provided" in err - utils.assert_not_called(get_dns_name_mock) - utils.assert_called(gethostbyname_ex_mock) + utils.assert_called(getaddrinfo_mock) + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_correct_descriptors_cname_dns_primary(mocker): + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=("fs-deadbeef.efs.us-east-1.amazonaws.com", None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return( + "fs-deadbeef.efs.us-east-1.amazonaws.com" + ), + ) + config = _get_mock_config() + for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: + assert (fs_id, path, az) == mount_efs.match_device( + config, device, DEFAULT_NFS_OPTIONS + ) + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_correct_descriptors_cname_dns_secondary(mocker): + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=("fs-deadbeef.efs.us-east-1.amazonaws.com", None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return( + "fs-deadbeef.efs.us-east-1.amazonaws.com" + ), + ) + config = _get_mock_config() + for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: + assert (fs_id, path, az) == mount_efs.match_device( + config, device, DEFAULT_NFS_OPTIONS + ) + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_correct_descriptors_cname_dns_tertiary(mocker): + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=("fs-deadbeef.efs.us-east-1.amazonaws.com", None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return( + "fs-deadbeef.efs.us-east-1.amazonaws.com" + ), + ) + config = _get_mock_config() + for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: + assert (fs_id, path, az) == mount_efs.match_device( + config, device, DEFAULT_NFS_OPTIONS + ) + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_correct_descriptors_cname_dns_amongst_invalid(mocker): + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=("fs-deadbeef.efs.us-east-1.amazonaws.com", None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return( + "fs-deadbeef.efs.us-east-1.amazonaws.com" + ), + ) + config = _get_mock_config() + for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: + assert (fs_id, path, az) == mount_efs.match_device( + config, device, DEFAULT_NFS_OPTIONS + ) + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_unresolvable_domain(mocker, capsys): + mocker.patch("socket.getaddrinfo", side_effect=socket.gaierror) + config = _get_mock_config() + with pytest.raises(SystemExit) as ex: + mount_efs.match_device(config, "custom-cname.example.com", DEFAULT_NFS_OPTIONS) + + assert 0 != ex.value.code + out, err = capsys.readouterr() + assert "Failed to resolve" in err + + +def test_match_device_no_hostnames(mocker, capsys): + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return(""), + ) + config = _get_mock_config() + with pytest.raises(SystemExit) as ex: + mount_efs.match_device(config, "custom-cname.example.com", DEFAULT_NFS_OPTIONS) + + assert 0 != ex.value.code + out, err = capsys.readouterr() + assert "did not resolve" in err + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_no_hostnames2(mocker, capsys): + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=[ + (socket.AF_INET6, socket.SOCK_STREAM, 6, "", ("::1", 0, 0, 0)), + (socket.AF_INET6, socket.SOCK_STREAM, 6, "", ("::2", 0, 0, 0)), + ], + ) + config = _get_mock_config() + with pytest.raises(SystemExit) as ex: + mount_efs.match_device(config, "custom-cname.example.com", DEFAULT_NFS_OPTIONS) + + assert 0 != ex.value.code + out, err = capsys.readouterr() + assert "did not resolve" in err + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_resolve_to_invalid_efs_dns_name(mocker, capsys): + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return("invalid-efs-name.example.com"), + ) + config = _get_mock_config() + with pytest.raises(SystemExit) as ex: + mount_efs.match_device(config, "custom-cname.example.com", DEFAULT_NFS_OPTIONS) + + assert 0 != ex.value.code + out, err = capsys.readouterr() + assert "did not resolve to a valid DNS name" in err + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_resolve_to_unexpected_efs_dns_name(mocker, capsys): + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=("fs-deadbeef.efs.us-west-1.amazonaws.com", None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return( + "fs-deadbeef.efs.us-east-1.amazonaws.com" + ), + ) + config = _get_mock_config() + with pytest.raises(SystemExit) as ex: + mount_efs.match_device(config, "custom-cname.example.com", DEFAULT_NFS_OPTIONS) + + assert 0 != ex.value.code + out, err = capsys.readouterr() + assert "did not resolve to a valid DNS name" in err + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_fqdn_same_as_dns_name(mocker, capsys): + dns_name = "%s.efs.us-east-1.amazonaws.com" % FS_ID + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return(dns_name), + ) + efs_fqdn_match = mount_efs.EFS_FQDN_RE.match(dns_name) + assert efs_fqdn_match + assert FS_ID == efs_fqdn_match.group("fs_id") + + config = _get_mock_config() + ( + expected_dns_name, + ip_address, + ) = mount_efs.get_dns_name_and_fallback_mount_target_ip_address( + config, FS_ID, DEFAULT_NFS_OPTIONS + ) + assert dns_name == expected_dns_name + assert None == ip_address + + for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: + assert (fs_id, path, az) == mount_efs.match_device( + config, device, DEFAULT_NFS_OPTIONS + ) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_fqdn_same_as_dns_name_with_az(mocker, capsys): + dns_name = "%s.%s.efs.us-east-1.amazonaws.com" % (DEFAULT_AZ, FS_ID) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return(dns_name), + ) + efs_fqdn_match = mount_efs.EFS_FQDN_RE.match(dns_name) + assert efs_fqdn_match + assert FS_ID == efs_fqdn_match.group("fs_id") + + config = _get_mock_config() + ( + expected_dns_name, + ip_address, + ) = mount_efs.get_dns_name_and_fallback_mount_target_ip_address( + config, FS_ID, OPTIONS_WITH_AZ + ) + assert dns_name == expected_dns_name + assert None == ip_address + for device, (fs_id, path, az) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS_WITH_AZ: + assert (fs_id, path, az) == mount_efs.match_device( + config, device, OPTIONS_WITH_AZ + ) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_with_az_dns_name_mount_az_not_in_option(mocker): + # When dns_name is provided for mounting, if the az is not provided in the mount option, also dns_name contains az + # info, verify that the az info returned is equal to the az info in the dns name + dns_name = "us-east-1a.fs-deadbeef.efs.us-east-1.amazonaws.com" + config = _get_mock_config() + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=(dns_name, None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return(dns_name), + ) + fsid, path, az = mount_efs.match_device(config, dns_name, DEFAULT_NFS_OPTIONS) + + assert az == "us-east-1a" + + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_with_az_dns_name_mount_az_in_option(mocker): + # When dns_name is provided for mounting, if the az is provided in the mount option, also dns_name contains az + # info, verify that the az info returned is equal to the az info in the dns name + dns_name = "us-east-1a.fs-deadbeef.efs.us-east-1.amazonaws.com" + config = _get_mock_config() + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=(dns_name, None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return(dns_name), + ) + fsid, path, az = mount_efs.match_device(config, dns_name, OPTIONS_WITH_AZ) + + assert az == "us-east-1a" + + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_with_dns_name_mount_az_in_option(mocker): + # When dns_name is mapping to the az_dns_name, and the az field is provided to the option, verify that the az info returned is + # equal to the az info in the dns name + dns_name = "example.random.com" + az_dns_name = "us-east-1a.fs-deadbeef.efs.us-east-1.amazonaws.com" + config = _get_mock_config() + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=(az_dns_name, None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return(az_dns_name), + ) + fsid, path, az = mount_efs.match_device(config, dns_name, OPTIONS_WITH_AZ) + + assert az == "us-east-1a" + + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_with_dns_name_mount_az_in_option_not_match(mocker, capsys): + # When dns_name is mapping to the az_dns_name, and the az field is provided to the option, while the two az value is not + # the same, verify that exception is thrown + dns_name = "example.random.com" + az_dns_name = "us-east-1b.fs-deadbeef.efs.us-east-1.amazonaws.com" + config = _get_mock_config() + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=(az_dns_name, None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return(az_dns_name), + ) + + with pytest.raises(SystemExit) as ex: + mount_efs.match_device(config, dns_name, OPTIONS_WITH_AZ) + + assert 0 != ex.value.code + out, err = capsys.readouterr() + assert "does not match the az provided" in err + utils.assert_not_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_ipv6_only_mount_target_resolves_via_fqdn(mocker): + """When an FQDN resolves to an IPv6-only mount target, match_device should + succeed using getaddrinfo (AF_INET6) instead of failing like gethostbyname_ex would. + """ + dns_name = "fs-deadbeef.efs.us-east-1.amazonaws.com" + ipv6_addr = "2600:1f16:1090:8802:228c:6404:76f8:e3c5" + config = _get_mock_config() + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=(dns_name, None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return( + dns_name, family=socket.AF_INET6, ip=ipv6_addr + ), + ) + + for device, ( + expected_fs_id, + expected_path, + _, + ) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS: + assert (expected_fs_id, expected_path, None) == mount_efs.match_device( + config, device, DEFAULT_NFS_OPTIONS + ) + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) + + +def test_match_device_ipv6_only_mount_target_with_az(mocker): + """IPv6-only mount target with AZ in the resolved FQDN.""" + dns_name = "us-east-1a.fs-deadbeef.efs.us-east-1.amazonaws.com" + ipv6_addr = "2600:1f16:1090:8802:228c:6404:76f8:e3c5" + config = _get_mock_config() + get_dns_name_mock = mocker.patch( + "mount_efs.get_dns_name_and_fallback_mount_target_ip_address", + return_value=(dns_name, None), + ) + getaddrinfo_mock = mocker.patch( + "socket.getaddrinfo", + return_value=_mock_getaddrinfo_return( + dns_name, family=socket.AF_INET6, ip=ipv6_addr + ), + ) + + for device, ( + expected_fs_id, + expected_path, + _, + ) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS_WITH_AZ: + assert (expected_fs_id, expected_path, DEFAULT_AZ) == mount_efs.match_device( + config, device, OPTIONS_WITH_AZ + ) + utils.assert_called(get_dns_name_mock) + utils.assert_called(getaddrinfo_mock) diff --git a/test/mount_efs_test/test_mount_nfs.py b/test/mount_efs_test/test_mount_nfs.py index d9070846..d3f06dd6 100644 --- a/test/mount_efs_test/test_mount_nfs.py +++ b/test/mount_efs_test/test_mount_nfs.py @@ -331,6 +331,50 @@ def test_mount_nfs_not_retry_on_non_retryable_failure(mocker): utils.assert_not_called(optimize_readahead_window_mock) +def test_mount_nfs_not_retry_access_denied_without_access_point(mocker): + optimize_readahead_window_mock = mocker.patch("mount_efs.optimize_readahead_window") + + mocker.patch( + "subprocess.Popen", side_effect=[common.ACCESS_DENIED_FAILURE_POPEN.mock] + ) + + with pytest.raises(SystemExit) as ex: + mount_efs.mount_nfs( + _get_config(), + DNS_NAME, + "/", + "/mnt", + DEFAULT_OPTIONS, + ) + + assert 0 != ex.value.code + utils.assert_not_called(optimize_readahead_window_mock) + + +def test_mount_nfs_retry_access_denied_with_access_point(mocker): + optimize_readahead_window_mock = mocker.patch("mount_efs.optimize_readahead_window") + + mocker.patch( + "subprocess.Popen", return_value=common.ACCESS_DENIED_FAILURE_POPEN.mock + ) + + options = dict(DEFAULT_OPTIONS) + options["accesspoint"] = "fsap-12345" + + with pytest.raises(SystemExit) as ex: + mount_efs.mount_nfs( + _get_config(), + DNS_NAME, + "/", + "/mnt", + options, + ) + + assert 0 != ex.value.code + assert subprocess.Popen.call_count > 1 + utils.assert_not_called(optimize_readahead_window_mock) + + def test_mount_nfs_failure_after_all_attempts_fail(mocker): optimize_readahead_window_mock = mocker.patch("mount_efs.optimize_readahead_window") mocker.patch( diff --git a/test/mount_efs_test/test_write_stunnel_config_file.py b/test/mount_efs_test/test_write_stunnel_config_file.py index feaaa83d..b8606d0c 100644 --- a/test/mount_efs_test/test_write_stunnel_config_file.py +++ b/test/mount_efs_test/test_write_stunnel_config_file.py @@ -850,3 +850,24 @@ def test_write_stunnel_config_with_ipv6_and_legacy_stunnel(mocker, tmpdir): efs_proxy_enabled=False, ), ) + + +def test_write_stunnel_config_efs_proxy_skips_stunnel_options(mocker, tmpdir): + get_stunnel_options_mock = mocker.patch("mount_efs.get_stunnel_options") + mocker.patch("mount_efs.add_tunnel_ca_options") + + mount_efs.write_stunnel_config_file( + _get_config(mocker), + str(tmpdir), + FS_ID, + MOUNT_POINT, + PORT, + DNS_NAME, + VERIFY_LEVEL, + OCSP_ENABLED, + _get_mount_options_tls(), + DEFAULT_REGION, + efs_proxy_enabled=True, + ) + + get_stunnel_options_mock.assert_not_called()