From c54031c9a8507847d60549f050df4bbedce16710 Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Mon, 5 Jan 2026 23:16:52 -0500 Subject: [PATCH] fix: use localhost for Docker integration tests Docker integration tests use container IPs (172.x.x.x) which are not routable from the host on macOS with Docker Desktop. Changed to use localhost with published ports instead. --- .../testdata/file_io_gcs/docker-compose.yaml | 2 ++ .../testdata/file_io_s3/docker-compose.yaml | 2 ++ crates/iceberg/tests/file_io_gcs_test.rs | 8 +------ crates/iceberg/tests/file_io_s3_test.rs | 24 ++++++------------- crates/integration_tests/src/lib.rs | 9 +++---- .../testdata/docker-compose.yaml | 1 + 6 files changed, 16 insertions(+), 30 deletions(-) diff --git a/crates/iceberg/testdata/file_io_gcs/docker-compose.yaml b/crates/iceberg/testdata/file_io_gcs/docker-compose.yaml index 6935a08644..5e168ca696 100644 --- a/crates/iceberg/testdata/file_io_gcs/docker-compose.yaml +++ b/crates/iceberg/testdata/file_io_gcs/docker-compose.yaml @@ -18,6 +18,8 @@ services: gcs-server: image: fsouza/fake-gcs-server@sha256:36b0116fae5236e8def76ccb07761a9ca323e476f366a5f4bf449cac19deaf2d + ports: + - 4443:4443 expose: - 4443 command: --scheme http diff --git a/crates/iceberg/testdata/file_io_s3/docker-compose.yaml b/crates/iceberg/testdata/file_io_s3/docker-compose.yaml index cbce31864e..c0d723530c 100644 --- a/crates/iceberg/testdata/file_io_s3/docker-compose.yaml +++ b/crates/iceberg/testdata/file_io_s3/docker-compose.yaml @@ -18,6 +18,8 @@ services: minio: image: minio/minio:RELEASE.2024-02-26T09-33-48Z + ports: + - 9002:9000 expose: - 9000 - 9001 diff --git a/crates/iceberg/tests/file_io_gcs_test.rs b/crates/iceberg/tests/file_io_gcs_test.rs index 9fbcdadd0e..d3ef22a4ac 100644 --- a/crates/iceberg/tests/file_io_gcs_test.rs +++ b/crates/iceberg/tests/file_io_gcs_test.rs @@ -53,13 +53,7 @@ mod tests { async fn get_file_io_gcs() -> FileIO { set_up(); - let ip = DOCKER_COMPOSE_ENV - .read() - .unwrap() - .as_ref() - .unwrap() - .get_container_ip("gcs-server"); - let addr = SocketAddr::new(ip, FAKE_GCS_PORT); + let addr = SocketAddr::new("127.0.0.1".parse().unwrap(), FAKE_GCS_PORT); // A bucket must exist for FileIO create_bucket(FAKE_GCS_BUCKET, addr.to_string()) diff --git a/crates/iceberg/tests/file_io_s3_test.rs b/crates/iceberg/tests/file_io_s3_test.rs index b044128327..6360a6ffd7 100644 --- a/crates/iceberg/tests/file_io_s3_test.rs +++ b/crates/iceberg/tests/file_io_s3_test.rs @@ -18,7 +18,7 @@ //! Integration tests for FileIO S3. #[cfg(all(test, feature = "storage-s3"))] mod tests { - use std::net::{IpAddr, SocketAddr}; + use std::net::SocketAddr; use std::sync::{Arc, RwLock}; use async_trait::async_trait; @@ -32,7 +32,7 @@ mod tests { use reqsign::{AwsCredential, AwsCredentialLoad}; use reqwest::Client; - const MINIO_PORT: u16 = 9000; + const MINIO_HOST_PORT: u16 = 9002; static DOCKER_COMPOSE_ENV: RwLock> = RwLock::new(None); #[ctor] @@ -55,8 +55,7 @@ mod tests { async fn get_file_io() -> FileIO { set_up(); - let container_ip = get_container_ip("minio"); - let minio_socket_addr = SocketAddr::new(container_ip, MINIO_PORT); + let minio_socket_addr = SocketAddr::new("127.0.0.1".parse().unwrap(), MINIO_HOST_PORT); FileIOBuilder::new("s3") .with_props(vec![ @@ -69,10 +68,8 @@ mod tests { .unwrap() } - fn get_container_ip(service_name: &str) -> IpAddr { - let guard = DOCKER_COMPOSE_ENV.read().unwrap(); - let docker_compose = guard.as_ref().unwrap(); - docker_compose.get_container_ip(service_name) + fn get_minio_socket_addr() -> SocketAddr { + SocketAddr::new("127.0.0.1".parse().unwrap(), MINIO_HOST_PORT) } #[tokio::test] @@ -200,11 +197,8 @@ mod tests { let mock_loader = MockCredentialLoader::new_minio(); let custom_loader = CustomAwsCredentialLoader::new(Arc::new(mock_loader)); - // Get container info for endpoint - let container_ip = get_container_ip("minio"); - let minio_socket_addr = SocketAddr::new(container_ip, MINIO_PORT); + let minio_socket_addr = get_minio_socket_addr(); - // Build FileIO with custom credential loader let file_io_with_custom_creds = FileIOBuilder::new("s3") .with_extension(custom_loader) .with_props(vec![ @@ -214,7 +208,6 @@ mod tests { .build() .unwrap(); - // Test that the FileIO was built successfully with the custom loader match file_io_with_custom_creds.exists("s3://bucket1/any").await { Ok(_) => {} Err(e) => panic!("Failed to check existence of bucket: {e}"), @@ -225,13 +218,10 @@ mod tests { async fn test_s3_with_custom_credential_loader_integration_failure() { let _file_io = get_file_io().await; - // Create a mock credential loader with no credentials let mock_loader = MockCredentialLoader::new(None); let custom_loader = CustomAwsCredentialLoader::new(Arc::new(mock_loader)); - // Get container info for endpoint - let container_ip = get_container_ip("minio"); - let minio_socket_addr = SocketAddr::new(container_ip, MINIO_PORT); + let minio_socket_addr = get_minio_socket_addr(); // Build FileIO with custom credential loader let file_io_with_custom_creds = FileIOBuilder::new("s3") diff --git a/crates/integration_tests/src/lib.rs b/crates/integration_tests/src/lib.rs index 44f6c30240..46ec4c126a 100644 --- a/crates/integration_tests/src/lib.rs +++ b/crates/integration_tests/src/lib.rs @@ -23,6 +23,7 @@ use iceberg_test_utils::docker::DockerCompose; use iceberg_test_utils::{normalize_test_name, set_up}; const REST_CATALOG_PORT: u16 = 8181; +const MINIO_PORT: u16 = 9000; pub struct TestFixture { pub _docker_compose: DockerCompose, @@ -36,21 +37,17 @@ pub fn set_test_fixture(func: &str) -> TestFixture { format!("{}/testdata", env!("CARGO_MANIFEST_DIR")), ); - // Stop any containers from previous runs and start new ones docker_compose.down(); docker_compose.up(); - let rest_catalog_ip = docker_compose.get_container_ip("rest"); - let minio_ip = docker_compose.get_container_ip("minio"); - let catalog_config = HashMap::from([ ( REST_CATALOG_PROP_URI.to_string(), - format!("http://{rest_catalog_ip}:{REST_CATALOG_PORT}"), + format!("http://127.0.0.1:{REST_CATALOG_PORT}"), ), ( S3_ENDPOINT.to_string(), - format!("http://{}:{}", minio_ip, 9000), + format!("http://127.0.0.1:{MINIO_PORT}"), ), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), diff --git a/crates/integration_tests/testdata/docker-compose.yaml b/crates/integration_tests/testdata/docker-compose.yaml index cf0240d1a5..04abca91c1 100644 --- a/crates/integration_tests/testdata/docker-compose.yaml +++ b/crates/integration_tests/testdata/docker-compose.yaml @@ -50,6 +50,7 @@ services: networks: rest_bridge: ports: + - 9000:9000 - 9001:9001 expose: - 9001