From 81fbfaddeb13dc927b24398a1763c4d572931d96 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 31 Oct 2025 21:09:55 +0000 Subject: [PATCH 01/41] feat(storage): signed urls --- Cargo.lock | 1 + Cargo.toml | 1 + src/auth/ID_TOKEN_DESIGN_DOC.md | 108 ++++++++++ src/auth/src/credentials.rs | 76 ++++++++ src/auth/src/credentials/impersonated.rs | 31 +++ src/auth/src/credentials/mds.rs | 70 ++++++- src/auth/src/credentials/service_account.rs | 15 ++ src/auth/src/credentials/user_account.rs | 15 ++ src/auth/src/lib.rs | 3 + src/auth/src/signer.rs | 206 ++++++++++++++++++++ src/integration-tests/Cargo.toml | 3 + src/integration-tests/src/storage.rs | 30 +++ src/integration-tests/tests/driver.rs | 17 ++ src/storage/Cargo.toml | 1 + src/storage/src/storage/client.rs | 86 ++++++++ 15 files changed, 660 insertions(+), 3 deletions(-) create mode 100644 src/auth/ID_TOKEN_DESIGN_DOC.md create mode 100644 src/auth/src/signer.rs diff --git a/Cargo.lock b/Cargo.lock index b3d4802629..a55fdb195b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4381,6 +4381,7 @@ dependencies = [ "async-trait", "base64", "bytes", + "chrono", "crc32c", "flate2", "futures", diff --git a/Cargo.toml b/Cargo.toml index 9d8d69f48c..402b3ce632 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -418,5 +418,6 @@ storage-samples = { path = "src/storage/examples" } unexpected_cfgs = { level = "deny", check-cfg = [ 'cfg(google_cloud_unstable_id_token)', 'cfg(google_cloud_unstable_storage_bidi)', + 'cfg(google_cloud_unstable_signer)', 'cfg(google_cloud_unstable_tracing)', ] } diff --git a/src/auth/ID_TOKEN_DESIGN_DOC.md b/src/auth/ID_TOKEN_DESIGN_DOC.md new file mode 100644 index 0000000000..b3fb930f87 --- /dev/null +++ b/src/auth/ID_TOKEN_DESIGN_DOC.md @@ -0,0 +1,108 @@ +# Design Doc: OIDC ID Token Verification + +## tl;dr + +This document describes the design for OIDC ID Token verification in our authentication library. We will introduce a `Verifier` to make it simple to validate tokens from Google and other providers. The design focuses on a flexible API and performance, using a cache for public keys. + +## Objective + +Our main goal is to offer a simple, secure, and efficient method for applications to verify OIDC ID tokens. This is important for service-to-service authentication, for example, with services behind Identity-Aware Proxy (IAP) or Cloud Run. + +## Background + +Verifying OIDC ID tokens is a critical security task. It involves fetching the correct public key, verifying the token's signature, and validating its claims. This process can be complex and easy to get wrong. By providing a `Verifier`, we offer a reliable and easy-to-use solution that handles these complexities and follows security best practices. + +## Detailed Design + +### API + +The main API for token verification is the `Verifier` struct. It uses a builder pattern for configuration. + +```rust +pub struct Verifier { + // ... +} + +impl Verifier { + /// Creates a new Verifier with default settings. + pub fn new() -> Self { + // ... + } + + /// Sets the expected audience for the token. + /// This is a required field for validation. + pub fn with_audience>(mut self, audience: S) -> Self { + // ... + } + + /// Sets the expected email claim for the token. + /// If set, the verifier will also check if the `email_verified` claim is true. + pub fn with_email>(mut self, email: S) -> Self { + // ... + } + + /// Sets a custom JWKS URL to fetch the public keys from. + /// If not set, the URL is determined based on the token's `alg` header. + pub fn with_jwks_url>(mut self, jwks_url: S) -> Self { + // ... + } + + /// Sets the allowed clock skew for validating the token's expiration time. + /// Defaults to 10 seconds. + pub fn with_clock_skew(mut self, clock_skew: Duration) -> Self { + // ... + } + + /// Verifies the ID token and returns the claims if valid. + pub async fn verify>(&self, token: S) -> Result> { + // ... + } +} +``` + +### Implementation Details + +#### Verification Process + +The `Verifier::verify` method follows these steps: + +1. **Decode Header**: First, we decode the JWT header to get the `kid` (Key ID) and `alg` (algorithm). The `kid` is needed to select the correct key, and the `alg` helps determine the default JWKS URL if not provided. +2. **Fetch Public Key**: The `JwkClient` gets the public key. It checks a local cache first using the `kid`. If the key is not cached, it downloads the JWK set from the correct URL, finds the key with the matching `kid`, and caches it. +3. **Validate Signature and Claims**: We use the `jsonwebtoken` crate to verify the token's signature. It also validates standard claims like `iss` (issuer), `aud` (audience), and `exp` (expiration time). +4. **Validate Email**: If an email was configured in the `Verifier`, we also check that the `email` claim in the token matches and that the `email_verified` claim is `true`. + +#### JWK Client + +The `JwkClient` is responsible for getting the public keys. + +* **URL Resolution**: It determines the JWKS URL from the token's algorithm if a custom URL is not provided. It has default URLs for Google's `RS256` (OAuth2) and `ES256` (IAP) algorithms. +* **Caching**: It caches the `DecodingKey`s in memory, using the `kid` as the key in a `HashMap`. This cache is protected by a `tokio::sync::RwLock` for safe concurrent access. + +#### Proposed Cache Expiration + +Public keys can be rotated, so we should not cache them forever. Here is a proposal for cache expiration: + +1. **Store Expiration Time**: In the cache, we will store the key and its expiration time. A simple struct can be used: + ```rust + struct CacheEntry { + key: DecodingKey, + expires_at: Instant, + } + ``` +2. **Set a TTL**: Each cached key will have a "time-to-live" (TTL), for example, 1 hour. The expiration time will be `now + TTL`. +3. **Check on Access**: When we get a key from the cache, we will first check if it is expired. If it is, we will remove it and fetch a fresh one. + +This ensures that the application will use updated public keys. + +### Pros and Cons + +#### Pros + +* **Secure and Simple**: The `Verifier` handles complex security logic, making it easier for developers to use it correctly. +* **Good Performance**: Caching public keys avoids network requests and makes token verification faster. +* **Flexible**: The builder pattern and the option to use a custom JWKS URL make it adaptable for different use cases, including non-Google providers. + +#### Cons + +* **In-memory Cache**: The cache is not shared between processes. If an application runs on multiple instances, each will have its own cache. A distributed cache like Redis could be a solution but is not part of this design. +* **Cache Write Lock**: When updating the cache, a write lock is required, which blocks read access. This should be a rare event, so it is not expected to be a major performance issue. \ No newline at end of file diff --git a/src/auth/src/credentials.rs b/src/auth/src/credentials.rs index 01ca9df543..dbef7cd309 100644 --- a/src/auth/src/credentials.rs +++ b/src/auth/src/credentials.rs @@ -415,6 +415,20 @@ impl Builder { .or(self.quota_project_id); build_credentials(json_data, quota_project_id, self.scopes) } + + #[cfg(google_cloud_unstable_signer)] + pub fn signer(self) -> BuildResult { + let json_data = match load_adc()? { + AdcContents::Contents(contents) => { + Some(serde_json::from_str(&contents).map_err(BuilderError::parsing)?) + } + AdcContents::FallbackToMds => None, + }; + let quota_project_id = std::env::var(GOOGLE_CLOUD_QUOTA_PROJECT_VAR) + .ok() + .or(self.quota_project_id); + build_signer(json_data, quota_project_id, self.scopes) + } } #[derive(Debug, PartialEq)] @@ -458,6 +472,24 @@ macro_rules! config_builder { }}; } +/// Applies common optional configurations (quota project ID, scopes) to a +/// specific credential builder instance and then return a signer for it. +#[cfg(google_cloud_unstable_signer)] +macro_rules! config_signer { + ($builder_instance:expr, $quota_project_id_option:expr, $scopes_option:expr, $apply_scopes_closure:expr) => {{ + let builder = $builder_instance; + let builder = $quota_project_id_option + .into_iter() + .fold(builder, |b, qp| b.with_quota_project_id(qp)); + + let builder = $scopes_option + .into_iter() + .fold(builder, |b, s| $apply_scopes_closure(b, s)); + + builder.signer() + }}; +} + fn build_credentials( json: Option, quota_project_id: Option, @@ -508,6 +540,50 @@ fn build_credentials( } } +#[cfg(google_cloud_unstable_signer)] +fn build_signer( + json: Option, + quota_project_id: Option, + scopes: Option>, +) -> BuildResult { + match json { + None => config_signer!( + mds::Builder::from_adc(), + quota_project_id, + scopes, + |b: mds::Builder, s: Vec| b.with_scopes(s) + ), + Some(json) => { + let cred_type = extract_credential_type(&json)?; + match cred_type { + "authorized_user" => config_signer!( + user_account::Builder::new(json), + quota_project_id, + scopes, + |b: user_account::Builder, s: Vec| b.with_scopes(s) + ), + "service_account" => config_signer!( + service_account::Builder::new(json), + quota_project_id, + scopes, + |b: service_account::Builder, s: Vec| b + .with_access_specifier(service_account::AccessSpecifier::from_scopes(s)) + ), + "impersonated_service_account" => { + config_signer!( + impersonated::Builder::new(json), + quota_project_id, + scopes, + |b: impersonated::Builder, s: Vec| b.with_scopes(s) + ) + } + "external_account" => panic!("external account signer not supported yet"), + _ => Err(BuilderError::unknown_type(cred_type)), + } + } + } +} + fn path_not_found(path: String) -> BuilderError { BuilderError::loading(format!( "{path}. {}", diff --git a/src/auth/src/credentials/impersonated.rs b/src/auth/src/credentials/impersonated.rs index 18085062d9..784b2438c3 100644 --- a/src/auth/src/credentials/impersonated.rs +++ b/src/auth/src/credentials/impersonated.rs @@ -122,6 +122,7 @@ const IMPERSONATED_CREDENTIAL_TYPE: &str = "imp"; pub(crate) const DEFAULT_LIFETIME: Duration = Duration::from_secs(3600); const MSG: &str = "failed to fetch token"; +#[derive(Clone)] enum BuilderSource { FromJson(Value), FromCredentials(Credentials), @@ -430,6 +431,36 @@ impl Builder { }) } + #[cfg(google_cloud_unstable_signer)] + pub fn signer(self) -> BuildResult { + let source = self.source.clone(); + let components = match source { + BuilderSource::FromJson(json) => build_components_from_json(json)?, + BuilderSource::FromCredentials(source_credentials) => { + build_components_from_credentials( + source_credentials, + self.service_account_impersonation_url.clone(), + )? + } + }; + + // TODO: better use regex to extract email ? + let parts: Vec<&str> = components + .service_account_impersonation_url + .split("serviceAccounts/") + .collect(); + let client_email = parts[1] + .trim_end_matches(":generateAccessToken") + .to_string(); + + Ok(crate::signer::Signer { + inner: Arc::new(crate::signer::CredentialsSigner { + client_email, + inner: self.build()?, + }), + }) + } + fn build_components( self, ) -> BuildResult<( diff --git a/src/auth/src/credentials/mds.rs b/src/auth/src/credentials/mds.rs index c9d434b558..489f0bce73 100644 --- a/src/auth/src/credentials/mds.rs +++ b/src/auth/src/credentials/mds.rs @@ -259,16 +259,15 @@ impl Builder { } } - fn build_token_provider(self) -> TokenProviderWithRetry { + fn resolve_endpoint(&self) -> (String, bool) { let final_endpoint: String; let endpoint_overridden: bool; - // Determine the endpoint and whether it was overridden if let Ok(host_from_env) = std::env::var(GCE_METADATA_HOST_ENV_VAR) { // Check GCE_METADATA_HOST environment variable first final_endpoint = format!("http://{host_from_env}"); endpoint_overridden = true; - } else if let Some(builder_endpoint) = self.endpoint { + } else if let Some(builder_endpoint) = self.endpoint.clone() { // Else, check if an endpoint was provided to the mds::Builder final_endpoint = builder_endpoint; endpoint_overridden = true; @@ -277,6 +276,11 @@ impl Builder { final_endpoint = METADATA_ROOT.to_string(); endpoint_overridden = false; }; + (final_endpoint, endpoint_overridden) + } + + fn build_token_provider(self) -> TokenProviderWithRetry { + let (final_endpoint, endpoint_overridden) = self.resolve_endpoint(); let tp = MDSAccessTokenProvider::builder() .endpoint(final_endpoint) @@ -297,6 +301,66 @@ impl Builder { inner: Arc::new(mdsc), }) } + + #[cfg(google_cloud_unstable_signer)] + pub fn signer(self) -> BuildResult { + // TODO: have MDS specific impl that fetches email as needed + let (endpoint, _) = self.resolve_endpoint(); + + Ok(crate::signer::Signer { + inner: Arc::new(MDSCredentialsSigner { + endpoint, + inner: self.build()?, + }), + }) + } +} + +// Implements Signer for MDS that extends the existing CredentialsSigner by fetching +// email via MDS email endpoint. +#[derive(Clone, Debug)] +#[cfg(google_cloud_unstable_signer)] +struct MDSCredentialsSigner { + endpoint: String, + inner: Credentials, +} + +#[cfg(google_cloud_unstable_signer)] +#[async_trait::async_trait] +impl crate::signer::SigningProvider for MDSCredentialsSigner { + async fn requestor(&self) -> crate::signer::Result { + let client = Client::new(); + + let request = client + .get(format!("{}{}/email", self.endpoint, MDS_DEFAULT_URI)) + .header( + METADATA_FLAVOR, + HeaderValue::from_static(METADATA_FLAVOR_VALUE), + ); + + let response = request + .send() + .await + .map_err(crate::signer::SigningError::transport)?; + let client_email = response + .text() + .await + .map_err(crate::signer::SigningError::transport)?; + + Ok(client_email) + } + + async fn sign(&self, content: &str) -> crate::signer::Result { + // TODO: not efficient at all, recreating CredentialSigner and refetching email + let client_email = self.requestor().await?; + + let signer = crate::signer::CredentialsSigner { + client_email, + inner: self.inner.clone(), + }; + + signer.sign(content).await + } } #[async_trait::async_trait] diff --git a/src/auth/src/credentials/service_account.rs b/src/auth/src/credentials/service_account.rs index 08e3292de0..f13c9b221d 100644 --- a/src/auth/src/credentials/service_account.rs +++ b/src/auth/src/credentials/service_account.rs @@ -294,6 +294,21 @@ impl Builder { }), }) } + + #[cfg(google_cloud_unstable_signer)] + pub fn signer(self) -> BuildResult { + let service_account_key = + serde_json::from_value::(self.service_account_key.clone()) + .map_err(BuilderError::parsing)?; + let client_email = service_account_key.client_email; + + Ok(crate::signer::Signer { + inner: Arc::new(crate::signer::CredentialsSigner { + client_email, + inner: self.build()?, + }), + }) + } } /// A representation of a [service account key]. diff --git a/src/auth/src/credentials/user_account.rs b/src/auth/src/credentials/user_account.rs index 5cc093ccbd..7f9c9ab624 100644 --- a/src/auth/src/credentials/user_account.rs +++ b/src/auth/src/credentials/user_account.rs @@ -341,6 +341,21 @@ impl Builder { }), }) } + + #[cfg(google_cloud_unstable_signer)] + pub fn signer(self) -> BuildResult { + let authorized_user = + serde_json::from_value::(self.authorized_user.clone()) + .map_err(BuilderError::parsing)?; + let client_email = authorized_user.client_id; + + Ok(crate::signer::Signer { + inner: Arc::new(crate::signer::CredentialsSigner { + client_email, + inner: self.build()?, + }), + }) + } } #[derive(PartialEq)] diff --git a/src/auth/src/lib.rs b/src/auth/src/lib.rs index 29e8caaaa7..0261774975 100644 --- a/src/auth/src/lib.rs +++ b/src/auth/src/lib.rs @@ -57,3 +57,6 @@ pub(crate) mod retry; /// /// [Credentials]: https://cloud.google.com/docs/authentication#credentials pub(crate) mod headers_util; + +#[cfg(google_cloud_unstable_signer)] +pub mod signer; diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs new file mode 100644 index 0000000000..bf01ec65e6 --- /dev/null +++ b/src/auth/src/signer.rs @@ -0,0 +1,206 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::credentials::{CacheableResource, Credentials}; +use http::Extensions; +use reqwest::Client; +use std::sync::Arc; + +pub type Result = std::result::Result; + +/// An implementation of [crate::credentials::SigningProvider]. +#[derive(Clone, Debug)] +pub struct Signer { + pub(crate) inner: Arc, +} + +impl std::convert::From for Signer +where + T: SigningProvider + Send + Sync + 'static, +{ + fn from(value: T) -> Self { + Self { + inner: Arc::new(value), + } + } +} + +impl Signer { + pub async fn requestor(&self) -> Result { + self.inner.requestor().await + } + + pub async fn sign(self, content: &str) -> Result { + self.inner.sign(content).await + } +} + +// Implements Signer using IAM signBlob API and reusing using existing [Credentials] to +// authenticate to it. +#[derive(Clone, Debug)] +pub(crate) struct CredentialsSigner { + pub(crate) client_email: String, + pub(crate) inner: Credentials, +} + +#[derive(serde::Serialize)] +struct SignBlobRequest { + payload: String, +} + +#[derive(Debug, serde::Deserialize)] +struct SignBlobResponse { + #[serde(rename = "keyId")] + key_id: String, + #[serde(rename = "signedBlob")] + signed_blob: String, +} + +#[async_trait::async_trait] +impl SigningProvider for CredentialsSigner { + async fn requestor(&self) -> Result { + Ok(self.client_email.clone()) + } + + async fn sign(&self, content: &str) -> Result { + use base64::prelude::{BASE64_URL_SAFE_NO_PAD, Engine as _}; + + let source_headers = self + .inner + .headers(Extensions::new()) + .await + .map_err(SigningError::transport)?; + let source_headers = match source_headers { + CacheableResource::New { data, .. } => data, + CacheableResource::NotModified => { + unreachable!("requested source credentials without a caching etag") + } + }; + + let client_email = self.client_email.clone(); + let url = format!( + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:signBlob", + client_email + ); + + let client = Client::new(); + let payload = BASE64_URL_SAFE_NO_PAD.encode(content); + let body = SignBlobRequest { payload }; + + let response = client + .post(url) + .header("Content-Type", "application/json") + .headers(source_headers) + .json(&body) + .send() + .await + .map_err(|e| SigningError::transport(e))?; + + if !response.status().is_success() { + let err_text = response.text().await.map_err(SigningError::transport)?; + return Err(SigningError::transport(format!("err status: {err_text:?}"))); + } + + let res = response + .json::() + .await + .map_err(SigningError::parsing)?; + + println!("response: {res:?}"); + + let signature = BASE64_URL_SAFE_NO_PAD + .decode(res.signed_blob) + .map_err(SigningError::parsing)?; + let signature = String::from_utf8(signature).map_err(SigningError::parsing)?; + + Ok(signature) + } +} + +#[async_trait::async_trait] +pub trait SigningProvider: Send + Sync + std::fmt::Debug { + // represents the authorizer of the signed URL generation. + // It is typically the Google service account client email address from the Google Developers Console in the form of "xxx@developer.gserviceaccount.com". Required. + async fn requestor(&self) -> Result; + // creates a signed URL using the v4 schema. + async fn sign(&self, content: &str) -> Result; +} + +pub(crate) mod dynamic { + use super::Result; + + /// A dyn-compatible, crate-private version of `SigningProvider`. + #[async_trait::async_trait] + pub trait SigningProvider: Send + Sync + std::fmt::Debug { + async fn requestor(&self) -> Result; + async fn sign(&self, content: &str) -> Result; + } + + /// The public CredentialsProvider implements the dyn-compatible CredentialsProvider. + #[async_trait::async_trait] + impl SigningProvider for T + where + T: super::SigningProvider + Send + Sync, + { + async fn requestor(&self) -> Result { + T::requestor(self).await + } + + async fn sign(&self, content: &str) -> Result { + T::sign(self, content).await + } + } +} + +type BoxError = Box; + +#[derive(thiserror::Error, Debug)] +#[error(transparent)] +pub struct SigningError(SigningErrorKind); + +impl SigningError { + /// A problem using API to sign blob. + pub fn is_transport(&self) -> bool { + matches!(self.0, SigningErrorKind::Transport(_)) + } + + /// A problem parsing a credentials JSON specification. + pub fn is_parsing(&self) -> bool { + matches!(self.0, SigningErrorKind::Parsing(_)) + } + + /// A problem parsing a credentials specification. + pub(crate) fn parsing(source: T) -> SigningError + where + T: Into, + { + SigningError(SigningErrorKind::Parsing(source.into())) + } + + /// A problem using API to sign blob. + pub(crate) fn transport(source: T) -> SigningError + where + T: Into, + { + SigningError(SigningErrorKind::Transport(source.into())) + } +} + +#[derive(thiserror::Error, Debug)] +enum SigningErrorKind { + #[error("failed to generate signature via IAM API: {0}")] + Transport(#[source] BoxError), + #[error("failed to parse credentials: {0}")] + Parsing(#[source] BoxError), +} diff --git a/src/integration-tests/Cargo.toml b/src/integration-tests/Cargo.toml index d7e504f011..28d48a11c5 100644 --- a/src/integration-tests/Cargo.toml +++ b/src/integration-tests/Cargo.toml @@ -122,3 +122,6 @@ serde_with.workspace = true static_assertions.workspace = true test-case.workspace = true tokio.workspace = true + +[lints] +workspace = true \ No newline at end of file diff --git a/src/integration-tests/src/storage.rs b/src/integration-tests/src/storage.rs index 55885b1a42..56204e99e4 100644 --- a/src/integration-tests/src/storage.rs +++ b/src/integration-tests/src/storage.rs @@ -84,6 +84,36 @@ pub async fn objects( Ok(()) } +#[cfg(google_cloud_unstable_signer)] +pub async fn signed_urls( + builder: storage::builder::storage::ClientBuilder, + bucket_name: &str, + prefix: &str +) -> anyhow::Result<()> { + //let creds = auth::credentials::mds::Builder::default().build()?; + let client = builder.build().await?; + + // let signer = auth::credentials::mds::Builder::default().signer()?; + let signer = auth::credentials::Builder::default().signer()?; + + const CONTENTS: &str = "the quick brown fox jumps over the lazy dog"; + let insert = client + .write_object(bucket_name, format!("{prefix}/quick.text"), CONTENTS) + .set_content_type("text/plain") + .set_content_language("en") + .set_storage_class("STANDARD") + .send_unbuffered() + .await?; + tracing::info!("success with insert={insert:?}"); + + tracing::info!("testing signed_url()"); + let signed_url = client.signed_url(signer, bucket_name, &insert.name).await?; + + tracing::info!("signed_url={signed_url}"); + + Ok(()) +} + async fn read_all(mut response: ReadObjectResponse) -> Result> { let mut contents = Vec::new(); while let Some(b) = response.next().await.transpose()? { diff --git a/src/integration-tests/tests/driver.rs b/src/integration-tests/tests/driver.rs index fe98dce833..fd3297b654 100644 --- a/src/integration-tests/tests/driver.rs +++ b/src/integration-tests/tests/driver.rs @@ -216,6 +216,23 @@ mod driver { result } + #[cfg(all(test, google_cloud_unstable_signer))] + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn run_storage_signed_urls() -> integration_tests::Result<()> { + let _guard = integration_tests::enable_tracing(); + let (control, bucket) = integration_tests::storage::create_test_bucket().await?; + + let builder = Storage::builder(); + let result = integration_tests::storage::signed_urls(builder, &bucket.name, "default-endpoint") + .await + .map_err(integration_tests::report_error); + + if let Err(e) = storage_samples::cleanup_bucket(control, bucket.name.clone()).await { + tracing::error!("error cleaning up test bucket {}: {e:?}", bucket.name); + }; + result + } + #[test_case(Storage::builder(); "default")] #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn run_storage_read_object( diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index 41736a46be..cade85c90d 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -53,6 +53,7 @@ tokio = { workspace = true, features = ["io-util"] } tonic.workspace = true tracing.workspace = true uuid.workspace = true +chrono.workspace = true # Transitive dependencies. Used for minimal version selection. mime.workspace = true # Local crates diff --git a/src/storage/src/storage/client.rs b/src/storage/src/storage/client.rs index 181fe9f697..c60514cec0 100644 --- a/src/storage/src/storage/client.rs +++ b/src/storage/src/storage/client.rs @@ -225,6 +225,92 @@ where { ReadObject::new(self.stub.clone(), bucket, object, self.options.clone()) } + + #[cfg(google_cloud_unstable_signer)] + pub async fn signed_url(&self, signer: auth::signer::Signer, bucket: B, object: O) -> std::result::Result + where + B: Into, + O: Into, + { + use std::collections::BTreeMap; + use base64::{Engine, prelude::BASE64_STANDARD}; + use chrono::Utc; + use sha2::{Digest, Sha256}; + + // TODO: have builder for those parameters + let expiration = 7*24*60*60; // default is 7 days + let http_method="GET"; // default access http method + // important to use BTreeMap to keep ordered by keys + let mut headers: BTreeMap<&'static str, String> = BTreeMap::new(); + let mut query_parameters: BTreeMap<&'static str, String> = BTreeMap::new(); + + let canonical_uri = format!("/{}", object.into()); // TODO: escape object name + + let now = Utc::now(); + let request_timestamp = now.format("%Y%m%dT%H%M%SZ"); + let datestamp = now.format("%Y%m%d"); + let credential_scope = format!("{datestamp}/auto/storage/goog4_request"); + let requestor = signer.requestor().await.unwrap(); // TODO: map_err + let credential = format!("{requestor}/{credential_scope}"); + + let host = format!("{}.storage.googleapis.com", bucket.into()); + headers.insert("host", host.clone()); + let canonical_headers = "".to_string(); + let canonical_headers = headers + .iter() + .fold(canonical_headers, |acc, (k, v)| { + format!("{acc}\n{k}:{v}") + }); + + let signed_headers = "".to_string(); + let signed_headers = headers + .iter() + .fold(signed_headers, |acc, (k, _)| { + format!("{acc}{k};") + }); + let signed_headers = signed_headers.trim_end_matches(';').to_string(); + + query_parameters.insert("X-Goog-Algorithm", "GOOG4-RSA-SHA256".to_string()); + query_parameters.insert("X-Goog-Credential", credential); + query_parameters.insert("X-Goog-Expires", expiration.to_string()); + query_parameters.insert("X-Goog-SignedHeaders", signed_headers.clone()); + + let canonical_query_string = "".to_string(); + let canonical_query_string = query_parameters + .iter() + .fold(canonical_query_string, |acc, (k, v)| { + format!("{acc}{k}={v}&") + }); + let canonical_query_string = canonical_query_string.trim_start_matches('&').to_string(); + + let canonical_request = vec![ + http_method.to_string(), + canonical_uri.clone(), + canonical_query_string.clone(), + canonical_headers, + signed_headers, + "UNSIGNED-PAYLOAD".to_string(), + ].join("\n"); + + let canonical_request_hash = Sha256::digest(canonical_request.as_bytes()); + let canonical_request_hash = BASE64_STANDARD.encode(canonical_request_hash); + + let string_to_sign = vec![ + "GOOG4-RSA-SHA256".to_string(), + request_timestamp.to_string(), + credential_scope, + canonical_request_hash, + ].join("\n"); + + let signature = signer.sign(string_to_sign.as_str()).await.unwrap(); // TODO map_err + + let scheme_and_host = format!("https://{}", host); + let signed_url = format!("{}{}?{}&x-goog-signature={}", + scheme_and_host, canonical_uri, canonical_query_string, signature + ); + + Ok(signed_url) + } } impl Storage { From c1231d0b4c2d60eaa3bb6268b3721e5af32c2791 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 3 Nov 2025 17:56:42 +0000 Subject: [PATCH 02/41] fix: hex encode signatures --- Cargo.lock | 3 +++ Cargo.toml | 1 + src/auth/Cargo.toml | 1 + src/auth/src/signer.rs | 15 ++++++++------ src/integration-tests/src/storage.rs | 1 + src/storage/Cargo.toml | 2 ++ src/storage/src/storage/client.rs | 30 +++++++++++++++++----------- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a55fdb195b..989ca7201e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1222,6 +1222,7 @@ dependencies = [ "base64", "bon", "google-cloud-gax", + "hex", "http", "httptest", "mockall", @@ -4395,6 +4396,7 @@ dependencies = [ "google-cloud-storage", "google-cloud-type", "google-cloud-wkt", + "hex", "http", "http-body", "http-body-util", @@ -4422,6 +4424,7 @@ dependencies = [ "tokio-test", "tonic", "tracing", + "url", "uuid", ] diff --git a/Cargo.toml b/Cargo.toml index 402b3ce632..2c55999cf5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -328,6 +328,7 @@ uuid = { default-features = false, version = "1", features = ["v4" clap = { default-features = false, version = "4" } parse-size = { default-features = false, version = "1" } humantime = { default-features = false, version = "2" } +hex = { default-features = false, version = "0.4.3" } opentelemetry-semantic-conventions = { default-features = false, version = "0.31.0", features = [ "semconv_experimental", diff --git a/src/auth/Cargo.toml b/src/auth/Cargo.toml index 07004c0b4a..dc96de6fd5 100644 --- a/src/auth/Cargo.toml +++ b/src/auth/Cargo.toml @@ -46,6 +46,7 @@ thiserror.workspace = true time = { workspace = true, features = ["serde"] } tokio = { workspace = true, features = ["fs", "process"] } bon.workspace = true +hex = { workspace = true, features = ["std"] } # Local dependencies gax.workspace = true diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index bf01ec65e6..c97bd66ae6 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -61,8 +61,6 @@ struct SignBlobRequest { #[derive(Debug, serde::Deserialize)] struct SignBlobResponse { - #[serde(rename = "keyId")] - key_id: String, #[serde(rename = "signedBlob")] signed_blob: String, } @@ -74,7 +72,7 @@ impl SigningProvider for CredentialsSigner { } async fn sign(&self, content: &str) -> Result { - use base64::prelude::{BASE64_URL_SAFE_NO_PAD, Engine as _}; + use base64::{Engine, prelude::BASE64_STANDARD}; let source_headers = self .inner @@ -95,7 +93,7 @@ impl SigningProvider for CredentialsSigner { ); let client = Client::new(); - let payload = BASE64_URL_SAFE_NO_PAD.encode(content); + let payload = BASE64_STANDARD.encode(content); let body = SignBlobRequest { payload }; let response = client @@ -119,10 +117,15 @@ impl SigningProvider for CredentialsSigner { println!("response: {res:?}"); - let signature = BASE64_URL_SAFE_NO_PAD + let signature = BASE64_STANDARD .decode(res.signed_blob) .map_err(SigningError::parsing)?; - let signature = String::from_utf8(signature).map_err(SigningError::parsing)?; + + println!("signature base64 decode: {:?}", signature); + + let signature = hex::encode(signature); + + println!("signature hex encode: {:?}", signature); Ok(signature) } diff --git a/src/integration-tests/src/storage.rs b/src/integration-tests/src/storage.rs index 56204e99e4..51d86b2737 100644 --- a/src/integration-tests/src/storage.rs +++ b/src/integration-tests/src/storage.rs @@ -110,6 +110,7 @@ pub async fn signed_urls( let signed_url = client.signed_url(signer, bucket_name, &insert.name).await?; tracing::info!("signed_url={signed_url}"); + println!("signed_url={signed_url}"); Ok(()) } diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index cade85c90d..dd9576ee74 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -54,6 +54,8 @@ tonic.workspace = true tracing.workspace = true uuid.workspace = true chrono.workspace = true +url.workspace = true +hex = { workspace = true, features = ["std"] } # Transitive dependencies. Used for minimal version selection. mime.workspace = true # Local crates diff --git a/src/storage/src/storage/client.rs b/src/storage/src/storage/client.rs index c60514cec0..a45b642e2f 100644 --- a/src/storage/src/storage/client.rs +++ b/src/storage/src/storage/client.rs @@ -233,9 +233,10 @@ where O: Into, { use std::collections::BTreeMap; - use base64::{Engine, prelude::BASE64_STANDARD}; use chrono::Utc; + use hex; use sha2::{Digest, Sha256}; + use url::form_urlencoded; // TODO: have builder for those parameters let expiration = 7*24*60*60; // default is 7 days @@ -247,19 +248,21 @@ where let canonical_uri = format!("/{}", object.into()); // TODO: escape object name let now = Utc::now(); - let request_timestamp = now.format("%Y%m%dT%H%M%SZ"); + let request_timestamp = now.format("%Y%m%dT%H%M%SZ").to_string(); let datestamp = now.format("%Y%m%d"); let credential_scope = format!("{datestamp}/auto/storage/goog4_request"); let requestor = signer.requestor().await.unwrap(); // TODO: map_err let credential = format!("{requestor}/{credential_scope}"); - let host = format!("{}.storage.googleapis.com", bucket.into()); + let bucket = bucket.into(); + let bucket_name = bucket.trim_start_matches("projects/_/buckets/"); + let host = format!("{}.storage.googleapis.com", bucket_name); headers.insert("host", host.clone()); let canonical_headers = "".to_string(); let canonical_headers = headers .iter() .fold(canonical_headers, |acc, (k, v)| { - format!("{acc}\n{k}:{v}") + format!("{acc}{k}:{v}\n") }); let signed_headers = "".to_string(); @@ -272,16 +275,17 @@ where query_parameters.insert("X-Goog-Algorithm", "GOOG4-RSA-SHA256".to_string()); query_parameters.insert("X-Goog-Credential", credential); + query_parameters.insert("X-Goog-Date", request_timestamp.clone()); query_parameters.insert("X-Goog-Expires", expiration.to_string()); query_parameters.insert("X-Goog-SignedHeaders", signed_headers.clone()); - let canonical_query_string = "".to_string(); - let canonical_query_string = query_parameters + let mut canonical_query = form_urlencoded::Serializer::new("".to_string()); + query_parameters .iter() - .fold(canonical_query_string, |acc, (k, v)| { - format!("{acc}{k}={v}&") - }); - let canonical_query_string = canonical_query_string.trim_start_matches('&').to_string(); + .for_each(|(k, v)| { + canonical_query.append_pair(k, v); + }); + let canonical_query_string = canonical_query.finish(); let canonical_request = vec![ http_method.to_string(), @@ -292,12 +296,14 @@ where "UNSIGNED-PAYLOAD".to_string(), ].join("\n"); + println!("canonical_request: {canonical_request:?}"); + let canonical_request_hash = Sha256::digest(canonical_request.as_bytes()); - let canonical_request_hash = BASE64_STANDARD.encode(canonical_request_hash); + let canonical_request_hash = hex::encode(canonical_request_hash); let string_to_sign = vec![ "GOOG4-RSA-SHA256".to_string(), - request_timestamp.to_string(), + request_timestamp, credential_scope, canonical_request_hash, ].join("\n"); From d943c635e9fd0129243112bcb8e0c77e6bb5c2a9 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 3 Nov 2025 17:58:01 +0000 Subject: [PATCH 03/41] remove extra file --- src/auth/ID_TOKEN_DESIGN_DOC.md | 108 -------------------------------- 1 file changed, 108 deletions(-) delete mode 100644 src/auth/ID_TOKEN_DESIGN_DOC.md diff --git a/src/auth/ID_TOKEN_DESIGN_DOC.md b/src/auth/ID_TOKEN_DESIGN_DOC.md deleted file mode 100644 index b3fb930f87..0000000000 --- a/src/auth/ID_TOKEN_DESIGN_DOC.md +++ /dev/null @@ -1,108 +0,0 @@ -# Design Doc: OIDC ID Token Verification - -## tl;dr - -This document describes the design for OIDC ID Token verification in our authentication library. We will introduce a `Verifier` to make it simple to validate tokens from Google and other providers. The design focuses on a flexible API and performance, using a cache for public keys. - -## Objective - -Our main goal is to offer a simple, secure, and efficient method for applications to verify OIDC ID tokens. This is important for service-to-service authentication, for example, with services behind Identity-Aware Proxy (IAP) or Cloud Run. - -## Background - -Verifying OIDC ID tokens is a critical security task. It involves fetching the correct public key, verifying the token's signature, and validating its claims. This process can be complex and easy to get wrong. By providing a `Verifier`, we offer a reliable and easy-to-use solution that handles these complexities and follows security best practices. - -## Detailed Design - -### API - -The main API for token verification is the `Verifier` struct. It uses a builder pattern for configuration. - -```rust -pub struct Verifier { - // ... -} - -impl Verifier { - /// Creates a new Verifier with default settings. - pub fn new() -> Self { - // ... - } - - /// Sets the expected audience for the token. - /// This is a required field for validation. - pub fn with_audience>(mut self, audience: S) -> Self { - // ... - } - - /// Sets the expected email claim for the token. - /// If set, the verifier will also check if the `email_verified` claim is true. - pub fn with_email>(mut self, email: S) -> Self { - // ... - } - - /// Sets a custom JWKS URL to fetch the public keys from. - /// If not set, the URL is determined based on the token's `alg` header. - pub fn with_jwks_url>(mut self, jwks_url: S) -> Self { - // ... - } - - /// Sets the allowed clock skew for validating the token's expiration time. - /// Defaults to 10 seconds. - pub fn with_clock_skew(mut self, clock_skew: Duration) -> Self { - // ... - } - - /// Verifies the ID token and returns the claims if valid. - pub async fn verify>(&self, token: S) -> Result> { - // ... - } -} -``` - -### Implementation Details - -#### Verification Process - -The `Verifier::verify` method follows these steps: - -1. **Decode Header**: First, we decode the JWT header to get the `kid` (Key ID) and `alg` (algorithm). The `kid` is needed to select the correct key, and the `alg` helps determine the default JWKS URL if not provided. -2. **Fetch Public Key**: The `JwkClient` gets the public key. It checks a local cache first using the `kid`. If the key is not cached, it downloads the JWK set from the correct URL, finds the key with the matching `kid`, and caches it. -3. **Validate Signature and Claims**: We use the `jsonwebtoken` crate to verify the token's signature. It also validates standard claims like `iss` (issuer), `aud` (audience), and `exp` (expiration time). -4. **Validate Email**: If an email was configured in the `Verifier`, we also check that the `email` claim in the token matches and that the `email_verified` claim is `true`. - -#### JWK Client - -The `JwkClient` is responsible for getting the public keys. - -* **URL Resolution**: It determines the JWKS URL from the token's algorithm if a custom URL is not provided. It has default URLs for Google's `RS256` (OAuth2) and `ES256` (IAP) algorithms. -* **Caching**: It caches the `DecodingKey`s in memory, using the `kid` as the key in a `HashMap`. This cache is protected by a `tokio::sync::RwLock` for safe concurrent access. - -#### Proposed Cache Expiration - -Public keys can be rotated, so we should not cache them forever. Here is a proposal for cache expiration: - -1. **Store Expiration Time**: In the cache, we will store the key and its expiration time. A simple struct can be used: - ```rust - struct CacheEntry { - key: DecodingKey, - expires_at: Instant, - } - ``` -2. **Set a TTL**: Each cached key will have a "time-to-live" (TTL), for example, 1 hour. The expiration time will be `now + TTL`. -3. **Check on Access**: When we get a key from the cache, we will first check if it is expired. If it is, we will remove it and fetch a fresh one. - -This ensures that the application will use updated public keys. - -### Pros and Cons - -#### Pros - -* **Secure and Simple**: The `Verifier` handles complex security logic, making it easier for developers to use it correctly. -* **Good Performance**: Caching public keys avoids network requests and makes token verification faster. -* **Flexible**: The builder pattern and the option to use a custom JWKS URL make it adaptable for different use cases, including non-Google providers. - -#### Cons - -* **In-memory Cache**: The cache is not shared between processes. If an application runs on multiple instances, each will have its own cache. A distributed cache like Redis could be a solution but is not part of this design. -* **Cache Write Lock**: When updating the cache, a write lock is required, which blocks read access. This should be a rare event, so it is not expected to be a major performance issue. \ No newline at end of file From 4b91c2aadfa941cd64b082d3194ac7ea10e36fb9 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 17 Nov 2025 17:48:12 +0000 Subject: [PATCH 04/41] fix: add import back after merge --- Cargo.toml | 1 + src/auth/src/credentials/impersonated.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 430d8eea01..6e2eb207f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -298,6 +298,7 @@ clap = { default-features = false, version = "4" } crc32c = { default-features = false, version = "0.6.8" } futures = { default-features = false, version = "0.3" } http = { default-features = false, version = "1", features = ["std"] } +hex = { default-features = false, version = "0.4.3" } http-body = { default-features = false, version = "1" } http-body-util = { default-features = false, version = "0.1.3" } humantime = { default-features = false, version = "2" } diff --git a/src/auth/src/credentials/impersonated.rs b/src/auth/src/credentials/impersonated.rs index 8fcd49ca32..f4f00fd99a 100644 --- a/src/auth/src/credentials/impersonated.rs +++ b/src/auth/src/credentials/impersonated.rs @@ -122,6 +122,7 @@ pub(crate) const IMPERSONATED_CREDENTIAL_TYPE: &str = "imp"; pub(crate) const DEFAULT_LIFETIME: Duration = Duration::from_secs(3600); pub(crate) const MSG: &str = "failed to fetch token"; +#[derive(Clone)] pub(crate) enum BuilderSource { FromJson(Value), FromCredentials(Credentials), From 3a94b561d644ee2678c37a427864b1fd434a9716 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 17 Nov 2025 17:56:32 +0000 Subject: [PATCH 05/41] refactor: change to unstable_signed_url cfg --- Cargo.toml | 1 - src/auth/src/credentials.rs | 8 +-- src/auth/src/credentials/impersonated.rs | 2 +- src/auth/src/credentials/mds.rs | 8 +-- src/auth/src/credentials/service_account.rs | 4 +- src/auth/src/credentials/user_account.rs | 4 +- src/auth/src/lib.rs | 2 +- src/integration-tests/src/storage.rs | 6 +-- src/integration-tests/tests/driver.rs | 15 +++--- src/storage/src/storage/client.rs | 58 +++++++++++---------- 10 files changed, 55 insertions(+), 53 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6e2eb207f4..0201319847 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -434,6 +434,5 @@ unexpected_cfgs = { level = "deny", check-cfg = [ 'cfg(google_cloud_unstable_id_token)', 'cfg(google_cloud_unstable_signed_url)', 'cfg(google_cloud_unstable_storage_bidi)', - 'cfg(google_cloud_unstable_signer)', 'cfg(google_cloud_unstable_tracing)', ] } diff --git a/src/auth/src/credentials.rs b/src/auth/src/credentials.rs index e858a2ef0a..3a2e0919d3 100644 --- a/src/auth/src/credentials.rs +++ b/src/auth/src/credentials.rs @@ -416,8 +416,8 @@ impl Builder { build_credentials(json_data, quota_project_id, self.scopes) } - #[cfg(google_cloud_unstable_signer)] - pub fn signer(self) -> BuildResult { + #[cfg(google_cloud_unstable_signed_url)] + pub fn build_signer(self) -> BuildResult { let json_data = match load_adc()? { AdcContents::Contents(contents) => { Some(serde_json::from_str(&contents).map_err(BuilderError::parsing)?) @@ -474,7 +474,7 @@ macro_rules! config_builder { /// Applies common optional configurations (quota project ID, scopes) to a /// specific credential builder instance and then return a signer for it. -#[cfg(google_cloud_unstable_signer)] +#[cfg(google_cloud_unstable_signed_url)] macro_rules! config_signer { ($builder_instance:expr, $quota_project_id_option:expr, $scopes_option:expr, $apply_scopes_closure:expr) => {{ let builder = $builder_instance; @@ -540,7 +540,7 @@ fn build_credentials( } } -#[cfg(google_cloud_unstable_signer)] +#[cfg(google_cloud_unstable_signed_url)] fn build_signer( json: Option, quota_project_id: Option, diff --git a/src/auth/src/credentials/impersonated.rs b/src/auth/src/credentials/impersonated.rs index f4f00fd99a..e108f276d7 100644 --- a/src/auth/src/credentials/impersonated.rs +++ b/src/auth/src/credentials/impersonated.rs @@ -431,7 +431,7 @@ impl Builder { }) } - #[cfg(google_cloud_unstable_signer)] + #[cfg(google_cloud_unstable_signed_url)] pub fn signer(self) -> BuildResult { let source = self.source.clone(); let components = match source { diff --git a/src/auth/src/credentials/mds.rs b/src/auth/src/credentials/mds.rs index d402a7dd5d..395bb430c5 100644 --- a/src/auth/src/credentials/mds.rs +++ b/src/auth/src/credentials/mds.rs @@ -302,8 +302,8 @@ impl Builder { }) } - #[cfg(google_cloud_unstable_signer)] - pub fn signer(self) -> BuildResult { + #[cfg(google_cloud_unstable_signed_url)] + pub fn build_signer(self) -> BuildResult { // TODO: have MDS specific impl that fetches email as needed let (endpoint, _) = self.resolve_endpoint(); @@ -319,13 +319,13 @@ impl Builder { // Implements Signer for MDS that extends the existing CredentialsSigner by fetching // email via MDS email endpoint. #[derive(Clone, Debug)] -#[cfg(google_cloud_unstable_signer)] +#[cfg(google_cloud_unstable_signed_url)] struct MDSCredentialsSigner { endpoint: String, inner: Credentials, } -#[cfg(google_cloud_unstable_signer)] +#[cfg(google_cloud_unstable_signed_url)] #[async_trait::async_trait] impl crate::signer::SigningProvider for MDSCredentialsSigner { async fn requestor(&self) -> crate::signer::Result { diff --git a/src/auth/src/credentials/service_account.rs b/src/auth/src/credentials/service_account.rs index 5fa662f094..e8601ee9a6 100644 --- a/src/auth/src/credentials/service_account.rs +++ b/src/auth/src/credentials/service_account.rs @@ -295,8 +295,8 @@ impl Builder { }) } - #[cfg(google_cloud_unstable_signer)] - pub fn signer(self) -> BuildResult { + #[cfg(google_cloud_unstable_signed_url)] + pub fn build_signer(self) -> BuildResult { let service_account_key = serde_json::from_value::(self.service_account_key.clone()) .map_err(BuilderError::parsing)?; diff --git a/src/auth/src/credentials/user_account.rs b/src/auth/src/credentials/user_account.rs index 960accc623..1d00f19b1f 100644 --- a/src/auth/src/credentials/user_account.rs +++ b/src/auth/src/credentials/user_account.rs @@ -342,8 +342,8 @@ impl Builder { }) } - #[cfg(google_cloud_unstable_signer)] - pub fn signer(self) -> BuildResult { + #[cfg(google_cloud_unstable_signed_url)] + pub fn build_signer(self) -> BuildResult { let authorized_user = serde_json::from_value::(self.authorized_user.clone()) .map_err(BuilderError::parsing)?; diff --git a/src/auth/src/lib.rs b/src/auth/src/lib.rs index 0261774975..7224fb97e6 100644 --- a/src/auth/src/lib.rs +++ b/src/auth/src/lib.rs @@ -58,5 +58,5 @@ pub(crate) mod retry; /// [Credentials]: https://cloud.google.com/docs/authentication#credentials pub(crate) mod headers_util; -#[cfg(google_cloud_unstable_signer)] +#[cfg(google_cloud_unstable_signed_url)] pub mod signer; diff --git a/src/integration-tests/src/storage.rs b/src/integration-tests/src/storage.rs index 42b35cf980..6cfa8aa980 100644 --- a/src/integration-tests/src/storage.rs +++ b/src/integration-tests/src/storage.rs @@ -83,15 +83,15 @@ pub async fn objects( Ok(()) } -#[cfg(google_cloud_unstable_signer)] +#[cfg(google_cloud_unstable_signed_url)] pub async fn signed_urls( builder: storage::builder::storage::ClientBuilder, bucket_name: &str, - prefix: &str + prefix: &str, ) -> anyhow::Result<()> { //let creds = auth::credentials::mds::Builder::default().build()?; let client = builder.build().await?; - + // let signer = auth::credentials::mds::Builder::default().signer()?; let signer = auth::credentials::Builder::default().signer()?; diff --git a/src/integration-tests/tests/driver.rs b/src/integration-tests/tests/driver.rs index 97fcd09a07..9ef7003037 100644 --- a/src/integration-tests/tests/driver.rs +++ b/src/integration-tests/tests/driver.rs @@ -200,17 +200,18 @@ mod driver { result } - #[cfg(all(test, google_cloud_unstable_signer))] + #[cfg(all(test, google_cloud_unstable_signed_url))] #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn run_storage_signed_urls() -> integration_tests::Result<()> { let _guard = integration_tests::enable_tracing(); - let (control, bucket) = integration_tests::storage::create_test_bucket().await?; - + let (control, bucket) = integration_tests::storage::create_test_bucket().await?; + let builder = Storage::builder(); - let result = integration_tests::storage::signed_urls(builder, &bucket.name, "default-endpoint") - .await - .map_err(integration_tests::report_error); - + let result = + integration_tests::storage::signed_urls(builder, &bucket.name, "default-endpoint") + .await + .map_err(integration_tests::report_error); + if let Err(e) = storage_samples::cleanup_bucket(control, bucket.name.clone()).await { tracing::error!("error cleaning up test bucket {}: {e:?}", bucket.name); }; diff --git a/src/storage/src/storage/client.rs b/src/storage/src/storage/client.rs index f2e17dc980..0feb55fbfc 100644 --- a/src/storage/src/storage/client.rs +++ b/src/storage/src/storage/client.rs @@ -225,26 +225,31 @@ where { ReadObject::new(self.stub.clone(), bucket, object, self.options.clone()) } - - #[cfg(google_cloud_unstable_signer)] - pub async fn signed_url(&self, signer: auth::signer::Signer, bucket: B, object: O) -> std::result::Result + + #[cfg(google_cloud_unstable_signed_url)] + pub async fn signed_url( + &self, + signer: auth::signer::Signer, + bucket: B, + object: O, + ) -> std::result::Result where B: Into, O: Into, - { - use std::collections::BTreeMap; + { use chrono::Utc; use hex; use sha2::{Digest, Sha256}; + use std::collections::BTreeMap; use url::form_urlencoded; // TODO: have builder for those parameters - let expiration = 7*24*60*60; // default is 7 days - let http_method="GET"; // default access http method + let expiration = 7 * 24 * 60 * 60; // default is 7 days + let http_method = "GET"; // default access http method // important to use BTreeMap to keep ordered by keys - let mut headers: BTreeMap<&'static str, String> = BTreeMap::new(); - let mut query_parameters: BTreeMap<&'static str, String> = BTreeMap::new(); - + let mut headers: BTreeMap<&'static str, String> = BTreeMap::new(); + let mut query_parameters: BTreeMap<&'static str, String> = BTreeMap::new(); + let canonical_uri = format!("/{}", object.into()); // TODO: escape object name let now = Utc::now(); @@ -257,20 +262,16 @@ where let bucket = bucket.into(); let bucket_name = bucket.trim_start_matches("projects/_/buckets/"); let host = format!("{}.storage.googleapis.com", bucket_name); - headers.insert("host", host.clone()); + headers.insert("host", host.clone()); let canonical_headers = "".to_string(); let canonical_headers = headers .iter() - .fold(canonical_headers, |acc, (k, v)| { - format!("{acc}{k}:{v}\n") - }); + .fold(canonical_headers, |acc, (k, v)| format!("{acc}{k}:{v}\n")); let signed_headers = "".to_string(); let signed_headers = headers .iter() - .fold(signed_headers, |acc, (k, _)| { - format!("{acc}{k};") - }); + .fold(signed_headers, |acc, (k, _)| format!("{acc}{k};")); let signed_headers = signed_headers.trim_end_matches(';').to_string(); query_parameters.insert("X-Goog-Algorithm", "GOOG4-RSA-SHA256".to_string()); @@ -280,13 +281,11 @@ where query_parameters.insert("X-Goog-SignedHeaders", signed_headers.clone()); let mut canonical_query = form_urlencoded::Serializer::new("".to_string()); - query_parameters - .iter() - .for_each(|(k, v)| { - canonical_query.append_pair(k, v); - }); + query_parameters.iter().for_each(|(k, v)| { + canonical_query.append_pair(k, v); + }); let canonical_query_string = canonical_query.finish(); - + let canonical_request = vec![ http_method.to_string(), canonical_uri.clone(), @@ -294,11 +293,12 @@ where canonical_headers, signed_headers, "UNSIGNED-PAYLOAD".to_string(), - ].join("\n"); + ] + .join("\n"); println!("canonical_request: {canonical_request:?}"); - let canonical_request_hash = Sha256::digest(canonical_request.as_bytes()); + let canonical_request_hash = Sha256::digest(canonical_request.as_bytes()); let canonical_request_hash = hex::encode(canonical_request_hash); let string_to_sign = vec![ @@ -306,12 +306,14 @@ where request_timestamp, credential_scope, canonical_request_hash, - ].join("\n"); - + ] + .join("\n"); + let signature = signer.sign(string_to_sign.as_str()).await.unwrap(); // TODO map_err let scheme_and_host = format!("https://{}", host); - let signed_url = format!("{}{}?{}&x-goog-signature={}", + let signed_url = format!( + "{}{}?{}&x-goog-signature={}", scheme_and_host, canonical_uri, canonical_query_string, signature ); From 03e1522ad98b83e0b95def26d53b03f6b088edb3 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 17 Nov 2025 18:19:26 +0000 Subject: [PATCH 06/41] fix: change signer to build_signer --- src/auth/src/credentials.rs | 2 +- src/auth/src/credentials/impersonated.rs | 2 +- src/integration-tests/src/storage.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/auth/src/credentials.rs b/src/auth/src/credentials.rs index 3a2e0919d3..61f4270320 100644 --- a/src/auth/src/credentials.rs +++ b/src/auth/src/credentials.rs @@ -486,7 +486,7 @@ macro_rules! config_signer { .into_iter() .fold(builder, |b, s| $apply_scopes_closure(b, s)); - builder.signer() + builder.build_signer() }}; } diff --git a/src/auth/src/credentials/impersonated.rs b/src/auth/src/credentials/impersonated.rs index e108f276d7..d8f8897c7a 100644 --- a/src/auth/src/credentials/impersonated.rs +++ b/src/auth/src/credentials/impersonated.rs @@ -432,7 +432,7 @@ impl Builder { } #[cfg(google_cloud_unstable_signed_url)] - pub fn signer(self) -> BuildResult { + pub fn build_signer(self) -> BuildResult { let source = self.source.clone(); let components = match source { BuilderSource::FromJson(json) => build_components_from_json(json)?, diff --git a/src/integration-tests/src/storage.rs b/src/integration-tests/src/storage.rs index 6cfa8aa980..0f3ba22444 100644 --- a/src/integration-tests/src/storage.rs +++ b/src/integration-tests/src/storage.rs @@ -92,8 +92,8 @@ pub async fn signed_urls( //let creds = auth::credentials::mds::Builder::default().build()?; let client = builder.build().await?; - // let signer = auth::credentials::mds::Builder::default().signer()?; - let signer = auth::credentials::Builder::default().signer()?; + // let signer = auth::credentials::mds::Builder::default().build_signer()?; + let signer = auth::credentials::Builder::default().build_signer()?; const CONTENTS: &str = "the quick brown fox jumps over the lazy dog"; let insert = client From 56c808073bab863b5e2b07c93cb559664849d5e1 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 17 Nov 2025 20:19:40 +0000 Subject: [PATCH 07/41] refactor: add SignObject builder --- src/auth/src/signer.rs | 6 - src/integration-tests/src/storage.rs | 5 +- src/storage/src/lib.rs | 2 + src/storage/src/storage.rs | 2 + src/storage/src/storage/client.rs | 86 +-------------- src/storage/src/storage/sign_object.rs | 146 +++++++++++++++++++++++++ 6 files changed, 157 insertions(+), 90 deletions(-) create mode 100644 src/storage/src/storage/sign_object.rs diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index c97bd66ae6..fe24eecc32 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -115,18 +115,12 @@ impl SigningProvider for CredentialsSigner { .await .map_err(SigningError::parsing)?; - println!("response: {res:?}"); - let signature = BASE64_STANDARD .decode(res.signed_blob) .map_err(SigningError::parsing)?; - println!("signature base64 decode: {:?}", signature); - let signature = hex::encode(signature); - println!("signature hex encode: {:?}", signature); - Ok(signature) } } diff --git a/src/integration-tests/src/storage.rs b/src/integration-tests/src/storage.rs index 0f3ba22444..db42b6af61 100644 --- a/src/integration-tests/src/storage.rs +++ b/src/integration-tests/src/storage.rs @@ -106,7 +106,10 @@ pub async fn signed_urls( tracing::info!("success with insert={insert:?}"); tracing::info!("testing signed_url()"); - let signed_url = client.signed_url(signer, bucket_name, &insert.name).await?; + let signed_url = client + .signed_url(signer, bucket_name, &insert.name) + .send() + .await?; tracing::info!("signed_url={signed_url}"); println!("signed_url={signed_url}"); diff --git a/src/storage/src/lib.rs b/src/storage/src/lib.rs index 309670445c..e2de5a6d2b 100644 --- a/src/storage/src/lib.rs +++ b/src/storage/src/lib.rs @@ -53,6 +53,8 @@ pub mod builder { //! Request builders for [Storage][crate::client::Storage]. pub use crate::storage::client::ClientBuilder; pub use crate::storage::read_object::ReadObject; + #[cfg(google_cloud_unstable_signed_url)] + pub use crate::storage::sign_object::SignObject; pub use crate::storage::write_object::WriteObject; } pub mod storage_control { diff --git a/src/storage/src/storage.rs b/src/storage/src/storage.rs index f72b27042b..34301aac48 100644 --- a/src/storage/src/storage.rs +++ b/src/storage/src/storage.rs @@ -21,6 +21,8 @@ pub(crate) mod common_options; pub(crate) mod perform_upload; pub(crate) mod read_object; pub mod request_options; +#[cfg(google_cloud_unstable_signed_url)] +pub(crate) mod sign_object; pub mod streaming_source; pub mod stub; pub(crate) mod transport; diff --git a/src/storage/src/storage/client.rs b/src/storage/src/storage/client.rs index 0feb55fbfc..aa4fad0c65 100644 --- a/src/storage/src/storage/client.rs +++ b/src/storage/src/storage/client.rs @@ -227,97 +227,17 @@ where } #[cfg(google_cloud_unstable_signed_url)] - pub async fn signed_url( + pub fn signed_url( &self, signer: auth::signer::Signer, bucket: B, object: O, - ) -> std::result::Result + ) -> crate::builder::storage::SignObject where B: Into, O: Into, { - use chrono::Utc; - use hex; - use sha2::{Digest, Sha256}; - use std::collections::BTreeMap; - use url::form_urlencoded; - - // TODO: have builder for those parameters - let expiration = 7 * 24 * 60 * 60; // default is 7 days - let http_method = "GET"; // default access http method - // important to use BTreeMap to keep ordered by keys - let mut headers: BTreeMap<&'static str, String> = BTreeMap::new(); - let mut query_parameters: BTreeMap<&'static str, String> = BTreeMap::new(); - - let canonical_uri = format!("/{}", object.into()); // TODO: escape object name - - let now = Utc::now(); - let request_timestamp = now.format("%Y%m%dT%H%M%SZ").to_string(); - let datestamp = now.format("%Y%m%d"); - let credential_scope = format!("{datestamp}/auto/storage/goog4_request"); - let requestor = signer.requestor().await.unwrap(); // TODO: map_err - let credential = format!("{requestor}/{credential_scope}"); - - let bucket = bucket.into(); - let bucket_name = bucket.trim_start_matches("projects/_/buckets/"); - let host = format!("{}.storage.googleapis.com", bucket_name); - headers.insert("host", host.clone()); - let canonical_headers = "".to_string(); - let canonical_headers = headers - .iter() - .fold(canonical_headers, |acc, (k, v)| format!("{acc}{k}:{v}\n")); - - let signed_headers = "".to_string(); - let signed_headers = headers - .iter() - .fold(signed_headers, |acc, (k, _)| format!("{acc}{k};")); - let signed_headers = signed_headers.trim_end_matches(';').to_string(); - - query_parameters.insert("X-Goog-Algorithm", "GOOG4-RSA-SHA256".to_string()); - query_parameters.insert("X-Goog-Credential", credential); - query_parameters.insert("X-Goog-Date", request_timestamp.clone()); - query_parameters.insert("X-Goog-Expires", expiration.to_string()); - query_parameters.insert("X-Goog-SignedHeaders", signed_headers.clone()); - - let mut canonical_query = form_urlencoded::Serializer::new("".to_string()); - query_parameters.iter().for_each(|(k, v)| { - canonical_query.append_pair(k, v); - }); - let canonical_query_string = canonical_query.finish(); - - let canonical_request = vec![ - http_method.to_string(), - canonical_uri.clone(), - canonical_query_string.clone(), - canonical_headers, - signed_headers, - "UNSIGNED-PAYLOAD".to_string(), - ] - .join("\n"); - - println!("canonical_request: {canonical_request:?}"); - - let canonical_request_hash = Sha256::digest(canonical_request.as_bytes()); - let canonical_request_hash = hex::encode(canonical_request_hash); - - let string_to_sign = vec![ - "GOOG4-RSA-SHA256".to_string(), - request_timestamp, - credential_scope, - canonical_request_hash, - ] - .join("\n"); - - let signature = signer.sign(string_to_sign.as_str()).await.unwrap(); // TODO map_err - - let scheme_and_host = format!("https://{}", host); - let signed_url = format!( - "{}{}?{}&x-goog-signature={}", - scheme_and_host, canonical_uri, canonical_query_string, signature - ); - - Ok(signed_url) + crate::builder::storage::SignObject::new(signer, bucket.into(), object.into()) } } diff --git a/src/storage/src/storage/sign_object.rs b/src/storage/src/storage/sign_object.rs new file mode 100644 index 0000000000..f5ead6cd58 --- /dev/null +++ b/src/storage/src/storage/sign_object.rs @@ -0,0 +1,146 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Error, Result}; +use auth::signer::Signer; +use chrono::Utc; +use hex; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use url::form_urlencoded; + +/// A builder for creating signed URLs. +pub struct SignObject { + signer: Signer, + bucket: String, + object: String, + method: String, + expiration: std::time::Duration, + headers: BTreeMap<&'static str, String>, + query_parameters: BTreeMap<&'static str, String>, +} + +impl SignObject { + pub(crate) fn new(signer: Signer, bucket: String, object: String) -> Self { + Self { + signer, + bucket, + object, + method: "GET".to_string(), + expiration: std::time::Duration::from_secs(7 * 24 * 60 * 60), // 7 days + headers: BTreeMap::new(), + query_parameters: BTreeMap::new(), + } + } + + /// Sets the HTTP method for the signed URL. Default is "GET". + pub fn with_method>(mut self, method: S) -> Self { + self.method = method.into(); + self + } + + /// Sets the expiration time for the signed URL. Default is 7 days. + pub fn with_expiration(mut self, expiration: std::time::Duration) -> Self { + self.expiration = expiration; + self + } + + /// Adds a header to the signed URL. + /// Note: These headers must be present in the request when using the signed URL. + pub fn with_header>(mut self, key: &'static str, value: S) -> Self { + self.headers.insert(key, value.into()); + self + } + + /// Adds a query parameter to the signed URL. + pub fn with_query_param>(mut self, key: &'static str, value: S) -> Self { + self.query_parameters.insert(key, value.into()); + self + } + + /// Generates the signed URL. + pub async fn send(self) -> Result { + let canonical_uri = format!("/{}", self.object); // TODO: escape object name + + let now = Utc::now(); + let request_timestamp = now.format("%Y%m%dT%H%M%SZ").to_string(); + let datestamp = now.format("%Y%m%d"); + let credential_scope = format!("{datestamp}/auto/storage/goog4_request"); + let requestor = self.signer.requestor().await.map_err(Error::io)?; // TODO map to proper error + let credential = format!("{requestor}/{credential_scope}"); + + let bucket_name = self.bucket.trim_start_matches("projects/_/buckets/"); + let host = format!("{}.storage.googleapis.com", bucket_name); + + let mut headers = self.headers; + headers.insert("host", host.clone()); + + let canonical_headers = headers + .iter() + .fold("".to_string(), |acc, (k, v)| format!("{acc}{k}:{v}\n")); + + let signed_headers = headers + .iter() + .fold("".to_string(), |acc, (k, _)| format!("{acc}{k};")); + let signed_headers = signed_headers.trim_end_matches(';').to_string(); + + let mut query_parameters = self.query_parameters; + query_parameters.insert("X-Goog-Algorithm", "GOOG4-RSA-SHA256".to_string()); + query_parameters.insert("X-Goog-Credential", credential); + query_parameters.insert("X-Goog-Date", request_timestamp.clone()); + query_parameters.insert("X-Goog-Expires", self.expiration.as_secs().to_string()); + query_parameters.insert("X-Goog-SignedHeaders", signed_headers.clone()); + + let mut canonical_query = form_urlencoded::Serializer::new("".to_string()); + query_parameters.iter().for_each(|(k, v)| { + canonical_query.append_pair(k, v); + }); + let canonical_query_string = canonical_query.finish(); + + let canonical_request = vec![ + self.method, + canonical_uri.clone(), + canonical_query_string.clone(), + canonical_headers, + signed_headers, + "UNSIGNED-PAYLOAD".to_string(), + ] + .join("\n"); + + let canonical_request_hash = Sha256::digest(canonical_request.as_bytes()); + let canonical_request_hash = hex::encode(canonical_request_hash); + + let string_to_sign = vec![ + "GOOG4-RSA-SHA256".to_string(), + request_timestamp, + credential_scope, + canonical_request_hash, + ] + .join("\n"); + + let signature = self + .signer + .sign(string_to_sign.as_str()) + .await + .map_err(Error::io)?; // TODO map to proper error + + let scheme_and_host = format!("https://{}", host); + let signed_url = format!( + "{}{}?{}&x-goog-signature={}", + scheme_and_host, canonical_uri, canonical_query_string, signature + ); + + Ok(signed_url) + } +} From 005ee7c59ede25e0995661a50e7b7a39caaa0b44 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 17 Nov 2025 20:36:47 +0000 Subject: [PATCH 08/41] test: read signed_url to check if content matches --- src/integration-tests/Cargo.toml | 1 + src/integration-tests/src/storage.rs | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/integration-tests/Cargo.toml b/src/integration-tests/Cargo.toml index 26b1e82b6d..a968a03892 100644 --- a/src/integration-tests/Cargo.toml +++ b/src/integration-tests/Cargo.toml @@ -46,6 +46,7 @@ opentelemetry_sdk.workspace = true opentelemetry-otlp.workspace = true opentelemetry-proto.workspace = true rand = { workspace = true, features = ["thread_rng"] } +reqwest.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["process", "test-util"] } tokio-stream = { workspace = true, features = ["net"] } diff --git a/src/integration-tests/src/storage.rs b/src/integration-tests/src/storage.rs index db42b6af61..e07fa0cb77 100644 --- a/src/integration-tests/src/storage.rs +++ b/src/integration-tests/src/storage.rs @@ -112,7 +112,13 @@ pub async fn signed_urls( .await?; tracing::info!("signed_url={signed_url}"); - println!("signed_url={signed_url}"); + + // Download the contents of the object using the signed URL. + let client = reqwest::Client::new(); + let res = client.get(signed_url).send().await?; + let out = res.text().await?; + assert_eq!(out, CONTENTS); + tracing::info!("signed url works and can read contents={out:?}"); Ok(()) } From cf0e5b23f2ad4730fde9afba31a1a9af073cbdc9 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 17 Nov 2025 20:48:21 +0000 Subject: [PATCH 09/41] refactor: rename requestor to client_email --- src/auth/src/credentials/mds.rs | 4 ++-- src/auth/src/signer.rs | 14 +++++++------- src/storage/src/storage/sign_object.rs | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/auth/src/credentials/mds.rs b/src/auth/src/credentials/mds.rs index 395bb430c5..b56d885b06 100644 --- a/src/auth/src/credentials/mds.rs +++ b/src/auth/src/credentials/mds.rs @@ -328,7 +328,7 @@ struct MDSCredentialsSigner { #[cfg(google_cloud_unstable_signed_url)] #[async_trait::async_trait] impl crate::signer::SigningProvider for MDSCredentialsSigner { - async fn requestor(&self) -> crate::signer::Result { + async fn client_email(&self) -> crate::signer::Result { let client = Client::new(); let request = client @@ -352,7 +352,7 @@ impl crate::signer::SigningProvider for MDSCredentialsSigner { async fn sign(&self, content: &str) -> crate::signer::Result { // TODO: not efficient at all, recreating CredentialSigner and refetching email - let client_email = self.requestor().await?; + let client_email = self.client_email().await?; let signer = crate::signer::CredentialsSigner { client_email, diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index fe24eecc32..0e000fc7d7 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -37,8 +37,8 @@ where } impl Signer { - pub async fn requestor(&self) -> Result { - self.inner.requestor().await + pub async fn client_email(&self) -> Result { + self.inner.client_email().await } pub async fn sign(self, content: &str) -> Result { @@ -67,7 +67,7 @@ struct SignBlobResponse { #[async_trait::async_trait] impl SigningProvider for CredentialsSigner { - async fn requestor(&self) -> Result { + async fn client_email(&self) -> Result { Ok(self.client_email.clone()) } @@ -129,7 +129,7 @@ impl SigningProvider for CredentialsSigner { pub trait SigningProvider: Send + Sync + std::fmt::Debug { // represents the authorizer of the signed URL generation. // It is typically the Google service account client email address from the Google Developers Console in the form of "xxx@developer.gserviceaccount.com". Required. - async fn requestor(&self) -> Result; + async fn client_email(&self) -> Result; // creates a signed URL using the v4 schema. async fn sign(&self, content: &str) -> Result; } @@ -140,7 +140,7 @@ pub(crate) mod dynamic { /// A dyn-compatible, crate-private version of `SigningProvider`. #[async_trait::async_trait] pub trait SigningProvider: Send + Sync + std::fmt::Debug { - async fn requestor(&self) -> Result; + async fn client_email(&self) -> Result; async fn sign(&self, content: &str) -> Result; } @@ -150,8 +150,8 @@ pub(crate) mod dynamic { where T: super::SigningProvider + Send + Sync, { - async fn requestor(&self) -> Result { - T::requestor(self).await + async fn client_email(&self) -> Result { + T::client_email(self).await } async fn sign(&self, content: &str) -> Result { diff --git a/src/storage/src/storage/sign_object.rs b/src/storage/src/storage/sign_object.rs index f5ead6cd58..b82f8f7eb2 100644 --- a/src/storage/src/storage/sign_object.rs +++ b/src/storage/src/storage/sign_object.rs @@ -77,8 +77,8 @@ impl SignObject { let request_timestamp = now.format("%Y%m%dT%H%M%SZ").to_string(); let datestamp = now.format("%Y%m%d"); let credential_scope = format!("{datestamp}/auto/storage/goog4_request"); - let requestor = self.signer.requestor().await.map_err(Error::io)?; // TODO map to proper error - let credential = format!("{requestor}/{credential_scope}"); + let client_email = self.signer.client_email().await.map_err(Error::io)?; // TODO map to proper error + let credential = format!("{client_email}/{credential_scope}"); let bucket_name = self.bucket.trim_start_matches("projects/_/buckets/"); let host = format!("{}.storage.googleapis.com", bucket_name); From 9243db4e244c16ab6bca8ffc3368e6bbf195a04c Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 17 Nov 2025 20:56:12 +0000 Subject: [PATCH 10/41] fix: lint issues --- src/auth/src/signer.rs | 2 +- src/storage/src/storage/sign_object.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index 0e000fc7d7..8480e06416 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -103,7 +103,7 @@ impl SigningProvider for CredentialsSigner { .json(&body) .send() .await - .map_err(|e| SigningError::transport(e))?; + .map_err(SigningError::transport)?; if !response.status().is_success() { let err_text = response.text().await.map_err(SigningError::transport)?; diff --git a/src/storage/src/storage/sign_object.rs b/src/storage/src/storage/sign_object.rs index b82f8f7eb2..3df60751de 100644 --- a/src/storage/src/storage/sign_object.rs +++ b/src/storage/src/storage/sign_object.rs @@ -108,7 +108,7 @@ impl SignObject { }); let canonical_query_string = canonical_query.finish(); - let canonical_request = vec![ + let canonical_request = [ self.method, canonical_uri.clone(), canonical_query_string.clone(), @@ -121,7 +121,7 @@ impl SignObject { let canonical_request_hash = Sha256::digest(canonical_request.as_bytes()); let canonical_request_hash = hex::encode(canonical_request_hash); - let string_to_sign = vec![ + let string_to_sign = [ "GOOG4-RSA-SHA256".to_string(), request_timestamp, credential_scope, From c14a5373ebf57e942360502029d62a89c896a334 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 17 Nov 2025 21:03:22 +0000 Subject: [PATCH 11/41] test: add basic storage test with mock signer --- src/auth/src/signer.rs | 25 ++++++++++++++--- src/storage/Cargo.toml | 2 +- src/storage/src/storage/sign_object.rs | 37 ++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index 8480e06416..2e185e6c6a 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -19,7 +19,10 @@ use std::sync::Arc; pub type Result = std::result::Result; -/// An implementation of [crate::credentials::SigningProvider]. +/// An implementation of [crate::signer::SigningProvider] that wraps a dynamic provider. +/// +/// This struct is the primary entry point for signing operations. It can be created +/// from any type that implements [SigningProvider]. #[derive(Clone, Debug)] pub struct Signer { pub(crate) inner: Arc, @@ -37,10 +40,18 @@ where } impl Signer { + /// Returns the email address of the client performing the signing. + /// + /// This is typically the service account email. pub async fn client_email(&self) -> Result { self.inner.client_email().await } + /// Signs the provided content using the underlying provider. + /// + /// The content is typically a string-to-sign generated by the caller. + /// Returns the signature as a base64 encoded string (or other format depending on implementation, + /// but typically hex or base64). pub async fn sign(self, content: &str) -> Result { self.inner.sign(content).await } @@ -125,12 +136,18 @@ impl SigningProvider for CredentialsSigner { } } +/// A trait for types that can sign content. #[async_trait::async_trait] pub trait SigningProvider: Send + Sync + std::fmt::Debug { - // represents the authorizer of the signed URL generation. - // It is typically the Google service account client email address from the Google Developers Console in the form of "xxx@developer.gserviceaccount.com". Required. + /// Returns the email address of the authorizer. + /// + /// It is typically the Google service account client email address from the Google Developers Console + /// in the form of "xxx@developer.gserviceaccount.com". Required. async fn client_email(&self) -> Result; - // creates a signed URL using the v4 schema. + + /// Signs the content. + /// + /// Returns the signature. async fn sign(&self, content: &str) -> Result; } diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index a87a4e1a26..ed5ba124e5 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -54,7 +54,7 @@ tokio = { workspace = true, features = ["io-util"] } tonic.workspace = true tracing.workspace = true uuid.workspace = true -chrono.workspace = true +chrono = { workspace = true, features = ["clock"] } url.workspace = true hex = { workspace = true, features = ["std"] } # Transitive dependencies. Used for minimal version selection. diff --git a/src/storage/src/storage/sign_object.rs b/src/storage/src/storage/sign_object.rs index 3df60751de..9aafe00103 100644 --- a/src/storage/src/storage/sign_object.rs +++ b/src/storage/src/storage/sign_object.rs @@ -144,3 +144,40 @@ impl SignObject { Ok(signed_url) } } + +#[cfg(test)] +mod tests { + use super::*; + use auth::signer::{Signer, SigningProvider}; + + #[derive(Debug)] + struct MockSigner; + + #[async_trait::async_trait] + impl SigningProvider for MockSigner { + async fn client_email(&self) -> auth::signer::Result { + Ok("test@example.com".to_string()) + } + + async fn sign(&self, _content: &str) -> auth::signer::Result { + Ok("test-signature".to_string()) + } + } + + #[tokio::test] + async fn test_signed_url_generation() { + let signer = Signer::from(MockSigner); + let url = SignObject::new(signer, "test-bucket".to_string(), "test-object".to_string()) + .with_method("PUT") + .with_expiration(std::time::Duration::from_secs(3600)) + .with_header("x-goog-meta-test", "value") + .send() + .await + .unwrap(); + + assert!(url.starts_with("https://test-bucket.storage.googleapis.com/test-object")); + assert!(url.contains("x-goog-signature=test-signature")); + assert!(url.contains("X-Goog-Algorithm=GOOG4-RSA-SHA256")); + assert!(url.contains("X-Goog-Credential=test%40example.com")); + } +} From a03b17fbba30a322c03ceb7f5ee7270e7a8b0d75 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 17 Nov 2025 21:08:43 +0000 Subject: [PATCH 12/41] refactor: accept asRef in signer --- src/auth/src/credentials/mds.rs | 2 +- src/auth/src/signer.rs | 15 +++++++++------ src/storage/src/storage/sign_object.rs | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/auth/src/credentials/mds.rs b/src/auth/src/credentials/mds.rs index b56d885b06..a09a0b321b 100644 --- a/src/auth/src/credentials/mds.rs +++ b/src/auth/src/credentials/mds.rs @@ -350,7 +350,7 @@ impl crate::signer::SigningProvider for MDSCredentialsSigner { Ok(client_email) } - async fn sign(&self, content: &str) -> crate::signer::Result { + async fn sign(&self, content: &[u8]) -> crate::signer::Result { // TODO: not efficient at all, recreating CredentialSigner and refetching email let client_email = self.client_email().await?; diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index 2e185e6c6a..d2594593c3 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -52,8 +52,11 @@ impl Signer { /// The content is typically a string-to-sign generated by the caller. /// Returns the signature as a base64 encoded string (or other format depending on implementation, /// but typically hex or base64). - pub async fn sign(self, content: &str) -> Result { - self.inner.sign(content).await + pub async fn sign(&self, content: T) -> Result + where + T: AsRef<[u8]> + Send + Sync, + { + self.inner.sign(content.as_ref()).await } } @@ -82,7 +85,7 @@ impl SigningProvider for CredentialsSigner { Ok(self.client_email.clone()) } - async fn sign(&self, content: &str) -> Result { + async fn sign(&self, content: &[u8]) -> Result { use base64::{Engine, prelude::BASE64_STANDARD}; let source_headers = self @@ -148,7 +151,7 @@ pub trait SigningProvider: Send + Sync + std::fmt::Debug { /// Signs the content. /// /// Returns the signature. - async fn sign(&self, content: &str) -> Result; + async fn sign(&self, content: &[u8]) -> Result; } pub(crate) mod dynamic { @@ -158,7 +161,7 @@ pub(crate) mod dynamic { #[async_trait::async_trait] pub trait SigningProvider: Send + Sync + std::fmt::Debug { async fn client_email(&self) -> Result; - async fn sign(&self, content: &str) -> Result; + async fn sign(&self, content: &[u8]) -> Result; } /// The public CredentialsProvider implements the dyn-compatible CredentialsProvider. @@ -171,7 +174,7 @@ pub(crate) mod dynamic { T::client_email(self).await } - async fn sign(&self, content: &str) -> Result { + async fn sign(&self, content: &[u8]) -> Result { T::sign(self, content).await } } diff --git a/src/storage/src/storage/sign_object.rs b/src/storage/src/storage/sign_object.rs index 9aafe00103..8758fed73c 100644 --- a/src/storage/src/storage/sign_object.rs +++ b/src/storage/src/storage/sign_object.rs @@ -159,7 +159,7 @@ mod tests { Ok("test@example.com".to_string()) } - async fn sign(&self, _content: &str) -> auth::signer::Result { + async fn sign(&self, _content: &[u8]) -> auth::signer::Result { Ok("test-signature".to_string()) } } From 437f5f3d64cfea9a55006602d137b2a24fd81d7f Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Tue, 18 Nov 2025 16:23:31 +0000 Subject: [PATCH 13/41] refactor: move signing out of gcs client --- src/integration-tests/src/storage.rs | 5 +-- src/storage/src/lib.rs | 2 +- src/storage/src/storage/client.rs | 14 ------ src/storage/src/storage/sign_object.rs | 59 +++++++++++++++++++------- 4 files changed, 46 insertions(+), 34 deletions(-) diff --git a/src/integration-tests/src/storage.rs b/src/integration-tests/src/storage.rs index e07fa0cb77..915f5318df 100644 --- a/src/integration-tests/src/storage.rs +++ b/src/integration-tests/src/storage.rs @@ -106,9 +106,8 @@ pub async fn signed_urls( tracing::info!("success with insert={insert:?}"); tracing::info!("testing signed_url()"); - let signed_url = client - .signed_url(signer, bucket_name, &insert.name) - .send() + let signed_url = storage::builder::storage::SignedUrlBuilder::new(bucket_name, &insert.name) + .sign_with(&signer) .await?; tracing::info!("signed_url={signed_url}"); diff --git a/src/storage/src/lib.rs b/src/storage/src/lib.rs index e2de5a6d2b..29d99ab256 100644 --- a/src/storage/src/lib.rs +++ b/src/storage/src/lib.rs @@ -54,7 +54,7 @@ pub mod builder { pub use crate::storage::client::ClientBuilder; pub use crate::storage::read_object::ReadObject; #[cfg(google_cloud_unstable_signed_url)] - pub use crate::storage::sign_object::SignObject; + pub use crate::storage::sign_object::SignedUrlBuilder; pub use crate::storage::write_object::WriteObject; } pub mod storage_control { diff --git a/src/storage/src/storage/client.rs b/src/storage/src/storage/client.rs index aa4fad0c65..a61b24ee9a 100644 --- a/src/storage/src/storage/client.rs +++ b/src/storage/src/storage/client.rs @@ -225,20 +225,6 @@ where { ReadObject::new(self.stub.clone(), bucket, object, self.options.clone()) } - - #[cfg(google_cloud_unstable_signed_url)] - pub fn signed_url( - &self, - signer: auth::signer::Signer, - bucket: B, - object: O, - ) -> crate::builder::storage::SignObject - where - B: Into, - O: Into, - { - crate::builder::storage::SignObject::new(signer, bucket.into(), object.into()) - } } impl Storage { diff --git a/src/storage/src/storage/sign_object.rs b/src/storage/src/storage/sign_object.rs index 8758fed73c..b2e23039dc 100644 --- a/src/storage/src/storage/sign_object.rs +++ b/src/storage/src/storage/sign_object.rs @@ -21,26 +21,32 @@ use std::collections::BTreeMap; use url::form_urlencoded; /// A builder for creating signed URLs. -pub struct SignObject { - signer: Signer, +pub struct SignedUrlBuilder { bucket: String, object: String, method: String, expiration: std::time::Duration, headers: BTreeMap<&'static str, String>, query_parameters: BTreeMap<&'static str, String>, + endpoint: String, + client_email: Option, } -impl SignObject { - pub(crate) fn new(signer: Signer, bucket: String, object: String) -> Self { +impl SignedUrlBuilder { + pub fn new(bucket: B, object: O) -> Self + where + B: Into, + O: Into, + { Self { - signer, - bucket, - object, + bucket: bucket.into(), + object: object.into(), method: "GET".to_string(), expiration: std::time::Duration::from_secs(7 * 24 * 60 * 60), // 7 days headers: BTreeMap::new(), query_parameters: BTreeMap::new(), + endpoint: "https://storage.googleapis.com".to_string(), + client_email: None, } } @@ -69,19 +75,40 @@ impl SignObject { self } - /// Generates the signed URL. - pub async fn send(self) -> Result { + /// Sets the endpoint for the signed URL. Default is "https://storage.googleapis.com". + pub fn with_endpoint>(mut self, endpoint: S) -> Self { + self.endpoint = endpoint.into(); + self + } + + /// Sets the client email for the signed URL. + /// If not set, the email will be fetched from the signer. + pub fn with_client_email>(mut self, client_email: S) -> Self { + self.client_email = Some(client_email.into()); + self + } + + /// Generates the signed URL using the provided signer. + pub async fn sign_with(self, signer: &Signer) -> Result { let canonical_uri = format!("/{}", self.object); // TODO: escape object name let now = Utc::now(); let request_timestamp = now.format("%Y%m%dT%H%M%SZ").to_string(); let datestamp = now.format("%Y%m%d"); let credential_scope = format!("{datestamp}/auto/storage/goog4_request"); - let client_email = self.signer.client_email().await.map_err(Error::io)?; // TODO map to proper error + let client_email = if let Some(email) = self.client_email { + email + } else { + signer.client_email().await.map_err(Error::io)? // TODO map to proper error + }; let credential = format!("{client_email}/{credential_scope}"); + let endpoint_url = url::Url::parse(&self.endpoint).map_err(Error::io)?; // TODO map to proper error + let endpoint_host = endpoint_url + .host_str() + .ok_or(Error::io("Invalid endpoint URL"))?; // TODO map to proper error let bucket_name = self.bucket.trim_start_matches("projects/_/buckets/"); - let host = format!("{}.storage.googleapis.com", bucket_name); + let host = format!("{}.{}", bucket_name, endpoint_host); let mut headers = self.headers; headers.insert("host", host.clone()); @@ -129,13 +156,13 @@ impl SignObject { ] .join("\n"); - let signature = self - .signer + let signature = signer .sign(string_to_sign.as_str()) .await .map_err(Error::io)?; // TODO map to proper error - let scheme_and_host = format!("https://{}", host); + let scheme_and_host = format!("{}://{}", endpoint_url.scheme(), host); + let signed_url = format!( "{}{}?{}&x-goog-signature={}", scheme_and_host, canonical_uri, canonical_query_string, signature @@ -167,11 +194,11 @@ mod tests { #[tokio::test] async fn test_signed_url_generation() { let signer = Signer::from(MockSigner); - let url = SignObject::new(signer, "test-bucket".to_string(), "test-object".to_string()) + let url = SignedUrlBuilder::new("test-bucket", "test-object") .with_method("PUT") .with_expiration(std::time::Duration::from_secs(3600)) .with_header("x-goog-meta-test", "value") - .send() + .sign_with(&signer) .await .unwrap(); From c6ea5a029e935698c5ba21d18a011cf3a97ba618 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Tue, 18 Nov 2025 20:59:24 +0000 Subject: [PATCH 14/41] feat: add error handling and tests for it --- src/auth/src/signer.rs | 21 ++++++ src/storage/src/error.rs | 14 ++++ src/storage/src/storage/sign_object.rs | 97 +++++++++++++++++++++++--- 3 files changed, 124 insertions(+), 8 deletions(-) diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index d2594593c3..b8880a4858 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -197,6 +197,12 @@ impl SigningError { matches!(self.0, SigningErrorKind::Parsing(_)) } + /// Mocked error. + #[doc(hidden)] + pub fn is_mock(&self) -> bool { + matches!(self.0, SigningErrorKind::Mock(_)) + } + /// A problem parsing a credentials specification. pub(crate) fn parsing(source: T) -> SigningError where @@ -212,6 +218,19 @@ impl SigningError { { SigningError(SigningErrorKind::Transport(source.into())) } + + /// Creates a new fake `SigningError`. + /// + /// This function is only intended for use in the client libraries + /// implementation. Application may use this in mocks, though we do not + /// recommend that you write tests for specific error cases. + /// + /// # Parameters + /// * `message` - The underlying error that caused the signing failure. + #[doc(hidden)] + pub fn mock>(message: T) -> Self { + SigningError(SigningErrorKind::Mock(message.into())) + } } #[derive(thiserror::Error, Debug)] @@ -220,4 +239,6 @@ enum SigningErrorKind { Transport(#[source] BoxError), #[error("failed to parse credentials: {0}")] Parsing(#[source] BoxError), + #[error("mocked error when signing blob: {0}")] + Mock(#[source] BoxError), } diff --git a/src/storage/src/error.rs b/src/storage/src/error.rs index ae2455c6e0..68add2e602 100644 --- a/src/storage/src/error.rs +++ b/src/storage/src/error.rs @@ -300,6 +300,20 @@ pub enum WriteError { }, } +/// Represents an error that can occur when signing a URL. +#[cfg(google_cloud_unstable_signed_url)] +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum SigningError { + /// The signing operation failed. + #[error("signing failed: {0}")] + Signing(#[source] auth::signer::SigningError), + + /// The endpoint URL is invalid. + #[error("invalid endpoint: {0}")] + InvalidEndpoint(#[source] Box), +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/storage/src/storage/sign_object.rs b/src/storage/src/storage/sign_object.rs index b2e23039dc..666d728b23 100644 --- a/src/storage/src/storage/sign_object.rs +++ b/src/storage/src/storage/sign_object.rs @@ -12,14 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{Error, Result}; +use crate::error::SigningError; use auth::signer::Signer; use chrono::Utc; use hex; +use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode}; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use url::form_urlencoded; +/// https://cloud.google.com/storage/docs/request-endpoints#encoding +const PATH_ENCODE_SET: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'"') + .add(b'#') + .add(b'<') + .add(b'>') + .add(b'?') + .add(b'`') + .add(b'{') + .add(b'}'); + /// A builder for creating signed URLs. pub struct SignedUrlBuilder { bucket: String, @@ -89,8 +102,9 @@ impl SignedUrlBuilder { } /// Generates the signed URL using the provided signer. - pub async fn sign_with(self, signer: &Signer) -> Result { - let canonical_uri = format!("/{}", self.object); // TODO: escape object name + pub async fn sign_with(self, signer: &Signer) -> std::result::Result { + let encoded_object = utf8_percent_encode(&self.object, PATH_ENCODE_SET).to_string(); + let canonical_uri = format!("/{}", encoded_object); let now = Utc::now(); let request_timestamp = now.format("%Y%m%dT%H%M%SZ").to_string(); @@ -99,14 +113,15 @@ impl SignedUrlBuilder { let client_email = if let Some(email) = self.client_email { email } else { - signer.client_email().await.map_err(Error::io)? // TODO map to proper error + signer.client_email().await.map_err(SigningError::Signing)? }; let credential = format!("{client_email}/{credential_scope}"); - let endpoint_url = url::Url::parse(&self.endpoint).map_err(Error::io)?; // TODO map to proper error + let endpoint_url = + url::Url::parse(&self.endpoint).map_err(|e| SigningError::InvalidEndpoint(e.into()))?; let endpoint_host = endpoint_url .host_str() - .ok_or(Error::io("Invalid endpoint URL"))?; // TODO map to proper error + .ok_or_else(|| SigningError::InvalidEndpoint("invalid endpoint host".into()))?; let bucket_name = self.bucket.trim_start_matches("projects/_/buckets/"); let host = format!("{}.{}", bucket_name, endpoint_host); @@ -159,7 +174,7 @@ impl SignedUrlBuilder { let signature = signer .sign(string_to_sign.as_str()) .await - .map_err(Error::io)?; // TODO map to proper error + .map_err(SigningError::Signing)?; let scheme_and_host = format!("{}://{}", endpoint_url.scheme(), host); @@ -177,6 +192,8 @@ mod tests { use super::*; use auth::signer::{Signer, SigningProvider}; + type TestResult = anyhow::Result<()>; + #[derive(Debug)] struct MockSigner; @@ -192,7 +209,7 @@ mod tests { } #[tokio::test] - async fn test_signed_url_generation() { + async fn test_signed_url_generation() -> TestResult { let signer = Signer::from(MockSigner); let url = SignedUrlBuilder::new("test-bucket", "test-object") .with_method("PUT") @@ -206,5 +223,69 @@ mod tests { assert!(url.contains("x-goog-signature=test-signature")); assert!(url.contains("X-Goog-Algorithm=GOOG4-RSA-SHA256")); assert!(url.contains("X-Goog-Credential=test%40example.com")); + + Ok(()) + } + + #[tokio::test] + async fn test_signed_url_generation_escaping() -> TestResult { + let signer: Signer = Signer::from(MockSigner); + let url = SignedUrlBuilder::new("test-bucket", "folder/test object.txt") + .with_method("PUT") + .with_header("content-type", "text/plain") + .sign_with(&signer) + .await + .unwrap(); + + assert!( + url.starts_with("https://test-bucket.storage.googleapis.com/folder/test%20object.txt?") + ); + assert!(url.contains("x-goog-signature=")); + + Ok(()) + } + + #[tokio::test] + async fn test_signed_url_error_signing() -> TestResult { + #[derive(Debug)] + struct FailSigner; + #[async_trait::async_trait] + impl SigningProvider for FailSigner { + async fn client_email(&self) -> auth::signer::Result { + Ok("test@example.com".to_string()) + } + async fn sign(&self, _content: &[u8]) -> auth::signer::Result { + Err(auth::signer::SigningError::mock("test".to_string())) + } + } + let signer = Signer::from(FailSigner); + let err = SignedUrlBuilder::new("b", "o") + .sign_with(&signer) + .await + .unwrap_err(); + + match err { + SigningError::Signing(e) => assert!(e.is_mock()), + _ => panic!("unexpected error type: {:?}", err), + } + + Ok(()) + } + + #[tokio::test] + async fn test_signed_url_error_endpoint() -> TestResult { + let signer: Signer = Signer::from(MockSigner); + let err = SignedUrlBuilder::new("b", "o") + .with_endpoint("invalid-url") + .sign_with(&signer) + .await + .unwrap_err(); + + match err { + SigningError::InvalidEndpoint(_) => {} + _ => panic!("unexpected error type: {:?}", err), + } + + Ok(()) } } From 0758f0c109f40e05b1f36eb3dd4739dc67d1409a Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 19 Nov 2025 17:14:49 +0000 Subject: [PATCH 15/41] refactor: rename sign object mod --- src/storage/src/lib.rs | 2 +- src/storage/src/storage.rs | 2 +- src/storage/src/storage/{sign_object.rs => signed_url.rs} | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) rename src/storage/src/storage/{sign_object.rs => signed_url.rs} (99%) diff --git a/src/storage/src/lib.rs b/src/storage/src/lib.rs index 29d99ab256..e71defdfc6 100644 --- a/src/storage/src/lib.rs +++ b/src/storage/src/lib.rs @@ -54,7 +54,7 @@ pub mod builder { pub use crate::storage::client::ClientBuilder; pub use crate::storage::read_object::ReadObject; #[cfg(google_cloud_unstable_signed_url)] - pub use crate::storage::sign_object::SignedUrlBuilder; + pub use crate::storage::signed_url::SignedUrlBuilder; pub use crate::storage::write_object::WriteObject; } pub mod storage_control { diff --git a/src/storage/src/storage.rs b/src/storage/src/storage.rs index 34301aac48..72f4f4535a 100644 --- a/src/storage/src/storage.rs +++ b/src/storage/src/storage.rs @@ -22,7 +22,7 @@ pub(crate) mod perform_upload; pub(crate) mod read_object; pub mod request_options; #[cfg(google_cloud_unstable_signed_url)] -pub(crate) mod sign_object; +pub(crate) mod signed_url; pub mod streaming_source; pub mod stub; pub(crate) mod transport; diff --git a/src/storage/src/storage/sign_object.rs b/src/storage/src/storage/signed_url.rs similarity index 99% rename from src/storage/src/storage/sign_object.rs rename to src/storage/src/storage/signed_url.rs index 666d728b23..788e2a26d6 100644 --- a/src/storage/src/storage/sign_object.rs +++ b/src/storage/src/storage/signed_url.rs @@ -15,7 +15,6 @@ use crate::error::SigningError; use auth::signer::Signer; use chrono::Utc; -use hex; use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode}; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; From c1a907e21c9bfa403396918e4b003bf42ce50b68 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 19 Nov 2025 20:49:00 +0000 Subject: [PATCH 16/41] impl(auth): local signing with SA and MDS cache email --- src/auth/src/credentials/impersonated.rs | 83 +++++++++++++++------ src/auth/src/credentials/mds.rs | 56 +------------- src/auth/src/credentials/service_account.rs | 80 ++++++++++---------- src/auth/src/signer.rs | 3 + src/auth/src/signer/mds.rs | 82 ++++++++++++++++++++ src/auth/src/signer/service_account.rs | 64 ++++++++++++++++ 6 files changed, 253 insertions(+), 115 deletions(-) create mode 100644 src/auth/src/signer/mds.rs create mode 100644 src/auth/src/signer/service_account.rs diff --git a/src/auth/src/credentials/impersonated.rs b/src/auth/src/credentials/impersonated.rs index d8f8897c7a..7efad9bfa9 100644 --- a/src/auth/src/credentials/impersonated.rs +++ b/src/auth/src/credentials/impersonated.rs @@ -434,31 +434,37 @@ impl Builder { #[cfg(google_cloud_unstable_signed_url)] pub fn build_signer(self) -> BuildResult { let source = self.source.clone(); - let components = match source { - BuilderSource::FromJson(json) => build_components_from_json(json)?, + match source { + BuilderSource::FromJson(json) => { + let signer = build_signer_from_json(json.clone())?; + if let Some(signer) = signer { + return Ok(signer); + } + let components = build_components_from_json(json)?; + let client_email = + extract_client_email(&components.service_account_impersonation_url)?; + Ok(crate::signer::Signer { + inner: Arc::new(crate::signer::CredentialsSigner { + client_email, + inner: self.build()?, + }), + }) + } BuilderSource::FromCredentials(source_credentials) => { - build_components_from_credentials( + let components = build_components_from_credentials( source_credentials, self.service_account_impersonation_url.clone(), - )? + )?; + let client_email = + extract_client_email(&components.service_account_impersonation_url)?; + Ok(crate::signer::Signer { + inner: Arc::new(crate::signer::CredentialsSigner { + client_email, + inner: self.build()?, + }), + }) } - }; - - // TODO: better use regex to extract email ? - let parts: Vec<&str> = components - .service_account_impersonation_url - .split("serviceAccounts/") - .collect(); - let client_email = parts[1] - .trim_end_matches(":generateAccessToken") - .to_string(); - - Ok(crate::signer::Signer { - inner: Arc::new(crate::signer::CredentialsSigner { - client_email, - inner: self.build()?, - }), - }) + } } fn build_components( @@ -535,6 +541,41 @@ pub(crate) fn build_components_from_json( }) } +#[cfg(google_cloud_unstable_signed_url)] +fn build_signer_from_json(json: Value) -> BuildResult> { + use crate::credentials::service_account::ServiceAccountKey; + use crate::signer::service_account::ServiceAccountSigner; + + let config = + serde_json::from_value::(json).map_err(BuilderError::parsing)?; + + let client_email = extract_client_email(&config.service_account_impersonation_url)?; + let source_credential_type = extract_credential_type(&config.source_credentials)?; + if source_credential_type == "service_account" { + let service_account_key = + serde_json::from_value::(config.source_credentials) + .map_err(BuilderError::parsing)?; + let signer = ServiceAccountSigner::from_impersonated_service_account( + service_account_key, + client_email, + ); + return Ok(Some(signer.into())); + } + Ok(None) +} + +#[cfg(google_cloud_unstable_signed_url)] +fn extract_client_email(service_account_impersonation_url: &str) -> BuildResult { + // TODO: better use regex to extract email ? + let parts: Vec<&str> = service_account_impersonation_url + .split("serviceAccounts/") + .collect(); + let client_email = parts[1] + .trim_end_matches(":generateAccessToken") + .to_string(); + Ok(client_email) +} + pub(crate) fn build_components_from_credentials( source_credentials: Credentials, service_account_impersonation_url: Option, diff --git a/src/auth/src/credentials/mds.rs b/src/auth/src/credentials/mds.rs index a09a0b321b..b6b6583eac 100644 --- a/src/auth/src/credentials/mds.rs +++ b/src/auth/src/credentials/mds.rs @@ -304,65 +304,15 @@ impl Builder { #[cfg(google_cloud_unstable_signed_url)] pub fn build_signer(self) -> BuildResult { - // TODO: have MDS specific impl that fetches email as needed let (endpoint, _) = self.resolve_endpoint(); - + let credentials = self.build()?; + let signing_provider = crate::signer::mds::MDSCredentialsSigner::new(endpoint, credentials); Ok(crate::signer::Signer { - inner: Arc::new(MDSCredentialsSigner { - endpoint, - inner: self.build()?, - }), + inner: Arc::new(signing_provider), }) } } -// Implements Signer for MDS that extends the existing CredentialsSigner by fetching -// email via MDS email endpoint. -#[derive(Clone, Debug)] -#[cfg(google_cloud_unstable_signed_url)] -struct MDSCredentialsSigner { - endpoint: String, - inner: Credentials, -} - -#[cfg(google_cloud_unstable_signed_url)] -#[async_trait::async_trait] -impl crate::signer::SigningProvider for MDSCredentialsSigner { - async fn client_email(&self) -> crate::signer::Result { - let client = Client::new(); - - let request = client - .get(format!("{}{}/email", self.endpoint, MDS_DEFAULT_URI)) - .header( - METADATA_FLAVOR, - HeaderValue::from_static(METADATA_FLAVOR_VALUE), - ); - - let response = request - .send() - .await - .map_err(crate::signer::SigningError::transport)?; - let client_email = response - .text() - .await - .map_err(crate::signer::SigningError::transport)?; - - Ok(client_email) - } - - async fn sign(&self, content: &[u8]) -> crate::signer::Result { - // TODO: not efficient at all, recreating CredentialSigner and refetching email - let client_email = self.client_email().await?; - - let signer = crate::signer::CredentialsSigner { - client_email, - inner: self.inner.clone(), - }; - - signer.sign(content).await - } -} - #[async_trait::async_trait] impl CredentialsProvider for MDSCredentials where diff --git a/src/auth/src/credentials/service_account.rs b/src/auth/src/credentials/service_account.rs index e8601ee9a6..49ce874109 100644 --- a/src/auth/src/credentials/service_account.rs +++ b/src/auth/src/credentials/service_account.rs @@ -300,13 +300,10 @@ impl Builder { let service_account_key = serde_json::from_value::(self.service_account_key.clone()) .map_err(BuilderError::parsing)?; - let client_email = service_account_key.client_email; - + let signing_provider = + crate::signer::service_account::ServiceAccountSigner::new(service_account_key); Ok(crate::signer::Signer { - inner: Arc::new(crate::signer::CredentialsSigner { - client_email, - inner: self.build()?, - }), + inner: Arc::new(signing_provider), }) } } @@ -318,7 +315,7 @@ impl Builder { pub(crate) struct ServiceAccountKey { /// The client email address of the service account. /// (e.g., "my-sa@my-project.iam.gserviceaccount.com"). - client_email: String, + pub(crate) client_email: String, /// ID of the service account's private key. private_key_id: String, /// The PEM-encoded PKCS#8 private key string associated with the service account. @@ -330,6 +327,38 @@ pub(crate) struct ServiceAccountKey { universe_domain: Option, } +impl ServiceAccountKey { + // Creates a signer using the private key stored in the service account file. + pub(crate) fn signer(&self) -> Result> { + let private_key = self.private_key.clone(); + let key_provider = CryptoProvider::get_default().map_or_else( + || rustls::crypto::ring::default_provider().key_provider, + |p| p.key_provider, + ); + + let private_key = rustls_pemfile::read_one(&mut private_key.as_bytes()) + .map_err(errors::non_retryable)? + .ok_or_else(|| { + errors::non_retryable_from_str("missing PEM section in service account key") + })?; + let pk = match private_key { + Item::Pkcs8Key(item) => key_provider.load_private_key(item.into()), + other => { + return Err(Self::unexpected_private_key_error(other)); + } + }; + let sk = pk.map_err(errors::non_retryable)?; + sk.choose_scheme(&[rustls::SignatureScheme::RSA_PKCS1_SHA256]) + .ok_or_else(|| errors::non_retryable_from_str("Unable to choose RSA_PKCS1_SHA256 signing scheme as it is not supported by current signer")) + } + + fn unexpected_private_key_error(private_key_format: Item) -> CredentialsError { + errors::non_retryable_from_str(format!( + "expected key to be in form of PKCS8, found {private_key_format:?}", + )) + } +} + impl std::fmt::Debug for ServiceAccountKey { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ServiceAccountKey") @@ -415,7 +444,7 @@ impl ServiceAccountTokenGenerator { } pub(crate) fn generate(&self) -> Result { - let signer = self.signer(&self.service_account_key.private_key)?; + let signer = self.service_account_key.signer()?; // The claims encode a unix timestamp. `std::time::Instant` has no // epoch, so we use `time::OffsetDateTime`, which reads system time, in @@ -451,35 +480,6 @@ impl ServiceAccountTokenGenerator { Ok(token) } - - // Creates a signer using the private key stored in the service account file. - fn signer(&self, private_key: &String) -> Result> { - let key_provider = CryptoProvider::get_default().map_or_else( - || rustls::crypto::ring::default_provider().key_provider, - |p| p.key_provider, - ); - - let private_key = rustls_pemfile::read_one(&mut private_key.as_bytes()) - .map_err(errors::non_retryable)? - .ok_or_else(|| { - errors::non_retryable_from_str("missing PEM section in service account key") - })?; - let pk = match private_key { - Item::Pkcs8Key(item) => key_provider.load_private_key(item.into()), - other => { - return Err(Self::unexpected_private_key_error(other)); - } - }; - let sk = pk.map_err(errors::non_retryable)?; - sk.choose_scheme(&[rustls::SignatureScheme::RSA_PKCS1_SHA256]) - .ok_or_else(|| errors::non_retryable_from_str("Unable to choose RSA_PKCS1_SHA256 signing scheme as it is not supported by current signer")) - } - - fn unexpected_private_key_error(private_key_format: Item) -> CredentialsError { - errors::non_retryable_from_str(format!( - "expected key to be in form of PKCS8, found {private_key_format:?}", - )) - } } #[async_trait::async_trait] @@ -768,7 +768,7 @@ mod tests { ..Default::default() }; - let signer = tg.signer(&tg.service_account_key.private_key); + let signer = tg.service_account_key.signer(); let expected_error_message = "missing PEM section in service account key"; assert!(signer.is_err_and(|e| e.to_string().contains(expected_error_message))); Ok(()) @@ -781,9 +781,7 @@ mod tests { Item::Crl(Vec::new().into()) // Example unsupported key type ); - let error = ServiceAccountTokenGenerator::unexpected_private_key_error(Item::Crl( - Vec::new().into(), - )); + let error = ServiceAccountKey::unexpected_private_key_error(Item::Crl(Vec::new().into())); assert!(error.to_string().contains(&expected_message)); Ok(()) } diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index b8880a4858..738ac3dd97 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -17,6 +17,9 @@ use http::Extensions; use reqwest::Client; use std::sync::Arc; +pub(crate) mod mds; +pub(crate) mod service_account; + pub type Result = std::result::Result; /// An implementation of [crate::signer::SigningProvider] that wraps a dynamic provider. diff --git a/src/auth/src/signer/mds.rs b/src/auth/src/signer/mds.rs new file mode 100644 index 0000000000..29011622df --- /dev/null +++ b/src/auth/src/signer/mds.rs @@ -0,0 +1,82 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::credentials::Credentials; +use crate::credentials::mds::{MDS_DEFAULT_URI, METADATA_FLAVOR, METADATA_FLAVOR_VALUE}; +use crate::signer::{Result, SigningError, SigningProvider}; +use http::HeaderValue; +use reqwest::Client; +use std::sync::Arc; +use tokio::sync::RwLock; + +// Implements Signer for MDS that extends the existing CredentialsSigner by fetching +// email via MDS email endpoint. +#[derive(Clone, Debug)] +pub(crate) struct MDSCredentialsSigner { + endpoint: String, + client_email: Arc>, + inner: Credentials, +} + +impl MDSCredentialsSigner { + pub(crate) fn new(endpoint: String, inner: Credentials) -> Self { + Self { + endpoint, + client_email: Arc::new(RwLock::new(String::new())), + inner, + } + } +} + +#[async_trait::async_trait] +impl SigningProvider for MDSCredentialsSigner { + async fn client_email(&self) -> Result { + let mut client_email = self + .client_email + .try_write() + .map_err(|_e| SigningError::transport("failed to obtain lock to read client email"))?; + + if client_email.is_empty() { + let client = Client::new(); + + let request = client + .get(format!("{}{}/email", self.endpoint, MDS_DEFAULT_URI)) + .header( + METADATA_FLAVOR, + HeaderValue::from_static(METADATA_FLAVOR_VALUE), + ); + + let response = request.send().await.map_err(SigningError::transport)?; + let email = response.text().await.map_err(SigningError::transport)?; + + *client_email = email.clone(); + } + + Ok(client_email.clone()) + } + + async fn sign(&self, content: &[u8]) -> Result { + let client_email = self.client_email().await?; + + let signer = crate::signer::CredentialsSigner { + client_email, + inner: self.inner.clone(), + }; + + signer.sign(content).await + } +} + +#[cfg(test)] +mod tests {} diff --git a/src/auth/src/signer/service_account.rs b/src/auth/src/signer/service_account.rs new file mode 100644 index 0000000000..aebcae49ec --- /dev/null +++ b/src/auth/src/signer/service_account.rs @@ -0,0 +1,64 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::credentials::service_account::ServiceAccountKey; +use crate::signer::{Result, SigningError, SigningProvider}; + +// Implements a local Signer using Service Account private key. +#[derive(Clone, Debug)] +pub(crate) struct ServiceAccountSigner { + service_account_key: ServiceAccountKey, + client_email: String, +} + +impl ServiceAccountSigner { + pub(crate) fn new(service_account_key: ServiceAccountKey) -> Self { + Self { + service_account_key: service_account_key.clone(), + client_email: service_account_key.client_email.clone(), + } + } + pub(crate) fn from_impersonated_service_account( + service_account_key: ServiceAccountKey, + client_email: String, + ) -> Self { + Self { + service_account_key, + client_email, + } + } +} + +#[async_trait::async_trait] +impl SigningProvider for ServiceAccountSigner { + async fn client_email(&self) -> Result { + Ok(self.client_email.clone()) + } + + async fn sign(&self, content: &[u8]) -> Result { + let signer = self + .service_account_key + .signer() + .map_err(SigningError::parsing)?; + + let signature = signer.sign(content).map_err(SigningError::parsing)?; + + let signature = hex::encode(signature); + + Ok(signature) + } +} + +#[cfg(test)] +mod tests {} From ad60935ffee057b801f05cebcbd88df3cd85f85a Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 21 Nov 2025 15:16:38 +0000 Subject: [PATCH 17/41] impl: remove user_account signer --- src/auth/src/credentials.rs | 13 ++++++------- src/auth/src/credentials/user_account.rs | 15 --------------- 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/src/auth/src/credentials.rs b/src/auth/src/credentials.rs index 61f4270320..2fea817384 100644 --- a/src/auth/src/credentials.rs +++ b/src/auth/src/credentials.rs @@ -556,12 +556,9 @@ fn build_signer( Some(json) => { let cred_type = extract_credential_type(&json)?; match cred_type { - "authorized_user" => config_signer!( - user_account::Builder::new(json), - quota_project_id, - scopes, - |b: user_account::Builder, s: Vec| b.with_scopes(s) - ), + "authorized_user" => Err(BuilderError::not_supported( + "authorized_user signer is not supported", + )), "service_account" => config_signer!( service_account::Builder::new(json), quota_project_id, @@ -577,7 +574,9 @@ fn build_signer( |b: impersonated::Builder, s: Vec| b.with_scopes(s) ) } - "external_account" => panic!("external account signer not supported yet"), + "external_account" => Err(BuilderError::not_supported( + "external_account signer is not supported", + )), _ => Err(BuilderError::unknown_type(cred_type)), } } diff --git a/src/auth/src/credentials/user_account.rs b/src/auth/src/credentials/user_account.rs index 1d00f19b1f..e5f6241c16 100644 --- a/src/auth/src/credentials/user_account.rs +++ b/src/auth/src/credentials/user_account.rs @@ -341,21 +341,6 @@ impl Builder { }), }) } - - #[cfg(google_cloud_unstable_signed_url)] - pub fn build_signer(self) -> BuildResult { - let authorized_user = - serde_json::from_value::(self.authorized_user.clone()) - .map_err(BuilderError::parsing)?; - let client_email = authorized_user.client_id; - - Ok(crate::signer::Signer { - inner: Arc::new(crate::signer::CredentialsSigner { - client_email, - inner: self.build()?, - }), - }) - } } #[derive(PartialEq)] From 48f2dae40e009d66a63946005282195a06b9a567 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 21 Nov 2025 15:51:21 +0000 Subject: [PATCH 18/41] impl: improve error handling --- src/auth/src/signer.rs | 38 ++++++++++++++++---------- src/auth/src/signer/service_account.rs | 2 +- src/storage/src/error.rs | 7 +++-- src/storage/src/storage/signed_url.rs | 4 +-- 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index 738ac3dd97..a091d18a2b 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -130,11 +130,11 @@ impl SigningProvider for CredentialsSigner { let res = response .json::() .await - .map_err(SigningError::parsing)?; + .map_err(SigningError::transport)?; let signature = BASE64_STANDARD .decode(res.signed_blob) - .map_err(SigningError::parsing)?; + .map_err(SigningError::transport)?; let signature = hex::encode(signature); @@ -195,18 +195,17 @@ impl SigningError { matches!(self.0, SigningErrorKind::Transport(_)) } - /// A problem parsing a credentials JSON specification. + /// A problem parsing a private key for local signing. pub fn is_parsing(&self) -> bool { matches!(self.0, SigningErrorKind::Parsing(_)) } - /// Mocked error. - #[doc(hidden)] - pub fn is_mock(&self) -> bool { - matches!(self.0, SigningErrorKind::Mock(_)) + /// A problem signing content. + pub fn is_sign(&self) -> bool { + matches!(self.0, SigningErrorKind::Sign(_)) } - /// A problem parsing a credentials specification. + /// A problem parsing a private key for local signing. pub(crate) fn parsing(source: T) -> SigningError where T: Into, @@ -222,7 +221,15 @@ impl SigningError { SigningError(SigningErrorKind::Transport(source.into())) } - /// Creates a new fake `SigningError`. + /// A problem signing content. + pub(crate) fn sign(source: T) -> SigningError + where + T: Into, + { + SigningError(SigningErrorKind::Sign(source.into())) + } + + /// Creates a new `SigningError`. /// /// This function is only intended for use in the client libraries /// implementation. Application may use this in mocks, though we do not @@ -231,8 +238,11 @@ impl SigningError { /// # Parameters /// * `message` - The underlying error that caused the signing failure. #[doc(hidden)] - pub fn mock>(message: T) -> Self { - SigningError(SigningErrorKind::Mock(message.into())) + pub fn from_msg(message: T) -> SigningError + where + T: Into, + { + SigningError(SigningErrorKind::Sign(message.into())) } } @@ -240,8 +250,8 @@ impl SigningError { enum SigningErrorKind { #[error("failed to generate signature via IAM API: {0}")] Transport(#[source] BoxError), - #[error("failed to parse credentials: {0}")] + #[error("failed to parse private key: {0}")] Parsing(#[source] BoxError), - #[error("mocked error when signing blob: {0}")] - Mock(#[source] BoxError), + #[error("failed to sign content: {0}")] + Sign(#[source] BoxError), } diff --git a/src/auth/src/signer/service_account.rs b/src/auth/src/signer/service_account.rs index aebcae49ec..1a37742975 100644 --- a/src/auth/src/signer/service_account.rs +++ b/src/auth/src/signer/service_account.rs @@ -52,7 +52,7 @@ impl SigningProvider for ServiceAccountSigner { .signer() .map_err(SigningError::parsing)?; - let signature = signer.sign(content).map_err(SigningError::parsing)?; + let signature = signer.sign(content).map_err(SigningError::sign)?; let signature = hex::encode(signature); diff --git a/src/storage/src/error.rs b/src/storage/src/error.rs index 68add2e602..f2b7933191 100644 --- a/src/storage/src/error.rs +++ b/src/storage/src/error.rs @@ -300,6 +300,9 @@ pub enum WriteError { }, } +#[cfg(google_cloud_unstable_signed_url)] +type BoxError = Box; + /// Represents an error that can occur when signing a URL. #[cfg(google_cloud_unstable_signed_url)] #[derive(thiserror::Error, Debug)] @@ -310,8 +313,8 @@ pub enum SigningError { Signing(#[source] auth::signer::SigningError), /// The endpoint URL is invalid. - #[error("invalid endpoint: {0}")] - InvalidEndpoint(#[source] Box), + #[error("invalid GCS endpoint: {0}")] + InvalidEndpoint(#[source] BoxError), } #[cfg(test)] diff --git a/src/storage/src/storage/signed_url.rs b/src/storage/src/storage/signed_url.rs index 788e2a26d6..2efcd6582b 100644 --- a/src/storage/src/storage/signed_url.rs +++ b/src/storage/src/storage/signed_url.rs @@ -254,7 +254,7 @@ mod tests { Ok("test@example.com".to_string()) } async fn sign(&self, _content: &[u8]) -> auth::signer::Result { - Err(auth::signer::SigningError::mock("test".to_string())) + Err(auth::signer::SigningError::from_msg("test".to_string())) } } let signer = Signer::from(FailSigner); @@ -264,7 +264,7 @@ mod tests { .unwrap_err(); match err { - SigningError::Signing(e) => assert!(e.is_mock()), + SigningError::Signing(e) => assert!(e.is_sign()), _ => panic!("unexpected error type: {:?}", err), } From 80996b82401453963f6822b2d55328c9b70372cf Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 21 Nov 2025 18:53:37 +0000 Subject: [PATCH 19/41] refactor: move signed url method to use http crate --- src/storage/src/storage/signed_url.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/storage/src/storage/signed_url.rs b/src/storage/src/storage/signed_url.rs index 2efcd6582b..cfaf10d68c 100644 --- a/src/storage/src/storage/signed_url.rs +++ b/src/storage/src/storage/signed_url.rs @@ -36,7 +36,7 @@ const PATH_ENCODE_SET: &AsciiSet = &CONTROLS pub struct SignedUrlBuilder { bucket: String, object: String, - method: String, + method: http::Method, expiration: std::time::Duration, headers: BTreeMap<&'static str, String>, query_parameters: BTreeMap<&'static str, String>, @@ -53,7 +53,7 @@ impl SignedUrlBuilder { Self { bucket: bucket.into(), object: object.into(), - method: "GET".to_string(), + method: http::Method::GET, expiration: std::time::Duration::from_secs(7 * 24 * 60 * 60), // 7 days headers: BTreeMap::new(), query_parameters: BTreeMap::new(), @@ -63,8 +63,8 @@ impl SignedUrlBuilder { } /// Sets the HTTP method for the signed URL. Default is "GET". - pub fn with_method>(mut self, method: S) -> Self { - self.method = method.into(); + pub fn with_method(mut self, method: http::Method) -> Self { + self.method = method; self } @@ -150,7 +150,7 @@ impl SignedUrlBuilder { let canonical_query_string = canonical_query.finish(); let canonical_request = [ - self.method, + self.method.to_string(), canonical_uri.clone(), canonical_query_string.clone(), canonical_headers, From f33bbaece4460e5795c2de86737fff83dea68fe0 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 24 Nov 2025 20:15:09 +0000 Subject: [PATCH 20/41] test: make it pass conformance tests --- Cargo.lock | 32 +- Cargo.toml | 1 + src/storage/Cargo.toml | 2 + .../test_service_account.not-a-test.json | 13 + .../storage/conformance/v4_signatures.json | 698 ++++++++++++++++++ src/storage/src/storage/signed_url.rs | 506 +++++++++++-- 6 files changed, 1178 insertions(+), 74 deletions(-) create mode 100644 src/storage/src/storage/conformance/test_service_account.not-a-test.json create mode 100644 src/storage/src/storage/conformance/v4_signatures.json diff --git a/Cargo.lock b/Cargo.lock index 6cd16e233b..0bc8020516 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -499,6 +499,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -649,7 +655,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4543,9 +4549,11 @@ dependencies = [ "paste", "percent-encoding", "pin-project", + "pretty_assertions", "prost", "prost-types", "reqwest", + "scoped-env", "serde", "serde_json", "serde_with", @@ -6150,6 +6158,16 @@ dependencies = [ "termtree", ] +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -6315,7 +6333,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -6561,7 +6579,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7074,7 +7092,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7939,6 +7957,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index 0201319847..044d5bd01c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -362,6 +362,7 @@ multer = { default-features = false, version = "3" } mutants = { default-features = false, version = "0.0.3" } once_cell = { default-features = false, version = "1.19.0" } paste = { default-features = false, version = "1" } +pretty_assertions = { default-features = false, version = "1.4.1", features = ["alloc"] } regex = { default-features = false, version = "1" } rsa = { default-features = false, version = "0.9.8" } rustc_version = { default-features = false, version = "0.4" } diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index ed5ba124e5..385b5f1be5 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -87,3 +87,5 @@ tempfile.workspace = true tokio-test.workspace = true tokio.workspace = true google-cloud-storage = { path = ".", features = ["unstable-stream"] } +pretty_assertions = { workspace = true } +scoped-env.workspace = true diff --git a/src/storage/src/storage/conformance/test_service_account.not-a-test.json b/src/storage/src/storage/conformance/test_service_account.not-a-test.json new file mode 100644 index 0000000000..63d33cb507 --- /dev/null +++ b/src/storage/src/storage/conformance/test_service_account.not-a-test.json @@ -0,0 +1,13 @@ +{ + "comment": "This is a dummy service account JSON file that is inactive. It's fine for it to be public.", + "type": "service_account", + "project_id": "dummy-project-id", + "private_key_id": "ffffffffffffffffffffffffffffffffffffffff", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCsPzMirIottfQ2\nryjQmPWocSEeGo7f7Q4/tMQXHlXFzo93AGgU2t+clEj9L5loNhLVq+vk+qmnyDz5\nQ04y8jVWyMYzzGNNrGRW/yaYqnqlKZCy1O3bmnNjV7EDbC/jE1ZLBY0U3HaSHfn6\nS9ND8MXdgD0/ulRTWwq6vU8/w6i5tYsU7n2LLlQTl1fQ7/emO9nYcCFJezHZVa0H\nmeWsdHwWsok0skwQYQNIzP3JF9BpR5gJT2gNge6KopDesJeLoLzaX7cUnDn+CAnn\nLuLDwwSsIVKyVxhBFsFXPplgpaQRwmGzwEbf/Xpt9qo26w2UMgn30jsOaKlSeAX8\ncS6ViF+tAgMBAAECggEACKRuJCP8leEOhQziUx8Nmls8wmYqO4WJJLyk5xUMUC22\nSI4CauN1e0V8aQmxnIc0CDkFT7qc9xBmsMoF+yvobbeKrFApvlyzNyM7tEa/exh8\nDGD/IzjbZ8VfWhDcUTwn5QE9DCoon9m1sG+MBNlokB3OVOt8LieAAREdEBG43kJu\nyQTOkY9BGR2AY1FnAl2VZ/jhNDyrme3tp1sW1BJrawzR7Ujo8DzlVcS2geKA9at7\n55ua5GbHz3hfzFgjVXDfnkWzId6aHypUyqHrSn1SqGEbyXTaleKTc6Pgv0PgkJjG\nhZazWWdSuf1T5Xbs0OhAK9qraoAzT6cXXvMEvvPt6QKBgQDXcZKqJAOnGEU4b9+v\nOdoh+nssdrIOBNMu1m8mYbUVYS1aakc1iDGIIWNM3qAwbG+yNEIi2xi80a2RMw2T\n9RyCNB7yqCXXVKLBiwg9FbKMai6Vpk2bWIrzahM9on7AhCax/X2AeOp+UyYhFEy6\nUFG4aHb8THscL7b515ukSuKb5QKBgQDMq+9PuaB0eHsrmL6q4vHNi3MLgijGg/zu\nAXaPygSYAwYW8KglcuLZPvWrL6OG0+CrfmaWTLsyIZO4Uhdj7MLvX6yK7IMnagvk\nL3xjgxSklEHJAwi5wFeJ8ai/1MIuCn8p2re3CbwISKpvf7Sgs/W4196P4vKvTiAz\njcTiSYFIKQKBgCjMpkS4O0TakMlGTmsFnqyOneLmu4NyIHgfPb9cA4n/9DHKLKAT\noaWxBPgatOVWs7RgtyGYsk+XubHkpC6f3X0+15mGhFwJ+CSE6tN+l2iF9zp52vqP\nQwkjzm7+pdhZbmaIpcq9m1K+9lqPWJRz/3XXuqi+5xWIZ7NaxGvRjqaNAoGAdK2b\nutZ2y48XoI3uPFsuP+A8kJX+CtWZrlE1NtmS7tnicdd19AtfmTuUL6fz0FwfW4Su\nlQZfPT/5B339CaEiq/Xd1kDor+J7rvUHM2+5p+1A54gMRGCLRv92FQ4EON0RC1o9\nm2I4SHysdO3XmjmdXmfp4BsgAKJIJzutvtbqlakCgYB+Cb10z37NJJ+WgjDt+yT2\nyUNH17EAYgWXryfRgTyi2POHuJitd64Xzuy6oBVs3wVveYFM6PIKXlj8/DahYX5I\nR2WIzoCNLL3bEZ+nC6Jofpb4kspoAeRporj29SgesK6QBYWHWX2H645RkRGYGpDo\n51gjy9m/hSNqBbH2zmh04A==\n-----END PRIVATE KEY-----\n", + "client_email": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com", + "client_id": "000000000000000000000", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com" +} \ No newline at end of file diff --git a/src/storage/src/storage/conformance/v4_signatures.json b/src/storage/src/storage/conformance/v4_signatures.json new file mode 100644 index 0000000000..40aa21cba3 --- /dev/null +++ b/src/storage/src/storage/conformance/v4_signatures.json @@ -0,0 +1,698 @@ +{ + "signingV4Tests": [ + { + "description": "Simple GET", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Simple PUT", + "bucket": "test-bucket", + "object": "test-object", + "method": "PUT", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=8adff1d4285739e31aa68e73767a46bc5511fde377497dbe08481bf5ceb34e29cc9a59921748d8ec3dd4085b7e9b7772a952afedfcdaecb3ae8352275b8b7c867f204e3db85076220a3127a8a9589302fc1181eae13b9b7fe41109ec8cdc93c1e8bac2d7a0cc32a109ca02d06957211326563ab3d3e678a0ba296e298b5fc5e14593c99d444c94724cc4be97015dbff1dca377b508fa0cb7169195de98d0e4ac96c42b918d28c8d92d33e1bd125ce0fb3cd7ad2c45dae65c22628378f6584971b8bf3945b26f2611eb651e9b6a8648970c1ecf386bb71327b082e7296c4e1ee2fc0bdd8983da80af375c817fb1ad491d0bc22c0f51dba0d66e2cffbc90803e47", + "scheme": "https", + "expectedCanonicalRequest": "PUT\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n78742860705da91404222d5d66ff89850292471199c3c2808d116ad12e6177b4" + }, + { + "description": "POST for resumable uploads", + "bucket": "test-bucket", + "object": "test-object", + "method": "POST", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-resumable&X-Goog-Signature=4a6d39b23343cedf4c30782aed4b384001828c79ffa3a080a481ea01a640dea0a0ceb58d67a12cef3b243c3f036bb3799c6ee88e8db3eaf7d0bdd4b70a228d0736e07eaa1ee076aff5c6ce09dff1f1f03a0d8ead0d2893408dd3604fdabff553aa6d7af2da67cdba6790006a70240f96717b98f1a6ccb24f00940749599be7ef72aaa5358db63ddd54b2de9e2d6d6a586eac4fe25f36d86fc6ab150418e9c6fa01b732cded226c6d62fc95b72473a4cc55a8257482583fe66d9ab6ede909eb41516a8690946c3e87b0f2052eb0e97e012a14b2f721c42e6e19b8a1cd5658ea36264f10b9b1ada66b8ed5bf7ed7d1708377ac6e5fe608ae361fb594d2e5b24c54", + "headers": { + "X-Goog-Resumable": "start" + }, + "scheme": "https", + "expectedCanonicalRequest": "POST\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-resumable\nhost:storage.googleapis.com\nx-goog-resumable:start\n\nhost;x-goog-resumable\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n877f8b40179d2753296f2fd6de815ab40503c7a3c446a7b44aa4e74422ff4daf" + }, + { + "description": "Vary expiration and timestamp", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 20, + "timestamp": "2019-03-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190301%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190301T090000Z&X-Goog-Expires=20&X-Goog-SignedHeaders=host&X-Goog-Signature=9669ed5b10664dc594c758296580662912cf4bcc5a4ba0b6bf055bcbf6f34eed7bdad664f534962174a924741a0c273a4f67bc1847cef20192a6beab44223bd9d4fbbd749c407b79997598c30f82ddc269ff47ec09fa3afe74e00616d438df0d96a7d8ad0adacfad1dc3286f864d924fe919fb0dce45d3d975c5afe8e13af2db9cc37ba77835f92f7669b61e94c6d562196c1274529e76cfff1564cc2cad7d5387dc8e12f7a5dfd925685fe92c30b43709eee29fa2f66067472cee5423d1a3a4182fe8cea75c9329d181dc6acad7c393cd04f8bf5bc0515127d8ebd65d80c08e19ad03316053ea60033fd1b1fd85a69c576415da3bf0a3718d9ea6d03e0d66f0", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190301%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190301T090000Z&X-Goog-Expires=20&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190301T090000Z\n20190301/auto/storage/goog4_request\n779f19fdb6fd381390e2d5af04947cf21750277ee3c20e0c97b7e46a1dff8907" + }, + { + "description": "Vary bucket and object", + "bucket": "test-bucket2", + "object": "test-object2", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket2/test-object2?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=36e3d58dfd3ec1d2dd2f24b5ee372a71e811ffaa2162a2b871d26728d0354270bc116face87127532969c4a3967ed05b7309af741e19c7202f3167aa8c2ac420b61417d6451442bb91d7c822cd17be8783f01e05372769c88913561d27e6660dd8259f0081a71f831be6c50283626cbf04494ac10c394b29bb3bce74ab91548f58a37118a452693cf0483d77561fc9cac8f1765d2c724994cca46a83517a10157ee0347a233a2aaeae6e6ab5e204ff8fc5f54f90a3efdb8301d9fff5475d58cd05b181affd657f48203f4fb133c3a3d355b8eefbd10d5a0a5fd70d06e9515460ad74e22334b2cba4b29cae4f6f285cdb92d8f3126d7a1479ca3bdb69c207d860", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket2/test-object2\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\na139afbf35ac30e9864f63197f79609731ab1b0ca166e2a456dba156fcd3f9ce" + }, + { + "description": "Slashes in object name should not be URL encoded", + "bucket": "test-bucket", + "object": "path/with/slashes/under_score/amper&sand/file.ext", + "headers": { + "header/name/with/slash": "should-be-encoded" + }, + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/path/with/slashes/under_score/amper%26sand/file.ext?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=header%2Fname%2Fwith%2Fslash%3Bhost&X-Goog-Signature=2a9a82e84e39f5d2c0d980514db17f8c3dece473c9a5743d54e8453f9811927b1b99ce548c534cababd8fa339183e75b410e12e32a4c72f5ff176e95651fabed0072e59e7e236eb7e26f52c0ce599db1c47ae07af1a98d20872b6fde23432c0a5fcf4fb2dda735169198c80cd5cc51be9904f7e5eef2cc489ff44ac5697c529e4b34ac08709a7d2e425619377212c64561ed8b4d2fcb70a26e4f9236f995ab4658d240ac85c7a353bae6b2d39d5fc0716afa435a1f6e100db5504612b5e610db370623ab4b8eba3c03c98f23dcb4b9ffd518f2212abb2f93649d25385d71603d470cff0b7631adb9d0849d38609dedb3097761c8f47ec0d57777bb063611c05b", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/path/with/slashes/under_score/amper%26sand/file.ext\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=header%2Fname%2Fwith%2Fslash%3Bhost\nheader/name/with/slash:should-be-encoded\nhost:storage.googleapis.com\n\nheader/name/with/slash;host\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\nf1d206dd8cbe1b892d4081ccddae0927d9f5fee5653fb2a2f43e7c20ed455cad" + }, + { + "description": "Forward Slashes should not be stripped", + "bucket": "test-bucket", + "object": "/path/with/slashes/under_score/amper&sand/file.ext", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket//path/with/slashes/under_score/amper%26sand/file.ext?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=2db8b70e3f85b39be7824f6d02be31af2e6a2eb63f6bb41254851f7ef51bdad8963a9d2b254f8379c1780c8e6898be002d4100a0abd3d45f1437687fed65d15dd237c3a6f3c399c64ffd4e4cea7ef1c2f0391d35ecbeeaf3e3148d23c6f24c839cfcd92c1496332f5bfbbf1ed1e957eb45fad57df24828c96cf243eec23fba014d277c22a572708beb355888c5a8c0047cb3015d7f62cc90285676e7e34626fd0ce9ba5e0da39fc3de0035cc3ad120c46cb73db87246ae123f7a342c235e9480bd7d7e00c13b1e1bb7be5e2bce74d59a53505172463b48aefeedb48281d90874aa4177c881d3596ed1067f02eaac13d810a7aed234c41978b1394d0ce3662f76", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket//path/with/slashes/under_score/amper%26sand/file.ext\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n63c601ecd6ccfec84f1113fc906609cbdf7651395f4300cecd96ddd2c35164f8" + }, + { + "description": "Simple headers", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=bar%3Bfoo%3Bhost&X-Goog-Signature=68ecd3b008328ed30d91e2fe37444ed7b9b03f28ed4424555b5161980531ef87db1c3a5bc0265aad5640af30f96014c94fb2dba7479c41bfe1c020eb90c0c6d387d4dd09d4a5df8b60ea50eb6b01cdd786a1e37020f5f95eb8f9b6cd3f65a1f8a8a65c9fcb61ea662959efd9cd73b683f8d8804ef4d6d9b2852419b013368842731359d7f9e6d1139032ceca75d5e67cee5fd0192ea2125e5f2955d38d3d50cf116f3a52e6a62de77f6207f5b95aaa1d7d0f8a46de89ea72e7ea30f21286318d7eba0142232b0deb3a1dc9e1e812a981c66b5ffda3c6b01a8a9d113155792309fd53a3acfd054ca7776e8eec28c26480cd1e3c812f67f91d14217f39a606669d", + "headers": { + "BAR": "BAR-value", + "foo": "foo-value" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=bar%3Bfoo%3Bhost\nbar:BAR-value\nfoo:foo-value\nhost:storage.googleapis.com\n\nbar;foo;host\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n59c1ac1a6ee7d773d5c4487ecc861d60b71c4871dd18fc7d8485fac09df1d296" + }, + { + "description": "Headers with colons", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=bar%3Bfoo%3Bhost&X-Goog-Signature=30b831c18b5cdef6dceaa476a395a28d80002ad70b4419af8fd63eaad02c2fbb4b4829b3a4e33e7796a9ce92735498dfc20e0fbc177172f7c8ab6a07736512c7c923ef2f28a2e72d727fd61ca89495c9e62d51b93a2f7061451240c909ed8d05a7bcf616c1ad90fa5cdbc27c4724dec6b29db04129b32402db4ddf7b5b554724481bfdbf41cb24c3c6b9e33bb411c864077d6a19a750a90eb5ad9370d2b171df2813c9a864b40b2ee215ae9790d7916155de863708aa5121bca42e4695def5322f3726f8e1a7ec56da7a1a4f6b959253513a10f7edf6594c02340021b8cc709b0177ec6bb127fc2fb705f508bde045ed94603471c19c1c6af165f559a3c4741b", + "headers": { + "BAR": "2023-02-10T03:", + "foo": "2023-02-10T02:00:00Z" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=bar%3Bfoo%3Bhost\nbar:2023-02-10T03:\nfoo:2023-02-10T02:00:00Z\nhost:storage.googleapis.com\n\nbar;foo;host\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\na2a6df7e6bd818894e1f60ac3c393901b512ca1cf1061ba602dace3fb38c19a6" + }, + { + "description": "Headers should be trimmed", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=collapsed%3Bhost%3Bleading%3Btabs%3Btrailing&X-Goog-Signature=75d77a3ed2d9b74ff7e1e23b2fd7cc714ad4cc32518c65f3a8197827cd87d302623bab990cf2ff3a633bfaae69b6c2d897add78c105aa68411229610421c4239579add4aff6bdbd5067a0fd61c3aa0029d7de0f8ae88fa3458fa70f875e841d6df9598597d9012b9f848c6857e08f2704ca2f332c71738490ffdda2ed928f9340549d7295745725062d28dc1696eab7cb3b88ac4fd445e951423f645d680a60dd8033d65b65f4c10286f59f4258dbb2bcf36a76ffdd40574104cbbf0b76901c24df5854f24c42e9192fcedc386d85704fec6a6bad3a5201e1fb6c491a4c43371b0913420743580daf3504e99204c6ec894b4d70cd27bc60c3fe2850e8bf3ed22", + "headers": { + "collapsed": "abc def", + "leading": " xyz", + "trailing": "abc ", + "tabs": "\tabc\t\t\t\tdef\t" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=collapsed%3Bhost%3Bleading%3Btabs%3Btrailing\ncollapsed:abc def\nhost:storage.googleapis.com\nleading:xyz\ntabs:abc def\ntrailing:abc\n\ncollapsed;host;leading;tabs;trailing\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n19153e83555808dbfeb8969043cc8ce8d5db0cce91dc11fb9df58b8130f09d42" + }, + { + "description": "Header value with multiple inline values", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bmultiple&X-Goog-Signature=5cc113735625341f59c7203f0c2c9febc95ba6af6b9c38814f8e523214712087dc0996e4960d273ae1889f248ac1e58d4d19cb3a69ad7670e9a8ca1b434e878f59339dc7006cf32dfd715337e9f593e0504371839174962a08294586e0c78160a7aa303397888c8350637c6af3b32ac310886cc4590bfda9ca561ee58fb5b8ec56bc606d2ada6e7df31f4276e9dcb96bcaea39dc2cd096f3fad774f9c4b30e317ad43736c05f76831437f44e8726c1e90d3f6c9827dc273f211f32fc85658dfc5d357eb606743a6b00a29e519eef1bebaf9db3e8f4b1f5f9afb648ad06e60bc42fa8b57025056697c874c9ea76f5a73201c9717ea43e54713ff3502ff3fc626b", + "headers": { + "multiple": " xyz , abc, def , xyz " + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bmultiple\nhost:storage.googleapis.com\nmultiple:xyz , abc, def , xyz\n\nhost;multiple\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n4df8e486146c31f1c8cd4e4c730554cde4326791ba48ec11fa969a3de064cd7f" + }, + { + "description": "Customer-supplied encryption key", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-encryption-algorithm%3Bx-goog-encryption-key%3Bx-goog-encryption-key-sha256&X-Goog-Signature=278a1c5a3bad248637054a047014760353942433955871031ed08f515b54588654ad033e91f046ab202b68673030e117d1b786c325e870238b035ba75b3feed560a17aff9bab6bddebd4a31a52cb68b214e27d3b0bd886502c6b36b164306fe88b5a07c6063592afe746b2a5d205dbe90dd5386b94f0a78f75d9f53ee884e18f476e8fc2eb1dd910ce0b4ae1f5d7b09876ef9bf983f539c028429e14bad3c75dbd4ed1ae37856f6d6f8a1805eaf8b52a0d6fc993902e4c1ee8de477661f7b67c3663000474cb00e178189789b2a3ed6bd21b4ade684fca8108ac4dd106acb17f5954d045775f7aa5a98ebda5d3075e11a8ea49c64c6ad1481e463e8c9f11f704", + "headers": { + "X-Goog-Encryption-Algorithm": "AES256", + "X-Goog-Encryption-Key": "key", + "X-Goog-Encryption-Key-Sha256": "key-hash" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-encryption-algorithm%3Bx-goog-encryption-key%3Bx-goog-encryption-key-sha256\nhost:storage.googleapis.com\nx-goog-encryption-algorithm:AES256\nx-goog-encryption-key:key\nx-goog-encryption-key-sha256:key-hash\n\nhost;x-goog-encryption-algorithm;x-goog-encryption-key;x-goog-encryption-key-sha256\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n66a45104eba8bdd9748723b45cbd54c3f0f6dba337a5deb9fb6a66334223dc06" + }, + { + "description": "List Objects", + "bucket": "test-bucket", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=6dbe94f8e52b2b8a9a476b1c857efa474e09944e2b52b925800316e094a7169d8dbe0df9c0ac08dabb22ac7e827470ceccd65f5a3eadba2a4fb9beebfe37f0d9bb1e552b851fa31a25045bdf019e507f5feb44f061551ef1aeb18dcec0e38ba2e2f77d560a46eaace9c56ed9aa642281301a9d848b0eb30749e34bc7f73a3d596240533466ff9b5f289cd0d4c845c7d96b82a35a5abd0c3aff83e4440ee6873e796087f43545544dc8c01afe1d79c726696b6f555371e491980e7ec145cca0803cf562c38f3fa1d724242f5dea25aac91d74ec9ddd739ff65523627763eaef25cd1f95ad985aaf0079b7c74eb5bcb2870a9b137a7b2c8e41fbe838c95872f75b", + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n51a7426c2a6c6ab80f336855fc629461ff182fb1d2cb552ac68e5ce8e25db487" + }, + { + "description": "Query Parameter Encoding", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&aA0%C3%A9%2F%3D%25-_.~=~%20._-%25%3D%2F%C3%A90Aa&X-Goog-Signature=0e4f289c28626a32af95d78f5a0d2c75e5f0bef1cfe5035a532a8185e3ad45b1d5e8ad5ae0fb1898420e40a1576479861c6579db7e3b28ef8386160995ac1374be85e780ac3dfcc62538e9b2e2c51555950fb6fd0d7ecc509d3ccd9c02af5a6c6eb930d21a7383792300eb50a093a597c1da2a290ed56b01844251f8271d0d5e61fc3f4273a0c26f80e061c06124d93346ea1c50388e3fe73494c05ac27a54caedc04d1476c276c7602554e4cc1933e41df31ea523f4009e879c92333f98b4313755470d7e400bbbde9f5bef36f40b2a1c4a85edbd62aaa84a516e3df055d5d3f575b9ea0f1c24f057240852d9e618403e1dd2cd0fa7fc16b071b07322172256", + "queryParameters": { + "aA0é/=%-_.~": "~ ._-%=/é0Aa" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&aA0%C3%A9%2F%3D%25-_.~=~%20._-%25%3D%2F%C3%A90Aa\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n448f96c23dafa8210900554e138b2b5fd55bc53ef53b8637cecc3edec45a8fcf" + }, + { + "description": "Query Parameter Ordering", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-Meta-Foo=bar&X-Goog-SignedHeaders=host&prefix=%2Ffoo&X-Goog-Signature=a07745e1b3d59b85cbe11aa766df72c22468959e7217615dccb7f030234f66b60b37e480f30725ed51f29816362ca8286c619ebb66448ff1d370be2a4a48aacf20d3d2d6200ed17341a5791baf2ee5cd9c2823adacc6264f66c8a54fa887e1bce3c55cf78fb2f6a52618cf09d6f945f63d148052a7b66a75e075ff5065828a806b84bdc49a42399be7483225c720d5e18a6160f79d815f433e7921694fe1d041099851793c2581db0e5ca503cfb566e414f900ceede5f9b22030edd32ab20b6f7f9fb2afba89098b9364e03397c03a94eac3a140c99979b8786844fb4f6c62c1985378939dd1bbaea8e41b9100dda85a27733171cc78d96ee362ea2c3432f4d8", + "queryParameters": { + "prefix": "/foo", + "X-Goog-Meta-Foo": "bar" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-Meta-Foo=bar&X-Goog-SignedHeaders=host&prefix=%2Ffoo\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n4dafe74ad142f32b7c25fc4e6b38fd3b8a6339d7f112247573fb0066f637db6c" + }, + { + "description": "Header Ordering", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-date&X-Goog-Signature=55a28435997457f1498291e878fd39c5f321973057d2541886020fdfd212b1467d9eeffdc70951ea952d634cb4193e657ed5b7860c46d37f7d904774680a16e518aa9dff273e8441d6893de615eb592e3113d682ad64a87eb0e0c48df17c30f899e7f940ba230530b30f725ab9ec38789682413752de6a026ae69dd858843100645f3ec986aed618d229f8844d378e0e66e907ede6dff7aac56723f51eb830e8877a56100c86a876173424602abefe6c22b6540a2b36634860b2e89137f297cca8f080bdf3433a9d614c5ab2ec84f65412b45516b30500886a2300f23c3423ae0e91546e3471ee08d06894bddc76203a418d46f35bf0b4574f7b24a693fb046c", + "headers": { + "X-Goog-Date": "20190201T090000Z" + }, + "scheme": "https", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-date\nhost:storage.googleapis.com\nx-goog-date:20190201T090000Z\n\nhost;x-goog-date\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n4052143280d90d5f4a8c878ff7418be6fee5d34e50b1da28d8081a094b88fa61" + }, + { + "description": "Signed Payload Instead of UNSIGNED-PAYLOAD", + "bucket": "test-bucket", + "object": "test-object", + "method": "PUT", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-content-sha256%3Bx-testcasemetadata-payload-value&X-Goog-Signature=3e5a9669e9aa162888dff1553d24c159bad4f16d444987f6a1b26d8ad0cb7927f15bfaf79c205324d2138fd1f62edb255430c77a03c0d6e9601399e2519014f9e1a7051d9be735cde530022c84602b1c4c25c86cb1e1584489e49d511c9a618a1a8443af31626ca5b2ad105eda1e4499f52b4043f3c1a3bd40c06c0cae36bb19a50ed8671e5d2cdbb148a196ce5a8c14d6970c08225da293e1ef400c92e7a3d5ba0a29ad0893827c96b203a04b04ebd51929bf99b323beba93097dfee700ee2c1bd97013779e5c8f156e56175d4d07e453b2eb0d616086f9f4753dde63507efe88b0dec29c872d25d9465f07778b16b532814148c578ee7e64ed8437006fa551", + "headers": { + "X-Goog-Content-SHA256": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b982", + "X-TestCaseMetadata-Payload-Value": "hello" + }, + "scheme": "https", + "expectedCanonicalRequest": "PUT\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host%3Bx-goog-content-sha256%3Bx-testcasemetadata-payload-value\nhost:storage.googleapis.com\nx-goog-content-sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b982\nx-testcasemetadata-payload-value:hello\n\nhost;x-goog-content-sha256;x-testcasemetadata-payload-value\n2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b982", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\nbe21a0841a897930ff5cf72e6e74ec5274efd76c3fe4cde6678f24a0a3d6dbec" + }, + { + "description": "Virtual Hosted Style", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://test-bucket.storage.googleapis.com/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=06c633ea060b0eda81ee58dd3337b01b0d243a44f18cb03ec948861f533a129e579c7fd4c856d187f1c7b86e5715ea0abf6a1c6ba32b69274d22b1b0406df6847dc87f0d289fe8dc0682351574849b8b13e4b66922f39441af96becb73ea4c56cd5e3eeb30bc91fe84e8bd205adca8639253bdb65b2fcaf2598a230c6d8f6d8177c9e58a61b6e826767f594056b490184d676897c4bbedc15d6fbf08c3fa82a406c62e74db661e6c5d7d3ced29e0619ee719dce4b8136360345b8dce120b9f1debd511c8dac3e6d874ee05bfda8c8f1c4fedd0c07fc6d98f5f18a349bb204d8ff401402a025194e2792df8a09282141157e4ca51d26a8d0d142a01c805321911", + "scheme": "https", + "urlStyle": "VIRTUAL_HOSTED_STYLE", + "expectedCanonicalRequest": "GET\n/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:test-bucket.storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n89eeae48258eccdcb1f592fb908008e3f5d36a949c002c1e614c94356dc18fc6" + }, + { + "description": "HTTP Bucket Bound Hostname Support", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "http://mydomain.tld/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7115a77f8c7ed1a8b74bca8b520904fca7f3bab90d69ea052687a94efd9b3a4e2a7fb7135d40e295e0a21958194c55da7e106227957c22ed6edc9d8b3d2a8133bc8af84fc9695dda8081d53f0db5ea9f28e5bfc225d78f873e9f571fd287bb7a95330e726aebd8eb4623cdb0b1a7ceb210b2ce1351b6be0191c2ad7b38f7ceb6c5ce2f98dbfb5a5a649050585e46e97f72f1f5407de657a56e34a3fdc80cdaa0598bd47f3e8af5ff22d0916b19b106890bff8c3f6587f1d3b076b16cd0ba0508607a672be33b9c75d537e15258450b43d22a21c4d528090acbb8e5bae7b31fc394e61394106ef1d6a8ed43074ab05bcec65674cd8113fb3de388da4d97e62f56", + "scheme": "http", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld", + "expectedCanonicalRequest": "GET\n/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:mydomain.tld\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\nd6c309924b51a5abbe4d6356f7bf29c2120c6b14649b1e97b3bc9309adca7d4b" + }, + { + "description": "HTTPS Bucket Bound Hostname Support", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://mydomain.tld/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7115a77f8c7ed1a8b74bca8b520904fca7f3bab90d69ea052687a94efd9b3a4e2a7fb7135d40e295e0a21958194c55da7e106227957c22ed6edc9d8b3d2a8133bc8af84fc9695dda8081d53f0db5ea9f28e5bfc225d78f873e9f571fd287bb7a95330e726aebd8eb4623cdb0b1a7ceb210b2ce1351b6be0191c2ad7b38f7ceb6c5ce2f98dbfb5a5a649050585e46e97f72f1f5407de657a56e34a3fdc80cdaa0598bd47f3e8af5ff22d0916b19b106890bff8c3f6587f1d3b076b16cd0ba0508607a672be33b9c75d537e15258450b43d22a21c4d528090acbb8e5bae7b31fc394e61394106ef1d6a8ed43074ab05bcec65674cd8113fb3de388da4d97e62f56", + "scheme": "https", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld", + "expectedCanonicalRequest": "GET\n/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:mydomain.tld\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\nd6c309924b51a5abbe4d6356f7bf29c2120c6b14649b1e97b3bc9309adca7d4b" + }, + { + "description": "Simple GET with hostname", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74", + "scheme": "https", + "hostname": "storage.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Simple GET with non-default hostname", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "http://localhost:8080/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=66c8c0371ca7d933a0d50f110abdf4fc7e3e379329134f272ebe4aa8100ccd5f21cd56ca5ccffae5ed36c8d6840e7cac80c2e7d786cd85b10d0faea34cddf09d2e7eb7f5c7c53934e4bf8f5cd654bc3c1b5ee9e3f8ca2189cd225b445bb866563fc4bd0d0b4d116111655611d12ec18f2d854fd7142d9afcc977dbd8f6d0524e4170506abf2b119bbe00d17697321d225162fabddb4ddae77781b4f3277a6b6fccfeb47d70b88537e5efb416001274aaeb1535b5aae757c997edc66d03898a5d08f767313d018d10992981d00e2a18ed9a6839b8a1ac7b3be1cab2e9511ba91e14a786443b59e9f21e1ae74a2c60106180646a764531fbe1fcd9c1e40550e56e", + "scheme": "http", + "hostname": "localhost:8080", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:localhost\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\ne47446edb8eed4c1797dfd31ce30272be89659a6ef38e91b549740c8f875d27b" + }, + { + "description": "Simple GET with endpoint on client", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.googleapis.com:443/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=95e6a13d43a1d1962e667f17397f2b80ac9bdd1669210d5e08e0135df9dff4e56113485dbe429ca2266487b9d1796ebdee2d7cf682a6ef3bb9fbb4c351686fba90d7b621cf1c4eb1fdf126460dd25fa0837dfdde0a9fd98662ce60844c458448fb2b352c203d9969cb74efa4bdb742287744a4f2308afa4af0e0773f55e32e92973619249214b97283b2daa14195244444e33f938138d1e5f561088ce8011f4986dda33a556412594db7c12fc40e1ff3f1bedeb7a42f5bcda0b9567f17f65855f65071fabb88ea12371877f3f77f10e1466fff6ff6973b74a933322ff0949ce357e20abe96c3dd5cfab42c9c83e740a4d32b9e11e146f0eb3404d2e975896f74", + "scheme": "https", + "clientEndpoint": "storage.googleapis.com:443", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n00e2fb794ea93d7adb703edaebdd509821fcc7d4f1a79ac5c8d2b394df109320" + }, + { + "description": "Endpoint on client with scheme", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "http://localhost:8080/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=66c8c0371ca7d933a0d50f110abdf4fc7e3e379329134f272ebe4aa8100ccd5f21cd56ca5ccffae5ed36c8d6840e7cac80c2e7d786cd85b10d0faea34cddf09d2e7eb7f5c7c53934e4bf8f5cd654bc3c1b5ee9e3f8ca2189cd225b445bb866563fc4bd0d0b4d116111655611d12ec18f2d854fd7142d9afcc977dbd8f6d0524e4170506abf2b119bbe00d17697321d225162fabddb4ddae77781b4f3277a6b6fccfeb47d70b88537e5efb416001274aaeb1535b5aae757c997edc66d03898a5d08f767313d018d10992981d00e2a18ed9a6839b8a1ac7b3be1cab2e9511ba91e14a786443b59e9f21e1ae74a2c60106180646a764531fbe1fcd9c1e40550e56e", + "scheme": "http", + "clientEndpoint": "http://localhost:8080", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:localhost\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\ne47446edb8eed4c1797dfd31ce30272be89659a6ef38e91b549740c8f875d27b" + }, + { + "description": "Emulator host", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://xyz.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=53b20003ff2c552b3194a6bccc25024663662392554b3334e989e2704f3a0308455eaacf45c335a78c0186a5cf8eef78bf5781a7267465d28a35c9e1291f87ff340e9ee40b3b9bdce70561bf000887ce38ccd7d2445a8749453960a8f11d37576dfd5942f92d6f4527bbeffb90526b5de9653b6ca16136e9f19bcb65d984ddaf22c4ade45d6a168bb4752a43de33ab121206f50d994612824407711bff720cb1b207b61b613c44c85d3ce16dc4fc6eba24e494e176b0780d0ab85a800b13fcbf31434ddf51992efae1efde330ebda0617d1c20078ef22a4f10a7bcbed961237442d9a8db78d7aeb777a4994b50efdd41e07c4966e912a30f92a7426f207e9545", + "emulatorHostname": "https://xyz.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:xyz.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n4f6f519cc03e25d19fcd476d7a45bffcccdba33d10e00214a0f2debc204e2386" + }, + { + "description": "Endpoint on client takes precedence over emulator", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "http://localhost:8080/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=66c8c0371ca7d933a0d50f110abdf4fc7e3e379329134f272ebe4aa8100ccd5f21cd56ca5ccffae5ed36c8d6840e7cac80c2e7d786cd85b10d0faea34cddf09d2e7eb7f5c7c53934e4bf8f5cd654bc3c1b5ee9e3f8ca2189cd225b445bb866563fc4bd0d0b4d116111655611d12ec18f2d854fd7142d9afcc977dbd8f6d0524e4170506abf2b119bbe00d17697321d225162fabddb4ddae77781b4f3277a6b6fccfeb47d70b88537e5efb416001274aaeb1535b5aae757c997edc66d03898a5d08f767313d018d10992981d00e2a18ed9a6839b8a1ac7b3be1cab2e9511ba91e14a786443b59e9f21e1ae74a2c60106180646a764531fbe1fcd9c1e40550e56e", + "scheme": "http", + "clientEndpoint": "http://localhost:8080", + "emulatorHostname": "https://xyz.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:localhost\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\ne47446edb8eed4c1797dfd31ce30272be89659a6ef38e91b549740c8f875d27b" + }, + { + "description": "Hostname takes precendence over endpoint and emulator", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://xyz.googleapis.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=53b20003ff2c552b3194a6bccc25024663662392554b3334e989e2704f3a0308455eaacf45c335a78c0186a5cf8eef78bf5781a7267465d28a35c9e1291f87ff340e9ee40b3b9bdce70561bf000887ce38ccd7d2445a8749453960a8f11d37576dfd5942f92d6f4527bbeffb90526b5de9653b6ca16136e9f19bcb65d984ddaf22c4ade45d6a168bb4752a43de33ab121206f50d994612824407711bff720cb1b207b61b613c44c85d3ce16dc4fc6eba24e494e176b0780d0ab85a800b13fcbf31434ddf51992efae1efde330ebda0617d1c20078ef22a4f10a7bcbed961237442d9a8db78d7aeb777a4994b50efdd41e07c4966e912a30f92a7426f207e9545", + "emulatorHostname": "http://localhost:9000", + "clientEndpoint": "http://localhost:8080", + "hostname": "xyz.googleapis.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:xyz.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n4f6f519cc03e25d19fcd476d7a45bffcccdba33d10e00214a0f2debc204e2386" + }, + { + "description": "Universe domain", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://storage.domain.com/test-bucket/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=8cd0d479a88fb7d791a2dcc8fc5b5f020ca817eeef5b5a5cb3260eb63cf47ecd271faa238d0fa31efca35bc2a9244bd122178c520749f922c0235726a5a6be099bf4f33a0d54187eee2e0208964c2a13104b03e235cdeb4f07b3eb566b8a33259cf7540a3fe823be601ace2a54a79acd6834cb646380c4cfc7ef0fd95d3ebbc1f97d840f6fe1dceed4269ecb4e91ff7e6633f38adab82049a965968367b9e7c362cec868d804bd42abbb6d2e837ce5d45ee9e1d92c7acc09623acaae3df6128ca15f9f80bb6543944e8c997f691c35113b9e9f44e86fd343524343b08dd8f887685588acc103e0b432f24912e7e1c63e086aeed1890e41b75beb64164fe6bfcf", + "universeDomain": "domain.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:storage.domain.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n31ff08f2cd5e6f02cc5ded6d74bb90ad97322b49b30d0cba130fcc473f85e822" + }, + { + "description": "Universe domain with virtual hosted style", + "bucket": "test-bucket", + "object": "test-object", + "urlStyle": "VIRTUAL_HOSTED_STYLE", + "method": "GET", + "expiration": 10, + "timestamp": "2019-02-01T09:00:00Z", + "expectedUrl": "https://test-bucket.storage.domain.com/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=25820e3a60856596cba594511d7d4039239b2728a9738f15d3a7acce8d70aa5435d0c91f99a9318f932afc73355ac562e014cb654e16ed5524b403536f1cba74489701fdc0c088b8826fccf20a648d3b2b704bd6661e01786d4132174c21441d0752be07e8af93e84e24b87799ee91fabef24a0a58d0889263280c3d37423fab677bd4d98469ab01aa36efaad62ff81ca27bf7fc92f14e20faa71e34de9ffbc5eb4ecf1b0361de42270665bb78367bd0a8cc6a604a8e347f0c864754bf14514aac3106fe73572a6c068ce2c380cc2a943b35502093d162ba9ae8de9abbbc9541ef765d5679857a89d36cc01be30cf1e04c4a477bbcd59a02955dcc1a903d8baa", + "universeDomain": "domain.com", + "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:test-bucket.storage.domain.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n6835c0cd7e63f2e34becade43beee99335c68c1455488da5b320cf13dc0a0ed5" + } + ], + "postPolicyV4Tests": [ + { + "description": "POST Policy Simple", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902670-h3q7wvodjor6bc7y", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z" + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902670-h3q7wvodjor6bc7y/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "9eb947e08891be62a6c26d214c5b639506a57c20e1242a135dce45326a821aa9d8eaf5a8b6961a7ba87ab4da09668eb303b615dabdd6344970a024d4697aa3c467ec527cb8318d6c971782c6f82555c8cdd791589226962f0c26d5c415bf9d257cf9ed266a43e1b74891c35f009be741cef42a1eec150ffa5783de217e3704e8f278209250423f62c57864c50c6e525e6e9b724fcafc3844898c6a219effedfaf1204cb95eb135f63307db86726b8d6c12b8a75df61f93143b5e798acdffee0b5ad481075fa89a9d49a470cea339e5e43b61a0d87ef54f355dc401a659b583e7906c20bf84b8da7941d2e932acab21ffe92416985262fe772a117483b6e70410", + "policy": "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjcwLWgzcTd3dm9kam9yNmJjN3kifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"bucket\":\"rsaposttest-1579902670-h3q7wvodjor6bc7y\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Simple Virtual Hosted Style", + "policyInput": { + "scheme": "https", + "urlStyle": "VIRTUAL_HOSTED_STYLE", + "bucket": "rsaposttest-1579902670-h3q7wvodjor6bc7y", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z" + }, + "policyOutput": { + "url": "https://rsaposttest-1579902670-h3q7wvodjor6bc7y.storage.googleapis.com/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "9eb947e08891be62a6c26d214c5b639506a57c20e1242a135dce45326a821aa9d8eaf5a8b6961a7ba87ab4da09668eb303b615dabdd6344970a024d4697aa3c467ec527cb8318d6c971782c6f82555c8cdd791589226962f0c26d5c415bf9d257cf9ed266a43e1b74891c35f009be741cef42a1eec150ffa5783de217e3704e8f278209250423f62c57864c50c6e525e6e9b724fcafc3844898c6a219effedfaf1204cb95eb135f63307db86726b8d6c12b8a75df61f93143b5e798acdffee0b5ad481075fa89a9d49a470cea339e5e43b61a0d87ef54f355dc401a659b583e7906c20bf84b8da7941d2e932acab21ffe92416985262fe772a117483b6e70410", + "policy": "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjcwLWgzcTd3dm9kam9yNmJjN3kifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"bucket\":\"rsaposttest-1579902670-h3q7wvodjor6bc7y\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Simple Bucket Bound Hostname", + "policyInput": { + "scheme": "https", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld", + "bucket": "rsaposttest-1579902670-h3q7wvodjor6bc7y", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z" + }, + "policyOutput": { + "url": "https://mydomain.tld/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "9eb947e08891be62a6c26d214c5b639506a57c20e1242a135dce45326a821aa9d8eaf5a8b6961a7ba87ab4da09668eb303b615dabdd6344970a024d4697aa3c467ec527cb8318d6c971782c6f82555c8cdd791589226962f0c26d5c415bf9d257cf9ed266a43e1b74891c35f009be741cef42a1eec150ffa5783de217e3704e8f278209250423f62c57864c50c6e525e6e9b724fcafc3844898c6a219effedfaf1204cb95eb135f63307db86726b8d6c12b8a75df61f93143b5e798acdffee0b5ad481075fa89a9d49a470cea339e5e43b61a0d87ef54f355dc401a659b583e7906c20bf84b8da7941d2e932acab21ffe92416985262fe772a117483b6e70410", + "policy": "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjcwLWgzcTd3dm9kam9yNmJjN3kifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"bucket\":\"rsaposttest-1579902670-h3q7wvodjor6bc7y\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Simple Bucket Bound Hostname HTTP", + "policyInput": { + "scheme": "http", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld", + "bucket": "rsaposttest-1579902670-h3q7wvodjor6bc7y", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z" + }, + "policyOutput": { + "url": "http://mydomain.tld/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "9eb947e08891be62a6c26d214c5b639506a57c20e1242a135dce45326a821aa9d8eaf5a8b6961a7ba87ab4da09668eb303b615dabdd6344970a024d4697aa3c467ec527cb8318d6c971782c6f82555c8cdd791589226962f0c26d5c415bf9d257cf9ed266a43e1b74891c35f009be741cef42a1eec150ffa5783de217e3704e8f278209250423f62c57864c50c6e525e6e9b724fcafc3844898c6a219effedfaf1204cb95eb135f63307db86726b8d6c12b8a75df61f93143b5e798acdffee0b5ad481075fa89a9d49a470cea339e5e43b61a0d87ef54f355dc401a659b583e7906c20bf84b8da7941d2e932acab21ffe92416985262fe772a117483b6e70410", + "policy": "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjcwLWgzcTd3dm9kam9yNmJjN3kifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"bucket\":\"rsaposttest-1579902670-h3q7wvodjor6bc7y\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy ACL matching", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902662-x2kd7kjwh2w5izcw", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "conditions": { + "startsWith": [ + "$acl", + "public" + ] + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902662-x2kd7kjwh2w5izcw/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "134b88d3d7ed57fd7ba79fe53b52191f2ccf46bde0ed1b1fe1557025dc76388fab367b55e887e7a2a23425c98c72d5d1095700568ca5e71f0bd38042b8d94ace6d75bce2c17760b53a8af7706944fd9f382b3a78c9651d9214a5d6252f099c18a759cd98f8dd3e32ac024d5e1b63d4c01d44954d4e943254f3a4cc4cab74cd251a733d4794a22e5840993b6d2970aa050f403c68c25019e91d133d47ff7188facf13560f918ae8efe49ec9ebcc4080d141154554f65cc6d9d6ef0e8bc12119c7491800d79769b5f27707ea9fe78c7af3c39df82608ca78f6f60b638510fd45a14404ed0224365c7ea45b839d91db99a7f8af50a64b754817318fae7bb94b3574", + "policy": "eyJjb25kaXRpb25zIjpbWyJzdGFydHMtd2l0aCIsIiRhY2wiLCJwdWJsaWMiXSx7ImJ1Y2tldCI6InJzYXBvc3R0ZXN0LTE1Nzk5MDI2NjIteDJrZDdrandoMnc1aXpjdyJ9LHsia2V5IjoidGVzdC1vYmplY3QifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAxMjNUMDQzNTMwWiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0LWlhbS1jcmVkZW50aWFsc0BkdW1teS1wcm9qZWN0LWlkLmlhbS5nc2VydmljZWFjY291bnQuY29tLzIwMjAwMTIzL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDEtMjNUMDQ6MzU6NDBaIn0=" + }, + "expectedDecodedPolicy": "{\"conditions\":[[\"starts-with\",\"$acl\",\"public\"],{\"bucket\":\"rsaposttest-1579902662-x2kd7kjwh2w5izcw\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Within Content-Range", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902672-lpd47iogn6hx4sle", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "conditions": { + "contentLengthRange": [ + 246, + 266 + ] + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902672-lpd47iogn6hx4sle/", + "fields": { + "key": "test-object", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "7f81e41a67c60dda3578c3e4f6457692e9cdb7b920da30204de893dac10e625fabea5f8002c8e1891bedc47e6136c8a012dbb8df4e532aea3e186b6c98050cdaf8d82d6cf01495ed87d97af6909589e456cd4d1a624462166cdd4c3bb3a6f945a69d69768bef4d8542add6c2971c3d9993af805f9e4cf2ad8abc69cf8dc3a99eb658eb51030d84037583a5991de29cad34edf7c0d88f6e904f8a00b86022394cc35e9552ed7bdcec3f04e46165952f78cd8bfcabb2def569b6076d0009f6ccc79b94fd67497481711dea1351e82f9e2626c9de374c81aa418000bff039f97367d021afb85d228230b4f3cd5ffe58ccb140bebc62a34e45fc42ba75aec4335035", + "policy": "eyJjb25kaXRpb25zIjpbWyJjb250ZW50LWxlbmd0aC1yYW5nZSIsMjQ2LDI2Nl0seyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjcyLWxwZDQ3aW9nbjZoeDRzbGUifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[[\"content-length-range\",246,266],{\"bucket\":\"rsaposttest-1579902672-lpd47iogn6hx4sle\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Cache-Control File Header", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902669-nwk5s7vvfjgdjs62", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "fields": { + "acl": "public-read", + "cache-control": "public,max-age=86400" + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902669-nwk5s7vvfjgdjs62/", + "fields": { + "key": "test-object", + "acl": "public-read", + "cache-control": "public,max-age=86400", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "99f29892dca68c13a1a9939392efa0891e68def63376ee6de4cf978e1537fcf79e1f8ef8ed068bc00a1c6a311fafbcc95eee5ad59aee502fe443b06905a6942f04e9f516e6bdd4162571b989a45378850d0721956a1808d5f4e3d531b6c20c654886b6910acd4c334127f78f8e6bfcb38ed82c65ecd2b0283e4e17275cbae40c43619ccecfe47cea81dad5ec180bbebc239c1c323af6719df4916e85db2b0a7f9a931ccb8ffe4d6f23899359339593c92f246be884324a1959327a5108a88f48da5be22444c943ff58493b3d1579f4dc734a7b14b3759b8e4a9350666e55e187a3b14f8b6388cf474ec8b7c7ed67cd5e21d0e13c1cf09884fdf5deb045b8bb6f", + "policy": "eyJjb25kaXRpb25zIjpbeyJhY2wiOiJwdWJsaWMtcmVhZCJ9LHsiY2FjaGUtY29udHJvbCI6InB1YmxpYyxtYXgtYWdlPTg2NDAwIn0seyJidWNrZXQiOiJyc2Fwb3N0dGVzdC0xNTc5OTAyNjY5LW53azVzN3Z2ZmpnZGpzNjIifSx7ImtleSI6InRlc3Qtb2JqZWN0In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"acl\":\"public-read\"},{\"cache-control\":\"public,max-age=86400\"},{\"bucket\":\"rsaposttest-1579902669-nwk5s7vvfjgdjs62\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Success With Status", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902678-pt5yms55j47r6qy4", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "fields": { + "success_action_status": "200" + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902678-pt5yms55j47r6qy4/", + "fields": { + "key": "test-object", + "success_action_status": "200", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "4b93fb02fa09f8eca92fc27eaa79028ae56c4b61fef83715833ebf7601607dbee627d6101e078881e4b24d795e9e2062ddb7b01470a742b9f6b1aac5c7b7c86d17bf298259189fd6ae0d6ae952b993a6a9f4eaf218bcc462d8dbc0b8553ca4d00349714e1143655a8eed18b02c71e9e53558055976cf3dc58f5946c9e9d6bda9305eed0575f7be80abff41d7a02fe2ab9a2abe87ab7040314734c1179e3a8edb0a024f227509391ca1ef4705140252a1a0bd6022096e9ef0ef5789639bce5953d5c4595b81b262768dbbfe2b7f68e3ebd2cf42f746897fe7c0ac8ec08c6cb85db8b1737a98d25bfa4a4022be72c4e17a1687856c1020b4fdd9438e91949437be", + "policy": "eyJjb25kaXRpb25zIjpbeyJzdWNjZXNzX2FjdGlvbl9zdGF0dXMiOiIyMDAifSx7ImJ1Y2tldCI6InJzYXBvc3R0ZXN0LTE1Nzk5MDI2NzgtcHQ1eW1zNTVqNDdyNnF5NCJ9LHsia2V5IjoidGVzdC1vYmplY3QifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAxMjNUMDQzNTMwWiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0LWlhbS1jcmVkZW50aWFsc0BkdW1teS1wcm9qZWN0LWlkLmlhbS5nc2VydmljZWFjY291bnQuY29tLzIwMjAwMTIzL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDEtMjNUMDQ6MzU6NDBaIn0=" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"success_action_status\":\"200\"},{\"bucket\":\"rsaposttest-1579902678-pt5yms55j47r6qy4\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Success With Redirect", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902671-6ldm6caw4se52vrx", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "fields": { + "success_action_redirect": "http://www.google.com/" + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902671-6ldm6caw4se52vrx/", + "fields": { + "key": "test-object", + "success_action_redirect": "http://www.google.com/", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "6e83c57e73b1794eb6c5903a1eebdb5c29b0adc333010a5cd623fcb1a9716a31c680c94d43fd9f1f18134b864c380c78f8c1ab048038e743f9da080148365acaa01374dc7aa626cc93c73010a67b79c6776faf5edb8eb7ad56c8f9b9c998a1dab7ea1de675f2b315951c4ca2f54a3d21570896aaa66d8980ed09adf4e4240b49478bdabdaf51b720124569e94b1918856893c14c119c529fcb2e01838198b5d18042994d180fd4b9e26aef1d97fe5646c328e15a05decf6005e1c64cb7783811811f4cd5a720cbd6aa4cfc27ac81fc0b163ee9719c53af5019fd2be83b87e0da6d285f0270bc94f1e8788993794c309745c22709ee0dbad0e463f06830aabbbf", + "policy": "eyJjb25kaXRpb25zIjpbeyJzdWNjZXNzX2FjdGlvbl9yZWRpcmVjdCI6Imh0dHA6Ly93d3cuZ29vZ2xlLmNvbS8ifSx7ImJ1Y2tldCI6InJzYXBvc3R0ZXN0LTE1Nzk5MDI2NzEtNmxkbTZjYXc0c2U1MnZyeCJ9LHsia2V5IjoidGVzdC1vYmplY3QifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAxMjNUMDQzNTMwWiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0LWlhbS1jcmVkZW50aWFsc0BkdW1teS1wcm9qZWN0LWlkLmlhbS5nc2VydmljZWFjY291bnQuY29tLzIwMjAwMTIzL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDEtMjNUMDQ6MzU6NDBaIn0=" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"success_action_redirect\":\"http://www.google.com/\"},{\"bucket\":\"rsaposttest-1579902671-6ldm6caw4se52vrx\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy Character Escaping", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902671-6ldm6caw4se52vrx", + "object": "$test-object-é", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "fields": { + "success_action_redirect": "http://www.google.com/", + "x-goog-meta-custom-1": "$test-object-é-metadata" + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902671-6ldm6caw4se52vrx/", + "fields": { + "key": "$test-object-é", + "success_action_redirect": "http://www.google.com/", + "x-goog-meta-custom-1": "$test-object-é-metadata", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "a79ab836c4a36c3cd1dd3253c9482e2bb65ab461ac61fccee8757169da32c77f07a0da4c47745314fc9fba24d93679c864bb197e6eff26caddf3099b72db962451131afa969bf901d6d4ef63db3e36d48af4040b743b37ab8a08174e63cb5da39082490c03a8a28f7ede43f847a8f4447bb82b73434a1bcd365c8e6f62ae09a7b30a0706745787542a919096632840925d5677f668800220e6dbe83c8a42dc8343c85c16499b7179b96a677cfb35af6cf0face1b0409f40f41fd159df50d9fe4dd915439bd34d98ae22f4e2376e6b6c86654abe147083f2766fa75cc2cee9241f0ea5bcb8daa431712952f1038e7c596568500d80834957988be69560de5ce5d", + "policy": "eyJjb25kaXRpb25zIjpbeyJzdWNjZXNzX2FjdGlvbl9yZWRpcmVjdCI6Imh0dHA6Ly93d3cuZ29vZ2xlLmNvbS8ifSx7IngtZ29vZy1tZXRhLWN1c3RvbS0xIjoiJHRlc3Qtb2JqZWN0LVx1MDBlOS1tZXRhZGF0YSJ9LHsiYnVja2V0IjoicnNhcG9zdHRlc3QtMTU3OTkwMjY3MS02bGRtNmNhdzRzZTUydnJ4In0seyJrZXkiOiIkdGVzdC1vYmplY3QtXHUwMGU5In0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMTIzVDA0MzUzMFoifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdC1pYW0tY3JlZGVudGlhbHNAZHVtbXktcHJvamVjdC1pZC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbS8yMDIwMDEyMy9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAxLTIzVDA0OjM1OjQwWiJ9" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"success_action_redirect\":\"http://www.google.com/\"},{\"x-goog-meta-custom-1\":\"$test-object-\u00e9-metadata\"},{\"bucket\":\"rsaposttest-1579902671-6ldm6caw4se52vrx\"},{\"key\":\"$test-object-\u00e9\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + }, + { + "description": "POST Policy With Additional Metadata", + "policyInput": { + "scheme": "https", + "bucket": "rsaposttest-1579902671-6ldm6caw4se52vrx", + "object": "test-object", + "expiration": 10, + "timestamp": "2020-01-23T04:35:30Z", + "fields": { + "content-disposition": "attachment; filename=\"~._-%=/é0Aa\"", + "content-encoding": "gzip", + "content-type": "text/plain", + "success_action_redirect": "http://www.google.com/" + } + }, + "policyOutput": { + "url": "https://storage.googleapis.com/rsaposttest-1579902671-6ldm6caw4se52vrx/", + "fields": { + "content-disposition": "attachment; filename=\"~._-%=/é0Aa\"", + "content-encoding": "gzip", + "content-type": "text/plain", + "key": "test-object", + "success_action_redirect": "http://www.google.com/", + "x-goog-algorithm": "GOOG4-RSA-SHA256", + "x-goog-credential": "test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request", + "x-goog-date": "20200123T043530Z", + "x-goog-signature": "10e881a484d257672192a50892f7373ef243e1ff0e9043e47b3487d8280e4a27e85b0b16a60e5f9f539fc04c5b6141ca8a568fd2b66555000061cad696d6841cb31dc78862dbf0f66b7d55e72156c21a2ffa116923f86df523e4b16ef686acb46bc2665a7827c5dfafc26d7a6919ffea7f2d7803aa61f93d6389731adface622a848e663b5106858754e06e1a63d55feca12d814e1bcbcf5c42cd573950f53c0e9aa9bf2e746aa1287d0a293e07c24cf15698d42f11639cbd385ba8d9fc7db17dffdcab6d4b4be2e2219f7b98a58303294087858c120a0bc550bad31e4f101615066b9e946f0d54bcd7ae8e1306608b539213c809c13deae16a2a5d62b2e9cb7", + "policy": "eyJjb25kaXRpb25zIjpbeyJjb250ZW50LWRpc3Bvc2l0aW9uIjoiYXR0YWNobWVudDsgZmlsZW5hbWU9XCJ+Ll8tJT0vXHUwMGU5MEFhXCIifSx7ImNvbnRlbnQtZW5jb2RpbmciOiJnemlwIn0seyJjb250ZW50LXR5cGUiOiJ0ZXh0L3BsYWluIn0seyJzdWNjZXNzX2FjdGlvbl9yZWRpcmVjdCI6Imh0dHA6Ly93d3cuZ29vZ2xlLmNvbS8ifSx7ImJ1Y2tldCI6InJzYXBvc3R0ZXN0LTE1Nzk5MDI2NzEtNmxkbTZjYXc0c2U1MnZyeCJ9LHsia2V5IjoidGVzdC1vYmplY3QifSx7IngtZ29vZy1kYXRlIjoiMjAyMDAxMjNUMDQzNTMwWiJ9LHsieC1nb29nLWNyZWRlbnRpYWwiOiJ0ZXN0LWlhbS1jcmVkZW50aWFsc0BkdW1teS1wcm9qZWN0LWlkLmlhbS5nc2VydmljZWFjY291bnQuY29tLzIwMjAwMTIzL2F1dG8vc3RvcmFnZS9nb29nNF9yZXF1ZXN0In0seyJ4LWdvb2ctYWxnb3JpdGhtIjoiR09PRzQtUlNBLVNIQTI1NiJ9XSwiZXhwaXJhdGlvbiI6IjIwMjAtMDEtMjNUMDQ6MzU6NDBaIn0=" + }, + "expectedDecodedPolicy": "{\"conditions\":[{\"content-disposition\":\"attachment; filename=\\\"~._-%=/é0Aa\\\"\"},{\"content-encoding\":\"gzip\"},{\"content-type\":\"text/plain\"},{\"success_action_redirect\":\"http://www.google.com/\"},{\"bucket\":\"rsaposttest-1579902671-6ldm6caw4se52vrx\"},{\"key\":\"test-object\"},{\"x-goog-date\":\"20200123T043530Z\"},{\"x-goog-credential\":\"test-iam-credentials@dummy-project-id.iam.gserviceaccount.com/20200123/auto/storage/goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}],\"expiration\":\"2020-01-23T04:35:40Z\"}" + } + } + ] +} \ No newline at end of file diff --git a/src/storage/src/storage/signed_url.rs b/src/storage/src/storage/signed_url.rs index cfaf10d68c..8c2571d594 100644 --- a/src/storage/src/storage/signed_url.rs +++ b/src/storage/src/storage/signed_url.rs @@ -14,54 +14,170 @@ use crate::error::SigningError; use auth::signer::Signer; -use chrono::Utc; -use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode}; +use chrono::{DateTime, Utc}; +use percent_encoding::{AsciiSet, utf8_percent_encode}; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; -use url::form_urlencoded; /// https://cloud.google.com/storage/docs/request-endpoints#encoding -const PATH_ENCODE_SET: &AsciiSet = &CONTROLS +/// !, #, $, &, ', (, ), *, +, ,, /, :, ;, =, ?, @, [, ] +const PATH_ENCODE_SET: AsciiSet = AsciiSet::EMPTY .add(b' ') - .add(b'"') + .add(b'!') .add(b'#') - .add(b'<') - .add(b'>') + .add(b'$') + .add(b'&') + .add(b'\'') + .add(b'(') + .add(b')') + .add(b'*') + .add(b'+') + .add(b',') + .add(b':') + .add(b';') + .add(b'=') .add(b'?') - .add(b'`') - .add(b'{') - .add(b'}'); + .add(b'@') + .add(b'[') + .add(b']'); /// A builder for creating signed URLs. +#[derive(Debug)] pub struct SignedUrlBuilder { - bucket: String, - object: String, + scope: SigningScope, method: http::Method, expiration: std::time::Duration, - headers: BTreeMap<&'static str, String>, - query_parameters: BTreeMap<&'static str, String>, - endpoint: String, + headers: BTreeMap, + query_parameters: BTreeMap, + endpoint: Option, + universe_domain: String, client_email: Option, + timestamp: DateTime, + url_style: UrlStyle, +} + +#[derive(Debug, Clone)] +pub enum UrlStyle { + PathStyle, + BucketBoundHostname, + VirtualHostedStyle, +} + +impl Default for UrlStyle { + fn default() -> Self { + UrlStyle::PathStyle + } +} + +#[derive(Debug)] +pub enum SigningScope +where + B: Into, + O: Into, +{ + Bucket(B), + Object(B, O), +} + +impl SigningScope { + fn bucket_name(&self) -> String { + let bucket = match self { + SigningScope::Bucket(bucket) => bucket, + SigningScope::Object(bucket, _) => bucket, + }; + + bucket.trim_start_matches("projects/_/buckets/").to_string() + } + + fn bucket_endpoint(&self, endpoint: &str, url_style: UrlStyle) -> String { + let bucket_name = self.bucket_name(); + let scheme = if endpoint.starts_with("http://") { + "http" + } else { + "https" + }; + let endpoint = endpoint + .trim_start_matches("http://") + .trim_start_matches("https://"); + match url_style { + UrlStyle::PathStyle => { + format!("{scheme}://{endpoint}") + } + UrlStyle::BucketBoundHostname => { + format!("{scheme}://{endpoint}") + } + UrlStyle::VirtualHostedStyle => { + format!("{scheme}://{bucket_name}.{endpoint}") + } + } + } + + fn canonical_uri(&self, url_style: UrlStyle) -> String { + let bucket_name = self.bucket_name(); + match self { + SigningScope::Object(_, object) => { + let encoded_object = utf8_percent_encode(&object, &PATH_ENCODE_SET).to_string(); + match url_style { + UrlStyle::PathStyle => { + format!("/{bucket_name}/{encoded_object}") + } + UrlStyle::BucketBoundHostname => { + format!("/{encoded_object}") + } + UrlStyle::VirtualHostedStyle => { + format!("/{encoded_object}") + } + } + } + SigningScope::Bucket(_) => match url_style { + UrlStyle::PathStyle => { + format!("/{bucket_name}") + } + UrlStyle::BucketBoundHostname => "".to_string(), + UrlStyle::VirtualHostedStyle => "".to_string(), + }, + } + } + + fn canonical_url(&self, endpoint: &str, url_style: UrlStyle) -> String { + let bucket_endpoint = self.bucket_endpoint(endpoint, url_style.clone()); + let uri = self.canonical_uri(url_style.clone()); + format!("{bucket_endpoint}{uri}") + } } impl SignedUrlBuilder { - pub fn new(bucket: B, object: O) -> Self + pub fn new(scope: SigningScope) -> Self where B: Into, O: Into, { Self { - bucket: bucket.into(), - object: object.into(), + scope: match scope { + SigningScope::Bucket(bucket) => SigningScope::Bucket(bucket.into()), + SigningScope::Object(bucket, object) => { + SigningScope::Object(bucket.into(), object.into()) + } + }, method: http::Method::GET, expiration: std::time::Duration::from_secs(7 * 24 * 60 * 60), // 7 days headers: BTreeMap::new(), query_parameters: BTreeMap::new(), - endpoint: "https://storage.googleapis.com".to_string(), + endpoint: None, + universe_domain: "googleapis.com".to_string(), client_email: None, + timestamp: Utc::now(), + url_style: UrlStyle::PathStyle, } } + #[cfg(test)] + /// Sets the timestamp for the signed URL. Only used in tests. + pub fn with_timestamp(mut self, timestamp: DateTime) -> Self { + self.timestamp = timestamp; + self + } + /// Sets the HTTP method for the signed URL. Default is "GET". pub fn with_method(mut self, method: http::Method) -> Self { self.method = method; @@ -74,22 +190,34 @@ impl SignedUrlBuilder { self } + /// Sets the URL style for the signed URL. Default is `UrlStyle::PathStyle`. + pub fn with_url_style(mut self, url_style: UrlStyle) -> Self { + self.url_style = url_style; + self + } + /// Adds a header to the signed URL. /// Note: These headers must be present in the request when using the signed URL. - pub fn with_header>(mut self, key: &'static str, value: S) -> Self { - self.headers.insert(key, value.into()); + pub fn with_header, V: Into>(mut self, key: K, value: V) -> Self { + self.headers.insert(key.into(), value.into()); self } /// Adds a query parameter to the signed URL. - pub fn with_query_param>(mut self, key: &'static str, value: S) -> Self { - self.query_parameters.insert(key, value.into()); + pub fn with_query_param, V: Into>(mut self, key: K, value: V) -> Self { + self.query_parameters.insert(key.into(), value.into()); self } /// Sets the endpoint for the signed URL. Default is "https://storage.googleapis.com". pub fn with_endpoint>(mut self, endpoint: S) -> Self { - self.endpoint = endpoint.into(); + self.endpoint = Some(endpoint.into()); + self + } + + /// Sets the universe domain for the signed URL. Default is "googleapis.com". + pub fn with_universe_domain>(mut self, universe_domain: S) -> Self { + self.universe_domain = universe_domain.into(); self } @@ -100,62 +228,131 @@ impl SignedUrlBuilder { self } + fn resolve_endpoint(&self) -> String { + if let Some(endpoint) = self.endpoint.clone() { + if !endpoint.starts_with("http") { + return format!("https://{}", endpoint); + } + return endpoint; + } + + let emulator_host = std::env::var("STORAGE_EMULATOR_HOST"); + if let Ok(host) = emulator_host + && !host.is_empty() + { + if host.starts_with("http") { + return host; + } + return format!("http://{host}"); + } + + format!("https://storage.{}", self.universe_domain.clone()) + } + /// Generates the signed URL using the provided signer. pub async fn sign_with(self, signer: &Signer) -> std::result::Result { - let encoded_object = utf8_percent_encode(&self.object, PATH_ENCODE_SET).to_string(); - let canonical_uri = format!("/{}", encoded_object); + let (url, _, _) = self.sign_internal(signer).await?; + Ok(url) + } + + fn canonicalize_header_value(value: &str) -> String { + let clean_value = value.replace("\t", " ").trim().to_string(); + clean_value.split_whitespace().collect::>().join(" ") + } - let now = Utc::now(); + /// Generates the signed URL using the provided signer. + /// Returns the signed URL, the string to sign, and the canonical request. + /// Used to check conformance test expectations. + async fn sign_internal( + self, + signer: &Signer, + ) -> std::result::Result<(String, String, String), SigningError> { + let now = self.timestamp; let request_timestamp = now.format("%Y%m%dT%H%M%SZ").to_string(); let datestamp = now.format("%Y%m%d"); let credential_scope = format!("{datestamp}/auto/storage/goog4_request"); - let client_email = if let Some(email) = self.client_email { + let client_email = if let Some(email) = self.client_email.clone() { email } else { signer.client_email().await.map_err(SigningError::Signing)? }; let credential = format!("{client_email}/{credential_scope}"); + let endpoint = self.resolve_endpoint(); + let canonical_url = self.scope.canonical_url(&endpoint, self.url_style.clone()); let endpoint_url = - url::Url::parse(&self.endpoint).map_err(|e| SigningError::InvalidEndpoint(e.into()))?; + url::Url::parse(&canonical_url).map_err(|e| SigningError::InvalidEndpoint(e.into()))?; let endpoint_host = endpoint_url .host_str() .ok_or_else(|| SigningError::InvalidEndpoint("invalid endpoint host".into()))?; - let bucket_name = self.bucket.trim_start_matches("projects/_/buckets/"); - let host = format!("{}.{}", bucket_name, endpoint_host); let mut headers = self.headers; - headers.insert("host", host.clone()); + headers.insert("host".to_string(), endpoint_host.to_string()); - let canonical_headers = headers - .iter() - .fold("".to_string(), |acc, (k, v)| format!("{acc}{k}:{v}\n")); + let mut sorted_headers = headers.keys().collect::>(); + sorted_headers.sort_by_key(|k| k.to_lowercase()); - let signed_headers = headers - .iter() - .fold("".to_string(), |acc, (k, _)| format!("{acc}{k};")); + let signed_headers = sorted_headers.iter().fold("".to_string(), |acc, k| { + format!("{acc}{};", k.to_lowercase()) + }); let signed_headers = signed_headers.trim_end_matches(';').to_string(); let mut query_parameters = self.query_parameters; - query_parameters.insert("X-Goog-Algorithm", "GOOG4-RSA-SHA256".to_string()); - query_parameters.insert("X-Goog-Credential", credential); - query_parameters.insert("X-Goog-Date", request_timestamp.clone()); - query_parameters.insert("X-Goog-Expires", self.expiration.as_secs().to_string()); - query_parameters.insert("X-Goog-SignedHeaders", signed_headers.clone()); - - let mut canonical_query = form_urlencoded::Serializer::new("".to_string()); - query_parameters.iter().for_each(|(k, v)| { - canonical_query.append_pair(k, v); + query_parameters.insert( + "X-Goog-Algorithm".to_string(), + "GOOG4-RSA-SHA256".to_string(), + ); + query_parameters.insert("X-Goog-Credential".to_string(), credential); + query_parameters.insert("X-Goog-Date".to_string(), request_timestamp.clone()); + query_parameters.insert( + "X-Goog-Expires".to_string(), + self.expiration.as_secs().to_string(), + ); + query_parameters.insert("X-Goog-SignedHeaders".to_string(), signed_headers.clone()); + + let mut canonical_query = url::form_urlencoded::Serializer::new("".to_string()); + let mut sorted_query_parameters_keys = query_parameters.keys().collect::>(); + sorted_query_parameters_keys.sort_by_key(|k| k.to_string()); + sorted_query_parameters_keys.iter().for_each(|k| { + let value = query_parameters.get(k.as_str()); + if value.is_none() { + return; + } + let value = value.unwrap(); + canonical_query.append_pair(k, value); + }); + let canonical_query = canonical_query.finish(); + let canonical_query = canonical_query + .replace("%7E", "~") // rollback to ~ + .replace("+", "%20"); // missing %20 in + + + let canonical_headers = sorted_headers.iter().fold("".to_string(), |acc, k| { + let header_value = headers.get(k.as_str()); + if header_value.is_none() { + return acc; + } + let header_value = Self::canonicalize_header_value(&header_value.unwrap()); + format!("{acc}{}:{}\n", k.to_lowercase(), header_value) + }); + + // If the user provides a value for X-Goog-Content-SHA256, we must use + // that value in the request string. If not, we use UNSIGNED-PAYLOAD. + let signature = "UNSIGNED-PAYLOAD".to_string(); + let signature = headers.iter().fold(signature, |acc, (k, v)| { + if k.to_lowercase().eq("x-goog-content-sha256") { + return v.clone(); + } + acc }); - let canonical_query_string = canonical_query.finish(); + let canonical_uri = self.scope.canonical_uri(self.url_style); let canonical_request = [ self.method.to_string(), canonical_uri.clone(), - canonical_query_string.clone(), + canonical_query.clone(), canonical_headers, signed_headers, - "UNSIGNED-PAYLOAD".to_string(), + signature, ] .join("\n"); @@ -175,21 +372,25 @@ impl SignedUrlBuilder { .await .map_err(SigningError::Signing)?; - let scheme_and_host = format!("{}://{}", endpoint_url.scheme(), host); - let signed_url = format!( - "{}{}?{}&x-goog-signature={}", - scheme_and_host, canonical_uri, canonical_query_string, signature + "{}?{}&X-Goog-Signature={}", + canonical_url, canonical_query, signature ); - Ok(signed_url) + Ok((signed_url, string_to_sign, canonical_request)) } } #[cfg(test)] mod tests { use super::*; + use auth::credentials::service_account::Builder as ServiceAccount; use auth::signer::{Signer, SigningProvider}; + use chrono::DateTime; + use scoped_env::ScopedEnv; + use serde::Deserialize; + use std::collections::HashMap; + use tokio::time::Duration; type TestResult = anyhow::Result<()>; @@ -210,16 +411,16 @@ mod tests { #[tokio::test] async fn test_signed_url_generation() -> TestResult { let signer = Signer::from(MockSigner); - let url = SignedUrlBuilder::new("test-bucket", "test-object") - .with_method("PUT") + let url = SignedUrlBuilder::new(SigningScope::Object("test-bucket", "test-object")) + .with_method(http::Method::PUT) .with_expiration(std::time::Duration::from_secs(3600)) .with_header("x-goog-meta-test", "value") .sign_with(&signer) .await .unwrap(); - assert!(url.starts_with("https://test-bucket.storage.googleapis.com/test-object")); - assert!(url.contains("x-goog-signature=test-signature")); + assert!(url.starts_with("https://storage.googleapis.com/test-bucket/test-object")); + assert!(url.contains("X-Goog-Signature=test-signature")); assert!(url.contains("X-Goog-Algorithm=GOOG4-RSA-SHA256")); assert!(url.contains("X-Goog-Credential=test%40example.com")); @@ -229,17 +430,20 @@ mod tests { #[tokio::test] async fn test_signed_url_generation_escaping() -> TestResult { let signer: Signer = Signer::from(MockSigner); - let url = SignedUrlBuilder::new("test-bucket", "folder/test object.txt") - .with_method("PUT") - .with_header("content-type", "text/plain") - .sign_with(&signer) - .await - .unwrap(); + let url = SignedUrlBuilder::new(SigningScope::Object( + "test-bucket", + "folder/test object.txt", + )) + .with_method(http::Method::PUT) + .with_header("content-type", "text/plain") + .sign_with(&signer) + .await + .unwrap(); assert!( - url.starts_with("https://test-bucket.storage.googleapis.com/folder/test%20object.txt?") + url.starts_with("https://storage.googleapis.com/test-bucket/folder/test%20object.txt?") ); - assert!(url.contains("x-goog-signature=")); + assert!(url.contains("X-Goog-Signature=")); Ok(()) } @@ -258,7 +462,7 @@ mod tests { } } let signer = Signer::from(FailSigner); - let err = SignedUrlBuilder::new("b", "o") + let err = SignedUrlBuilder::new(SigningScope::Object("b", "o")) .sign_with(&signer) .await .unwrap_err(); @@ -274,8 +478,8 @@ mod tests { #[tokio::test] async fn test_signed_url_error_endpoint() -> TestResult { let signer: Signer = Signer::from(MockSigner); - let err = SignedUrlBuilder::new("b", "o") - .with_endpoint("invalid-url") + let err = SignedUrlBuilder::new(SigningScope::Object("b", "o")) + .with_endpoint("invalid url") .sign_with(&signer) .await .unwrap_err(); @@ -287,4 +491,166 @@ mod tests { Ok(()) } + + #[derive(Deserialize)] + struct SignedUrlTestSuite { + #[serde(rename = "signingV4Tests")] + signing_v4_tests: Vec, + } + + #[derive(Deserialize)] + struct SignedUrlTest { + description: String, + bucket: String, + object: Option, + method: String, + expiration: u64, + timestamp: String, + #[serde(rename = "expectedUrl")] + expected_url: String, + headers: Option>, + #[serde(rename = "queryParameters")] + query_parameters: Option>, + scheme: Option, + #[serde(rename = "urlStyle")] + url_style: Option, + #[serde(rename = "bucketBoundHostname")] + bucket_bound_hostname: Option, + #[serde(rename = "expectedCanonicalRequest")] + expected_canonical_request: String, + #[serde(rename = "expectedStringToSign")] + expected_string_to_sign: String, + hostname: Option, + #[serde(rename = "clientEndpoint")] + client_endpoint: Option, + #[serde(rename = "emulatorHostname")] + emulator_hostname: Option, + #[serde(rename = "universeDomain")] + universe_domain: Option, + } + + #[tokio::test] + async fn signed_url_conformance() -> anyhow::Result<()> { + let service_account_key = serde_json::from_slice(include_bytes!( + "conformance/test_service_account.not-a-test.json" + ))?; + + let signer = ServiceAccount::new(service_account_key) + .build_signer() + .expect("failed to build signer"); + + let suite: SignedUrlTestSuite = + serde_json::from_slice(include_bytes!("conformance/v4_signatures.json"))?; + + let mut failed_tests = Vec::new(); + let mut passed_tests = Vec::new(); + // let mut skipped_tests = Vec::new(); + let total_tests = suite.signing_v4_tests.len(); + for test in suite.signing_v4_tests { + let timestamp = + DateTime::parse_from_rfc3339(&test.timestamp).expect("invalid timestamp"); + let method = http::Method::from_bytes(test.method.as_bytes()).expect("invalid method"); + let scheme = test.scheme.unwrap_or("https".to_string()); + let url_style = match test.url_style { + Some(url_style) => match url_style.as_str() { + "VIRTUAL_HOSTED_STYLE" => UrlStyle::VirtualHostedStyle, + "BUCKET_BOUND_HOSTNAME" => UrlStyle::BucketBoundHostname, + _ => UrlStyle::PathStyle, + }, + None => UrlStyle::PathStyle, + }; + let scope = match test.object { + Some(object) => SigningScope::Object(test.bucket, object), + None => SigningScope::Bucket(test.bucket), + }; + + let emulator_hostname = test.emulator_hostname.unwrap_or_default(); + let _e = ScopedEnv::set("STORAGE_EMULATOR_HOST", emulator_hostname.as_str()); + + let builder = SignedUrlBuilder::new(scope) + .with_method(method) + .with_url_style(url_style) + .with_expiration(Duration::from_secs(test.expiration)) + .with_timestamp(timestamp.into()); + + let builder = test + .universe_domain + .iter() + .fold(builder, |builder, universe_domain| { + builder.with_universe_domain(universe_domain) + }); + let builder = test + .client_endpoint + .iter() + .fold(builder, |builder, client_endpoint| { + builder.with_endpoint(client_endpoint) + }); + let builder = test + .bucket_bound_hostname + .iter() + .fold(builder, |builder, hostname| { + builder.with_endpoint(format!("{}://{}", scheme, hostname)) + }); + let builder = test.hostname.iter().fold(builder, |builder, hostname| { + builder.with_endpoint(format!("{}://{}", scheme, hostname)) + }); + let builder = test.headers.iter().fold(builder, |builder, headers| { + headers.iter().fold(builder, |builder, (k, v)| { + builder.with_header(k.clone(), v.clone()) + }) + }); + let builder = test + .query_parameters + .iter() + .fold(builder, |builder, query_params| { + query_params.iter().fold(builder, |builder, (k, v)| { + builder.with_query_param(k.clone(), v.clone()) + }) + }); + + let (signed_url, string_to_sign, canonical_request) = + builder.sign_internal(&signer).await?; + + if canonical_request != test.expected_canonical_request + || string_to_sign != test.expected_string_to_sign + || signed_url != test.expected_url + { + println!("❌ Failed test: {}", test.description); + let diff = pretty_assertions::StrComparison::new( + &canonical_request, + &test.expected_canonical_request, + ); + println!("Canonical request diff: {}", diff); + let diff = pretty_assertions::StrComparison::new( + &string_to_sign, + &test.expected_string_to_sign, + ); + println!("String to sign diff: {}", diff); + let diff = pretty_assertions::StrComparison::new(&signed_url, &test.expected_url); + println!("Signed URL diff: {}", diff); + failed_tests.push(test.description); + continue; + } + passed_tests.push(test.description); + } + + let failed = !failed_tests.is_empty(); + let total_passed = passed_tests.len(); + /*for test in skipped_tests { + println!("⚠️ Skipped test: {}", test); + }*/ + for test in passed_tests { + println!("✅ Passed test: {}", test); + } + for test in failed_tests { + println!("❌ Failed test: {}", test); + } + println!("{}/{} tests passed", total_passed, total_tests); + + if failed { + Err(anyhow::anyhow!("Some tests failed")) + } else { + Ok(()) + } + } } From e0b566cd6e973e34a0694fab42061571e89999b4 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 24 Nov 2025 20:15:27 +0000 Subject: [PATCH 21/41] fix: conformance test virtual hosted style case --- src/storage/src/storage/conformance/v4_signatures.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/src/storage/conformance/v4_signatures.json b/src/storage/src/storage/conformance/v4_signatures.json index 40aa21cba3..2d5457eb34 100644 --- a/src/storage/src/storage/conformance/v4_signatures.json +++ b/src/storage/src/storage/conformance/v4_signatures.json @@ -400,7 +400,7 @@ "timestamp": "2019-02-01T09:00:00Z", "expectedUrl": "https://test-bucket.storage.domain.com/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=25820e3a60856596cba594511d7d4039239b2728a9738f15d3a7acce8d70aa5435d0c91f99a9318f932afc73355ac562e014cb654e16ed5524b403536f1cba74489701fdc0c088b8826fccf20a648d3b2b704bd6661e01786d4132174c21441d0752be07e8af93e84e24b87799ee91fabef24a0a58d0889263280c3d37423fab677bd4d98469ab01aa36efaad62ff81ca27bf7fc92f14e20faa71e34de9ffbc5eb4ecf1b0361de42270665bb78367bd0a8cc6a604a8e347f0c864754bf14514aac3106fe73572a6c068ce2c380cc2a943b35502093d162ba9ae8de9abbbc9541ef765d5679857a89d36cc01be30cf1e04c4a477bbcd59a02955dcc1a903d8baa", "universeDomain": "domain.com", - "expectedCanonicalRequest": "GET\n/test-bucket/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:test-bucket.storage.domain.com\n\nhost\nUNSIGNED-PAYLOAD", + "expectedCanonicalRequest": "GET\n/test-object\nX-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host\nhost:test-bucket.storage.domain.com\n\nhost\nUNSIGNED-PAYLOAD", "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\n6835c0cd7e63f2e34becade43beee99335c68c1455488da5b320cf13dc0a0ed5" } ], From 006a52e8e0ed1913d8f94c6d2c0e5729f67e5ed8 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 26 Nov 2025 18:27:52 +0000 Subject: [PATCH 22/41] fix: bad merge --- src/storage/Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index 663b99e58d..4eb104e7bb 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -88,7 +88,6 @@ test-case.workspace = true tempfile.workspace = true tokio-test.workspace = true tokio.workspace = true +pretty_assertions = { workspace = true } # The tests use `unstable-stream`, which is not enabled by default. google-cloud-storage = { path = ".", features = ["unstable-stream"] } -pretty_assertions = { workspace = true } -scoped-env.workspace = true From 98d86395398f4751d76f1e7c7dc219ea55254ef1 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 28 Nov 2025 17:33:21 +0000 Subject: [PATCH 23/41] fix: typo in conformance test --- src/storage/src/storage/conformance/v4_signatures.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/src/storage/conformance/v4_signatures.json b/src/storage/src/storage/conformance/v4_signatures.json index 2d5457eb34..8f3dfbce7a 100644 --- a/src/storage/src/storage/conformance/v4_signatures.json +++ b/src/storage/src/storage/conformance/v4_signatures.json @@ -365,7 +365,7 @@ "expectedStringToSign": "GOOG4-RSA-SHA256\n20190201T090000Z\n20190201/auto/storage/goog4_request\ne47446edb8eed4c1797dfd31ce30272be89659a6ef38e91b549740c8f875d27b" }, { - "description": "Hostname takes precendence over endpoint and emulator", + "description": "Hostname takes precedence over endpoint and emulator", "bucket": "test-bucket", "object": "test-object", "method": "GET", From 3cc98f4a86747b06f54fffbb06d7d9ebdf7ca64c Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 28 Nov 2025 17:42:44 +0000 Subject: [PATCH 24/41] fix: lint issue on cargo.toml --- src/storage/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index 4eb104e7bb..8e6e004220 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -90,4 +90,4 @@ tokio-test.workspace = true tokio.workspace = true pretty_assertions = { workspace = true } # The tests use `unstable-stream`, which is not enabled by default. -google-cloud-storage = { path = ".", features = ["unstable-stream"] } +google-cloud-storage = { path = ".", features = ["unstable-stream"] } From 4f11901d6c24f0e268331e615663f29b5fc70806 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 28 Nov 2025 17:55:25 +0000 Subject: [PATCH 25/41] fix: integration test to use signing scopes --- src/storage/src/storage/signed_url.rs | 29 ++++++++++++++++++--------- tests/integration/src/storage.rs | 7 ++++--- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/storage/src/storage/signed_url.rs b/src/storage/src/storage/signed_url.rs index 8c2571d594..d2f946d1af 100644 --- a/src/storage/src/storage/signed_url.rs +++ b/src/storage/src/storage/signed_url.rs @@ -147,7 +147,7 @@ impl SigningScope { } impl SignedUrlBuilder { - pub fn new(scope: SigningScope) -> Self + fn new(scope: SigningScope) -> Self where B: Into, O: Into, @@ -171,6 +171,21 @@ impl SignedUrlBuilder { } } + pub fn for_object(bucket: B, object: O) -> Self + where + B: Into, + O: Into, + { + Self::new(SigningScope::Object(bucket, object)) + } + + pub fn for_bucket(bucket: B) -> Self + where + B: Into, + { + Self::new(SigningScope::Bucket(bucket)) + } + #[cfg(test)] /// Sets the timestamp for the signed URL. Only used in tests. pub fn with_timestamp(mut self, timestamp: DateTime) -> Self { @@ -544,7 +559,6 @@ mod tests { let mut failed_tests = Vec::new(); let mut passed_tests = Vec::new(); - // let mut skipped_tests = Vec::new(); let total_tests = suite.signing_v4_tests.len(); for test in suite.signing_v4_tests { let timestamp = @@ -559,15 +573,15 @@ mod tests { }, None => UrlStyle::PathStyle, }; - let scope = match test.object { - Some(object) => SigningScope::Object(test.bucket, object), - None => SigningScope::Bucket(test.bucket), + let builder = match test.object { + Some(object) => SignedUrlBuilder::for_object(test.bucket, object), + None => SignedUrlBuilder::for_bucket(test.bucket), }; let emulator_hostname = test.emulator_hostname.unwrap_or_default(); let _e = ScopedEnv::set("STORAGE_EMULATOR_HOST", emulator_hostname.as_str()); - let builder = SignedUrlBuilder::new(scope) + let builder = builder .with_method(method) .with_url_style(url_style) .with_expiration(Duration::from_secs(test.expiration)) @@ -636,9 +650,6 @@ mod tests { let failed = !failed_tests.is_empty(); let total_passed = passed_tests.len(); - /*for test in skipped_tests { - println!("⚠️ Skipped test: {}", test); - }*/ for test in passed_tests { println!("✅ Passed test: {}", test); } diff --git a/tests/integration/src/storage.rs b/tests/integration/src/storage.rs index 915f5318df..022243820f 100644 --- a/tests/integration/src/storage.rs +++ b/tests/integration/src/storage.rs @@ -106,9 +106,10 @@ pub async fn signed_urls( tracing::info!("success with insert={insert:?}"); tracing::info!("testing signed_url()"); - let signed_url = storage::builder::storage::SignedUrlBuilder::new(bucket_name, &insert.name) - .sign_with(&signer) - .await?; + let signed_url = + storage::builder::storage::SignedUrlBuilder::for_object(bucket_name, &insert.name) + .sign_with(&signer) + .await?; tracing::info!("signed_url={signed_url}"); From 67231e461520541aded37e73a033afe5cfb53551 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 3 Dec 2025 16:31:33 +0000 Subject: [PATCH 26/41] fix: remove async_trait from pub SigningProvider --- src/auth/src/credentials/impersonated.rs | 7 +- src/auth/src/signer.rs | 11 +-- src/auth/src/signer/mds.rs | 4 +- src/auth/src/signer/service_account.rs | 4 +- src/storage/src/storage/signed_url.rs | 111 +++++++++++------------ 5 files changed, 66 insertions(+), 71 deletions(-) diff --git a/src/auth/src/credentials/impersonated.rs b/src/auth/src/credentials/impersonated.rs index a0a9b7e75d..b2250ecac6 100644 --- a/src/auth/src/credentials/impersonated.rs +++ b/src/auth/src/credentials/impersonated.rs @@ -601,11 +601,14 @@ fn build_signer_from_json(json: Value) -> BuildResult(config.source_credentials) .map_err(BuilderError::parsing)?; - let signer = ServiceAccountSigner::from_impersonated_service_account( + let signing_provider = ServiceAccountSigner::from_impersonated_service_account( service_account_key, client_email, ); - return Ok(Some(signer.into())); + let signer = crate::signer::Signer { + inner: Arc::new(signing_provider), + }; + return Ok(Some(signer)); } Ok(None) } diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index a091d18a2b..a8a519cb1e 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -83,7 +83,7 @@ struct SignBlobResponse { } #[async_trait::async_trait] -impl SigningProvider for CredentialsSigner { +impl dynamic::SigningProvider for CredentialsSigner { async fn client_email(&self) -> Result { Ok(self.client_email.clone()) } @@ -143,18 +143,17 @@ impl SigningProvider for CredentialsSigner { } /// A trait for types that can sign content. -#[async_trait::async_trait] -pub trait SigningProvider: Send + Sync + std::fmt::Debug { +pub trait SigningProvider: std::fmt::Debug { /// Returns the email address of the authorizer. /// /// It is typically the Google service account client email address from the Google Developers Console /// in the form of "xxx@developer.gserviceaccount.com". Required. - async fn client_email(&self) -> Result; + fn client_email(&self) -> impl Future> + Send; /// Signs the content. /// /// Returns the signature. - async fn sign(&self, content: &[u8]) -> Result; + fn sign(&self, content: &[u8]) -> impl Future> + Send; } pub(crate) mod dynamic { diff --git a/src/auth/src/signer/mds.rs b/src/auth/src/signer/mds.rs index 29011622df..584b3c852b 100644 --- a/src/auth/src/signer/mds.rs +++ b/src/auth/src/signer/mds.rs @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ use crate::credentials::Credentials; use crate::credentials::mds::{MDS_DEFAULT_URI, METADATA_FLAVOR, METADATA_FLAVOR_VALUE}; -use crate::signer::{Result, SigningError, SigningProvider}; +use crate::signer::{Result, SigningError, dynamic::SigningProvider}; use http::HeaderValue; use reqwest::Client; use std::sync::Arc; diff --git a/src/auth/src/signer/service_account.rs b/src/auth/src/signer/service_account.rs index 1a37742975..118fc479e8 100644 --- a/src/auth/src/signer/service_account.rs +++ b/src/auth/src/signer/service_account.rs @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ // limitations under the License. use crate::credentials::service_account::ServiceAccountKey; -use crate::signer::{Result, SigningError, SigningProvider}; +use crate::signer::{Result, SigningError, dynamic::SigningProvider}; // Implements a local Signer using Service Account private key. #[derive(Clone, Debug)] diff --git a/src/storage/src/storage/signed_url.rs b/src/storage/src/storage/signed_url.rs index d2f946d1af..b89ca73876 100644 --- a/src/storage/src/storage/signed_url.rs +++ b/src/storage/src/storage/signed_url.rs @@ -44,7 +44,7 @@ const PATH_ENCODE_SET: AsciiSet = AsciiSet::EMPTY /// A builder for creating signed URLs. #[derive(Debug)] pub struct SignedUrlBuilder { - scope: SigningScope, + scope: SigningScope, method: http::Method, expiration: std::time::Duration, headers: BTreeMap, @@ -70,16 +70,12 @@ impl Default for UrlStyle { } #[derive(Debug)] -pub enum SigningScope -where - B: Into, - O: Into, -{ - Bucket(B), - Object(B, O), +enum SigningScope { + Bucket(String), + Object(String, String), } -impl SigningScope { +impl SigningScope { fn bucket_name(&self) -> String { let bucket = match self { SigningScope::Bucket(bucket) => bucket, @@ -147,18 +143,9 @@ impl SigningScope { } impl SignedUrlBuilder { - fn new(scope: SigningScope) -> Self - where - B: Into, - O: Into, - { + fn new(scope: SigningScope) -> Self { Self { - scope: match scope { - SigningScope::Bucket(bucket) => SigningScope::Bucket(bucket.into()), - SigningScope::Object(bucket, object) => { - SigningScope::Object(bucket.into(), object.into()) - } - }, + scope, method: http::Method::GET, expiration: std::time::Duration::from_secs(7 * 24 * 60 * 60), // 7 days headers: BTreeMap::new(), @@ -176,14 +163,14 @@ impl SignedUrlBuilder { B: Into, O: Into, { - Self::new(SigningScope::Object(bucket, object)) + Self::new(SigningScope::Object(bucket.into(), object.into())) } pub fn for_bucket(bucket: B) -> Self where B: Into, { - Self::new(SigningScope::Bucket(bucket)) + Self::new(SigningScope::Bucket(bucket.into())) } #[cfg(test)] @@ -409,24 +396,26 @@ mod tests { type TestResult = anyhow::Result<()>; - #[derive(Debug)] - struct MockSigner; - - #[async_trait::async_trait] - impl SigningProvider for MockSigner { - async fn client_email(&self) -> auth::signer::Result { - Ok("test@example.com".to_string()) - } + mockall::mock! { + #[derive(Debug)] + Signer {} - async fn sign(&self, _content: &[u8]) -> auth::signer::Result { - Ok("test-signature".to_string()) + impl SigningProvider for Signer { + async fn client_email(&self) -> auth::signer::Result; + async fn sign(&self, _content: &[u8]) -> auth::signer::Result; } } #[tokio::test] async fn test_signed_url_generation() -> TestResult { - let signer = Signer::from(MockSigner); - let url = SignedUrlBuilder::new(SigningScope::Object("test-bucket", "test-object")) + let mut mock = MockSigner::new(); + mock.expect_client_email() + .return_once(|| Ok("test@example.com".to_string())); + mock.expect_sign() + .return_once(|_content| Ok("test-signature".to_string())); + + let signer = Signer::from(mock); + let url = SignedUrlBuilder::for_object("test-bucket", "test-object") .with_method(http::Method::PUT) .with_expiration(std::time::Duration::from_secs(3600)) .with_header("x-goog-meta-test", "value") @@ -444,16 +433,19 @@ mod tests { #[tokio::test] async fn test_signed_url_generation_escaping() -> TestResult { - let signer: Signer = Signer::from(MockSigner); - let url = SignedUrlBuilder::new(SigningScope::Object( - "test-bucket", - "folder/test object.txt", - )) - .with_method(http::Method::PUT) - .with_header("content-type", "text/plain") - .sign_with(&signer) - .await - .unwrap(); + let mut mock = MockSigner::new(); + mock.expect_client_email() + .return_once(|| Ok("test@example.com".to_string())); + mock.expect_sign() + .return_once(|_content| Ok("test-signature".to_string())); + + let signer = Signer::from(mock); + let url = SignedUrlBuilder::for_object("test-bucket", "folder/test object.txt") + .with_method(http::Method::PUT) + .with_header("content-type", "text/plain") + .sign_with(&signer) + .await + .unwrap(); assert!( url.starts_with("https://storage.googleapis.com/test-bucket/folder/test%20object.txt?") @@ -465,19 +457,14 @@ mod tests { #[tokio::test] async fn test_signed_url_error_signing() -> TestResult { - #[derive(Debug)] - struct FailSigner; - #[async_trait::async_trait] - impl SigningProvider for FailSigner { - async fn client_email(&self) -> auth::signer::Result { - Ok("test@example.com".to_string()) - } - async fn sign(&self, _content: &[u8]) -> auth::signer::Result { - Err(auth::signer::SigningError::from_msg("test".to_string())) - } - } - let signer = Signer::from(FailSigner); - let err = SignedUrlBuilder::new(SigningScope::Object("b", "o")) + let mut mock = MockSigner::new(); + mock.expect_client_email() + .return_once(|| Ok("test@example.com".to_string())); + mock.expect_sign() + .return_once(|_content| Err(auth::signer::SigningError::from_msg("test".to_string()))); + + let signer = Signer::from(mock); + let err = SignedUrlBuilder::for_object("b", "o") .sign_with(&signer) .await .unwrap_err(); @@ -492,8 +479,14 @@ mod tests { #[tokio::test] async fn test_signed_url_error_endpoint() -> TestResult { - let signer: Signer = Signer::from(MockSigner); - let err = SignedUrlBuilder::new(SigningScope::Object("b", "o")) + let mut mock = MockSigner::new(); + mock.expect_client_email() + .return_once(|| Ok("test@example.com".to_string())); + mock.expect_sign() + .return_once(|_content| Ok("test-signature".to_string())); + + let signer = Signer::from(mock); + let err = SignedUrlBuilder::for_object("b", "o") .with_endpoint("invalid url") .sign_with(&signer) .await From 5bbd5587e61c8ba91fde7930f17b68d470b4f8ab Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 3 Dec 2025 17:05:13 +0000 Subject: [PATCH 27/41] fix: not_supported behind feature flag --- src/auth/src/build_errors.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/auth/src/build_errors.rs b/src/auth/src/build_errors.rs index af99ca8174..423b75df7a 100644 --- a/src/auth/src/build_errors.rs +++ b/src/auth/src/build_errors.rs @@ -84,7 +84,6 @@ impl Error { Error(ErrorKind::MissingField(field)) } - #[cfg(feature = "idtoken")] /// The given credential type is not supported. pub(crate) fn not_supported(credential_type: T) -> Error where @@ -103,8 +102,7 @@ enum ErrorKind { #[error("unknown or invalid credentials type {0}")] UnknownType(#[source] BoxError), #[error("missing required field: {0}")] - MissingField(&'static str), - #[cfg(feature = "idtoken")] + MissingField(&'static str), #[error("credentials type not supported: {0}")] NotSupported(#[source] BoxError), } From 6b0656b8be2a52c6f9ea22233db3224a028abe19 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 3 Dec 2025 18:14:43 +0000 Subject: [PATCH 28/41] docs: reflow --- src/auth/src/signer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index 5d13b38ac0..47571e5abe 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -53,8 +53,7 @@ impl Signer { /// Signs the provided content using the underlying provider. /// /// The content is typically a string-to-sign generated by the caller. - /// Returns the signature as a base64 encoded string (or other format depending on implementation, - /// but typically hex or base64). + /// Returns the signature as a hex encoded string. pub async fn sign(&self, content: T) -> Result where T: AsRef<[u8]> + Send + Sync, @@ -146,8 +145,9 @@ impl dynamic::SigningProvider for CredentialsSigner { pub trait SigningProvider: std::fmt::Debug { /// Returns the email address of the authorizer. /// - /// It is typically the Google service account client email address from the Google Developers Console - /// in the form of "xxx@developer.gserviceaccount.com". Required. + /// It is typically the Google service account client email address + /// from the Google Developers Console in the form of + /// "xxx@developer.gserviceaccount.com". fn client_email(&self) -> impl Future> + Send; /// Signs the content. From 4c080321b6732b0e284009e696918c2d9ca593e6 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 3 Dec 2025 18:23:00 +0000 Subject: [PATCH 29/41] fix: clippy all the things --- src/auth/src/signer.rs | 6 +++--- src/storage/src/storage/signed_url.rs | 13 ++++--------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index 47571e5abe..f6e069abd6 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -145,9 +145,9 @@ impl dynamic::SigningProvider for CredentialsSigner { pub trait SigningProvider: std::fmt::Debug { /// Returns the email address of the authorizer. /// - /// It is typically the Google service account client email address - /// from the Google Developers Console in the form of - /// "xxx@developer.gserviceaccount.com". + /// It is typically the Google service account client email address + /// from the Google Developers Console in the form of + /// "xxx@developer.gserviceaccount.com". fn client_email(&self) -> impl Future> + Send; /// Signs the content. diff --git a/src/storage/src/storage/signed_url.rs b/src/storage/src/storage/signed_url.rs index b89ca73876..67c0c69662 100644 --- a/src/storage/src/storage/signed_url.rs +++ b/src/storage/src/storage/signed_url.rs @@ -56,19 +56,14 @@ pub struct SignedUrlBuilder { url_style: UrlStyle, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub enum UrlStyle { + #[default] PathStyle, BucketBoundHostname, VirtualHostedStyle, } -impl Default for UrlStyle { - fn default() -> Self { - UrlStyle::PathStyle - } -} - #[derive(Debug)] enum SigningScope { Bucket(String), @@ -112,7 +107,7 @@ impl SigningScope { let bucket_name = self.bucket_name(); match self { SigningScope::Object(_, object) => { - let encoded_object = utf8_percent_encode(&object, &PATH_ENCODE_SET).to_string(); + let encoded_object = utf8_percent_encode(object, &PATH_ENCODE_SET).to_string(); match url_style { UrlStyle::PathStyle => { format!("/{bucket_name}/{encoded_object}") @@ -333,7 +328,7 @@ impl SignedUrlBuilder { if header_value.is_none() { return acc; } - let header_value = Self::canonicalize_header_value(&header_value.unwrap()); + let header_value = Self::canonicalize_header_value(header_value.unwrap()); format!("{acc}{}:{}\n", k.to_lowercase(), header_value) }); From e67ea668e74854432bcbf87df7b9047c6c028252 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 3 Dec 2025 19:48:45 +0000 Subject: [PATCH 30/41] refactor: add signed url components --- src/storage/src/storage/signed_url.rs | 30 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/storage/src/storage/signed_url.rs b/src/storage/src/storage/signed_url.rs index 67c0c69662..889c2de433 100644 --- a/src/storage/src/storage/signed_url.rs +++ b/src/storage/src/storage/signed_url.rs @@ -137,6 +137,14 @@ impl SigningScope { } } +struct SigningComponents { + #[allow(dead_code)] + canonical_request: String, + #[allow(dead_code)] + string_to_sign: String, + signed_url: String, +} + impl SignedUrlBuilder { fn new(scope: SigningScope) -> Self { Self { @@ -233,8 +241,8 @@ impl SignedUrlBuilder { return endpoint; } - let emulator_host = std::env::var("STORAGE_EMULATOR_HOST"); - if let Ok(host) = emulator_host + let emulator_host = std::env::var("STORAGE_EMULATOR_HOST").ok(); + if let Some(host) = emulator_host && !host.is_empty() { if host.starts_with("http") { @@ -248,8 +256,8 @@ impl SignedUrlBuilder { /// Generates the signed URL using the provided signer. pub async fn sign_with(self, signer: &Signer) -> std::result::Result { - let (url, _, _) = self.sign_internal(signer).await?; - Ok(url) + let components = self.sign_internal(signer).await?; + Ok(components.signed_url) } fn canonicalize_header_value(value: &str) -> String { @@ -263,7 +271,7 @@ impl SignedUrlBuilder { async fn sign_internal( self, signer: &Signer, - ) -> std::result::Result<(String, String, String), SigningError> { + ) -> std::result::Result { let now = self.timestamp; let request_timestamp = now.format("%Y%m%dT%H%M%SZ").to_string(); let datestamp = now.format("%Y%m%d"); @@ -374,7 +382,11 @@ impl SignedUrlBuilder { canonical_url, canonical_query, signature ); - Ok((signed_url, string_to_sign, canonical_request)) + Ok(SigningComponents { + canonical_request, + string_to_sign, + signed_url, + }) } } @@ -610,8 +622,10 @@ mod tests { }) }); - let (signed_url, string_to_sign, canonical_request) = - builder.sign_internal(&signer).await?; + let components = builder.sign_internal(&signer).await?; + let canonical_request = components.canonical_request; + let string_to_sign = components.string_to_sign; + let signed_url = components.signed_url; if canonical_request != test.expected_canonical_request || string_to_sign != test.expected_string_to_sign From 4d766592f2ca4d9a59cd70b6c1c8ca84be53e76d Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 3 Dec 2025 19:49:57 +0000 Subject: [PATCH 31/41] refactor: move CredentialsSigner to signer folder --- src/auth/src/credentials/impersonated.rs | 4 +- src/auth/src/credentials/mds.rs | 2 +- src/auth/src/signer.rs | 83 +------------------- src/auth/src/signer/iam.rs | 97 ++++++++++++++++++++++++ src/auth/src/signer/mds.rs | 41 +++++----- 5 files changed, 125 insertions(+), 102 deletions(-) create mode 100644 src/auth/src/signer/iam.rs diff --git a/src/auth/src/credentials/impersonated.rs b/src/auth/src/credentials/impersonated.rs index b2250ecac6..7110f268f3 100644 --- a/src/auth/src/credentials/impersonated.rs +++ b/src/auth/src/credentials/impersonated.rs @@ -490,7 +490,7 @@ impl Builder { let client_email = extract_client_email(&components.service_account_impersonation_url)?; Ok(crate::signer::Signer { - inner: Arc::new(crate::signer::CredentialsSigner { + inner: Arc::new(crate::signer::iam::IamSigner { client_email, inner: self.build()?, }), @@ -504,7 +504,7 @@ impl Builder { let client_email = extract_client_email(&components.service_account_impersonation_url)?; Ok(crate::signer::Signer { - inner: Arc::new(crate::signer::CredentialsSigner { + inner: Arc::new(crate::signer::iam::IamSigner { client_email, inner: self.build()?, }), diff --git a/src/auth/src/credentials/mds.rs b/src/auth/src/credentials/mds.rs index 52e04101e7..b16a2bc713 100644 --- a/src/auth/src/credentials/mds.rs +++ b/src/auth/src/credentials/mds.rs @@ -326,7 +326,7 @@ impl Builder { pub fn build_signer(self) -> BuildResult { let (endpoint, _) = self.resolve_endpoint(); let credentials = self.build()?; - let signing_provider = crate::signer::mds::MDSCredentialsSigner::new(endpoint, credentials); + let signing_provider = crate::signer::mds::MDSSigner::new(endpoint, credentials); Ok(crate::signer::Signer { inner: Arc::new(signing_provider), }) diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index f6e069abd6..0b39f789de 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -12,11 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::credentials::{CacheableResource, Credentials}; -use http::Extensions; -use reqwest::Client; use std::sync::Arc; +pub(crate) mod iam; pub(crate) mod mds; pub(crate) mod service_account; @@ -62,85 +60,6 @@ impl Signer { } } -// Implements Signer using IAM signBlob API and reusing using existing [Credentials] to -// authenticate to it. -#[derive(Clone, Debug)] -pub(crate) struct CredentialsSigner { - pub(crate) client_email: String, - pub(crate) inner: Credentials, -} - -#[derive(serde::Serialize)] -struct SignBlobRequest { - payload: String, -} - -#[derive(Debug, serde::Deserialize)] -struct SignBlobResponse { - #[serde(rename = "signedBlob")] - signed_blob: String, -} - -#[async_trait::async_trait] -impl dynamic::SigningProvider for CredentialsSigner { - async fn client_email(&self) -> Result { - Ok(self.client_email.clone()) - } - - async fn sign(&self, content: &[u8]) -> Result { - use base64::{Engine, prelude::BASE64_STANDARD}; - - let source_headers = self - .inner - .headers(Extensions::new()) - .await - .map_err(SigningError::transport)?; - let source_headers = match source_headers { - CacheableResource::New { data, .. } => data, - CacheableResource::NotModified => { - unreachable!("requested source credentials without a caching etag") - } - }; - - let client_email = self.client_email.clone(); - let url = format!( - "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:signBlob", - client_email - ); - - let client = Client::new(); - let payload = BASE64_STANDARD.encode(content); - let body = SignBlobRequest { payload }; - - let response = client - .post(url) - .header("Content-Type", "application/json") - .headers(source_headers) - .json(&body) - .send() - .await - .map_err(SigningError::transport)?; - - if !response.status().is_success() { - let err_text = response.text().await.map_err(SigningError::transport)?; - return Err(SigningError::transport(format!("err status: {err_text:?}"))); - } - - let res = response - .json::() - .await - .map_err(SigningError::transport)?; - - let signature = BASE64_STANDARD - .decode(res.signed_blob) - .map_err(SigningError::transport)?; - - let signature = hex::encode(signature); - - Ok(signature) - } -} - /// A trait for types that can sign content. pub trait SigningProvider: std::fmt::Debug { /// Returns the email address of the authorizer. diff --git a/src/auth/src/signer/iam.rs b/src/auth/src/signer/iam.rs new file mode 100644 index 0000000000..e6e9afaaa2 --- /dev/null +++ b/src/auth/src/signer/iam.rs @@ -0,0 +1,97 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::credentials::{CacheableResource, Credentials}; +use crate::signer::{Result, SigningError, dynamic::SigningProvider}; +use http::Extensions; +use reqwest::Client; + +// Implements Signer using IAM signBlob API and reusing using existing [Credentials] to +// authenticate to it. +#[derive(Clone, Debug)] +pub(crate) struct IamSigner { + pub(crate) client_email: String, + pub(crate) inner: Credentials, +} + +#[derive(serde::Serialize)] +struct SignBlobRequest { + payload: String, +} + +#[derive(Debug, serde::Deserialize)] +struct SignBlobResponse { + #[serde(rename = "signedBlob")] + signed_blob: String, +} + +#[async_trait::async_trait] +impl SigningProvider for IamSigner { + async fn client_email(&self) -> Result { + Ok(self.client_email.clone()) + } + + async fn sign(&self, content: &[u8]) -> Result { + use base64::{Engine, prelude::BASE64_STANDARD}; + + let source_headers = self + .inner + .headers(Extensions::new()) + .await + .map_err(SigningError::transport)?; + let source_headers = match source_headers { + CacheableResource::New { data, .. } => data, + CacheableResource::NotModified => { + unreachable!("requested source credentials without a caching etag") + } + }; + + let client_email = self.client_email.clone(); + let url = format!( + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:signBlob", + client_email + ); + + let client = Client::new(); + let payload = BASE64_STANDARD.encode(content); + let body = SignBlobRequest { payload }; + + let response = client + .post(url) + .header("Content-Type", "application/json") + .headers(source_headers) + .json(&body) + .send() + .await + .map_err(SigningError::transport)?; + + if !response.status().is_success() { + let err_text = response.text().await.map_err(SigningError::transport)?; + return Err(SigningError::transport(format!("err status: {err_text:?}"))); + } + + let res = response + .json::() + .await + .map_err(SigningError::transport)?; + + let signature = BASE64_STANDARD + .decode(res.signed_blob) + .map_err(SigningError::transport)?; + + let signature = hex::encode(signature); + + Ok(signature) + } +} diff --git a/src/auth/src/signer/mds.rs b/src/auth/src/signer/mds.rs index 584b3c852b..ecadc440f7 100644 --- a/src/auth/src/signer/mds.rs +++ b/src/auth/src/signer/mds.rs @@ -20,16 +20,16 @@ use reqwest::Client; use std::sync::Arc; use tokio::sync::RwLock; -// Implements Signer for MDS that extends the existing CredentialsSigner by fetching +// Implements Signer for MDS that extends the existing IamSigner by fetching // email via MDS email endpoint. #[derive(Clone, Debug)] -pub(crate) struct MDSCredentialsSigner { +pub(crate) struct MDSSigner { endpoint: String, client_email: Arc>, inner: Credentials, } -impl MDSCredentialsSigner { +impl MDSSigner { pub(crate) fn new(endpoint: String, inner: Credentials) -> Self { Self { endpoint, @@ -40,7 +40,7 @@ impl MDSCredentialsSigner { } #[async_trait::async_trait] -impl SigningProvider for MDSCredentialsSigner { +impl SigningProvider for MDSSigner { async fn client_email(&self) -> Result { let mut client_email = self .client_email @@ -48,18 +48,7 @@ impl SigningProvider for MDSCredentialsSigner { .map_err(|_e| SigningError::transport("failed to obtain lock to read client email"))?; if client_email.is_empty() { - let client = Client::new(); - - let request = client - .get(format!("{}{}/email", self.endpoint, MDS_DEFAULT_URI)) - .header( - METADATA_FLAVOR, - HeaderValue::from_static(METADATA_FLAVOR_VALUE), - ); - - let response = request.send().await.map_err(SigningError::transport)?; - let email = response.text().await.map_err(SigningError::transport)?; - + let email = self.fetch_client_email().await?; *client_email = email.clone(); } @@ -69,7 +58,7 @@ impl SigningProvider for MDSCredentialsSigner { async fn sign(&self, content: &[u8]) -> Result { let client_email = self.client_email().await?; - let signer = crate::signer::CredentialsSigner { + let signer = crate::signer::iam::IamSigner { client_email, inner: self.inner.clone(), }; @@ -78,5 +67,23 @@ impl SigningProvider for MDSCredentialsSigner { } } +impl MDSSigner { + async fn fetch_client_email(&self) -> Result { + let client = Client::new(); + + let request = client + .get(format!("{}{}/email", self.endpoint, MDS_DEFAULT_URI)) + .header( + METADATA_FLAVOR, + HeaderValue::from_static(METADATA_FLAVOR_VALUE), + ); + + let response = request.send().await.map_err(SigningError::transport)?; + let email = response.text().await.map_err(SigningError::transport)?; + + Ok(email) + } +} + #[cfg(test)] mod tests {} From d56068189e1daf2f4570742d5e793b35847df616 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Thu, 4 Dec 2025 15:23:45 +0000 Subject: [PATCH 32/41] test: add IamSigner tests --- src/auth/src/credentials/impersonated.rs | 14 ++- src/auth/src/signer/iam.rs | 127 ++++++++++++++++++++++- src/auth/src/signer/mds.rs | 5 +- 3 files changed, 129 insertions(+), 17 deletions(-) diff --git a/src/auth/src/credentials/impersonated.rs b/src/auth/src/credentials/impersonated.rs index 7110f268f3..d0f92df7ac 100644 --- a/src/auth/src/credentials/impersonated.rs +++ b/src/auth/src/credentials/impersonated.rs @@ -489,11 +489,10 @@ impl Builder { let components = build_components_from_json(json)?; let client_email = extract_client_email(&components.service_account_impersonation_url)?; + let creds = self.build()?; + let signer = crate::signer::iam::IamSigner::new(client_email, creds); Ok(crate::signer::Signer { - inner: Arc::new(crate::signer::iam::IamSigner { - client_email, - inner: self.build()?, - }), + inner: Arc::new(signer), }) } BuilderSource::FromCredentials(source_credentials) => { @@ -503,11 +502,10 @@ impl Builder { )?; let client_email = extract_client_email(&components.service_account_impersonation_url)?; + let creds = self.build()?; + let signer = crate::signer::iam::IamSigner::new(client_email, creds); Ok(crate::signer::Signer { - inner: Arc::new(crate::signer::iam::IamSigner { - client_email, - inner: self.build()?, - }), + inner: Arc::new(signer), }) } } diff --git a/src/auth/src/signer/iam.rs b/src/auth/src/signer/iam.rs index e6e9afaaa2..6945a279dc 100644 --- a/src/auth/src/signer/iam.rs +++ b/src/auth/src/signer/iam.rs @@ -21,11 +21,12 @@ use reqwest::Client; // authenticate to it. #[derive(Clone, Debug)] pub(crate) struct IamSigner { - pub(crate) client_email: String, - pub(crate) inner: Credentials, + client_email: String, + inner: Credentials, + endpoint: String, } -#[derive(serde::Serialize)] +#[derive(Debug, serde::Serialize)] struct SignBlobRequest { payload: String, } @@ -36,6 +37,16 @@ struct SignBlobResponse { signed_blob: String, } +impl IamSigner { + pub(crate) fn new(client_email: String, inner: Credentials) -> Self { + Self { + client_email, + inner, + endpoint: "https://iamcredentials.googleapis.com".to_string(), + } + } +} + #[async_trait::async_trait] impl SigningProvider for IamSigner { async fn client_email(&self) -> Result { @@ -59,8 +70,8 @@ impl SigningProvider for IamSigner { let client_email = self.client_email.clone(); let url = format!( - "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:signBlob", - client_email + "{}/v1/projects/-/serviceAccounts/{}:signBlob", + self.endpoint, client_email ); let client = Client::new(); @@ -95,3 +106,109 @@ impl SigningProvider for IamSigner { Ok(signature) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::credentials::{Credentials, CredentialsProvider, EntityTag}; + use crate::errors::CredentialsError; + use base64::{Engine, prelude::BASE64_STANDARD}; + use http::header::{HeaderName, HeaderValue}; + use http::{Extensions, HeaderMap}; + use httptest::matchers::{all_of, contains, eq, json_decoded, request}; + use httptest::responders::json_encoded; + use httptest::{Expectation, Server}; + use serde_json::json; + + type TestResult = anyhow::Result<()>; + + mockall::mock! { + #[derive(Debug)] + Credentials {} + + impl CredentialsProvider for Credentials { + async fn headers(&self, extensions: Extensions) -> std::result::Result, CredentialsError>; + async fn universe_domain(&self) -> Option; + } + } + + #[tokio::test] + async fn test_iam_sign() -> TestResult { + let server = Server::run(); + let payload = BASE64_STANDARD.encode("test"); + let signed_blob = BASE64_STANDARD.encode("signed_blob"); + server.expect( + Expectation::matching(all_of![ + request::method_path( + "POST", + format!("/v1/projects/-/serviceAccounts/test@example.com:signBlob") + ), + request::headers(contains(("authorization", "Bearer test-value"))), + request::body(json_decoded(eq(json!({ + "payload": payload, + })))) + ]) + .respond_with(json_encoded(json!({ + "signedBlob": signed_blob, + }))), + ); + let endpoint = server.url("").to_string().trim_end_matches('/').to_string(); + + let mut mock = MockCredentials::new(); + mock.expect_headers().return_once(|_extensions| { + let headers = HeaderMap::from_iter([( + HeaderName::from_static("authorization"), + HeaderValue::from_static("Bearer test-value"), + )]); + Ok(CacheableResource::New { + entity_tag: EntityTag::default(), + data: headers, + }) + }); + let creds = Credentials::from(mock); + + let mut signer = IamSigner::new("test@example.com".to_string(), creds); + signer.endpoint = endpoint; + let signature = signer.sign(b"test").await.unwrap(); + + assert_eq!(signature, hex::encode("signed_blob")); + + Ok(()) + } + + #[tokio::test] + async fn test_iam_sign_api_error() -> TestResult { + let server = Server::run(); + let payload = BASE64_STANDARD.encode("test"); + server.expect( + Expectation::matching(all_of![request::method_path( + "POST", + format!("/v1/projects/-/serviceAccounts/test@example.com:signBlob") + ),]) + .respond_with(json_encoded(json!({ + "error": { + "code": 400, + "message": "test-error", + }, + }))), + ); + let endpoint = server.url("").to_string().trim_end_matches('/').to_string(); + + let mut mock = MockCredentials::new(); + mock.expect_headers().return_once(|_extensions| { + Ok(CacheableResource::New { + entity_tag: EntityTag::default(), + data: HeaderMap::new(), + }) + }); + let creds = Credentials::from(mock); + + let mut signer = IamSigner::new("test@example.com".to_string(), creds); + signer.endpoint = endpoint; + let err = signer.sign(b"test").await.unwrap_err(); + + assert!(err.is_transport()); + + Ok(()) + } +} diff --git a/src/auth/src/signer/mds.rs b/src/auth/src/signer/mds.rs index ecadc440f7..b1f7986bfc 100644 --- a/src/auth/src/signer/mds.rs +++ b/src/auth/src/signer/mds.rs @@ -58,10 +58,7 @@ impl SigningProvider for MDSSigner { async fn sign(&self, content: &[u8]) -> Result { let client_email = self.client_email().await?; - let signer = crate::signer::iam::IamSigner { - client_email, - inner: self.inner.clone(), - }; + let signer = crate::signer::iam::IamSigner::new(client_email, self.inner.clone()); signer.sign(content).await } From 6f69d7cf8774dea708750ef3b236d0099b4baab6 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 5 Dec 2025 17:22:27 +0000 Subject: [PATCH 33/41] fix: iam signer retry logic --- src/auth/src/signer/iam.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/auth/src/signer/iam.rs b/src/auth/src/signer/iam.rs index 78f827fcdc..91f99dab86 100644 --- a/src/auth/src/signer/iam.rs +++ b/src/auth/src/signer/iam.rs @@ -108,7 +108,6 @@ async fn sign_blob_call_with_retry( let backoff_policy: Arc = backoff_policy.into(); let retry_throttler: RetryThrottlerArg = AdaptiveThrottler::default().into(); let retry_throttler: SharedRetryThrottler = retry_throttler.into(); - retry_loop( async move |_| { let source_headers = credentials @@ -141,14 +140,30 @@ async fn sign_blob_call( } }; - client + let response = client .post(url) .header("Content-Type", "application/json") .headers(source_headers.clone()) .json(&body) .send() .await - .map_err(|e| gax::error::Error::transport(source_headers, e)) + .map_err(gax::error::Error::io)?; + + let status = response.status(); + if !status.is_success() { + let err_headers = response.headers().clone(); + let err_payload = response + .bytes() + .await + .map_err(|e| gax::error::Error::transport(err_headers.clone(), e))?; + return Err(gax::error::Error::http( + status.as_u16(), + err_headers, + err_payload, + )); + } + + Ok(response) } #[cfg(test)] @@ -263,6 +278,7 @@ mod tests { Ok(()) } + #[tokio::test] async fn test_iam_sign_retry() -> TestResult { let server = Server::run(); let signed_blob = BASE64_STANDARD.encode("signed_blob"); @@ -283,7 +299,7 @@ mod tests { let endpoint = server.url("").to_string().trim_end_matches('/').to_string(); let mut mock = MockCredentials::new(); - mock.expect_headers().return_once(|_extensions| { + mock.expect_headers().returning(|_extensions| { Ok(CacheableResource::New { entity_tag: EntityTag::default(), data: HeaderMap::new(), From e7bbb8a42476fe26af444433423579fe34166fa7 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 5 Dec 2025 19:34:34 +0000 Subject: [PATCH 34/41] fix: move SigningProvider trait to return Bytes --- Cargo.lock | 1 + src/auth/Cargo.toml | 1 + src/auth/src/signer.rs | 14 +++++++------- src/auth/src/signer/iam.rs | 10 ++++------ src/auth/src/signer/mds.rs | 2 +- src/auth/src/signer/service_account.rs | 8 +++----- src/storage/src/storage/signed_url.rs | 15 ++++++++++----- 7 files changed, 27 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9286a73d47..e2c4ce1c84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1477,6 +1477,7 @@ dependencies = [ "async-trait", "base64", "bon", + "bytes", "google-cloud-gax", "hex", "http", diff --git a/src/auth/Cargo.toml b/src/auth/Cargo.toml index 829174afcb..502e6f6a7d 100644 --- a/src/auth/Cargo.toml +++ b/src/auth/Cargo.toml @@ -36,6 +36,7 @@ rustc_version.workspace = true [dependencies] async-trait.workspace = true base64.workspace = true +bytes.workspace = true http.workspace = true reqwest = { workspace = true, features = ["json", "rustls-tls"] } rustls = { workspace = true, features = ["logging", "ring", "std", "tls12"] } diff --git a/src/auth/src/signer.rs b/src/auth/src/signer.rs index 0b39f789de..67ab32dfe6 100644 --- a/src/auth/src/signer.rs +++ b/src/auth/src/signer.rs @@ -51,8 +51,7 @@ impl Signer { /// Signs the provided content using the underlying provider. /// /// The content is typically a string-to-sign generated by the caller. - /// Returns the signature as a hex encoded string. - pub async fn sign(&self, content: T) -> Result + pub async fn sign(&self, content: T) -> Result where T: AsRef<[u8]> + Send + Sync, { @@ -72,7 +71,7 @@ pub trait SigningProvider: std::fmt::Debug { /// Signs the content. /// /// Returns the signature. - fn sign(&self, content: &[u8]) -> impl Future> + Send; + fn sign(&self, content: &[u8]) -> impl Future> + Send; } pub(crate) mod dynamic { @@ -82,7 +81,7 @@ pub(crate) mod dynamic { #[async_trait::async_trait] pub trait SigningProvider: Send + Sync + std::fmt::Debug { async fn client_email(&self) -> Result; - async fn sign(&self, content: &[u8]) -> Result; + async fn sign(&self, content: &[u8]) -> Result; } /// The public CredentialsProvider implements the dyn-compatible CredentialsProvider. @@ -95,7 +94,7 @@ pub(crate) mod dynamic { T::client_email(self).await } - async fn sign(&self, content: &[u8]) -> Result { + async fn sign(&self, content: &[u8]) -> Result { T::sign(self, content).await } } @@ -186,7 +185,7 @@ mod tests { impl SigningProvider for Signer { async fn client_email(&self) -> Result; - async fn sign(&self, content: &[u8]) -> Result; + async fn sign(&self, content: &[u8]) -> Result; } } @@ -195,7 +194,8 @@ mod tests { let mut mock = MockSigner::new(); mock.expect_client_email() .returning(|| Ok("test".to_string())); - mock.expect_sign().returning(|_| Ok("test".to_string())); + mock.expect_sign() + .returning(|_| Ok(bytes::Bytes::from("test"))); let signer = Signer::from(mock); let result = signer.client_email().await?; diff --git a/src/auth/src/signer/iam.rs b/src/auth/src/signer/iam.rs index 91f99dab86..e7c5bc1373 100644 --- a/src/auth/src/signer/iam.rs +++ b/src/auth/src/signer/iam.rs @@ -61,7 +61,7 @@ impl SigningProvider for IamSigner { Ok(self.client_email.clone()) } - async fn sign(&self, content: &[u8]) -> Result { + async fn sign(&self, content: &[u8]) -> Result { use base64::{Engine, prelude::BASE64_STANDARD}; let payload = BASE64_STANDARD.encode(content); @@ -89,9 +89,7 @@ impl SigningProvider for IamSigner { .decode(res.signed_blob) .map_err(SigningError::transport)?; - let signature = hex::encode(signature); - - Ok(signature) + Ok(bytes::Bytes::from(signature)) } } @@ -231,7 +229,7 @@ mod tests { signer.endpoint = endpoint; let signature = signer.sign(b"test").await.unwrap(); - assert_eq!(signature, hex::encode("signed_blob")); + assert_eq!(signature.as_ref(), b"signed_blob"); Ok(()) } @@ -311,7 +309,7 @@ mod tests { signer.endpoint = endpoint; let signature = signer.sign(b"test").await.unwrap(); - assert_eq!(signature, hex::encode("signed_blob")); + assert_eq!(signature.as_ref(), b"signed_blob"); Ok(()) } diff --git a/src/auth/src/signer/mds.rs b/src/auth/src/signer/mds.rs index b1f7986bfc..d1d3bf206c 100644 --- a/src/auth/src/signer/mds.rs +++ b/src/auth/src/signer/mds.rs @@ -55,7 +55,7 @@ impl SigningProvider for MDSSigner { Ok(client_email.clone()) } - async fn sign(&self, content: &[u8]) -> Result { + async fn sign(&self, content: &[u8]) -> Result { let client_email = self.client_email().await?; let signer = crate::signer::iam::IamSigner::new(client_email, self.inner.clone()); diff --git a/src/auth/src/signer/service_account.rs b/src/auth/src/signer/service_account.rs index 52f5fd9c5f..5994b6bfdd 100644 --- a/src/auth/src/signer/service_account.rs +++ b/src/auth/src/signer/service_account.rs @@ -46,7 +46,7 @@ impl SigningProvider for ServiceAccountSigner { Ok(self.client_email.clone()) } - async fn sign(&self, content: &[u8]) -> Result { + async fn sign(&self, content: &[u8]) -> Result { let signer = self .service_account_key .signer() @@ -54,9 +54,7 @@ impl SigningProvider for ServiceAccountSigner { let signature = signer.sign(content).map_err(SigningError::sign)?; - let signature = hex::encode(signature); - - Ok(signature) + Ok(bytes::Bytes::from(signature)) } } @@ -93,7 +91,7 @@ mod tests { let inner_signer = service_account_key.signer().unwrap(); let inner_result = inner_signer.sign(b"test")?; - assert_eq!(result, hex::encode(inner_result)); + assert_eq!(result.as_ref(), inner_result); Ok(()) } } diff --git a/src/storage/src/storage/signed_url.rs b/src/storage/src/storage/signed_url.rs index 889c2de433..3bd383d5fd 100644 --- a/src/storage/src/storage/signed_url.rs +++ b/src/storage/src/storage/signed_url.rs @@ -377,6 +377,8 @@ impl SignedUrlBuilder { .await .map_err(SigningError::Signing)?; + let signature = hex::encode(signature); + let signed_url = format!( "{}?{}&X-Goog-Signature={}", canonical_url, canonical_query, signature @@ -409,7 +411,7 @@ mod tests { impl SigningProvider for Signer { async fn client_email(&self) -> auth::signer::Result; - async fn sign(&self, _content: &[u8]) -> auth::signer::Result; + async fn sign(&self, _content: &[u8]) -> auth::signer::Result; } } @@ -419,7 +421,7 @@ mod tests { mock.expect_client_email() .return_once(|| Ok("test@example.com".to_string())); mock.expect_sign() - .return_once(|_content| Ok("test-signature".to_string())); + .return_once(|_content| Ok(bytes::Bytes::from("test-signature"))); let signer = Signer::from(mock); let url = SignedUrlBuilder::for_object("test-bucket", "test-object") @@ -430,8 +432,11 @@ mod tests { .await .unwrap(); + let signature = hex::encode(b"test-signature"); + let x_goog_signature = format!("X-Goog-Signature={signature}"); + assert!(url.starts_with("https://storage.googleapis.com/test-bucket/test-object")); - assert!(url.contains("X-Goog-Signature=test-signature")); + assert!(url.contains(&x_goog_signature)); assert!(url.contains("X-Goog-Algorithm=GOOG4-RSA-SHA256")); assert!(url.contains("X-Goog-Credential=test%40example.com")); @@ -444,7 +449,7 @@ mod tests { mock.expect_client_email() .return_once(|| Ok("test@example.com".to_string())); mock.expect_sign() - .return_once(|_content| Ok("test-signature".to_string())); + .return_once(|_content| Ok(bytes::Bytes::from("test-signature"))); let signer = Signer::from(mock); let url = SignedUrlBuilder::for_object("test-bucket", "folder/test object.txt") @@ -490,7 +495,7 @@ mod tests { mock.expect_client_email() .return_once(|| Ok("test@example.com".to_string())); mock.expect_sign() - .return_once(|_content| Ok("test-signature".to_string())); + .return_once(|_content| Ok(bytes::Bytes::from("test-signature"))); let signer = Signer::from(mock); let err = SignedUrlBuilder::for_object("b", "o") From 673de8a589bb6d0ea7b80395142bd4e59269d06c Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 5 Dec 2025 19:39:36 +0000 Subject: [PATCH 35/41] fix: remove extra line --- src/auth/src/signer/iam.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/auth/src/signer/iam.rs b/src/auth/src/signer/iam.rs index e7c5bc1373..037126f1e9 100644 --- a/src/auth/src/signer/iam.rs +++ b/src/auth/src/signer/iam.rs @@ -106,6 +106,7 @@ async fn sign_blob_call_with_retry( let backoff_policy: Arc = backoff_policy.into(); let retry_throttler: RetryThrottlerArg = AdaptiveThrottler::default().into(); let retry_throttler: SharedRetryThrottler = retry_throttler.into(); + retry_loop( async move |_| { let source_headers = credentials From 798b9134b16967f43dbf5579a742b67b26fa2d54 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 5 Dec 2025 21:07:00 +0000 Subject: [PATCH 36/41] test: improve test with io error --- src/auth/src/signer/iam.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/auth/src/signer/iam.rs b/src/auth/src/signer/iam.rs index 037126f1e9..888a7e82a1 100644 --- a/src/auth/src/signer/iam.rs +++ b/src/auth/src/signer/iam.rs @@ -281,6 +281,11 @@ mod tests { async fn test_iam_sign_retry() -> TestResult { let server = Server::run(); let signed_blob = BASE64_STANDARD.encode("signed_blob"); + let invalid_res = http::Response::builder() + .version(http::Version::HTTP_3) // unsupported version + .status(204) + .body(Vec::new()) + .unwrap(); server.expect( Expectation::matching(all_of![request::method_path( "POST", @@ -288,7 +293,7 @@ mod tests { ),]) .times(3) .respond_with(cycle![ - status_code(503).body("try-again"), + invalid_res, // forces i/o error status_code(503).body("try-again"), json_encoded(json!({ "signedBlob": signed_blob, From bfc9cb2ceb325dc9b68758b8b46adab96cd1f9a8 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 8 Dec 2025 20:16:19 +0000 Subject: [PATCH 37/41] fix: add hex as workspace dep --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index f2604db910..4525c4c792 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -308,6 +308,7 @@ crates_io_api = { default-features = false, version = "0.12" } clap = { default-features = false, version = "4" } crc32c = { default-features = false, version = "0.6.8" } futures = { default-features = false, version = "0.3" } +hex = { default-features = false, version = "0.4" } http = { default-features = false, version = "1", features = ["std"] } http-body = { default-features = false, version = "1" } http-body-util = { default-features = false, version = "0.1.3" } From c40fca062da0efe28931c44458a31547f9f35f71 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 10 Dec 2025 17:01:04 +0000 Subject: [PATCH 38/41] fix: changes from mds branch --- src/auth/src/credentials/impersonated.rs | 4 ++-- src/auth/src/signer/mds.rs | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/auth/src/credentials/impersonated.rs b/src/auth/src/credentials/impersonated.rs index d0f92df7ac..4c776135b4 100644 --- a/src/auth/src/credentials/impersonated.rs +++ b/src/auth/src/credentials/impersonated.rs @@ -490,7 +490,7 @@ impl Builder { let client_email = extract_client_email(&components.service_account_impersonation_url)?; let creds = self.build()?; - let signer = crate::signer::iam::IamSigner::new(client_email, creds); + let signer = crate::signer::iam::IamSigner::new(client_email, creds, None); Ok(crate::signer::Signer { inner: Arc::new(signer), }) @@ -503,7 +503,7 @@ impl Builder { let client_email = extract_client_email(&components.service_account_impersonation_url)?; let creds = self.build()?; - let signer = crate::signer::iam::IamSigner::new(client_email, creds); + let signer = crate::signer::iam::IamSigner::new(client_email, creds, None); Ok(crate::signer::Signer { inner: Arc::new(signer), }) diff --git a/src/auth/src/signer/mds.rs b/src/auth/src/signer/mds.rs index 938b0a2a35..e17a0f74a4 100644 --- a/src/auth/src/signer/mds.rs +++ b/src/auth/src/signer/mds.rs @@ -93,6 +93,9 @@ impl MDSSigner { #[cfg(test)] mod tests { + use super::*; + use crate::credentials::{CacheableResource, Credentials, CredentialsProvider, EntityTag}; + use crate::errors::CredentialsError; use base64::{Engine, prelude::BASE64_STANDARD}; use http::header::{HeaderName, HeaderValue}; use http::{Extensions, HeaderMap}; From 9b4eb05889aae65859c36735327e22bada3ecd55 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 10 Dec 2025 17:11:52 +0000 Subject: [PATCH 39/41] refactor: credential builder macro --- src/auth/src/credentials.rs | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/auth/src/credentials.rs b/src/auth/src/credentials.rs index 2e0dbcad21..58d0bb31fe 100644 --- a/src/auth/src/credentials.rs +++ b/src/auth/src/credentials.rs @@ -599,15 +599,12 @@ fn extract_credential_type(json: &Value) -> BuildResult<&str> { /// It helps avoid repetitive code in the `build_credentials` function. macro_rules! config_builder { ($builder_instance:expr, $quota_project_id_option:expr, $scopes_option:expr, $apply_scopes_closure:expr) => {{ - let builder = $builder_instance; - let builder = $quota_project_id_option - .into_iter() - .fold(builder, |b, qp| b.with_quota_project_id(qp)); - - let builder = $scopes_option - .into_iter() - .fold(builder, |b, s| $apply_scopes_closure(b, s)); - + let builder = config_common_builder!( + $builder_instance, + $quota_project_id_option, + $scopes_option, + $apply_scopes_closure + ); builder.build_access_token_credentials() }}; } @@ -616,6 +613,18 @@ macro_rules! config_builder { /// specific credential builder instance and then return a signer for it. #[cfg(google_cloud_unstable_signed_url)] macro_rules! config_signer { + ($builder_instance:expr, $quota_project_id_option:expr, $scopes_option:expr, $apply_scopes_closure:expr) => {{ + let builder = config_common_builder!( + $builder_instance, + $quota_project_id_option, + $scopes_option, + $apply_scopes_closure + ); + builder.build_signer() + }}; +} + +macro_rules! config_common_builder { ($builder_instance:expr, $quota_project_id_option:expr, $scopes_option:expr, $apply_scopes_closure:expr) => {{ let builder = $builder_instance; let builder = $quota_project_id_option @@ -626,7 +635,7 @@ macro_rules! config_signer { .into_iter() .fold(builder, |b, s| $apply_scopes_closure(b, s)); - builder.build_signer() + builder }}; } From 2716b8efc054c0d22f566149f111c623fad4a40e Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Tue, 30 Dec 2025 15:11:00 +0000 Subject: [PATCH 40/41] feat: leave test only fields to exist only in tests --- src/storage/src/storage/signed_url.rs | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/storage/src/storage/signed_url.rs b/src/storage/src/storage/signed_url.rs index 518cc63f61..851a21a198 100644 --- a/src/storage/src/storage/signed_url.rs +++ b/src/storage/src/storage/signed_url.rs @@ -130,9 +130,9 @@ impl SigningScope { } struct SigningComponents { - #[allow(dead_code)] + #[cfg(test)] canonical_request: String, - #[allow(dead_code)] + #[cfg(test)] string_to_sign: String, signed_url: String, } @@ -209,7 +209,7 @@ impl SignedUrlBuilder { #[cfg(test)] /// Sets the timestamp for the signed URL. Only used in tests. - pub fn with_timestamp(mut self, timestamp: DateTime) -> Self { + fn with_timestamp(mut self, timestamp: DateTime) -> Self { self.timestamp = timestamp; self } @@ -551,7 +551,9 @@ impl SignedUrlBuilder { ); Ok(SigningComponents { + #[cfg(test)] canonical_request, + #[cfg(test)] string_to_sign, signed_url, }) @@ -768,12 +770,13 @@ mod tests { } #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] struct SignedUrlTestSuite { - #[serde(rename = "signingV4Tests")] signing_v4_tests: Vec, } #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] struct SignedUrlTest { description: String, bucket: String, @@ -781,26 +784,17 @@ mod tests { method: String, expiration: u64, timestamp: String, - #[serde(rename = "expectedUrl")] expected_url: String, headers: Option>, - #[serde(rename = "queryParameters")] query_parameters: Option>, scheme: Option, - #[serde(rename = "urlStyle")] url_style: Option, - #[serde(rename = "bucketBoundHostname")] bucket_bound_hostname: Option, - #[serde(rename = "expectedCanonicalRequest")] expected_canonical_request: String, - #[serde(rename = "expectedStringToSign")] expected_string_to_sign: String, hostname: Option, - #[serde(rename = "clientEndpoint")] client_endpoint: Option, - #[serde(rename = "emulatorHostname")] emulator_hostname: Option, - #[serde(rename = "universeDomain")] universe_domain: Option, } From db5b45889d9b80a499c09974cb52ef503833c889 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Tue, 6 Jan 2026 17:19:30 +0000 Subject: [PATCH 41/41] fix: remove extra tests --- src/storage/src/storage/signed_url.rs | 55 ++------------------------- 1 file changed, 3 insertions(+), 52 deletions(-) diff --git a/src/storage/src/storage/signed_url.rs b/src/storage/src/storage/signed_url.rs index 38d5fc0b86..c4d429acf3 100644 --- a/src/storage/src/storage/signed_url.rs +++ b/src/storage/src/storage/signed_url.rs @@ -491,6 +491,9 @@ impl SignedUrlBuilder { let header_value = Self::canonicalize_header_value(v); format!("{acc}{}:{}\n", k, header_value) }); + + // If the user provides a value for X-Goog-Content-SHA256, we must use + // that value in the request string. If not, we use UNSIGNED-PAYLOAD. let signature = headers .get("x-goog-content-sha256") .cloned() @@ -617,58 +620,6 @@ mod tests { Ok(()) } - #[tokio::test] - async fn test_signed_url_generation() -> TestResult { - let mut mock = MockSigner::new(); - mock.expect_client_email() - .return_once(|| Ok("test@example.com".to_string())); - mock.expect_sign() - .return_once(|_content| Ok(bytes::Bytes::from("test-signature"))); - - let signer = Signer::from(mock); - let url = SignedUrlBuilder::for_object("test-bucket", "test-object") - .with_method(http::Method::PUT) - .with_expiration(std::time::Duration::from_secs(3600)) - .with_header("x-goog-meta-test", "value") - .sign_with(&signer) - .await - .unwrap(); - - let signature = hex::encode(b"test-signature"); - let x_goog_signature = format!("X-Goog-Signature={signature}"); - - assert!(url.starts_with("https://storage.googleapis.com/test-bucket/test-object")); - assert!(url.contains(&x_goog_signature)); - assert!(url.contains("X-Goog-Algorithm=GOOG4-RSA-SHA256")); - assert!(url.contains("X-Goog-Credential=test%40example.com")); - - Ok(()) - } - - #[tokio::test] - async fn test_signed_url_generation_escaping() -> TestResult { - let mut mock = MockSigner::new(); - mock.expect_client_email() - .return_once(|| Ok("test@example.com".to_string())); - mock.expect_sign() - .return_once(|_content| Ok(bytes::Bytes::from("test-signature"))); - - let signer = Signer::from(mock); - let url = SignedUrlBuilder::for_object("test-bucket", "folder/test object.txt") - .with_method(http::Method::PUT) - .with_header("content-type", "text/plain") - .sign_with(&signer) - .await - .unwrap(); - - assert!( - url.starts_with("https://storage.googleapis.com/test-bucket/folder/test%20object.txt?") - ); - assert!(url.contains("X-Goog-Signature=")); - - Ok(()) - } - #[tokio::test] async fn test_signed_url_error_signing() -> TestResult { let mut mock = MockSigner::new();