From d4e0ec87d8044b7d7d349b7c947bc12db0835788 Mon Sep 17 00:00:00 2001 From: Ragim Ibragimov Date: Mon, 19 May 2025 15:17:18 +0300 Subject: [PATCH] feat: vault check - check vault server availability at startup - add option to completely disable vault source - test fixes - update README.md --- CHANGELOG.md | 7 ++++ README.md | 9 ++++- cmd/goconfig/goconfig_test.go | 27 +------------ pkg/config/config.go | 18 ++++++--- pkg/config/config_private_test.go | 8 ++-- .../config_test/testdata/config/vault.toml | 4 -- .../testdata/vault/config/vault.toml | 1 + .../testdata/vault/disabled/default.toml | 2 + .../testdata/vault/disabled/production.toml | 2 + .../testdata/vault/disabled/vault.toml | 5 +++ .../config_test/testdata/vault/token/env.toml | 2 +- .../config_test/vault_auth_token_test.go | 38 ++++++++++++++++++- .../{vault_static_test.go => vault_test.go} | 29 ++------------ pkg/config/vault.go | 6 +++ pkg/entry/vault.go | 5 +++ pkg/entry/vault_test.go | 18 +++++---- pkg/vault/server.go | 10 +++++ 17 files changed, 117 insertions(+), 74 deletions(-) delete mode 100644 pkg/config/config_test/testdata/config/vault.toml create mode 100644 pkg/config/config_test/testdata/vault/disabled/default.toml create mode 100644 pkg/config/config_test/testdata/vault/disabled/production.toml create mode 100644 pkg/config/config_test/testdata/vault/disabled/vault.toml rename pkg/config/config_test/{vault_static_test.go => vault_test.go} (68%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03fc527..83f5abe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# v1.2.0 + +- check vault server availability at startup +- add option to completely disable vault source +- test fixes +- update README.md + # v1.1.0 - add ability to find from certain file sources diff --git a/README.md b/README.md index 8354c46..3fc77ef 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,9 @@ Environment file may be any supported file extension - `.json`, `.yaml` (`.yml`) #### Vault -If you create `vault.EXT` file then fields from that file will proceed to lookup from vault server. Loading vault variables is dynamic too both the same with environment. The field has special syntax `mount_path,secret_path,secret_key` where are listed vault secret mount path, secret path, and secret key separated by comma. Suppose we have vault server with configured postgresql secret and started on `http://localhost:8200`. Lets load for example `username` and `password` keys to our config. And suppose for our simple example that vault setup has the `root` token to access. All again: mount - secret, path - postgresql, keys - username, password. Create directory `config` and place `vault.toml` file in to it with contents: +If you create `vault.EXT` file then fields from that file will proceed to lookup from vault server. When *goconfig* instance was created then vault domain will be checkup. If you want to disable vault lookup on some configurations then use `goconfig.vault.enable` system option. All vault system options will be explained below. + +Loading vault variables is dynamic too both the same with environment. The field has special syntax `mount_path,secret_path,secret_key` where are listed vault secret mount path, secret path, and secret key separated by comma. Suppose we have vault server with configured postgresql secret and started on `http://localhost:8200`. Lets load for example `username` and `password` keys to our config. And suppose for our simple example that vault setup has the `root` token to access. All again: mount - secret, path - postgresql, keys - username, password. Create directory `config` and place `vault.toml` file in to it with contents: ```toml [postgresql] @@ -245,12 +247,15 @@ So, before we can use vault keys we must configure vault client. There are two w ```toml [goconfig.vault] address = "http://127.0.0.1:8200" +enable = false min_retry_wait = "3" max_retry_wait = "5" max_retries = "10" timeout = "30" ``` +You can specify optional `enable` param to turn off vault on specific configuration environment. + Params `min_retry_wait`, `max_retry_wait`, `max_retries` and `timeout` must be specified as strings. Their values treated as seconds when specified without postfix. Otherwise the value will be supplied to `time.ParseDuration` function. ##### Vault Authorization @@ -286,7 +291,7 @@ It is not acceptable in many cases to provide credentials directly to file. So y ```toml [goconfig.vault.auth] -token = "VAULT_TOKEN" +token = "CUSTOM_VAULT_TOKEN" ``` And so on for other cases. diff --git a/cmd/goconfig/goconfig_test.go b/cmd/goconfig/goconfig_test.go index ee278b0..13d313d 100644 --- a/cmd/goconfig/goconfig_test.go +++ b/cmd/goconfig/goconfig_test.go @@ -551,24 +551,8 @@ func TestGoconfigVerboseError(t *testing.T) { func TestGoconfigVaultTokenError(t *testing.T) { t.Parallel() - ctx := context.Background() - vaultServer := vaultMock.NewServer("root") t.Cleanup(vaultServer.Close) - vaultClient := vaultMock.NewClient(vaultServer.URL, "root", vaultServer.Client()) - - err := vaultClient.WriteSecret(ctx, "secret", "goconfig_cmd_secret", map[string]any{ - "password": "abc123", - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - err = vaultClient.DeleteSecret(ctx, "secret", "goconfig_cmd_secret") - if err != nil { - t.Fatal(err) - } - }) d := TmpConfigDir(t) @@ -577,7 +561,7 @@ func TestGoconfigVaultTokenError(t *testing.T) { CreateConfigFile(d, "vault.toml", `password="secret,goconfig_cmd_secret"`) testCases := [][]string{ - {"go", "run", "./goconfig.go", "--config=" + d, "--get=password", "--token", "root1"}, + {"go", "run", "./goconfig.go", "--config=" + d, "--get=password", "--token", "broken_token"}, } for i, testCase := range testCases { @@ -589,16 +573,9 @@ func TestGoconfigVaultTokenError(t *testing.T) { cmd.Stderr = &stderr err := cmd.Run() - if err != nil { + if err == nil { t.Fatal(err, stderr.String()) } - - if stdout.String() != `qwerty123456` { - t.Fatal(stdout.String()) - } - if stderr.String() != "" { - t.Fatal(stderr.String()) - } }) } } diff --git a/pkg/config/config.go b/pkg/config/config.go index 7b2c079..fde32e5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/boolka/goconfig/pkg/entry" + tokenRoleAuth "github.com/boolka/goconfig/pkg/vault" vault "github.com/hashicorp/vault/api" appRoleAuth "github.com/hashicorp/vault/api/auth/approle" userPassAuth "github.com/hashicorp/vault/api/auth/userpass" @@ -150,6 +151,10 @@ func New(ctx context.Context, options Options) (cfg *Config, err error) { sources = append(sources, dirSources...) } + slices.SortFunc(sources, func(a, b configEntry) int { + return int(b.source) - int(a.source) + }) + for i, source := range sources { var replaceEntry entry.Entry var src cfgSource @@ -167,6 +172,13 @@ func New(ctx context.Context, options Options) (cfg *Config, err error) { // load vault config from source files vaultCfg := loadVaultConfig(ctx, vault.DefaultConfig(), sources) + // vault disabled + if vaultCfg == nil { + // remove correspond file from sources + sources = slices.Delete(sources, i, i+1) + continue + } + // set goconfig logger if options.Logger != nil { vaultCfg.Logger = options.Logger @@ -186,7 +198,7 @@ func New(ctx context.Context, options Options) (cfg *Config, err error) { switch authType { case tokenVaultConfigAuthType: - vaultClient.SetToken(creds[0]) + vaultAuth = tokenRoleAuth.NewTokenAuth(creds[0]) case appRoleVaultConfigAuthType: roleId := creds[0] secretId := creds[1] @@ -232,10 +244,6 @@ func New(ctx context.Context, options Options) (cfg *Config, err error) { } } - slices.SortFunc(sources, func(a, b configEntry) int { - return int(b.source) - int(a.source) - }) - if logger != nil && logger.Enabled(ctx, slog.LevelDebug) { for i, source := range sources { logger.DebugContext(ctx, fmt.Sprintf(`%d loaded source, file: "%s", cfgSource: %d`, i, source.file, source.source)) diff --git a/pkg/config/config_private_test.go b/pkg/config/config_private_test.go index 1c46cd2..99f7748 100644 --- a/pkg/config/config_private_test.go +++ b/pkg/config/config_private_test.go @@ -29,11 +29,11 @@ func TestConfigSourcesPrecedence(t *testing.T) { t.Fatal("incorrect order of sources") } - if len(cfg.sources) != 14 { + if len(cfg.sources) != 13 { for i, source := range cfg.sources { t.Error(i, source.file) } - t.Fatal("expected", 14, "found", len(cfg.sources)) + t.Fatal("expected", 13, "found", len(cfg.sources)) } } @@ -59,10 +59,10 @@ func TestConfigFileSystem(t *testing.T) { t.Fatal("incorrect order of sources") } - if len(cfg.sources) != 14 { + if len(cfg.sources) != 13 { for i, source := range cfg.sources { t.Error(i, source.file) } - t.Fatal("expected", 14, "found", len(cfg.sources)) + t.Fatal("expected", 13, "found", len(cfg.sources)) } } diff --git a/pkg/config/config_test/testdata/config/vault.toml b/pkg/config/config_test/testdata/config/vault.toml deleted file mode 100644 index 097ccbb..0000000 --- a/pkg/config/config_test/testdata/config/vault.toml +++ /dev/null @@ -1,4 +0,0 @@ -vault = "vault.toml" - -[goconfig.vault.auth] -token = "root" diff --git a/pkg/config/config_test/testdata/vault/config/vault.toml b/pkg/config/config_test/testdata/vault/config/vault.toml index d46fd11..9940541 100644 --- a/pkg/config/config_test/testdata/vault/config/vault.toml +++ b/pkg/config/config_test/testdata/vault/config/vault.toml @@ -1,4 +1,5 @@ password1 = "secret,goconfig_secret_1" +broken_field = "broken_value" [userpass] password2 = "secret,goconfig_secret_1,password2" diff --git a/pkg/config/config_test/testdata/vault/disabled/default.toml b/pkg/config/config_test/testdata/vault/disabled/default.toml new file mode 100644 index 0000000..0cd69c1 --- /dev/null +++ b/pkg/config/config_test/testdata/vault/disabled/default.toml @@ -0,0 +1,2 @@ +[goconfig.vault] +enable = false diff --git a/pkg/config/config_test/testdata/vault/disabled/production.toml b/pkg/config/config_test/testdata/vault/disabled/production.toml new file mode 100644 index 0000000..08e9fb0 --- /dev/null +++ b/pkg/config/config_test/testdata/vault/disabled/production.toml @@ -0,0 +1,2 @@ +[goconfig.vault] +enable = true diff --git a/pkg/config/config_test/testdata/vault/disabled/vault.toml b/pkg/config/config_test/testdata/vault/disabled/vault.toml new file mode 100644 index 0000000..e07e2af --- /dev/null +++ b/pkg/config/config_test/testdata/vault/disabled/vault.toml @@ -0,0 +1,5 @@ +[goconfig.vault] +address = "http://127.0.0.1:8200" + +[goconfig.vault.auth] +token = "invalid_token" diff --git a/pkg/config/config_test/testdata/vault/token/env.toml b/pkg/config/config_test/testdata/vault/token/env.toml index 7544400..56e744a 100644 --- a/pkg/config/config_test/testdata/vault/token/env.toml +++ b/pkg/config/config_test/testdata/vault/token/env.toml @@ -1,2 +1,2 @@ [goconfig.vault.auth] -token = "VAULT_TOKEN" +token = "CUSTOM_VAULT_TOKEN" diff --git a/pkg/config/config_test/vault_auth_token_test.go b/pkg/config/config_test/vault_auth_token_test.go index 154d70a..89e486c 100644 --- a/pkg/config/config_test/vault_auth_token_test.go +++ b/pkg/config/config_test/vault_auth_token_test.go @@ -73,7 +73,7 @@ func TestVaultTokenAuthFromFile(t *testing.T) { ctx := context.Background() prepareVaultSecret(ctx, t, "root") - t.Setenv("VAULT_TOKEN", "root") + t.Setenv("CUSTOM_VAULT_TOKEN", "root") cfg, err := config.New(ctx, config.Options{ Directory: "testdata/vault/token", @@ -142,3 +142,39 @@ func TestVaultMustGetCertainSource(t *testing.T) { cfg.MustGet(ctx, "password1", "vault") } + +func TestVaultInitializationInvalidToken(t *testing.T) { + ctx := context.Background() + + t.Setenv("CUSTOM_VAULT_TOKEN", "invalid_token") + + _, err := config.New(ctx, config.Options{ + Directory: "testdata/vault/token", + }) + if err == nil { + t.Fatal(err) + } +} + +func TestVaultInitializationDisableVault(t *testing.T) { + ctx := context.Background() + + _, err := config.New(ctx, config.Options{ + Directory: "testdata/vault/disabled", + }) + if err != nil { + t.Fatal(err) + } +} + +func TestVaultInitializationEnableVault(t *testing.T) { + ctx := context.Background() + + _, err := config.New(ctx, config.Options{ + Directory: "testdata/vault/disabled", + Deployment: "production", + }) + if err == nil { + t.Fatal(err) + } +} diff --git a/pkg/config/config_test/vault_static_test.go b/pkg/config/config_test/vault_test.go similarity index 68% rename from pkg/config/config_test/vault_static_test.go rename to pkg/config/config_test/vault_test.go index 519e0fa..73f6e2f 100644 --- a/pkg/config/config_test/vault_static_test.go +++ b/pkg/config/config_test/vault_test.go @@ -1,10 +1,9 @@ +//go:build vault + package config_test import ( - "bytes" "context" - "log/slog" - "strings" "testing" "time" @@ -13,8 +12,6 @@ import ( ) func TestVaultConfig(t *testing.T) { - t.Parallel() - ctx := context.Background() cfg, err := config.New(ctx, config.Options{ @@ -53,17 +50,10 @@ func TestVaultConfig(t *testing.T) { } func TestUnavailableServer(t *testing.T) { - t.Parallel() - - buf := bytes.NewBuffer([]byte{}) - ctx := context.Background() _, err := config.New(ctx, config.Options{ Directory: "testdata/vault/unauthorized", - Logger: slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{ - Level: slog.LevelDebug, - })), }) if err != entry.ErrVaultUnauthorized { t.Fatal(err) @@ -71,27 +61,16 @@ func TestUnavailableServer(t *testing.T) { } func TestBrokenPath(t *testing.T) { - t.Setenv("TEST_FILE_VAULT", "config_file_vault_custom") - - buf := bytes.NewBuffer([]byte{}) - ctx := context.Background() cfg, err := config.New(ctx, config.Options{ - Directory: "testdata/config", - Logger: slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{ - Level: slog.LevelDebug, - })), + Directory: "testdata/vault/config", }) if err != nil { t.Fatal(err) } - if v, ok := cfg.Get(ctx, "vault"); !ok || v != "config_file_vault_custom" { + if v, ok := cfg.Get(ctx, "broken_field"); ok && v != entry.ErrVaultInvalidPath { t.Fatal(v, ok) } - - if !strings.Contains(buf.String(), "invalid vault path") { - t.Fatal("valid path") - } } diff --git a/pkg/config/vault.go b/pkg/config/vault.go index 18ac46b..68aef80 100644 --- a/pkg/config/vault.go +++ b/pkg/config/vault.go @@ -8,6 +8,12 @@ import ( ) func loadVaultConfig(ctx context.Context, config *vault.Config, entries []configEntry) *vault.Config { + if isEnable, ok := searchThroughSources(ctx, entries, "goconfig.vault.enable"); ok { + if isEnable, ok := isEnable.(bool); ok && !isEnable { + return nil + } + } + if address, ok := searchThroughSources(ctx, entries, "goconfig.vault.address"); ok { if address, ok := address.(string); ok { config.Address = address diff --git a/pkg/entry/vault.go b/pkg/entry/vault.go index c7f19df..e7b18d1 100644 --- a/pkg/entry/vault.go +++ b/pkg/entry/vault.go @@ -47,6 +47,11 @@ func NewVault(ctx context.Context, entry Entry, client *vault.Client, auth vault } } + _, err := client.Auth().Token().LookupSelf() + if err != nil { + return nil, err + } + return &VaultEntry{ client: client, entry: entry, diff --git a/pkg/entry/vault_test.go b/pkg/entry/vault_test.go index 1dcd4c2..3cbe8bc 100644 --- a/pkg/entry/vault_test.go +++ b/pkg/entry/vault_test.go @@ -190,14 +190,10 @@ func TestVaultBrokenTokenAuth(t *testing.T) { client.SetToken("broken") - v, err := NewVault(ctx, tomlEntry, client, nil) - if err != nil { + _, err = NewVault(ctx, tomlEntry, client, nil) + if err == nil { t.Fatal(err) } - - if v, ok := v.Get(ctx, "password1"); ok { - t.Fatal(v, ok) - } } func TestVaultUserPassAuth(t *testing.T) { @@ -544,6 +540,14 @@ func TestVaultAppRoleAuthDenied(t *testing.T) { func TestVaultBrokenPath(t *testing.T) { ctx := context.Background() + vaultServer := vaultMock.NewServer("root") + t.Cleanup(func() { + vaultServer.Close() + }) + + vaultCfg := vault.DefaultConfig() + vaultCfg.Address = vaultServer.URL + f, err := os.Open("./testdata/vault.toml") if err != nil { t.Fatal(err) @@ -557,7 +561,7 @@ func TestVaultBrokenPath(t *testing.T) { t.Fatal(err) } - client, err := vault.NewClient(nil) + client, err := vault.NewClient(vaultCfg) if err != nil { t.Fatal(err) } diff --git a/pkg/vault/server.go b/pkg/vault/server.go index 45acd90..21d3db0 100644 --- a/pkg/vault/server.go +++ b/pkg/vault/server.go @@ -89,6 +89,16 @@ func NewServer(token string) *httptest.Server { rw.WriteHeader(http.StatusOK) }) + // lookup self token + mux.HandleFunc("GET /v1/auth/token/lookup-self", func(rw http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Vault-Token") != token { + rw.WriteHeader(http.StatusForbidden) + return + } + + rw.WriteHeader(http.StatusOK) + }) + // create user mux.HandleFunc("POST /v1/auth/userpass/users/{username}", func(rw http.ResponseWriter, r *http.Request) { if r.Header.Get("X-Vault-Token") != token {