From 1a8e3824c712a66bfb108773377219b3a2ef91bd Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Mon, 25 Jun 2018 19:56:31 +0100 Subject: [PATCH 1/7] Attempts to read separate cluster config from directory --- pkg/tarmak/config/config.go | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/pkg/tarmak/config/config.go b/pkg/tarmak/config/config.go index 2b8eaee973..183974f6a5 100644 --- a/pkg/tarmak/config/config.go +++ b/pkg/tarmak/config/config.go @@ -10,6 +10,7 @@ import ( "regexp" "strings" + "github.com/hashicorp/go-multierror" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -336,10 +337,56 @@ func (c *Config) ReadConfig() (*tarmakv1alpha1.Config, error) { return nil, fmt.Errorf("got unexpected config type: %v", gvk) } + _, err = c.readClusters() + if err != nil { + return nil, fmt.Errorf("failed to read cluster configs: %v", err) + } + c.conf = config return config, nil } +func (c *Config) readClusters() ([]*clusterv1alpha1.Cluster, error) { + dir, err := ioutil.ReadDir(c.tarmak.ConfigPath()) + if err != nil { + return nil, nil + } + + var clusterFiles []os.FileInfo + for _, f := range dir { + if !f.IsDir() && strings.HasPrefix(f.Name(), "cluster") && strings.HasSuffix(f.Name(), ".yaml") { + clusterFiles = append(clusterFiles, f) + } + } + + var result *multierror.Error + var clusterConfigs []*clusterv1alpha1.Cluster + for _, f := range clusterFiles { + b, err := ioutil.ReadFile(filepath.Join(c.tarmak.ConfigPath(), f.Name())) + if err != nil { + result = multierror.Append(result, err) + continue + } + + configObj, gvk, err := c.codecs.UniversalDeserializer().Decode(b, nil, nil) + if err != nil { + err = fmt.Errorf("failed to decode cluster config: %v", err) + result = multierror.Append(result, err) + continue + } + + clusterConfig, ok := configObj.(*clusterv1alpha1.Cluster) + if !ok { + result = multierror.Append(result, fmt.Errorf("got unexpected config type: %v", gvk)) + continue + } + + clusterConfigs = append(clusterConfigs, clusterConfig) + } + + return clusterConfigs, result.ErrorOrNil() +} + func (c *Config) Contact() string { return c.conf.Contact } From 429544119f0af90854a96b42a6fa4310ce0cfeb0 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Tue, 26 Jun 2018 16:32:32 +0100 Subject: [PATCH 2/7] First attempt at splitting configuration --- pkg/tarmak/config/config.go | 66 ++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/pkg/tarmak/config/config.go b/pkg/tarmak/config/config.go index 183974f6a5..4a384a8595 100644 --- a/pkg/tarmak/config/config.go +++ b/pkg/tarmak/config/config.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/go-multierror" "github.com/sirupsen/logrus" + //"gopkg.in/yaml.v2" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/runtime/serializer/json" @@ -320,71 +321,92 @@ func (c *Config) configPath() string { } func (c *Config) ReadConfig() (*tarmakv1alpha1.Config, error) { - path := c.configPath() + var config *tarmakv1alpha1.Config - configBytes, err := ioutil.ReadFile(path) + mainConfigs, err := c.readConfigFragment("tarmak") if err != nil { - return nil, err + return nil, fmt.Errorf("faield to read main tarmak configs: %v", err) } - configObj, gvk, err := c.codecs.UniversalDecoder(tarmakv1alpha1.SchemeGroupVersion).Decode(configBytes, nil, nil) - if err != nil { - return nil, err + if len(mainConfigs) == 0 { + return nil, errors.New("failed to read configuration, at least a single configuration file must exist with prefix 'tarmak'") } - config, ok := configObj.(*tarmakv1alpha1.Config) - if !ok { - return nil, fmt.Errorf("got unexpected config type: %v", gvk) + config = mainConfigs[0].DeepCopy() + for _, m := range mainConfigs[1:] { + config.Clusters = append(config.Clusters, m.Clusters...) + config.Environments = append(config.Environments, m.Environments...) + config.Providers = append(config.Providers, m.Providers...) } - _, err = c.readClusters() + clustersConfigs, err := c.readConfigFragment("cluster") if err != nil { return nil, fmt.Errorf("failed to read cluster configs: %v", err) } + for _, clusterConfig := range clustersConfigs { + config.Clusters = append(config.Clusters, clusterConfig.Clusters...) + } + + environmentsConfigs, err := c.readConfigFragment("environment") + if err != nil { + return nil, fmt.Errorf("failed to read environments configs: %v", err) + } + for _, environmentConfig := range environmentsConfigs { + config.Environments = append(config.Environments, environmentConfig.Environments...) + } + + providersConfigs, err := c.readConfigFragment("providers") + if err != nil { + return nil, fmt.Errorf("failed to read providers configs: %v", err) + } + for _, providerConfig := range providersConfigs { + config.Providers = append(config.Providers, providerConfig.Providers...) + } c.conf = config return config, nil } -func (c *Config) readClusters() ([]*clusterv1alpha1.Cluster, error) { +func (c *Config) readConfigFragment(fragment string) ([]*tarmakv1alpha1.Config, error) { dir, err := ioutil.ReadDir(c.tarmak.ConfigPath()) if err != nil { return nil, nil } - var clusterFiles []os.FileInfo + var fragFiles []os.FileInfo for _, f := range dir { - if !f.IsDir() && strings.HasPrefix(f.Name(), "cluster") && strings.HasSuffix(f.Name(), ".yaml") { - clusterFiles = append(clusterFiles, f) + if !f.IsDir() && strings.HasPrefix(f.Name(), fragment) && strings.HasSuffix(f.Name(), ".yaml") { + fragFiles = append(fragFiles, f) } } var result *multierror.Error - var clusterConfigs []*clusterv1alpha1.Cluster - for _, f := range clusterFiles { + var fragConfigs []*tarmakv1alpha1.Config + for _, f := range fragFiles { b, err := ioutil.ReadFile(filepath.Join(c.tarmak.ConfigPath(), f.Name())) if err != nil { result = multierror.Append(result, err) continue } - configObj, gvk, err := c.codecs.UniversalDeserializer().Decode(b, nil, nil) + configObj, gvk, err := c.codecs.UniversalDecoder(tarmakv1alpha1.SchemeGroupVersion).Decode(b, nil, nil) if err != nil { - err = fmt.Errorf("failed to decode cluster config: %v", err) + err = fmt.Errorf("faild to decode config file '%s': %v", f.Name(), err) result = multierror.Append(result, err) continue } - clusterConfig, ok := configObj.(*clusterv1alpha1.Cluster) + config, ok := configObj.(*tarmakv1alpha1.Config) if !ok { - result = multierror.Append(result, fmt.Errorf("got unexpected config type: %v", gvk)) + err := fmt.Errorf("got unexpected config type: %v", gvk) + result = multierror.Append(result, err) continue } - clusterConfigs = append(clusterConfigs, clusterConfig) + fragConfigs = append(fragConfigs, config) } - return clusterConfigs, result.ErrorOrNil() + return fragConfigs, result.ErrorOrNil() } func (c *Config) Contact() string { From b42a563e1ac31f80c8c6faa257dfd08afc3540f2 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Tue, 26 Jun 2018 16:52:49 +0100 Subject: [PATCH 3/7] Provides suffixes flag configuration file names to merge --- cmd/tarmak/cmd/root.go | 12 ++++++ pkg/apis/tarmak/v1alpha1/types.go | 2 + pkg/tarmak/config/config.go | 71 +++++++++++-------------------- 3 files changed, 38 insertions(+), 47 deletions(-) diff --git a/cmd/tarmak/cmd/root.go b/cmd/tarmak/cmd/root.go index 53f3b0be75..a23488d86d 100644 --- a/cmd/tarmak/cmd/root.go +++ b/cmd/tarmak/cmd/root.go @@ -105,6 +105,18 @@ func init() { "override the current cluster set in the config", ) + RootCmd.PersistentFlags().StringSliceVar( + &globalFlags.ConfigSuffixes, + "config-suffixes", + []string{ + "tarmak", + "cluster", + "environment", + "provider", + }, + "set suffixes of files containing tarmak configuration", + ) + if version == "dev" { RootCmd.PersistentFlags().BoolVar( &globalFlags.WingDevMode, diff --git a/pkg/apis/tarmak/v1alpha1/types.go b/pkg/apis/tarmak/v1alpha1/types.go index a235ed67d0..ba28a4215f 100644 --- a/pkg/apis/tarmak/v1alpha1/types.go +++ b/pkg/apis/tarmak/v1alpha1/types.go @@ -127,6 +127,8 @@ type Flags struct { Version string // expose tarmak's build time version WingDevMode bool // use a bundled wing version rather than a tagged release from GitHub + + ConfigSuffixes []string // the suffixes used for imported configuration fragments } // This contains the cluster specifc operation flags diff --git a/pkg/tarmak/config/config.go b/pkg/tarmak/config/config.go index 4a384a8595..e37a081133 100644 --- a/pkg/tarmak/config/config.go +++ b/pkg/tarmak/config/config.go @@ -2,6 +2,7 @@ package config import ( + "bytes" "errors" "fmt" "io/ioutil" @@ -12,7 +13,6 @@ import ( "github.com/hashicorp/go-multierror" "github.com/sirupsen/logrus" - //"gopkg.in/yaml.v2" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/runtime/serializer/json" @@ -322,52 +322,41 @@ func (c *Config) configPath() string { func (c *Config) ReadConfig() (*tarmakv1alpha1.Config, error) { var config *tarmakv1alpha1.Config + var configBytes bytes.Buffer + var result *multierror.Error - mainConfigs, err := c.readConfigFragment("tarmak") - if err != nil { - return nil, fmt.Errorf("faield to read main tarmak configs: %v", err) - } + for _, suffix := range c.flags.ConfigSuffixes { + b, err := c.readConfigFragment(suffix) + if err != nil { + result = multierror.Append(result, fmt.Errorf("failed to read main tarmak configs: %v", err)) + continue + } - if len(mainConfigs) == 0 { - return nil, errors.New("failed to read configuration, at least a single configuration file must exist with prefix 'tarmak'") + if _, err := configBytes.Write(b); err != nil { + result = multierror.Append(result, err) + continue + } } - config = mainConfigs[0].DeepCopy() - for _, m := range mainConfigs[1:] { - config.Clusters = append(config.Clusters, m.Clusters...) - config.Environments = append(config.Environments, m.Environments...) - config.Providers = append(config.Providers, m.Providers...) + if result.ErrorOrNil() != nil { + return nil, result.ErrorOrNil() } - clustersConfigs, err := c.readConfigFragment("cluster") + configObj, gvk, err := c.codecs.UniversalDecoder(tarmakv1alpha1.SchemeGroupVersion).Decode(configBytes.Bytes(), nil, nil) if err != nil { - return nil, fmt.Errorf("failed to read cluster configs: %v", err) - } - for _, clusterConfig := range clustersConfigs { - config.Clusters = append(config.Clusters, clusterConfig.Clusters...) + return nil, fmt.Errorf("failed to decode config bytes from file: %v", err) } - environmentsConfigs, err := c.readConfigFragment("environment") - if err != nil { - return nil, fmt.Errorf("failed to read environments configs: %v", err) - } - for _, environmentConfig := range environmentsConfigs { - config.Environments = append(config.Environments, environmentConfig.Environments...) - } - - providersConfigs, err := c.readConfigFragment("providers") - if err != nil { - return nil, fmt.Errorf("failed to read providers configs: %v", err) - } - for _, providerConfig := range providersConfigs { - config.Providers = append(config.Providers, providerConfig.Providers...) + config, ok := configObj.(*tarmakv1alpha1.Config) + if !ok { + return nil, fmt.Errorf("got unexpected config type: %v", gvk) } c.conf = config return config, nil } -func (c *Config) readConfigFragment(fragment string) ([]*tarmakv1alpha1.Config, error) { +func (c *Config) readConfigFragment(fragment string) ([]byte, error) { dir, err := ioutil.ReadDir(c.tarmak.ConfigPath()) if err != nil { return nil, nil @@ -380,8 +369,8 @@ func (c *Config) readConfigFragment(fragment string) ([]*tarmakv1alpha1.Config, } } + var buff bytes.Buffer var result *multierror.Error - var fragConfigs []*tarmakv1alpha1.Config for _, f := range fragFiles { b, err := ioutil.ReadFile(filepath.Join(c.tarmak.ConfigPath(), f.Name())) if err != nil { @@ -389,24 +378,12 @@ func (c *Config) readConfigFragment(fragment string) ([]*tarmakv1alpha1.Config, continue } - configObj, gvk, err := c.codecs.UniversalDecoder(tarmakv1alpha1.SchemeGroupVersion).Decode(b, nil, nil) - if err != nil { - err = fmt.Errorf("faild to decode config file '%s': %v", f.Name(), err) + if _, err := buff.Write(b); err != nil { result = multierror.Append(result, err) - continue } - - config, ok := configObj.(*tarmakv1alpha1.Config) - if !ok { - err := fmt.Errorf("got unexpected config type: %v", gvk) - result = multierror.Append(result, err) - continue - } - - fragConfigs = append(fragConfigs, config) } - return fragConfigs, result.ErrorOrNil() + return buff.Bytes(), result.ErrorOrNil() } func (c *Config) Contact() string { From a0df3d3b5098ff81ac5bc801788b9c735769fa01 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Tue, 26 Jun 2018 17:02:26 +0100 Subject: [PATCH 4/7] Improve file access --- pkg/tarmak/config/config.go | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/pkg/tarmak/config/config.go b/pkg/tarmak/config/config.go index e37a081133..ac3fa7e7a7 100644 --- a/pkg/tarmak/config/config.go +++ b/pkg/tarmak/config/config.go @@ -322,27 +322,13 @@ func (c *Config) configPath() string { func (c *Config) ReadConfig() (*tarmakv1alpha1.Config, error) { var config *tarmakv1alpha1.Config - var configBytes bytes.Buffer - var result *multierror.Error - - for _, suffix := range c.flags.ConfigSuffixes { - b, err := c.readConfigFragment(suffix) - if err != nil { - result = multierror.Append(result, fmt.Errorf("failed to read main tarmak configs: %v", err)) - continue - } - - if _, err := configBytes.Write(b); err != nil { - result = multierror.Append(result, err) - continue - } - } - if result.ErrorOrNil() != nil { - return nil, result.ErrorOrNil() + configBytes, err := c.readConfigFragments(c.flags.ConfigSuffixes) + if err != nil { + return nil, fmt.Errorf("failed to read tarmak configs: %v", err) } - configObj, gvk, err := c.codecs.UniversalDecoder(tarmakv1alpha1.SchemeGroupVersion).Decode(configBytes.Bytes(), nil, nil) + configObj, gvk, err := c.codecs.UniversalDecoder(tarmakv1alpha1.SchemeGroupVersion).Decode(configBytes, nil, nil) if err != nil { return nil, fmt.Errorf("failed to decode config bytes from file: %v", err) } @@ -356,16 +342,23 @@ func (c *Config) ReadConfig() (*tarmakv1alpha1.Config, error) { return config, nil } -func (c *Config) readConfigFragment(fragment string) ([]byte, error) { +func (c *Config) readConfigFragments(suffixes []string) ([]byte, error) { dir, err := ioutil.ReadDir(c.tarmak.ConfigPath()) if err != nil { - return nil, nil + return nil, err } var fragFiles []os.FileInfo for _, f := range dir { - if !f.IsDir() && strings.HasPrefix(f.Name(), fragment) && strings.HasSuffix(f.Name(), ".yaml") { - fragFiles = append(fragFiles, f) + if !f.IsDir() && strings.HasSuffix(f.Name(), ".yaml") { + + for _, suffix := range suffixes { + if strings.HasPrefix(f.Name(), suffix) { + fragFiles = append(fragFiles, f) + break + } + } + } } From 2cc535ed9472c53cb4031c72853a2fd34b5ca904 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Tue, 26 Jun 2018 17:11:34 +0100 Subject: [PATCH 5/7] Fixes config_test.go --- pkg/tarmak/config/config_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/tarmak/config/config_test.go b/pkg/tarmak/config/config_test.go index 19fe18080a..5da469b3ee 100644 --- a/pkg/tarmak/config/config_test.go +++ b/pkg/tarmak/config/config_test.go @@ -55,7 +55,9 @@ func newFakeConfig(t *testing.T) *fakeConfig { } c.fakeTarmak.EXPECT().Log().AnyTimes().Return(logger.WithField("app", "tarmak")) - c.Config, err = New(c.fakeTarmak, &tarmakv1alpha1.Flags{}) + c.Config, err = New(c.fakeTarmak, &tarmakv1alpha1.Flags{ + ConfigSuffixes: []string{"tarmak"}, + }) if err != nil { t.Error("unexpected error: ", err) From f3f9f3f408f31ce91a8fd7fff1a831ff72c3a5ce Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Wed, 27 Jun 2018 14:38:33 +0100 Subject: [PATCH 6/7] Suffix -> prefix --- cmd/tarmak/cmd/root.go | 6 +++--- pkg/apis/tarmak/v1alpha1/types.go | 2 +- pkg/tarmak/config/config.go | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/tarmak/cmd/root.go b/cmd/tarmak/cmd/root.go index a23488d86d..ad3654d6e0 100644 --- a/cmd/tarmak/cmd/root.go +++ b/cmd/tarmak/cmd/root.go @@ -106,15 +106,15 @@ func init() { ) RootCmd.PersistentFlags().StringSliceVar( - &globalFlags.ConfigSuffixes, - "config-suffixes", + &globalFlags.ConfigPrefixes, + "config-prefixes", []string{ "tarmak", "cluster", "environment", "provider", }, - "set suffixes of files containing tarmak configuration", + "set prefixes of files containing tarmak configuration to be used", ) if version == "dev" { diff --git a/pkg/apis/tarmak/v1alpha1/types.go b/pkg/apis/tarmak/v1alpha1/types.go index ba28a4215f..d16c6e0984 100644 --- a/pkg/apis/tarmak/v1alpha1/types.go +++ b/pkg/apis/tarmak/v1alpha1/types.go @@ -128,7 +128,7 @@ type Flags struct { WingDevMode bool // use a bundled wing version rather than a tagged release from GitHub - ConfigSuffixes []string // the suffixes used for imported configuration fragments + ConfigPrefixes []string // the prefixes used for imported configuration fragments } // This contains the cluster specifc operation flags diff --git a/pkg/tarmak/config/config.go b/pkg/tarmak/config/config.go index ac3fa7e7a7..871e461372 100644 --- a/pkg/tarmak/config/config.go +++ b/pkg/tarmak/config/config.go @@ -323,7 +323,7 @@ func (c *Config) configPath() string { func (c *Config) ReadConfig() (*tarmakv1alpha1.Config, error) { var config *tarmakv1alpha1.Config - configBytes, err := c.readConfigFragments(c.flags.ConfigSuffixes) + configBytes, err := c.readConfigFragments(c.flags.ConfigPrefixes) if err != nil { return nil, fmt.Errorf("failed to read tarmak configs: %v", err) } @@ -342,7 +342,7 @@ func (c *Config) ReadConfig() (*tarmakv1alpha1.Config, error) { return config, nil } -func (c *Config) readConfigFragments(suffixes []string) ([]byte, error) { +func (c *Config) readConfigFragments(prefixes []string) ([]byte, error) { dir, err := ioutil.ReadDir(c.tarmak.ConfigPath()) if err != nil { return nil, err @@ -352,8 +352,8 @@ func (c *Config) readConfigFragments(suffixes []string) ([]byte, error) { for _, f := range dir { if !f.IsDir() && strings.HasSuffix(f.Name(), ".yaml") { - for _, suffix := range suffixes { - if strings.HasPrefix(f.Name(), suffix) { + for _, prefix := range prefixes { + if strings.HasPrefix(f.Name(), prefix) { fragFiles = append(fragFiles, f) break } From 9e5536669c09cb08830b521d15cde06815c2d629 Mon Sep 17 00:00:00 2001 From: JoshVanL Date: Wed, 27 Jun 2018 15:39:19 +0100 Subject: [PATCH 7/7] Fixes config_test.go --- pkg/tarmak/config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tarmak/config/config_test.go b/pkg/tarmak/config/config_test.go index 5da469b3ee..4bc0266cdc 100644 --- a/pkg/tarmak/config/config_test.go +++ b/pkg/tarmak/config/config_test.go @@ -56,7 +56,7 @@ func newFakeConfig(t *testing.T) *fakeConfig { c.fakeTarmak.EXPECT().Log().AnyTimes().Return(logger.WithField("app", "tarmak")) c.Config, err = New(c.fakeTarmak, &tarmakv1alpha1.Flags{ - ConfigSuffixes: []string{"tarmak"}, + ConfigPrefixes: []string{"tarmak"}, }) if err != nil {