diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2b87cc4..68b6bf5 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -44,6 +44,6 @@ jobs: - name: Package binaries run: 'make package' - name: Create GitHub Release - run: gh release create "v$(grep -v '#' ./VERSION)" --generate-notes --verify-tag --latest ./dist/zipped/* + run: gh release create "v$(grep -v '#' ./VERSION)" --draft --generate-notes --verify-tag --latest ./dist/compressed/* env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 3d4ec61..87a4bcf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,7 @@ *.log *.out *cache* +!internal/archive/cache +!internal/archive/cache/** build/ deps/ diff --git a/Makefile b/Makefile index 37cc469..faa5e4c 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ all: ci package package-debian ci: clean @bash ./scripts/ci.sh + @make -s clean # test is just an alias for ci test: ci @@ -38,15 +39,14 @@ clean: *cache* \ .*cache* \ ./build/ \ - ./dist/zipped/*.tar.gz \ - ./dist/zipped/*.zip \ + ./dist/compressed/*.tar.gz \ + ./dist/compressed/*.zip \ ./dist/debian/vdm.deb \ *.out @sudo rm -rf ./dist/debian/vdm/usr -# TODO: until I sort out the tests to write test data consistently, these deps/ -# directories etc. can kind of show up anywhere +# TODO: until I (maybe) sort out the tests to write test data consistently, +# these deps/ directories etc. can kind of show up anywhere @find . -type d -name '*deps*' -exec rm -rf {} + - @find . -type f -name '*VDMMETA*' -delete bump-versions: clean @bash ./scripts/bump-versions.sh "$${old_version:-}" @@ -63,3 +63,9 @@ add-local-symlinks: @mkdir -p "$${HOME}"/.local/bin @ln -fs $$(realpath build/$$(go env GOOS)-$$(go env GOARCH)/$(BINNAME)) "$${HOME}"/.local/bin/$(BINNAME) @printf 'Symlinked vdm to %s\n' "$${HOME}"/.local/bin/$(BINNAME) + +install: + @go install ./... + +docs: + @go run golang.org/x/pkgsite/cmd/pkgsite@latest -open . diff --git a/README.md b/README.md index a6b1ff6..852552c 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,9 @@ retrieve them whenever you need them. ### Installation `vdm` can be installed from [its GitHub Releases -page](https://github.com/opensourcecorp/vdm/releases). There is a zipped binary -for major platforms & architectures, and those are indicated in the Asset file -name. For example, if you have an M2 macOS laptop, you would download the +page](https://github.com/opensourcecorp/vdm/releases). There is a compressed +binary for major platforms & architectures, and those are indicated in the Asset +file name. For example, if you have an M2 macOS laptop, you would download the `vdm_darwin-arm64.tar.gz` file, and extract it to somewhere on your `$PATH`. If you have a recent version of the Go toolchain available, you can also install @@ -43,50 +43,52 @@ go run github.com/opensourcecorp/vdm@ ... ### Usage -To get started, you'll need a `vdm` spec file, which is just a YAML (or JSON) +To get started, you'll need a `vdm` specfile, which is just a YAML (or JSON) file specifying all your external dependencies along with (usually) their revisions & where you want them to live on your filesystem: ```yaml remotes: - - type: "git" # the default, and so can be omitted if desired - remote: "https://github.com/opensourcecorp/go-common" # can specify as 'git@...' to use SSH instead - local_path: "./deps/go-common" - version: "v0.2.0" # tag example; can also be a branch, commit hash, or the word 'latest' + - type: "git" + source: "https://github.com/opensourcecorp/vdm" # can specify the protocol in any way that you would usually run 'git clone' + version: "v0.2.0" # tag example; can also be a branch, a commit hash, or anything else supported by 'git checkout' + destination: "./deps/" # git types are themselves directories, so to prevent duplicating their top-level names you probably want to just specify the root destination - - type: "file" # the 'file' type assumes the version is in the remote field itself somehow, so 'version' can be omitted - remote: "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto" - local_path: "./deps/proto/http/http.proto" + - type: "git" + source: "https://github.com/opensourcecorp/osc-infra" + version: "main" + destination: "./deps/" ``` You can have as many dependency specifications in that array as you want, and -they can be stored wherever you want. By default, this spec file is called +they can be stored wherever you want. By default, this specfile is called `vdm.yaml` and lives at the calling location (which is probably your repo's root), but you can call it whatever you want and point to it using the -`--spec-file` flag to `vdm`. +`--specfile|-f` flag to `vdm`. -Once you have a spec file, just run: +Once you have a specfile, just run: ```sh vdm sync ``` -and `vdm` will process the spec file, retrieve your dependencies as specified, -and put them where you told them to go. By default, `vdm sync` also removes the -local `.git` directories for each `git` remote, so as to not upset your local -Git tree. If you want to change the version/revision of a remote, just update -your spec file and run `vdm sync` again. +and `vdm` will process the specfile, retrieve your dependencies as specified, +and put them where you told them to go. `vdm sync` also removes the local `.git` +directories for each `git` remote, so as to not upset your local Git tree. If +you want to change the version/revision of a remote, just update your specfile +and run `vdm sync` again. -After running `vdm sync` with the above example spec file, your directory tree +After running `vdm sync` with the above example specfile, your directory tree would look something like this: ```txt ./vdm.yaml ./deps/ - go-common/ + vdm/ + + osc-infra/ - http.proto ``` ## Dependencies @@ -99,18 +101,8 @@ types. `vdm` will fail with an informative error if it can't find `git` on your ## A note about auth -`vdm` has zero goals to be an authN/authZ manager. If a remote in your spec file +`vdm` has zero goals to be an authN/authZ manager. If a remote in your specfile depends on a certain auth setup (an SSH key, something for HTTP basic auth like a `.netrc` file, an `.npmrc` config file, etc.), that setup is out of `vdm`'s scope. If required, you will need to ensure proper auth is configured before running `vdm` commands. - -## Future work - -- Make the sync mechanism more robust, such that if your spec file changes to - remove remotes, they'll get cleaned up automatically. - -- Add `--keep-git-dir` flag so that `git` remote types don't wipe the `.git` - directory at clone-time. - -- Support more than just `git` and `file` types, and make `file` better diff --git a/VERSION b/VERSION index e9ab468..a4fb4e6 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ # Version number used as an anchor for listed versions in various files in the # tree, orchestrated by scripts/bump-versions.sh -0.2.1 +0.3.0 diff --git a/cmd/doc.go b/cmd/doc.go index 669e3cf..2f75f76 100644 --- a/cmd/doc.go +++ b/cmd/doc.go @@ -1,4 +1,2 @@ -/* -Package cmd calls the implementation logic for vdm. -*/ +// Package cmd calls the implementation logic for vdm. package cmd diff --git a/cmd/flagsupport.go b/cmd/flagsupport.go index 31a122e..053a8cf 100644 --- a/cmd/flagsupport.go +++ b/cmd/flagsupport.go @@ -3,17 +3,18 @@ package cmd import ( "os" + "github.com/opensourcecorp/vdm/cmd/vars" "github.com/opensourcecorp/vdm/internal/message" "github.com/spf13/viper" ) -// MaybeSetDebug sets the DEBUG environment variable if it was set as a flag by +// maybeSetDebug sets the DEBUG environment variable if it was set as a flag by // the caller. -func MaybeSetDebug() { +func maybeSetDebug() { if viper.GetBool(debugFlagKey) { - err := os.Setenv("DEBUG", "true") + err := os.Setenv(vars.Debug, "true") if err != nil { - message.Fatalf("internal error: unable to set environment variable DEBUG") + message.Fatalf("internal error: unable to set environment variable %s", vars.Debug) } } } diff --git a/cmd/root.go b/cmd/root.go index baee032..e8726b8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,72 +1,86 @@ package cmd import ( + "errors" "fmt" - "os" "github.com/opensourcecorp/vdm/internal/message" + "github.com/opensourcecorp/vdm/internal/vdminit" "github.com/spf13/cobra" "github.com/spf13/viper" ) // !!! DO NOT TOUCH, the version-bumper script handles updating this !!! -const vdmVersion string = "v0.2.1" - -var rootCmd = cobra.Command{ - Use: "vdm", - Short: "vdm -- a Versioned-Dependency Manager", - Long: "vdm is used to manage arbitrary remote dependencies", - TraverseChildren: true, - Version: vdmVersion, - Run: func(cmd *cobra.Command, args []string) { - MaybeSetDebug() - if len(args) == 0 { - err := cmd.Help() - if err != nil { - message.Fatalf("failed to print help message, somehow") - } - os.Exit(0) - } - }, -} +const vdmVersion string = "v0.3.0" +// rootFlags defines the CLI flags for the root command. type rootFlags struct { SpecFilePath string Debug bool } -// RootFlagValues contains an initalized [rootFlags] struct with populated +// rootFlagValues contains an initalized [rootFlags] struct with populated // values. -var RootFlagValues rootFlags +var rootFlagValues rootFlags // Flag name keys const ( - specFilePathFlagKey string = "specfile-path" + specFilePathFlagKey string = "specfile" debugFlagKey string = "debug" ) -func init() { +func newRootCommand() *cobra.Command { var err error + cmd := &cobra.Command{ + Use: "vdm", + Short: "vdm -- a Versioned-Dependency Manager", + Long: "vdm is used to manage retrieval of arbitrary remote dependencies", + TraverseChildren: true, + Version: vdmVersion, + SilenceUsage: true, + SilenceErrors: true, + RunE: executeRootCommand, + } + + cmd.PersistentFlags().StringVarP(&rootFlagValues.SpecFilePath, specFilePathFlagKey, "f", "./vdm.yaml", "Path to vdm specfile") + err = viper.BindPFlag(specFilePathFlagKey, cmd.PersistentFlags().Lookup(specFilePathFlagKey)) + if err != nil { + message.Fatalf("internal error: unable to bind state of flag --%s: %v", specFilePathFlagKey, err) + } - rootCmd.PersistentFlags().StringVar(&RootFlagValues.SpecFilePath, specFilePathFlagKey, "./vdm.yaml", "Path to vdm specfile") - err = viper.BindPFlag(specFilePathFlagKey, rootCmd.PersistentFlags().Lookup(specFilePathFlagKey)) + cmd.PersistentFlags().BoolVar(&rootFlagValues.Debug, debugFlagKey, false, "Show debug messages during runtime") + err = viper.BindPFlag(debugFlagKey, cmd.PersistentFlags().Lookup(debugFlagKey)) if err != nil { - message.Fatalf("internal error: unable to bind state of flag --%s", specFilePathFlagKey) + message.Fatalf("internal error: unable to bind state of flag --%s: %v", debugFlagKey, err) + } + + cmd.AddCommand(newSyncCommand()) + + return cmd +} + +func executeRootCommand(cmd *cobra.Command, args []string) error { + maybeSetDebug() + if len(args) == 0 { + err := cmd.Help() + if err != nil { + return errors.New("failed to print help message, somehow") + } } - rootCmd.PersistentFlags().BoolVar(&RootFlagValues.Debug, debugFlagKey, false, "Show debug messages during runtime") - err = viper.BindPFlag(debugFlagKey, rootCmd.PersistentFlags().Lookup(debugFlagKey)) + err := vdminit.Paths() if err != nil { - message.Fatalf("internal error: unable to bind state of flag --%s", debugFlagKey) + return fmt.Errorf("initializing vdm: %w", err) } - rootCmd.AddCommand(syncCmd) + return errors.New("you must provide a subcommand to vdm") } // Execute wraps the primary execution logic for vdm's root command, and returns // any errors encountered to the caller. func Execute() error { - if err := rootCmd.Execute(); err != nil { + cmd := newRootCommand() + if err := cmd.Execute(); err != nil { return fmt.Errorf("executing root command: %w", err) } diff --git a/cmd/sync.go b/cmd/sync.go index cbf43ea..27db9ec 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -1,80 +1,84 @@ package cmd import ( + "errors" "fmt" + "path/filepath" "github.com/opensourcecorp/vdm/internal/message" "github.com/opensourcecorp/vdm/internal/remotes" + "github.com/opensourcecorp/vdm/internal/vdminit" "github.com/opensourcecorp/vdm/internal/vdmspec" "github.com/spf13/cobra" ) -var syncCmd = &cobra.Command{ - Use: "sync", - Short: "Sync remotes based on specfile", - RunE: syncExecute, +func newSyncCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "sync", + Short: "Sync remotes based on specfile", + RunE: executeSyncSubCommand, + } + + return cmd } -func syncExecute(_ *cobra.Command, _ []string) error { - MaybeSetDebug() +func executeSyncSubCommand(_ *cobra.Command, _ []string) error { + maybeSetDebug() + + err := vdminit.Paths() + if err != nil { + return fmt.Errorf("initializing vdm: %w", err) + } + if err := sync(); err != nil { return fmt.Errorf("executing sync command: %w", err) } + return nil } // sync does the heavy lifting to ensure that the local directory tree(s) match // the desired state as defined in the specfile. func sync() error { - spec, err := vdmspec.GetSpecFromFile(RootFlagValues.SpecFilePath) + spec, err := vdmspec.GetSpecFromFile(rootFlagValues.SpecFilePath) if err != nil { - return fmt.Errorf("getting specs from spec file: %w", err) + return fmt.Errorf("getting specs from specfile: %w", err) } err = spec.Validate() if err != nil { - return fmt.Errorf("your vdm spec file is malformed: %w", err) + return fmt.Errorf("your vdm specfile is malformed: %w", err) } -SpecLoop: for _, remote := range spec.Remotes { - // process stored vdm metafile so we know what operations to actually - // perform for existing directories - vdmMeta, err := remote.GetVDMMeta() - if err != nil { - return fmt.Errorf("getting vdm metadata file for sync: %w", err) + var determinedRemote vdmspec.Remoter + switch remote.Type { + case vdmspec.GitType: + determinedRemote = remotes.Git{RemoteTemplate: remote} + case vdmspec.ArchiveType: + return errors.New("cannot process 'archive' remote types, as they are not yet fully implemented") + case vdmspec.LocalType: + return errors.New("cannot process 'local' remote types, as they are not yet fully implemented") + default: + return fmt.Errorf("unrecognized remote type %q", remote.Type) } - if vdmMeta == (vdmspec.Remote{}) { - message.Infof("%s: %s not found at local path, will be created", remote.OpMsg(), vdmspec.MetaFileName) - } else { - if vdmMeta.Version != remote.Version && vdmMeta.Remote != remote.Remote { - message.Infof("%s: Will change '%s' from current local version spec '%s' to '%s'...", remote.OpMsg(), remote.Remote, vdmMeta.Version, remote.Version) - panic("not implemented") - } - message.Infof("%s: version unchanged in spec file, skipping", remote.OpMsg()) - continue SpecLoop + cachePath, err := determinedRemote.Cache() + if err != nil { + return fmt.Errorf("caching %q remote: %w", remote.Type, err) } - switch remote.Type { - case vdmspec.GitType, "": - if err := remotes.SyncGit(remote); err != nil { - return fmt.Errorf("syncing git remote: %w", err) - } - case vdmspec.FileType: - if err := remotes.SyncFile(remote); err != nil { - return fmt.Errorf("syncing file remote: %w", err) - } - default: - return fmt.Errorf("unrecognized remote type '%s'", remote.Type) + absDestination, err := filepath.Abs(remote.Destination) + if err != nil { + return fmt.Errorf("determining abspath of remote's destination %q: %w", remote.Destination, err) } - err = remote.WriteVDMMeta() + err = determinedRemote.Sync(cachePath, absDestination) if err != nil { - return fmt.Errorf("could not write %s file to disk: %w", vdmspec.MetaFileName, err) + return fmt.Errorf("syncing %q remote: %w", remote.Type, err) } - message.Infof("%s: Done.", remote.OpMsg()) + remote.OpMsg("Done.") } message.Infof("All done!") diff --git a/cmd/sync_test.go b/cmd/sync_test.go index 8708f01..05f0dc9 100644 --- a/cmd/sync_test.go +++ b/cmd/sync_test.go @@ -1,10 +1,11 @@ package cmd import ( + "os" "path/filepath" "testing" - "github.com/opensourcecorp/vdm/internal/vdmspec" + "github.com/opensourcecorp/vdm/internal/vdminit" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,52 +17,48 @@ var ( ) func TestSync(t *testing.T) { - spec, err := vdmspec.GetSpecFromFile(testSpecFilePath) - require.NoError(t, err) + // This runs as part of the outer test container, because we want to inspect + // the filesystem state as we go + _, cleanup := vdminit.SetupVDMForTest(t) + t.Cleanup(cleanup) - // Need to override for test - RootFlagValues.SpecFilePath = testSpecFilePath - err = sync() - require.NoError(t, err) - - // defer t.Cleanup(func() { - // for _, remote := range spec.Remotes { - // err := os.RemoveAll(remote.LocalPath) - // require.NoError(t, err) - // } - // }) - - t.Run("SyncGit", func(t *testing.T) { - t.Run("spec[0] used a tag", func(t *testing.T) { - vdmMeta, err := spec.Remotes[0].GetVDMMeta() - require.NoError(t, err) - assert.Equal(t, "v0.2.0", vdmMeta.Version) - }) + cmd := newRootCommand() + cmd.SetArgs([]string{ + "--debug", + "--specfile", testSpecFilePath, + "sync", + }) - t.Run("spec[1] used 'latest'", func(t *testing.T) { - vdmMeta, err := spec.Remotes[1].GetVDMMeta() - require.NoError(t, err) - assert.Equal(t, "latest", vdmMeta.Version) - }) + t.Run("sync works without throwing any errors", func(t *testing.T) { + err := cmd.Execute() + assert.NoError(t, err) + }) - t.Run("spec[2] used a branch", func(t *testing.T) { - vdmMeta, err := spec.Remotes[2].GetVDMMeta() - require.NoError(t, err) - assert.Equal(t, "main", vdmMeta.Version) - }) + t.Run("filesytem state is as expected", func(t *testing.T) { + expectedGitDirs := map[string]string{ + "git-tag": "vdm", + "git-branch": "osc-infra", + "git-hash": "go-common", + } + for topDir, secondDir := range expectedGitDirs { + sourceRoot := filepath.Join("deps", topDir, secondDir) + t.Run("source directory exists at its destination", func(t *testing.T) { + gitSource, err := os.Stat(sourceRoot) + require.NoError(t, err) + assert.True(t, gitSource.IsDir()) + }) - t.Run("spec[3] used a hash", func(t *testing.T) { - vdmMeta, err := spec.Remotes[3].GetVDMMeta() - require.NoError(t, err) - assert.Equal(t, "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f", vdmMeta.Version) - }) - }) + t.Run(".git directory was removed", func(t *testing.T) { + dotGitDir := filepath.Join(sourceRoot, ".git") + _, err := os.Stat(dotGitDir) + assert.ErrorIs(t, err, os.ErrNotExist) + }) - t.Run("SyncFile", func(t *testing.T) { - t.Run("spec[4] had an implicit version", func(t *testing.T) { - vdmMeta, err := spec.Remotes[4].GetVDMMeta() - require.NoError(t, err) - assert.Equal(t, "", vdmMeta.Version) - }) + t.Run("a known file in the remote exists, and is a file", func(t *testing.T) { + readmePath := filepath.Join(sourceRoot, "README.md") + _, err := os.Stat(readmePath) + assert.NoError(t, err) + }) + } }) } diff --git a/cmd/vars/vars.go b/cmd/vars/vars.go new file mode 100644 index 0000000..85ea435 --- /dev/null +++ b/cmd/vars/vars.go @@ -0,0 +1,46 @@ +// Package vars houses constants etc. for working with command-line flag values +// across packages. These helpers are pushed down to their own package in order +// to avoid import cycles. +package vars + +import ( + "fmt" + "os" + "path/filepath" +) + +const ( + // Debug anchors to the DEBUG env var + Debug = "DEBUG" + // TryLocalSources anchors to the TRY_LOCAL_SOURCES env var + TryLocalSources = "TRY_LOCAL_SOURCES" + // VDMHomeEnvVarName anchors to the VDM_HOME env var + VDMHomeEnvVarName = "VDM_HOME" +) + +func GetVDMHomeDir() (string, error) { + homedir, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("determining home directory: %w", err) + } + + var vdmHome string + vdmHomeOverride, ok := os.LookupEnv(VDMHomeEnvVarName) + if ok { + vdmHome = filepath.Join(vdmHomeOverride, ".vdm") + } else { + vdmHome = filepath.Join(homedir, ".vdm") + } + + return vdmHome, nil +} + +// GetVDMCacheDir returns the determined path to vdm's cache directory. +func GetVDMCacheDir() (string, error) { + vdmHome, err := GetVDMHomeDir() + if err != nil { + return "", fmt.Errorf("getting vdm home directory: %w", err) + } + + return filepath.Join(vdmHome, "cache"), nil +} diff --git a/dist/debian/vdm/DEBIAN/control b/dist/debian/vdm/DEBIAN/control index ad01abe..9920cde 100644 --- a/dist/debian/vdm/DEBIAN/control +++ b/dist/debian/vdm/DEBIAN/control @@ -1,5 +1,5 @@ Package: vdm -Version: 0.2.1 +Version: 0.3.0 Architecture: amd64 Maintainer: Ryan Price Description: Versioned Dependency Manager diff --git a/dist/man/man.1.md b/dist/man/man.1.md index b24b915..62ef18a 100644 --- a/dist/man/man.1.md +++ b/dist/man/man.1.md @@ -1,4 +1,4 @@ -% VDM(1) vdm 0.2.1 +% VDM(1) vdm 0.3.0 % Ryan J. Price % 2023 diff --git a/go.mod b/go.mod index b1b5ed1..ad12a32 100644 --- a/go.mod +++ b/go.mod @@ -7,17 +7,24 @@ require ( github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.31.1 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -27,8 +34,14 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index 365ddea..b992684 100644 --- a/go.sum +++ b/go.sum @@ -3,10 +3,17 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -15,13 +22,19 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -56,12 +69,15 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w= +golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= @@ -69,3 +85,23 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sqlite v1.31.1 h1:XVU0VyzxrYHlBhIs1DiEgSl0ZtdnPtbLVy8hSkzxGrs= +modernc.org/sqlite v1.31.1/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 0000000..b21f439 --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,200 @@ +package archive + +import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/opensourcecorp/vdm/internal/filetree" + "github.com/opensourcecorp/vdm/internal/message" +) + +var ( + tgzRegex = regexp.MustCompile(`(\.tar\.gz|\.tgz)$`) + zipRegex = regexp.MustCompile(`\.zip$`) +) + +// Much of the following functions taken from: +// https://www.arthurkoziel.com/writing-tar-gz-files-in-go/ + +// CreateArchive writes a gzipped tarball based on the provided root directory +// from which to construct the archive, and its target file name. It returns an +// open file handle to the archive, which should be closed by the caller. +func CreateArchive(root string, archivePath string) (err error) { + if !tgzRegex.MatchString(archivePath) { + return errors.New("provided archive path must have valid gzipped-tar extension") + } + + rootAbs, err := filepath.Abs(root) + if err != nil { + return fmt.Errorf("determining abspath of provided root dir %q: %w", root, err) + } + message.Debugf("root: %q, rootAbs: %q", root, rootAbs) + + archivePathAbs, err := filepath.Abs(archivePath) + if err != nil { + return fmt.Errorf("determining abspath of provided archive path %q: %w", archivePath, err) + } + + buf, err := os.Create(archivePathAbs) + if err != nil { + return fmt.Errorf("opening target archive path %q: %w", archivePathAbs, err) + } + defer func() { + if closeErr := buf.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("closing archive file buffer from %q: %w", archivePathAbs, closeErr)) + } + }() + + // gzip and tar get their own writers, and note that they are chained -- tar + // writes to gzip, which writes to the buffer. So, we only need to write to + // the tar writer + gzipWriter := gzip.NewWriter(buf) + defer func() { + if closeErr := gzipWriter.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("closing gzip writer: %w", closeErr)) + } + }() + + tarWriter := tar.NewWriter(gzipWriter) + defer func() { + if closeErr := tarWriter.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("closing tar writer: %w", closeErr)) + } + }() + + files, err := filetree.GetFilePathsInDirectory(rootAbs) + if err != nil { + return fmt.Errorf("populating list of files from %q: %w", root, err) + } + + for _, fileName := range files { + err := addToArchive(tarWriter, rootAbs, fileName) + if err != nil { + return fmt.Errorf("adding %q to archive: %w", fileName, err) + } + } + + return err +} + +func ExtractTGZArchive(src, dest string) error { + if !tgzRegex.MatchString(src) { + return errors.New("provided archive path must have valid gzipped-tar extension") + } + + gzipFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("opening archive file: %w", err) + } + + gzipReader, err := gzip.NewReader(gzipFile) + if err != nil { + log.Fatal("ExtractTarGz: NewReader failed") + } + + tarReader := tar.NewReader(gzipReader) + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } else if err != nil { + return fmt.Errorf("Next() failed in tar reader: %w", err) + } + + filePath := filepath.Join(dest, header.Name) + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(filePath, 0755); err != nil { + return fmt.Errorf("making directory during tar extraction: %w", err) + } + case tar.TypeReg: + dirPath := filepath.Dir(filePath) + if err := os.MkdirAll(dirPath, 0755); err != nil { + return fmt.Errorf("making directory during tar extraction: %w", err) + } + + outFile, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("creating output file %q during tar extraction: %w", filePath, err) + } + if _, err := io.Copy(outFile, tarReader); err != nil { + return fmt.Errorf("writing to output file %q during tar extraction: %w", filePath, err) + } + closeErr := outFile.Close() + if closeErr != nil { + return fmt.Errorf("closing output file used during tar extraction: %w", closeErr) + } + default: + return fmt.Errorf("unknown tar type '%b' in %q", header.Typeflag, filePath) + } + } + + return nil +} + +func addToArchive(tarWriter *tar.Writer, rootAbs string, fileName string) (err error) { + file, err := os.Open(fileName) + if err != nil { + return fmt.Errorf("opening file %q: %w", fileName, err) + } + defer func() { + if closeErr := file.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("closing file: %w", closeErr)) + } + }() + + info, err := file.Stat() + if err != nil { + return fmt.Errorf("getting file info for %q: %w", fileName, err) + } + + // Tar needs file headers, so create one from the file info + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + return fmt.Errorf("creating tar header for %q: %w", fileName, err) + } + + topLevelDir, err := maybeGetTopLevelDir(rootAbs) + if err != nil { + return fmt.Errorf("checking for top-level directory when adding to archive: %w", err) + } + fileNameClean := strings.ReplaceAll(fileName, rootAbs+string(filepath.Separator), "") + if topLevelDir != "" { + fileNameClean = filepath.Join(topLevelDir, fileNameClean) + } + + // Use full path as name (FileInfoHeader only takes the basename) + // If we don't do this the directory strucuture would + // not be preserved + // https://golang.org/src/archive/tar/common.go?#L626 + header.Name = fileNameClean + + // Write file header to the tar archive + err = tarWriter.WriteHeader(header) + if err != nil { + return fmt.Errorf("writing tar header for %q: %w", fileName, err) + } + + // Copy file content to tar archive + _, err = io.Copy(tarWriter, file) + if err != nil { + return fmt.Errorf("adding file %q to tar archive: %w", fileName, err) + } + + return err +} + +func maybeGetTopLevelDir(rootAbs string) (string, error) { + topLevelDir := filepath.Base(rootAbs) + return topLevelDir, nil +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go new file mode 100644 index 0000000..e5cc2bb --- /dev/null +++ b/internal/archive/archive_test.go @@ -0,0 +1,37 @@ +package archive + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateArchive(t *testing.T) { + archiveRoot := "../../testdata/filetree" + archivePath := filepath.Join(os.TempDir(), "archive-test.tar.gz") + t.Cleanup(func() { + err := os.RemoveAll(archivePath) + require.NoError(t, err) + }) + + err := CreateArchive(archiveRoot, archivePath) + require.NoError(t, err) + + // TODO: add test for inspecting contents once we implement an archive + // extractor -- as of now, I'm just checking on the CLI if the expected tree + // matches +} + +func TestMaybeGetTopLevelDir(t *testing.T) { + t.Run("works when rootDir is a directory", func(t *testing.T) { + rootDir := "../../testdata/filetree" + wantTopLevelDir := "filetree" + gotTopLevelDir, err := maybeGetTopLevelDir(rootDir) + + assert.NoError(t, err) + assert.Equal(t, wantTopLevelDir, gotTopLevelDir) + }) +} diff --git a/internal/archive/cache/cache.go b/internal/archive/cache/cache.go new file mode 100644 index 0000000..8f68dc9 --- /dev/null +++ b/internal/archive/cache/cache.go @@ -0,0 +1,60 @@ +package cache + +import ( + "fmt" + "math/rand" + "os" + "path/filepath" + + "github.com/opensourcecorp/vdm/cmd/vars" + "github.com/opensourcecorp/vdm/internal/archive" + "github.com/opensourcecorp/vdm/internal/message" + "github.com/opensourcecorp/vdm/internal/vdmspec" +) + +// AddRemote uses the provided [vdmspec.Remoter] information along with a file +// handle for a target archive to actually write the archive data. +func AddRemote(remote vdmspec.Remoter, cacheRoot string) (string, error) { + cachePath, err := GetPersistentCacheFilePath(remote) + if err != nil { + return "", fmt.Errorf("getting cache location for remote %q: %w", remote.GetSumDBKey(), err) + } + message.Debugf("persistent cache file path: %q", cachePath) + + err = archive.CreateArchive(cacheRoot, cachePath) + if err != nil { + return "", fmt.Errorf("creating archive while adding remote %q: %w", remote.GetSumDBKey(), err) + } + + err = AddToSumDB(remote, cachePath) + if err != nil { + return "", fmt.Errorf("adding %q details to sumdb: %w", cachePath, err) + } + + return cachePath, nil +} + +func GetPersistentCacheFilePath(remote vdmspec.Remoter) (string, error) { + b64 := StringToBase64(remote.GetSumDBKey()) + message.Debugf("remote %q base64'd to %q", remote.GetSumDBKey(), b64) + + cacheFileName := b64 + ".tar.gz" + + vdmCacheDir, err := vars.GetVDMCacheDir() + if err != nil { + return "", fmt.Errorf("determining vdm cache directory while adding remote: %w", err) + } + + cachePath := filepath.Join(vdmCacheDir, cacheFileName) + return cachePath, nil +} + +func GetTempCachePath(remote vdmspec.Remoter) string { + // tmpCachePath is where the actual retrieval is targeted, which should then + // be later archived to the persistent cache + randID := 100000000000 + rand.Intn(999999999999) + tmpCacheRoot := filepath.Join(os.TempDir(), fmt.Sprintf("vdm-tmp-%d", randID)) + tmpCachePath := filepath.Join(tmpCacheRoot, filepath.Base(remote.GetSource())) + + return tmpCachePath +} diff --git a/internal/archive/cache/doc.go b/internal/archive/cache/doc.go new file mode 100644 index 0000000..1dc7929 --- /dev/null +++ b/internal/archive/cache/doc.go @@ -0,0 +1,3 @@ +// Package cache provides tooling for handling the caching of data from Remotes, +// like writing the cache archives, calculating thier SHA sums, etc. +package cache diff --git a/internal/archive/cache/sumdb.go b/internal/archive/cache/sumdb.go new file mode 100644 index 0000000..c8a8824 --- /dev/null +++ b/internal/archive/cache/sumdb.go @@ -0,0 +1,228 @@ +package cache + +import ( + "crypto/sha256" + "database/sql" + "encoding/base64" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/opensourcecorp/vdm/cmd/vars" + "github.com/opensourcecorp/vdm/internal/message" + "github.com/opensourcecorp/vdm/internal/vdmspec" + + // Imports the SQLite driver + _ "modernc.org/sqlite" +) + +var ( + tableName = "sums" + createStatement = "CREATE" + insertStatement = "INSERT" + checkKeyExistsStatement = "CHECK_EXISTS" + dbStatements = map[string]string{ + createStatement: fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + key TEXT UNIQUE PRIMARY KEY, + source TEXT, + version TEXT, + sum TEXT UNIQUE + ); + `, tableName), + insertStatement: fmt.Sprintf(` + INSERT INTO %s ( + key, source, version, sum + ) VALUES ( + ?, ?, ?, ? + ); + `, tableName), + checkKeyExistsStatement: fmt.Sprintf(` + SELECT COUNT(*) + FROM %s + WHERE key = ? + ; + `, tableName), + } +) + +func CreateSumDB() (err error) { + sumDBPath, err := getSumDBPath() + if err != nil { + return fmt.Errorf("getting sumdb path %q: %w", sumDBPath, err) + } + + db, err := sql.Open("sqlite", sumDBPath) + if err != nil { + return fmt.Errorf("opening sumdb path %q: %w", sumDBPath, err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("closing sumdb file %q: %w", sumDBPath, closeErr)) + } + }() + + _, err = db.Exec(dbStatements[createStatement]) + if err != nil { + return fmt.Errorf("creating sums table: %w", err) + } + + return err +} + +func AddToSumDB(remote vdmspec.Remoter, cacheTargetPath string) (err error) { + sumDBPath, err := getSumDBPath() + if err != nil { + return fmt.Errorf("getting sumdb path %q: %w", sumDBPath, err) + } + + db, err := sql.Open("sqlite", sumDBPath) + if err != nil { + return fmt.Errorf("opening sumdb path %q: %w", sumDBPath, err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("closing sumdb file %q: %w", sumDBPath, closeErr)) + } + }() + + f, err := os.Open(cacheTargetPath) + if err != nil { + return fmt.Errorf("opening cache target path %q for hashing: %w", cacheTargetPath, err) + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("closing cache target file %q: %w", cacheTargetPath, closeErr)) + } + }() + + sum, err := calculateSHASum(f) + if err != nil { + return fmt.Errorf("calculating SHA sum for remote source %q: %w", remote.GetSource(), err) + } + message.Debugf("sum calculated for remote %q's archive file was %q", remote.GetSource(), sum) + + _, err = db.Exec( + dbStatements[insertStatement], + remote.GetSumDBKey(), + remote.GetSource(), + remote.GetVersion(), + sum, + ) + if err != nil { + return fmt.Errorf("inserting into sums table: %w", err) + } + + return err +} + +func CheckIfRemoteInSumDB(remote vdmspec.Remoter) (hasKey bool, err error) { + sumDBPath, err := getSumDBPath() + if err != nil { + return false, fmt.Errorf("getting sumdb path %q: %w", sumDBPath, err) + } + + db, err := sql.Open("sqlite", sumDBPath) + if err != nil { + return false, fmt.Errorf("opening sumdb path %q: %w", sumDBPath, err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + err = errors.Join(err, fmt.Errorf("closing sumdb file %q: %w", sumDBPath, closeErr)) + } + }() + + var numRows int + err = db.QueryRow(dbStatements[checkKeyExistsStatement], remote.GetSumDBKey()).Scan(&numRows) + if err != nil { + return false, fmt.Errorf("querying sumdb: %w", err) + } + message.Debugf("number of results from sumdb for remote key %q: %d", remote.GetSumDBKey(), numRows) + + if numRows > 0 { + hasKey = true + } + message.Debugf("sumdb query result checked for remote key %q, hasKey: %v", remote.GetSumDBKey(), hasKey) + + return hasKey, err +} + +// calculateSHASum takes an arbitrary [io.Reader] (such as an open +// file handle) and calculates the SHA256 checksum for it. +func calculateSHASum(reader io.Reader) (string, error) { + message.Debugf("reader address for calculating SHA sum: %v", reader) + hasher := sha256.New() + var n int64 + var err error + if n, err = io.Copy(hasher, reader); err != nil { + return "", fmt.Errorf("writing reader to hasher: %w", err) + } + message.Debugf("number of bytes copied to hasher: %d", n) + sum := fmt.Sprintf("%x", hasher.Sum(nil)) + return sum, nil +} + +// StringToBase64 does what it says on the tin. However, since this is intended +// for use as an archive file name, it replaces padding characters ('=') with +// '_PAD'. +func StringToBase64(s string) string { + // We use base64-encoded names for cache archives, because remote sources + // have slashes and that can break FS pathing + out := base64.StdEncoding.EncodeToString([]byte(s)) + out = replaceNonAlphaBase64Characters(out) + return out +} + +// StringFromBase64 does the inverse of [StringToBase64]. +func StringFromBase64(s string) (string, error) { + fixedString := restoreNonAlphaBase64Characters(s) + out, err := base64.StdEncoding.DecodeString(fixedString) + if err != nil { + return "", fmt.Errorf("decoding base64 string %q: %w", s, err) + } + return string(out), nil +} + +func getSumDBPath() (string, error) { + vdmCachePath, err := vars.GetVDMCacheDir() + if err != nil { + return "", fmt.Errorf("determining vdm cache path during sumdb creation: %w", err) + } + + message.Debugf("vdm cache directory during sumdb path determination was %q", vdmCachePath) + sumDBPath := filepath.Join(vdmCachePath, "sum.db") + message.Debugf("sumdb path to be used: %q", sumDBPath) + + return sumDBPath, nil +} + +// base64NonAlphaMap maps non-alphanumeric base-64 characters to their arbitrary +// replacement values +var base64NonAlphaMap = map[string]string{ + "=": "_EQ", + "/": "_SLASH", + "+": "_PLUS", +} + +// replaceNonAlphaBase64Characters takes a valid base64 string and replaces +// non-alphanumeric characters +func replaceNonAlphaBase64Characters(s string) string { + out := s + for nonAlpha, alpha := range base64NonAlphaMap { + out = strings.ReplaceAll(out, nonAlpha, alpha) + } + return out +} + +// restoreNonAlphaBase64Characters takes an invalid base64 string and restores +// non-alphanumeric characters +func restoreNonAlphaBase64Characters(s string) string { + out := s + for nonAlpha, alpha := range base64NonAlphaMap { + out = strings.ReplaceAll(out, alpha, nonAlpha) + } + return out +} diff --git a/internal/archive/cache/sumdb_test.go b/internal/archive/cache/sumdb_test.go new file mode 100644 index 0000000..45a5d04 --- /dev/null +++ b/internal/archive/cache/sumdb_test.go @@ -0,0 +1,96 @@ +package cache + +import ( + "os" + "path/filepath" + "testing" + + "github.com/opensourcecorp/vdm/internal/archive" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCalculateSHASum(t *testing.T) { + t.Run("works with an open file", func(t *testing.T) { + f, err := os.Open("../../../testdata/sumdb/sha256test.txt") + require.NoError(t, err) + t.Cleanup(func() { + closeErr := f.Close() + require.NoError(t, closeErr) + }) + + want := `25fce0ea957324f2fdab37fa2c35df8dc1c62703b1970a744a723603407b630b` + got, err := calculateSHASum(f) + + assert.NoError(t, err) + assert.Equal(t, want, got) + }) + + t.Run("works on a created archive file", func(t *testing.T) { + rootDir := "../../../testdata/filetree" + archivePath := filepath.Join(os.TempDir(), "archive-to-hash.tar.gz") + err := archive.CreateArchive(rootDir, archivePath) + require.NoError(t, err) + + f, err := os.Open(archivePath) + require.NoError(t, err) + t.Cleanup(func() { + err := os.RemoveAll(archivePath) + require.NoError(t, err) + }) + + want := `c1c5e8e5cd54819ea0243db2b203e581aae7308937fb575204914fc2e41ef2a7` + got, err := calculateSHASum(f) + + assert.NoError(t, err) + assert.Equal(t, want, got) + }) +} + +func TestStringAsBase64(t *testing.T) { + t.Run("works on string with no resulting padding", func(t *testing.T) { + src := "https://github.com/org/user" + want := "aHR0cHM6Ly9naXRodWIuY29tL29yZy91c2Vy" + got := StringToBase64(src) + assert.Equal(t, want, got) + }) + + t.Run("works on string WITH resulting padding", func(t *testing.T) { + src := "https://github.com/org/user12" + want := "aHR0cHM6Ly9naXRodWIuY29tL29yZy91c2VyMTI_EQ" + got := StringToBase64(src) + assert.Equal(t, want, got) + }) +} + +func TestStringFromBase64(t *testing.T) { + t.Run("works on string with no resulting padding", func(t *testing.T) { + src := "aHR0cHM6Ly9naXRodWIuY29tL29yZy91c2Vy" + want := "https://github.com/org/user" + got, err := StringFromBase64(src) + assert.NoError(t, err) + assert.Equal(t, want, got) + }) + + t.Run("works on string WITH resulting padding", func(t *testing.T) { + src := "aHR0cHM6Ly9naXRodWIuY29tL29yZy91c2VyMTI_EQ" + want := "https://github.com/org/user12" + got, err := StringFromBase64(src) + assert.NoError(t, err) + assert.Equal(t, want, got) + }) +} + +func TestNonAlphaBase64(t *testing.T) { + t.Run("replacer works", func(t *testing.T) { + want := "aHR0cHM6Ly9naXRodWIuY29tL29yZy91c2Vy_EQ" + got := replaceNonAlphaBase64Characters("aHR0cHM6Ly9naXRodWIuY29tL29yZy91c2Vy=") + assert.Equal(t, want, got) + }) + + t.Run("restorer works", func(t *testing.T) { + want := "aHR0cHM6Ly9naXRodWIuY29tL29yZy91c2Vy=" + got := restoreNonAlphaBase64Characters("aHR0cHM6Ly9naXRodWIuY29tL29yZy91c2Vy_EQ") + assert.Equal(t, want, got) + }) +} diff --git a/internal/archive/doc.go b/internal/archive/doc.go new file mode 100644 index 0000000..9e3c4d8 --- /dev/null +++ b/internal/archive/doc.go @@ -0,0 +1,3 @@ +// Package archive houses logic for interacting with archive file formats, and +// some of their specific uses within vdm. +package archive diff --git a/internal/filetree/doc.go b/internal/filetree/doc.go new file mode 100644 index 0000000..01ae666 --- /dev/null +++ b/internal/filetree/doc.go @@ -0,0 +1,3 @@ +// Package filetree contains functionality for determining & operating on file +// paths in a tree. +package filetree diff --git a/internal/filetree/filetree.go b/internal/filetree/filetree.go new file mode 100644 index 0000000..09598f5 --- /dev/null +++ b/internal/filetree/filetree.go @@ -0,0 +1,32 @@ +package filetree + +import ( + "fmt" + "os" + "path/filepath" +) + +// GetFilePathsInDirectory traverse a directory tree from the provided root, and +// returns a slice of the files in the tree. +func GetFilePathsInDirectory(root string) ([]string, error) { + rootAbs, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("determining abspath of root %q: %w", rootAbs, err) + } + + var files []string + err = filepath.Walk(rootAbs, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if !f.IsDir() { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("walking directory tree: %w", err) + } + + return files, nil +} diff --git a/internal/filetree/filetree_test.go b/internal/filetree/filetree_test.go new file mode 100644 index 0000000..eb5cd40 --- /dev/null +++ b/internal/filetree/filetree_test.go @@ -0,0 +1,32 @@ +package filetree + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetFilePathsInDirectory(t *testing.T) { + t.Run("works", func(t *testing.T) { + root, err := filepath.Abs("../../testdata/filetree") + require.NoError(t, err) + + want := []string{ + filepath.Join(root, "top-file"), + filepath.Join(root, "subdir", "subdir-file"), + filepath.Join(root, "subdir", "subdir-2", "subdir-2-file"), + } + got, err := GetFilePathsInDirectory(root) + require.NoError(t, err) + + assert.ElementsMatch(t, want, got) + }) + + t.Run("fails in some way, such as when given nonexistent root", func(t *testing.T) { + badRoot := "dir-that-doesnt-exist" + _, err := GetFilePathsInDirectory(badRoot) + assert.Error(t, err) + }) +} diff --git a/internal/message/message.go b/internal/message/message.go index f0f4f9c..f8d567f 100644 --- a/internal/message/message.go +++ b/internal/message/message.go @@ -1,16 +1,18 @@ -// Package message controls message printing. THis isn't a "logging" package per +// Package message controls message printing. This isn't a "logging" package per // se, but adds some niceties for log-like needs. package message import ( "fmt" "os" + + "github.com/opensourcecorp/vdm/cmd/vars" ) // Debugf prints out debug-level information messages with a formatting // directive. func Debugf(format string, args ...any) { - if os.Getenv("DEBUG") != "" { + if os.Getenv(vars.Debug) != "" { fmt.Printf("DEBUG: "+format+"\n", args...) } } @@ -18,7 +20,7 @@ func Debugf(format string, args ...any) { // Infof prints out debug-level information messages with a formatting // directive. func Infof(format string, args ...any) { - fmt.Printf(format+"\n", args...) + fmt.Printf("INFO: "+format+"\n", args...) } // Warnf prints out debug-level information messages with a formatting diff --git a/internal/remotes/file_test.go b/internal/remotes/archive.go similarity index 100% rename from internal/remotes/file_test.go rename to internal/remotes/archive.go diff --git a/internal/remotes/doc.go b/internal/remotes/doc.go index cc7572b..3ffcd76 100644 --- a/internal/remotes/doc.go +++ b/internal/remotes/doc.go @@ -1,4 +1,2 @@ -/* -Package remotes defines logic for the various types of remotes that vdm supports. -*/ +// Package remotes defines logic for the various types of remotes that vdm supports. package remotes diff --git a/internal/remotes/file.go b/internal/remotes/file.go deleted file mode 100644 index ed9bd59..0000000 --- a/internal/remotes/file.go +++ /dev/null @@ -1,107 +0,0 @@ -package remotes - -import ( - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - - "github.com/opensourcecorp/vdm/internal/message" - "github.com/opensourcecorp/vdm/internal/vdmspec" -) - -// SyncFile is the root of the sync operations for "file" remote types. -func SyncFile(remote vdmspec.Remote) error { - fileExists, err := checkFileExists(remote) - if err != nil { - return fmt.Errorf("checking if file exists locally: %w", err) - } - - if !fileExists { - message.Infof("File '%s' does not exist locally, retrieving", remote.LocalPath) - err = retrieveFile(remote) - if err != nil { - return fmt.Errorf("retrieving file: %w", err) - } - } else { - message.Infof("File '%s' already exists locally, skipping", remote.LocalPath) - } - - return nil -} - -func checkFileExists(remote vdmspec.Remote) (bool, error) { - fullPath, err := filepath.Abs(remote.LocalPath) - if err != nil { - return false, fmt.Errorf("determining abspath for file '%s': %w", remote.LocalPath, err) - } - - _, err = os.Stat(remote.LocalPath) - if errors.Is(err, os.ErrNotExist) { - return false, nil - } else if err != nil { - return false, fmt.Errorf("couldn't check if %s exists at '%s': %w", remote.LocalPath, fullPath, err) - } - - return true, nil -} - -func retrieveFile(remote vdmspec.Remote) (err error) { - resp, err := http.Get(remote.Remote) - if err != nil { - return fmt.Errorf("retrieving remote file '%s': %w", remote.Remote, err) - } - defer func() { - if closeErr := resp.Body.Close(); closeErr != nil { - err = errors.Join(fmt.Errorf("closing response body after remote file '%s' retrieval: %w", remote.Remote, err)) - } - }() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unsuccessful status code '%d' from server when retrieving remote file '%s'", resp.StatusCode, remote.Remote) - } - - err = ensureParentDirs(remote.LocalPath) - if err != nil { - return fmt.Errorf("creating parent directories for file: %w", err) - } - - // Note: I would normally use os.WriteFile() using the returned bytes - // directly, but the internet says this os.Create()/io.Copy() approach - // appears to be idiomatic - outFile, err := os.Create(remote.LocalPath) - if err != nil { - return fmt.Errorf("creating landing file '%s' for remote file: %w", remote.LocalPath, err) - } - defer func() { - if closeErr := outFile.Close(); closeErr != nil { - err = errors.Join(fmt.Errorf("closing local file '%s' after remote file '%s' retrieval: %w", remote.LocalPath, remote.Remote, err)) - } - }() - - bytesWritten, err := io.Copy(outFile, resp.Body) - if err != nil { - return fmt.Errorf("copying HTTP response to disk: ") - } - message.Debugf("wrote %d bytes to '%s'", bytesWritten, remote.LocalPath) - - return nil -} - -func ensureParentDirs(path string) error { - fullPath, err := filepath.Abs(path) - if err != nil { - return fmt.Errorf("determining abspath for file '%s': %w", path, err) - } - message.Debugf("absolute filepath for '%s' determined to be '%s'", path, fullPath) - dir := filepath.Dir(fullPath) - err = os.MkdirAll(dir, os.ModePerm) - if err != nil { - return fmt.Errorf("making directories: %w", err) - } - message.Debugf("created director(ies): %s", dir) - - return nil -} diff --git a/internal/remotes/git.go b/internal/remotes/git.go index 940a25c..b397633 100644 --- a/internal/remotes/git.go +++ b/internal/remotes/git.go @@ -7,36 +7,95 @@ import ( "os/exec" "path/filepath" + "github.com/opensourcecorp/vdm/internal/archive" + "github.com/opensourcecorp/vdm/internal/archive/cache" "github.com/opensourcecorp/vdm/internal/message" "github.com/opensourcecorp/vdm/internal/vdmspec" ) -// SyncGit is the root of the sync operations for "git" remote types. -func SyncGit(remote vdmspec.Remote) error { - err := gitClone(remote) +// Git defines the git remote type, and implements the [vdmspec.Remoter] +// interface. +type Git struct { + vdmspec.RemoteTemplate +} + +var _ vdmspec.Remoter = Git{} + +// Cache provides the [vdmspec.Remoter.Cache] operations for "git" remote types. +func (remote Git) Cache() (cachePath string, err error) { + tmpCachePath := cache.GetTempCachePath(remote) + message.Debugf("tmpCachePath: %q", tmpCachePath) + defer func() { + if rmErr := os.RemoveAll(filepath.Dir(tmpCachePath)); rmErr != nil { + err = errors.Join(err, fmt.Errorf("removing temporary cache path %q: %w", tmpCachePath, rmErr)) + } + }() + + remoteAlreadyInSumDB, err := cache.CheckIfRemoteInSumDB(remote) if err != nil { - return fmt.Errorf("cloing remote: %w", err) + return "", fmt.Errorf("checking if remote %q already in sumdb: %w", remote.GetSumDBKey(), err) } - if remote.Version != "latest" { - message.Infof("%s: Setting specified version...", remote.OpMsg()) - checkoutCmd := exec.Command("git", "-C", remote.LocalPath, "checkout", remote.Version) + if !remoteAlreadyInSumDB { + remote.OpMsg("Retrieving...") + err = gitClone(remote.Source, tmpCachePath) + if err != nil { + return "", fmt.Errorf("cloning git repository: %w", err) + } + + remote.OpMsg("Setting specified version...") + checkoutCmd := exec.Command("git", "-C", tmpCachePath, "checkout", remote.Version) checkoutOutput, err := checkoutCmd.CombinedOutput() if err != nil { - return fmt.Errorf("error checking out specified revision: exec error '%w', with output: %s", err, string(checkoutOutput)) + return "", fmt.Errorf("error checking out specified revision: exec error '%w', with output: %s", err, string(checkoutOutput)) + } + + message.Debugf("removing .git dir for local path %q", tmpCachePath) + dotGitPath := filepath.Join(tmpCachePath, ".git") + err = os.RemoveAll(dotGitPath) + if err != nil { + return "", fmt.Errorf("removing directory %q: %w", dotGitPath, err) + } + + cachePath, err = cache.AddRemote(remote, tmpCachePath) + if err != nil { + return "", fmt.Errorf("caching git remote %q: %w", remote.Source, err) + } + } else { + remote.OpMsg("Remote found in local cache; restoring...") + cachePath, err = cache.GetPersistentCacheFilePath(remote) + if err != nil { + return "", fmt.Errorf("getting existing cache path for remote %q: %w", remote.GetSumDBKey(), err) } } - message.Debugf("removing .git dir for local path '%s'", remote.LocalPath) - dotGitPath := filepath.Join(remote.LocalPath, ".git") - err = os.RemoveAll(dotGitPath) + return cachePath, err +} + +// Sync provides the [vdmspec.Remoter.Sync] operations for "git" remote types. +func (remote Git) Sync(src, dest string) error { + err := archive.ExtractTGZArchive(src, dest) if err != nil { - return fmt.Errorf("removing directory %s: %w", dotGitPath, err) + return fmt.Errorf("syncing git cache for remote %q: %w", remote.GetSource(), err) } - return nil } +// GetSource returns the Source field. +func (remote Git) GetSource() string { + return remote.Source +} + +// GetVersion returns the Version field. +func (remote Git) GetVersion() string { + return remote.Version +} + +// GetSumDBKey returns the Source & Version fields, concatenated with an '@'. +func (remote Git) GetSumDBKey() string { + return fmt.Sprintf("%s@%s", remote.Source, remote.Version) +} + func checkGitAvailable() error { cmd := exec.Command("git", "--version") sysOutput, err := cmd.CombinedOutput() @@ -48,27 +107,18 @@ func checkGitAvailable() error { return nil } -func gitClone(remote vdmspec.Remote) error { +func gitClone(src string, dest string) error { err := checkGitAvailable() if err != nil { - return fmt.Errorf("remote '%s' is a git type, but git may not installed/available on PATH: %w", remote.Remote, err) + return fmt.Errorf("remote %q is a git type, but git may not be installed/available on PATH: %w", src, err) } - // If users want "latest", then we can just do a depth-one clone and - // skip the checkout operation. But if they want non-latest, we need the - // full history to be able to find a specified revision - var cloneCmdArgs []string - if remote.Version == "latest" { - message.Debugf("%s: version specified as 'latest', so making shallow clone and skipping separate checkout operation", remote.OpMsg()) - cloneCmdArgs = []string{"clone", "--depth=1", remote.Remote, remote.LocalPath} - } else { - message.Debugf("%s: version specified as NOT latest, so making regular clone and will make separate checkout operation", remote.OpMsg()) - cloneCmdArgs = []string{"clone", remote.Remote, remote.LocalPath} - } + cloneCmdArgs := []string{"clone", src, dest} + message.Debugf("git args: %v", cloneCmdArgs) - message.Infof("%s: Retrieving...", remote.OpMsg()) cloneCmd := exec.Command("git", cloneCmdArgs...) cloneOutput, err := cloneCmd.CombinedOutput() + message.Debugf("git clone command output: %s", string(cloneOutput)) if err != nil { return fmt.Errorf("cloning remote: exec error '%w', with output: %s", err, string(cloneOutput)) } diff --git a/internal/remotes/git_test.go b/internal/remotes/git_test.go index 6b2bde4..af2e256 100644 --- a/internal/remotes/git_test.go +++ b/internal/remotes/git_test.go @@ -1,41 +1,12 @@ package remotes import ( - "os" "testing" - "github.com/opensourcecorp/vdm/internal/vdmspec" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func getTestGitSpec() vdmspec.Remote { - specLocalPath := "./deps/go-common" - return vdmspec.Remote{ - Type: "git", - Remote: "https://github.com/opensourcecorp/go-common", - Version: "v0.2.0", - LocalPath: specLocalPath, - } -} - -func TestSyncGit(t *testing.T) { - spec := getTestGitSpec() - err := SyncGit(spec) - require.NoError(t, err) - - defer t.Cleanup(func() { - if cleanupErr := os.RemoveAll(spec.LocalPath); cleanupErr != nil { - t.Fatalf("removing specLocalPath: %v", cleanupErr) - } - }) - - t.Run(".git directory was removed", func(t *testing.T) { - _, err := os.Stat("./deps/go-common-tag/.git") - assert.ErrorIs(t, err, os.ErrNotExist) - }) -} - func TestCheckGitAvailable(t *testing.T) { t.Run("checkGitAvailable", func(t *testing.T) { t.Run("no error when git is available", func(t *testing.T) { @@ -51,30 +22,3 @@ func TestCheckGitAvailable(t *testing.T) { }) }) } - -func TestGitClone(t *testing.T) { - spec := getTestGitSpec() - cloneErr := gitClone(spec) - - defer t.Cleanup(func() { - if cleanupErr := os.RemoveAll(spec.LocalPath); cleanupErr != nil { - t.Fatalf("removing specLocalPath: %v", cleanupErr) - } - }) - - t.Run("no error on success", func(t *testing.T) { - require.NoError(t, cloneErr) - }) - - t.Run("LocalPath is a directory, not a file", func(t *testing.T) { - outDir, err := os.Stat("./deps/go-common") - require.NoError(t, err) - assert.True(t, outDir.IsDir()) - }) - - t.Run("a known file in the remote exists, and is a file", func(t *testing.T) { - sampleFile, err := os.Stat("./deps/go-common/go.mod") - require.NoError(t, err) - assert.False(t, sampleFile.IsDir()) - }) -} diff --git a/internal/remotes/local.go b/internal/remotes/local.go new file mode 100644 index 0000000..a558bee --- /dev/null +++ b/internal/remotes/local.go @@ -0,0 +1 @@ +package remotes diff --git a/internal/vdminit/init.go b/internal/vdminit/init.go new file mode 100644 index 0000000..4a220c5 --- /dev/null +++ b/internal/vdminit/init.go @@ -0,0 +1,28 @@ +package vdminit + +import ( + "fmt" + "os" + + "github.com/opensourcecorp/vdm/cmd/vars" + "github.com/opensourcecorp/vdm/internal/archive/cache" +) + +func Paths() error { + vdmCacheDir, err := vars.GetVDMCacheDir() + if err != nil { + return fmt.Errorf("determining vdm cache directory while running init: %w", err) + } + + err = os.MkdirAll(vdmCacheDir, 0755) + if err != nil { + return fmt.Errorf("creating vdm cache directory %q during init: %w", vdmCacheDir, err) + } + + err = cache.CreateSumDB() + if err != nil { + return fmt.Errorf("creating sumdb during init: %w", err) + } + + return nil +} diff --git a/internal/vdminit/init_test.go b/internal/vdminit/init_test.go new file mode 100644 index 0000000..05d3699 --- /dev/null +++ b/internal/vdminit/init_test.go @@ -0,0 +1,51 @@ +package vdminit + +import ( + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/opensourcecorp/vdm/cmd/vars" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPaths(t *testing.T) { + // SetupVDMForTest itself calls Paths(), handily + _, cleanup := SetupVDMForTest(t) + t.Cleanup(cleanup) + + t.Run("env var is set right", func(t *testing.T) { + got := os.Getenv(vars.VDMHomeEnvVarName) + assert.Regexp(t, `vdm-tmp-\d+`, got) + }) + + t.Run("vdm cache is then returned as being under the test homedir", func(t *testing.T) { + vdmHome, err := vars.GetVDMHomeDir() + require.NoError(t, err) + want := filepath.Join(vdmHome, "cache") + + got, err := vars.GetVDMCacheDir() + require.NoError(t, err) + assert.Equal(t, want, got) + }) + + t.Run("sumdb exists and has sums table in it", func(t *testing.T) { + vdmCache, err := vars.GetVDMCacheDir() + require.NoError(t, err) + + sumDBPath := filepath.Join(vdmCache, "sum.db") + _, err = os.Stat(sumDBPath) + assert.NoError(t, err) + + db, err := sql.Open("sqlite", sumDBPath) + require.NoError(t, err) + + want := 1 + var got int + err = db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'sums';`).Scan(&got) + assert.NoError(t, err) + assert.Equal(t, want, got) + }) +} diff --git a/internal/vdminit/testhelpers.go b/internal/vdminit/testhelpers.go new file mode 100644 index 0000000..89af3fa --- /dev/null +++ b/internal/vdminit/testhelpers.go @@ -0,0 +1,38 @@ +package vdminit + +import ( + "fmt" + "math/rand" + "os" + "path/filepath" + "testing" + + "github.com/opensourcecorp/vdm/cmd/vars" +) + +// SetupVDMForTest is a helper function for setting up vdm's tests. Notably, vdm +// requires itself to be initialized with its own home directory etc. before +// first use, and tests need a clean way to do that, and an equally-clean way to +// tear down their mess. This function returns both the string path to vdm's +// test home directory, as well as a cleanup function to be called by the test. +func SetupVDMForTest(t *testing.T) (vdmHome string, cleanup func()) { + t.Helper() + + randID := 100000000000 + rand.Intn(999999999999) + vdmHome = filepath.Join(os.TempDir(), fmt.Sprintf("vdm-tmp-%d", randID)) + t.Setenv(vars.VDMHomeEnvVarName, vdmHome) + + err := Paths() + if err != nil { + t.Errorf("instantiating vdm paths for test: %v", err) + } + + cleanup = func() { + err := os.RemoveAll(vdmHome) + if err != nil { + t.Errorf("failed to clean up after test: %v", err) + } + } + + return vdmHome, cleanup +} diff --git a/internal/vdmspec/doc.go b/internal/vdmspec/doc.go index 0f9316c..d793ad3 100644 --- a/internal/vdmspec/doc.go +++ b/internal/vdmspec/doc.go @@ -1,5 +1,3 @@ -/* -Package vdmspec defines the [Spec] and [Remote] struct types, and their -associated methods. -*/ +// Package vdmspec defines the [Spec] and [RemoteTemplate] struct types, and +// their associated methods via the [Remoter] interface. package vdmspec diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index bfe2e3b..70f4051 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -1,6 +1,7 @@ package vdmspec import ( + "encoding/json" "errors" "fmt" "os" @@ -11,106 +12,86 @@ import ( "gopkg.in/yaml.v3" ) -// Spec defines the overall structure of the vmd specfile. -type Spec struct { - Remotes []Remote `json:"remotes" yaml:"remotes"` -} - -// Remote defines the structure of each remote configuration in the vdm -// specfile. -type Remote struct { - Type string `json:"type,omitempty" yaml:"type,omitempty"` - Remote string `json:"remote" yaml:"remote"` - Version string `json:"version,omitempty" yaml:"version,omitempty"` - LocalPath string `json:"local_path" yaml:"local_path"` -} - const ( - // MetaFileName is the name of the tracking file that vdm uses to record & - // track remote statuses on disk. - MetaFileName string = "VDMMETA" - - // GitType represents the string to match against for git remote types. + // GitType represents the string to match against for "git" remote types. GitType string = "git" - // FileType represents the string to match against for file remote types. - FileType string = "file" + // ArchiveType represents the string to match against for "archive" remote + // types. + ArchiveType string = "archive" + // LocalType represents the string to match against for "local" "remote" + // types. + LocalType string = "local" ) -// MakeMetaFilePath constructs the metafile path that vdm will use to track a -// remote's state on disk. -func (r Remote) MakeMetaFilePath() string { - metaFilePath := filepath.Join(r.LocalPath, MetaFileName) - // TODO: this is brittle, but it's the best I can think of right now - if r.Type == FileType { - fileDir := filepath.Dir(r.LocalPath) - fileName := filepath.Base(r.LocalPath) - // converts to e.g. 'VDMMETA_http.proto' - metaFilePath = filepath.Join(fileDir, fmt.Sprintf("%s_%s", MetaFileName, fileName)) - } - - return metaFilePath +// typeMap is used to house more programmatic access to the various remote +// types, such as in tests or [Spec.Validate] +var typeMap = map[string]int{ + GitType: 0, + ArchiveType: 1, + LocalType: 2, } -// WriteVDMMeta writes the metafile contents to disk, the path of which is -// determined by [Remote.MakeMetaFilePath]. -func (r Remote) WriteVDMMeta() error { - metaFilePath := r.MakeMetaFilePath() - vdmMetaContent, err := yaml.Marshal(r) - if err != nil { - return fmt.Errorf("writing %s: %w", metaFilePath, err) - } - - vdmMetaContent = append(vdmMetaContent, []byte("\n")...) - - message.Debugf("writing metadata file to '%s'", metaFilePath) - err = os.WriteFile(metaFilePath, vdmMetaContent, 0644) - if err != nil { - return fmt.Errorf("writing metadata file: %w", err) - } - - return nil +// Spec defines the overall structure of the vmd specfile. +type Spec struct { + Remotes []RemoteTemplate `json:"remotes" yaml:"remotes"` } -// GetVDMMeta reads the metafile from disk, and returns it for further -// processing. -func (r Remote) GetVDMMeta() (Remote, error) { - metaFilePath := r.MakeMetaFilePath() - _, err := os.Stat(metaFilePath) - if errors.Is(err, os.ErrNotExist) { - return Remote{}, nil // this is ok, because it might literally not exist yet - } else if err != nil { - return Remote{}, fmt.Errorf("couldn't check if %s exists at '%s': %w", MetaFileName, metaFilePath, err) - } - - vdmMetaFile, err := os.ReadFile(metaFilePath) - if err != nil { - message.Debugf("error reading VMDMMETA from disk: %w", err) - return Remote{}, fmt.Errorf("there was a problem reading the %s file from '%s': %w", MetaFileName, metaFilePath, err) - } - message.Debugf("%s contents read:\n%s", MetaFileName, string(vdmMetaFile)) - - var vdmMeta Remote - err = yaml.Unmarshal(vdmMetaFile, &vdmMeta) - if err != nil { - message.Debugf("error during %s unmarshal: w", MetaFileName, err) - return Remote{}, fmt.Errorf("there was a problem reading the contents of the %s file at '%s': %w", MetaFileName, metaFilePath, err) - } - message.Debugf("file %s unmarshalled: %+v", MetaFileName, vdmMeta) +// Remoter defines behavior that different remote types must exhibit +type Remoter interface { + // Cache should retrieve the remote, and cache it as an archive in + // VDM_HOME's cache + Cache() (string, error) + // Sync should unpack the archive from the cache in VDM_HOME to the + // specified destination + Sync(src, dest string) error + // GetRemote should return the [RemoteTemplate.Source] value + GetSource() string + // GetRemote should return the [RemoteTemplate.Version] value + GetVersion() string + // GetSumDBKey should concatenate the relevant values from the + // [RemoteTemplate] to produce a string value satisfying the primary key + // constraint for the sumdb + GetSumDBKey() string +} - return vdmMeta, nil +// RemoteTemplate defines the template structure for each potential remote +// configuration in the vdm specfile. This struct can be embedded into other +// remote type structs to provide the common fields between them. +type RemoteTemplate struct { + // Type is the type of Source, e.g. git, archive, file, etc. + Type string `json:"type" yaml:"type"` + // Source is the fully-qualifed location from which the Remote is retrieved, + // e.g. "https://github.com/some-org/some-repo" + Source string `json:"source" yaml:"source"` + // Version states the version requested from Source, and is then later used + // for tracking purposes. Version can be anything supported by the Type + // field -- for example, for the "git" Type, this can be a tag, a branch + // name, or a commit hash. + Version string `json:"version" yaml:"version"` + // Destination is the relative or absolute path on disk that Source will be + // placed at + Destination string `json:"destination" yaml:"destination"` + // TryLocalSource helps define behavior driven by the `try-local-sources` + // CLI flag, which allows checking for a local version of a + // [RemoteTemplate.Source], and falling back to the other Source field if + // the local path does not exist. This is especially useful for when you + // might be developing one of your Remotes in a nearby directory, and want + // to copy over that version of the Remote and not keep pushing-and-pulling + // to a Git upstream just to test the changes. + TryLocalSource string `json:"try_local_source" yaml:"try_local_source"` } -// GetSpecFromFile reads the specfile from disk (the path of which is determined -// by the user-supplied flag value), and returns it for further processing of -// remotes. +// GetSpecFromFile reads the specfile from disk (the path of which may be +// determined by the user-supplied flag value), and returns it for further +// processing of remotes. func GetSpecFromFile(specFilePath string) (Spec, error) { specFile, err := os.ReadFile(specFilePath) if err != nil { - message.Debugf("error reading specfile from disk: %w", err) + message.Debugf("error reading specfile from disk: %v", err) return Spec{}, fmt.Errorf( strings.Join([]string{ - "there was a problem reading your vdm file from '%s' -- does it not exist?", - "Either pass the --spec-file flag, or create one in the default location (details in the README).", + "there was a problem reading your vdm file from %q -- does it not exist?", + "Either pass the --specfile flag, or create one in the default location (details in the README).", "Error details: %w"}, " ", ), @@ -121,21 +102,31 @@ func GetSpecFromFile(specFilePath string) (Spec, error) { message.Debugf("specfile contents read:\n%s", string(specFile)) var spec Spec - err = yaml.Unmarshal(specFile, &spec) + specfileExtension := filepath.Ext(specFilePath) + switch specfileExtension { + case ".yaml", ".yml": + message.Debugf("specfile format is YAML") + err = yaml.Unmarshal(specFile, &spec) + case ".json": + message.Debugf("specfile format is JSON") + err = json.Unmarshal(specFile, &spec) + case ".toml": + message.Debugf("specfile format is TOML") + err = errors.New("TOML format for vdm specfile is not yet supported") + default: + err = fmt.Errorf("unsupported specfile extension %q", specfileExtension) + } if err != nil { - message.Debugf("error during specfile unmarshal: w", err) - return Spec{}, fmt.Errorf("there was a problem reading the contents of your vdm spec file: %w", err) + message.Debugf("error during specfile %s unmarshal: %v", specfileExtension, err) + return Spec{}, fmt.Errorf("there was a problem reading the contents of your vdm specfile %q: %w", specFilePath, err) } message.Debugf("vdmSpecs unmarshalled: %+v", spec) return spec, nil } -// OpMsg constructs a loggable message outlining the specific operation being -// performed at the moment -func (r Remote) OpMsg() string { - if r.Version != "" { - return fmt.Sprintf("%s@%s --> %s", r.Remote, r.Version, r.LocalPath) - } - return fmt.Sprintf("%s --> %s", r.Remote, r.LocalPath) +// OpMsg constructs a loggable message outlining the specific remote details +// being performed at the moment +func (r RemoteTemplate) OpMsg(msg string) { + message.Infof("%s@%s --> %s: %s", r.Source, r.Version, r.Destination, msg) } diff --git a/internal/vdmspec/spec_test.go b/internal/vdmspec/spec_test.go index 70918cc..115ce45 100644 --- a/internal/vdmspec/spec_test.go +++ b/internal/vdmspec/spec_test.go @@ -1,75 +1 @@ package vdmspec - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const testVDMRoot = "../../testdata" - -var ( - testVDMMetaFilePath = filepath.Join(testVDMRoot, MetaFileName) - - testRemote = Remote{ - Remote: "https://some-remote", - Version: "v1.0.0", - LocalPath: testVDMRoot, - } - - testSpecFilePath = filepath.Join(testVDMRoot, "vdm.yaml") - - testVDMMetaContents = fmt.Sprintf( - `{"remote": "https://some-remote", "version": "v1.0.0", "local_path": "%s"}`, - testVDMRoot, - ) -) - -func TestVDMMeta(t *testing.T) { - t.Run("GetVDMMeta", func(t *testing.T) { - err := os.WriteFile(testVDMMetaFilePath, []byte(testVDMMetaContents), 0644) - require.NoError(t, err) - - defer t.Cleanup(func() { - err := os.RemoveAll(testVDMMetaFilePath) - require.NoError(t, err) - }) - - got, err := testRemote.GetVDMMeta() - require.NoError(t, err) - assert.Equal(t, testRemote, got) - }) - - t.Run("WriteVDMMeta", func(t *testing.T) { - defer t.Cleanup(func() { - err := os.RemoveAll(testVDMMetaFilePath) - require.NoError(t, err) - }) - - // Needs to have parent dir(s) exist for write to work - err := os.MkdirAll(testRemote.LocalPath, 0644) - require.NoError(t, err) - - err = testRemote.WriteVDMMeta() - require.NoError(t, err) - - got, err := testRemote.GetVDMMeta() - require.NoError(t, err) - assert.Equal(t, testRemote, got) - }) - - t.Run("GetSpecsFromFile", func(t *testing.T) { - defer t.Cleanup(func() { - err := os.RemoveAll(testVDMMetaFilePath) - require.NoError(t, err) - }) - - spec, err := GetSpecFromFile(testSpecFilePath) - require.NoError(t, err) - assert.Equal(t, 5, len(spec.Remotes)) - }) -} diff --git a/internal/vdmspec/validate.go b/internal/vdmspec/validate.go index 2724244..1eba67b 100644 --- a/internal/vdmspec/validate.go +++ b/internal/vdmspec/validate.go @@ -13,44 +13,39 @@ import ( func (spec Spec) Validate() error { var allErrors []error + protocolRegex := regexp.MustCompile(`(http(s?)://|git://|git@|ftp(s?))`) for remoteIndex, remote := range spec.Remotes { - // Remote field - message.Debugf("Index #%d: validating field 'Remote' for %+v", remoteIndex, remote) - if len(remote.Remote) == 0 { - allErrors = append(allErrors, errors.New("all 'remote' fields must be non-zero length")) + // Source field + message.Debugf("Index #%d: validating field 'Source' for %+v", remoteIndex, remote) + if len(remote.Source) == 0 { + allErrors = append(allErrors, errors.New("all 'source' fields must be non-zero length")) } - protocolRegex := regexp.MustCompile(`(http(s?)://|git://|git@)`) - if !protocolRegex.MatchString(remote.Remote) { + if !protocolRegex.MatchString(remote.Source) { allErrors = append( allErrors, - fmt.Errorf("remote #%d provided as '%s', but all 'remote' fields must begin with a protocol specifier or other valid prefix (e.g. 'https://', '(user|git)@', etc.)", remoteIndex, remote.Remote), + fmt.Errorf("remote #%d provided as %q, but all 'source' fields must begin with a protocol specifier or other valid prefix (e.g. 'https://', '(user|git)@', etc.)", remoteIndex, remote.Source), ) } // Version field message.Debugf("Index #%d: validating field 'Version' for %+v", remoteIndex, remote) - if remote.Type == GitType && len(remote.Version) == 0 { - allErrors = append(allErrors, errors.New("all 'version' fields for the 'git' remote type must be non-zero length. If you don't care about the version (even though you probably should), then use 'latest'")) - } - if remote.Type == FileType && len(remote.Version) > 0 { - message.Warnf("NOTE: Remote #%d '%s' specified as type '%s', which does not take explicit version info (you provided '%s'); ignoring version field", remoteIndex, remote.Remote, remote.Type, remote.Version) + if remote.Type == GitType && remote.Version == "" { + allErrors = append(allErrors, errors.New("all 'version' fields for the 'git' remote type must be non-zero length")) } - // LocalPath field - message.Debugf("Index #%d: validating field 'LocalPath' for %+v", remoteIndex, remote) - if len(remote.LocalPath) == 0 { - allErrors = append(allErrors, errors.New("all 'local_path' fields must be non-zero length")) + // Destination field + message.Debugf("Index #%d: validating field 'Destination' for %+v", remoteIndex, remote) + if len(remote.Destination) == 0 { + allErrors = append(allErrors, errors.New("all 'destination' fields must be non-zero length")) } // Type field message.Debugf("Index #%d: validating field 'Type' for %+v", remoteIndex, remote) - typeMap := map[string]int{ - GitType: 1, - "": 2, // also git - FileType: 3, + if remote.Type == "" { + allErrors = append(allErrors, errors.New("all remotes must specify a 'type' field")) } if _, ok := typeMap[remote.Type]; !ok { - allErrors = append(allErrors, fmt.Errorf("unrecognized remote type '%s'", remote.Type)) + allErrors = append(allErrors, fmt.Errorf("unrecognized remote type %q", remote.Type)) } } @@ -58,7 +53,7 @@ func (spec Spec) Validate() error { for _, err := range allErrors { message.Errorf("validation failure: %s", err.Error()) } - return fmt.Errorf("%d validation failure(s) found in your vdm spec file", len(allErrors)) + return fmt.Errorf("%d validation failure(s) found in your vdm specfile", len(allErrors)) } return nil } diff --git a/internal/vdmspec/validate_test.go b/internal/vdmspec/validate_test.go index 7428cee..384536b 100644 --- a/internal/vdmspec/validate_test.go +++ b/internal/vdmspec/validate_test.go @@ -10,10 +10,11 @@ import ( func TestValidate(t *testing.T) { t.Run("passes", func(t *testing.T) { spec := Spec{ - Remotes: []Remote{{ - Remote: "https://some-remote", - Version: "v1.0.0", - LocalPath: "./deps/some-remote", + Remotes: []RemoteTemplate{{ + Type: GitType, + Source: "https://some-remote", + Version: "v1.0.0", + Destination: "./deps/some-remote", }}, } err := spec.Validate() @@ -22,10 +23,10 @@ func TestValidate(t *testing.T) { t.Run("fails on zero-length remote", func(t *testing.T) { spec := Spec{ - Remotes: []Remote{{ - Remote: "", - Version: "v1.0.0", - LocalPath: "./deps/some-remote", + Remotes: []RemoteTemplate{{ + Source: "", + Version: "v1.0.0", + Destination: "./deps/some-remote", }}, } err := spec.Validate() @@ -34,10 +35,10 @@ func TestValidate(t *testing.T) { t.Run("fails on remote without valid protocol", func(t *testing.T) { spec := Spec{ - Remotes: []Remote{{ - Remote: "some-remote", - Version: "v1.0.0", - LocalPath: "./deps/some-remote", + Remotes: []RemoteTemplate{{ + Source: "some-remote", + Version: "v1.0.0", + Destination: "./deps/some-remote", }}, } err := spec.Validate() @@ -46,11 +47,11 @@ func TestValidate(t *testing.T) { t.Run("fails on zero-length version for git remote type", func(t *testing.T) { spec := Spec{ - Remotes: []Remote{{ - Remote: "https://some-remote", - Version: "", - LocalPath: "./deps/some-remote", - Type: GitType, + Remotes: []RemoteTemplate{{ + Source: "https://some-remote", + Version: "", + Destination: "./deps/some-remote", + Type: GitType, }}, } err := spec.Validate() @@ -59,11 +60,11 @@ func TestValidate(t *testing.T) { t.Run("fails on unrecognized remote type", func(t *testing.T) { spec := Spec{ - Remotes: []Remote{{ - Remote: "https://some-remote", - Version: "", - LocalPath: "./deps/some-remote", - Type: "bad", + Remotes: []RemoteTemplate{{ + Source: "https://some-remote", + Version: "", + Destination: "./deps/some-remote", + Type: "bad", }}, } err := spec.Validate() @@ -72,10 +73,10 @@ func TestValidate(t *testing.T) { t.Run("fails on zero-length local path", func(t *testing.T) { spec := Spec{ - Remotes: []Remote{{ - Remote: "https://some-remote", - Version: "v1.0.0", - LocalPath: "", + Remotes: []RemoteTemplate{{ + Source: "https://some-remote", + Version: "v1.0.0", + Destination: "", }}, } err := spec.Validate() diff --git a/scripts/ci.sh b/scripts/ci.sh index 5bdab77..f66f8ec 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -11,19 +11,26 @@ if ! go vet ./... ; then failures+=('go-vet') fi -printf '>> Go linter\n' +printf '>> Go linter (staticcheck)\n' +if ! go run honnef.co/go/tools/cmd/staticcheck@latest ./... ; then + printf '>>> Failed go-lint-staticcheck\n' > /dev/stderr + failures+=('go-lint-staticcheck') +fi + +printf '>> Go linter (revive)\n' if ! go run github.com/mgechev/revive@latest --set_exit_status ./... ; then - printf '>>> Failed go-lint\n' > /dev/stderr - failures+=('go-lint') + printf '>>> Failed go-lint-revive\n' > /dev/stderr + failures+=('go-lint-revive') fi -printf '>> Go error checker\n' +printf '>> Go linter (errcheck)\n' if ! go run github.com/kisielk/errcheck@latest ./... ; then - printf '>>> Failed go-error-check\n' > /dev/stderr - failures+=('go-error-check') + printf '>>> Failed go-lint-errcheck\n' > /dev/stderr + failures+=('go-lint-errcheck') fi printf '>> Go test\n' +go clean -testcache if ! go test -cover -coverprofile=./cover.out ./... ; then printf '>>> Failed go-test check\n' > /dev/stderr failures+=('go-test') @@ -36,7 +43,8 @@ if ! make -s package ; then fi if [[ "${#failures[@]}" -gt 0 ]] ; then - printf '> One or more checks failed, see output above\n' > /dev/stderr + printf '> The following checks failed -- logs for each are above\n' > /dev/stderr + printf '%s\n' "${failures[@]}" exit 1 fi diff --git a/scripts/package.sh b/scripts/package.sh index 13daeda..0df51cc 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -2,16 +2,16 @@ set -euo pipefail # cd-jumps because it makes logs cleaner, not sorry -mkdir -p dist/zipped +mkdir -p dist/compressed cd build || exit 1 for built in * ; do - printf 'Packaging for %s into dist/zipped/\n' "${built}" + printf 'Packaging for %s into dist/compressed/\n' "${built}" cd "${built}" || exit 1 # Windows might like .zips better, otherwise make .tar.gzs if [[ "${built}" =~ 'windows' ]] ; then - zip -r9 ../../dist/zipped/vdm_"${built}".zip ./* + zip -r9 ../../dist/compressed/vdm_"${built}".zip ./* else - tar -czf ../../dist/zipped/vdm_"${built}".tar.gz ./* + tar -czf ../../dist/compressed/vdm_"${built}".tar.gz ./* fi cd - > /dev/null done diff --git a/testdata/filetree/subdir/subdir-2/subdir-2-file b/testdata/filetree/subdir/subdir-2/subdir-2-file new file mode 100644 index 0000000..e69de29 diff --git a/testdata/filetree/subdir/subdir-file b/testdata/filetree/subdir/subdir-file new file mode 100644 index 0000000..e69de29 diff --git a/testdata/filetree/top-file b/testdata/filetree/top-file new file mode 100644 index 0000000..e69de29 diff --git a/testdata/sumdb/sha256test.txt b/testdata/sumdb/sha256test.txt new file mode 100644 index 0000000..ecd37e2 --- /dev/null +++ b/testdata/sumdb/sha256test.txt @@ -0,0 +1 @@ +Contents here are irrelevant diff --git a/testdata/vdm.yaml b/testdata/vdm.yaml index 039d1e7..06a1e22 100644 --- a/testdata/vdm.yaml +++ b/testdata/vdm.yaml @@ -1,16 +1,13 @@ remotes: - - remote: "https://github.com/opensourcecorp/go-common" + - type: "git" + source: "https://github.com/opensourcecorp/vdm" version: "v0.2.0" - local_path: "./deps/go-common-tag" - - remote: "https://github.com/opensourcecorp/go-common" - version: "latest" - local_path: "./deps/go-common-latest" - - remote: "https://github.com/opensourcecorp/go-common" + destination: "./deps/git-tag" + - type: "git" + source: "https://github.com/opensourcecorp/osc-infra" version: "main" - local_path: "./deps/go-common-branch" - - remote: "https://github.com/opensourcecorp/go-common" + destination: "./deps/git-branch" + - type: "git" + source: "https://github.com/opensourcecorp/go-common" version: "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f" - local_path: "./deps/go-common-hash" - - type: "file" - remote: "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto" - local_path: "./deps/proto/http/http.proto" + destination: "./deps/git-hash"