From ae65b1248b03bfaf3c0d7ffa9f56c4a1fc3eb514 Mon Sep 17 00:00:00 2001 From: Zach Langbert Date: Fri, 5 Sep 2025 11:29:00 -0700 Subject: [PATCH] fix directories not being ignored for upload when running build --- go.mod | 3 +- go.sum | 6 ++- internal/archive/targz.go | 82 +++++++++++++------------------------- internal/commands/build.go | 3 +- 4 files changed, 35 insertions(+), 59 deletions(-) diff --git a/go.mod b/go.mod index 92c6475..c542b9d 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,12 @@ go 1.23.0 toolchain go1.23.9 require ( + github.com/boyter/gocodewalker v1.5.1 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.18.0 github.com/h2non/filetype v1.1.3 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hathora/cloud-sdk-go/hathoracloud v0.3.20 - github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 github.com/stretchr/testify v1.10.0 github.com/urfave/cli/v3 v3.0.0-alpha9 go.uber.org/zap v1.27.0 @@ -19,6 +19,7 @@ require ( ) require ( + github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/ericlagergren/decimal v0.0.0-20240411145413-00de7ca16731 // indirect github.com/mattn/go-colorable v0.1.14 // indirect diff --git a/go.sum b/go.sum index a834a6f..50730a7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/boyter/gocodewalker v1.5.1 h1:0YeK2QAkd+ymW5MsagMZapIXD3v9/vrZl0HkFSLpKsw= +github.com/boyter/gocodewalker v1.5.1/go.mod h1:9k+yM6+fIx61F0xI9ChXEGE5DYoLhggw8AxSOtW+kKo= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -16,8 +20,6 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= -github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= diff --git a/internal/archive/targz.go b/internal/archive/targz.go index e570e41..0e3b26b 100644 --- a/internal/archive/targz.go +++ b/internal/archive/targz.go @@ -10,8 +10,8 @@ import ( "slices" "strings" + "github.com/boyter/gocodewalker" "github.com/h2non/filetype" - "github.com/monochromegane/go-gitignore" "go.uber.org/zap" ) @@ -21,33 +21,6 @@ type TGZFile struct { Path string } -func getIgnoreMatchers(srcFolder string, filepaths ...string) ([]gitignore.IgnoreMatcher, error) { - var matchers []gitignore.IgnoreMatcher - for _, path := range filepaths { - matcher, err := gitignore.NewGitIgnore(filepath.Join(srcFolder, path), ".") - if err != nil { - zap.L().Debug("Could not find a " + path + " file. " + path + " matcher will not be used.") - continue - } - - matchers = append(matchers, matcher) - } - - return matchers, nil -} - -func shouldIgnoreFilepath(filepath string, isDir bool, matchers []gitignore.IgnoreMatcher) bool { - anyMatches := false - for _, matcher := range matchers { - if matcher.Match(filepath, isDir) { - anyMatches = true - break - } - } - - return anyMatches -} - func CreateTGZ(srcFolder string, ext string) (string, error) { fileName := filepath.Base(filepath.Clean(srcFolder)) destinationFile := fmt.Sprintf("%s.*.%s", fileName, ext) @@ -62,70 +35,69 @@ func CreateTGZ(srcFolder string, ext string) (string, error) { tarWriter := tar.NewWriter(gzipWriter) defer tarWriter.Close() - ignoreMatchers, err := getIgnoreMatchers( - srcFolder, - ".dockerignore", - ".gitignore", - ) + paths := make(chan *gocodewalker.File, 100) + + walker := gocodewalker.NewFileWalker(srcFolder, paths) + // .gitignore is ignored by default + walker.CustomIgnore = []string{".dockerignore"} + walker.SetErrorHandler(func(err error) bool { + zap.L().Error("Error walking directory", zap.Error(err)) + return false + }) + err = walker.Start() if err != nil { - return "", err + return "", fmt.Errorf("error walking directory: %w", err) } - err = filepath.Walk(srcFolder, func(filePath string, info os.FileInfo, err error) error { + for path := range paths { + + info, err := os.Stat(path.Location) if err != nil { - return err + return "", fmt.Errorf("error statting file: %w", err) } if !info.Mode().IsRegular() { - return nil + continue } - relPath, err := filepath.Rel(srcFolder, filePath) + relPath, err := filepath.Rel(srcFolder, path.Location) if err != nil { - return err + return "", fmt.Errorf("error getting relative path: %w", err) } - if shouldIgnoreFilepath(relPath, info.IsDir(), ignoreMatchers) { - zap.L().Debug("Ignoring file: " + relPath) - return nil - } + zap.L().Debug("Considering file: " + relPath) header, err := tar.FileInfoHeader(info, "") if err != nil { - return err + return "", fmt.Errorf("error creating tar file info header: %w", err) } header.Name = filepath.ToSlash(relPath) if err := tarWriter.WriteHeader(header); err != nil { - return err + return "", err } if info.IsDir() { zap.L().Debug("Including directory reference: " + relPath) - return nil + continue } - file, err := os.Open(filePath) + file, err := os.Open(path.Location) if err != nil { - return err + return "", fmt.Errorf("error opening file: %w", err) } defer file.Close() written, err := io.Copy(tarWriter, file) if err != nil { - return fmt.Errorf("error copying file content for %s: %w", filePath, err) + return "", fmt.Errorf("error copying file content for %s: %w", path.Location, err) } if written != info.Size() { - return fmt.Errorf("expected to write %d bytes but wrote %d bytes for file %s", info.Size(), written, filePath) + return "", fmt.Errorf("expected to write %d bytes but wrote %d bytes for file %s", info.Size(), written, path.Location) } zap.L().Debug("Including file: " + relPath) - return nil - }) - - if err != nil { - return "", err } return tarGzFile.Name(), nil diff --git a/internal/commands/build.go b/internal/commands/build.go index 6c4d648..86962b8 100644 --- a/internal/commands/build.go +++ b/internal/commands/build.go @@ -101,7 +101,7 @@ var Build = &cli.Command{ } if created.Status == components.BuildStatusFailed { - return fmt.Errorf("Build failed") + return fmt.Errorf("build failed") } return build.Output.Write(created, os.Stdout) @@ -236,6 +236,7 @@ func doBuildCreate(ctx context.Context, hathora *sdk.HathoraCloud, buildTag, bui if err != nil { return nil, fmt.Errorf("no build file available for run: %w", err) } + fileSize := int64(len(file.Content)) params := components.CreateMultipartBuildParams{BuildSizeInBytes: float64(fileSize)} if buildTag != "" {