From 9529eda0e55d551857b2d00c0e3087f576fe1d3d Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Fri, 9 Jan 2026 13:54:23 -0700 Subject: [PATCH] test minio in CircleCI --- .circleci/config.yml | 13 ++++- file/file_test.go | 101 ++------------------------------------ file/minio/client_test.go | 73 +++++++++++++++++---------- file/minio/glob_test.go | 64 ++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 124 deletions(-) create mode 100644 file/minio/glob_test.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 439017d5..5362e1fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,18 @@ executors: jobs: test: - executor: go-executor + docker: + - image: cimg/go:1.25 + - image: minio/minio + command: server /data + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + working_directory: ~/task-tools + environment: + MINIO_ENDPOINT: localhost:9000 + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin steps: - checkout - run: go install github.com/jstemmer/go-junit-report/v2@latest diff --git a/file/file_test.go b/file/file_test.go index ae312ba4..85d9de6d 100644 --- a/file/file_test.go +++ b/file/file_test.go @@ -1,9 +1,7 @@ package file import ( - "context" "errors" - "fmt" "log" "os" "strings" @@ -11,27 +9,11 @@ import ( "time" "github.com/hydronica/trial" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "github.com/pcelvng/task-tools/file/stat" ) -var ( - // minio test server credentials - // - // see: - // https://docs.minio.io/docs/golang-client-api-reference - testEndpoint = "play.minio.io:9000" - testBucket = "task-tools-test" - testClient *minio.Client - - wd string - opts = Options{ - AccessKey: "Q3AM3UQ867SPQQA43P2F", - SecretKey: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", - } -) +var wd string func TestMain(m *testing.M) { log.SetFlags(log.Lshortfile) @@ -39,26 +21,7 @@ func TestMain(m *testing.M) { // setup local files test wd, _ = os.Getwd() - // setup remote (minio/s3/gcs) test - - // s3 client - var err error - testClient, err = minio.New(testEndpoint, &minio.Options{ - Creds: credentials.NewStaticV4(opts.AccessKey, opts.SecretKey, ""), - Secure: true, - }) - if err != nil { - log.Println(err.Error()) - os.Exit(1) - } - - // bucket - err = createBucket(testBucket) - if err != nil { - log.Println(err) - } - - // create files + // create local test directories and files pths := []string{ "test/file-1.txt", "test/file2.txt", @@ -74,20 +37,15 @@ func TestMain(m *testing.M) { os.MkdirAll("./test/other/name/z1", 0750) os.MkdirAll("./test/other/name/z2", 0750) for _, pth := range pths { - if err := createFile("./"+pth, &opts); err != nil { - log.Fatal(err) - } // local - if err := createFile(fmt.Sprintf("mcs://%s/%s/%s", testEndpoint, testBucket, pth), &opts); err != nil { // remote + if err := createFile("./"+pth, nil); err != nil { log.Fatal(err) } - } code := m.Run() // cleanup os.RemoveAll("./test/") - rmBucket(testBucket) os.Exit(code) } @@ -141,46 +99,6 @@ func TestGlob_Local(t *testing.T) { trial.New(fn, cases).SubTest(t) } -func TestGlob_Minio(t *testing.T) { - path := "mcs://" + testEndpoint + "/" + testBucket - fn := func(input string) ([]string, error) { - sts, err := Glob(input, &opts) - files := make([]string, len(sts)) - for i := 0; i < len(sts); i++ { - files[i] = sts[i].Path - } - return files, err - } - cases := trial.Cases[string, []string]{ - "star.txt": { - Input: path + "/test/*.txt", - Expected: []string{path + "/test/file-1.txt", path + "/test/file2.txt"}, - }, - "file?.txt": { - Input: path + "/test/file?.txt", - Expected: []string{path + "/test/file2.txt"}, - }, - "file?.star": { - Input: path + "/test/file?.*", - Expected: []string{path + "/test/file2.txt", path + "/test/file3.gz"}, - }, - "folders": { - Input: path + "/test/*/*", - Expected: []string{path + "/test/f1/file4.gz", path + "/test/f3/file5.txt", path + "/test/f5/file-6.txt"}, - }, - "range": { - Input: path + "/test/f[1-3]/*", - Expected: []string{path + "/test/f1/file4.gz", path + "/test/f3/file5.txt"}, - }, - "folder/star.txt": { - Input: path + "/test/*/*.txt", - Expected: []string{path + "/test/f3/file5.txt", path + "/test/f5/file-6.txt"}, - }, - } - // TODO: test minio glob m3:localhost:9000/*/*.txt - trial.New(fn, cases).SubTest(t) -} - func TestIterStruct(t *testing.T) { fn := func(path string) (stat.Stats, error) { it := NewIterator(path, nil) @@ -228,16 +146,3 @@ func createFile(pth string, opt *Options) error { w.Close() return nil } - -func createBucket(bckt string) error { - exists, err := testClient.BucketExists(context.Background(), bckt) - if err != nil || exists { - return err - } - - return testClient.MakeBucket(context.Background(), bckt, minio.MakeBucketOptions{}) -} - -func rmBucket(bckt string) error { - return testClient.RemoveBucket(context.Background(), bckt) -} diff --git a/file/minio/client_test.go b/file/minio/client_test.go index b38731d8..e8f02c36 100644 --- a/file/minio/client_test.go +++ b/file/minio/client_test.go @@ -12,20 +12,24 @@ import ( ) var ( - // minio test server credentials - // - // see: - // https://docs.minio.io/docs/golang-client-api-reference testBucket = "task-tools-test" testClient *minio.Client - testOption = Option{ - Host: "play.minio.io:9000", - AccessKey: "Q3AM3UQ867SPQQA43P2F", - SecretKey: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", - Secure: true, - } ) +var testOption = Option{ + Host: getEnv("MINIO_ENDPOINT", "localhost:9000"), + AccessKey: getEnv("MINIO_ACCESS_KEY", "minioadmin"), + SecretKey: getEnv("MINIO_SECRET_KEY", "minioadmin"), + Secure: false, // local Docker instance uses HTTP +} + +func getEnv(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} + func TestMain(m *testing.M) { var err error @@ -33,35 +37,54 @@ func TestMain(m *testing.M) { // test client testClient, err = newTestClient() if err != nil { - log.Printf("SKIP: error with test minio client %v %v", testClient, err) + fmt.Println("\033[34mSKIP: Minio server not available - run 'docker run -p 9000:9000 minio/minio server /data' for local testing\033[0m") return } // make test bucket if err := createBucket(testBucket); err != nil { - log.Printf("SKIP: error with create bucket %v", err) + fmt.Printf("\033[34mSKIP: error creating bucket: %v\033[0m\n", err) + return } - // create two test files for reading - pth := fmt.Sprintf("mc://%v/read/test.txt", testBucket) - if err := createTestFile(pth); err != nil { - log.Printf("SKIP: error with create test.txt: %v", err) - return + // create test files for reading + readFiles := []string{ + fmt.Sprintf("mc://%v/read/test.txt", testBucket), + fmt.Sprintf("mc://%v/read/test.gz", testBucket), + } + for _, pth := range readFiles { + if err := createTestFile(pth); err != nil { + fmt.Printf("\033[34mSKIP: error creating %s: %v\033[0m\n", pth, err) + return + } } - // compressed read test file - gzPth := fmt.Sprintf("mc://%v/read/test.gz", testBucket) - if err := createTestFile(gzPth); err != nil { - log.Printf("SKIP: error with test.gz %v", err) - return + // create test files for glob testing + globFiles := []string{ + fmt.Sprintf("mc://%v/glob/file-1.txt", testBucket), + fmt.Sprintf("mc://%v/glob/file2.txt", testBucket), + fmt.Sprintf("mc://%v/glob/file3.gz", testBucket), + fmt.Sprintf("mc://%v/glob/f1/file4.gz", testBucket), + fmt.Sprintf("mc://%v/glob/f3/file5.txt", testBucket), + fmt.Sprintf("mc://%v/glob/f5/file-6.txt", testBucket), + } + for _, pth := range globFiles { + if err := createTestFile(pth); err != nil { + fmt.Printf("\033[34mSKIP: error creating %s: %v\033[0m\n", pth, err) + return + } } // run runRslt := m.Run() - // remove read objects - rmTestFile(pth) - rmTestFile(gzPth) + // remove test objects + for _, pth := range readFiles { + rmTestFile(pth) + } + for _, pth := range globFiles { + rmTestFile(pth) + } // remove test bucket rmBucket(testBucket) diff --git a/file/minio/glob_test.go b/file/minio/glob_test.go new file mode 100644 index 00000000..033077ad --- /dev/null +++ b/file/minio/glob_test.go @@ -0,0 +1,64 @@ +package minio_test + +import ( + "os" + "testing" + + "github.com/hydronica/trial" + "github.com/pcelvng/task-tools/file" +) + +func getEnv(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} + +func TestGlob(t *testing.T) { + endpoint := getEnv("MINIO_ENDPOINT", "localhost:9000") + accessKey := getEnv("MINIO_ACCESS_KEY", "minioadmin") + secretKey := getEnv("MINIO_SECRET_KEY", "minioadmin") + testBucket := "task-tools-test" + + path := "mc://" + endpoint + "/" + testBucket + opts := &file.Options{ + AccessKey: accessKey, + SecretKey: secretKey, + } + fn := func(input string) ([]string, error) { + sts, err := file.Glob(input, opts) + files := make([]string, len(sts)) + for i := 0; i < len(sts); i++ { + files[i] = sts[i].Path + } + return files, err + } + cases := trial.Cases[string, []string]{ + "star.txt": { + Input: path + "/glob/*.txt", + Expected: []string{path + "/glob/file-1.txt", path + "/glob/file2.txt"}, + }, + "file?.txt": { + Input: path + "/glob/file?.txt", + Expected: []string{path + "/glob/file2.txt"}, + }, + "file?.star": { + Input: path + "/glob/file?.*", + Expected: []string{path + "/glob/file2.txt", path + "/glob/file3.gz"}, + }, + "folders": { + Input: path + "/glob/*/*", + Expected: []string{path + "/glob/f1/file4.gz", path + "/glob/f3/file5.txt", path + "/glob/f5/file-6.txt"}, + }, + "range": { + Input: path + "/glob/f[1-3]/*", + Expected: []string{path + "/glob/f1/file4.gz", path + "/glob/f3/file5.txt"}, + }, + "folder/star.txt": { + Input: path + "/glob/*/*.txt", + Expected: []string{path + "/glob/f3/file5.txt", path + "/glob/f5/file-6.txt"}, + }, + } + trial.New(fn, cases).SubTest(t) +}