test minio in CircleCI

.circleci/config.yml

@@ -8,7 +8,18 @@ executors:
 
 jobs:
   test:
-    executor: go-executor 
+    docker:
+      - image: cimg/go:1.25
+      - image: minio/minio
+        command: server /data
+        environment:
+          MINIO_ROOT_USER: minioadmin
+          MINIO_ROOT_PASSWORD: minioadmin
+    working_directory: ~/task-tools
+    environment:
+      MINIO_ENDPOINT: localhost:9000
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
     steps:
       - checkout
       - run: go install github.com/jstemmer/go-junit-report/v2@latest

file/file_test.go

@@ -1,64 +1,27 @@
 package file
 
 import (
-	"context"
 	"errors"
-	"fmt"
 	"log"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/hydronica/trial"
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 
 	"github.com/pcelvng/task-tools/file/stat"
 )
 
-var (
-	// minio test server credentials
-	//
-	// see:
-	// https://docs.minio.io/docs/golang-client-api-reference
-	testEndpoint = "play.minio.io:9000"
-	testBucket   = "task-tools-test"
-	testClient   *minio.Client
-
-	wd   string
-	opts = Options{
-		AccessKey: "Q3AM3UQ867SPQQA43P2F",
-		SecretKey: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
-	}
-)
+var wd string
 
 func TestMain(m *testing.M) {
 	log.SetFlags(log.Lshortfile)
 
 	// setup local files test
 	wd, _ = os.Getwd()
 
-	// setup remote (minio/s3/gcs) test
-
-	// s3 client
-	var err error
-	testClient, err = minio.New(testEndpoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(opts.AccessKey, opts.SecretKey, ""),
-		Secure: true,
-	})
-	if err != nil {
-		log.Println(err.Error())
-		os.Exit(1)
-	}
-
-	// bucket
-	err = createBucket(testBucket)
-	if err != nil {
-		log.Println(err)
-	}
-
-	// create files
+	// create local test directories and files
 	pths := []string{
 		"test/file-1.txt",
 		"test/file2.txt",
@@ -74,20 +37,15 @@ func TestMain(m *testing.M) {
 	os.MkdirAll("./test/other/name/z1", 0750)
 	os.MkdirAll("./test/other/name/z2", 0750)
 	for _, pth := range pths {
-		if err := createFile("./"+pth, &opts); err != nil {
-			log.Fatal(err)
-		} // local
-		if err := createFile(fmt.Sprintf("mcs://%s/%s/%s", testEndpoint, testBucket, pth), &opts); err != nil { // remote
+		if err := createFile("./"+pth, nil); err != nil {
 			log.Fatal(err)
 		}
-
 	}
 
 	code := m.Run()
 
 	// cleanup
 	os.RemoveAll("./test/")
-	rmBucket(testBucket)
 	os.Exit(code)
 }
 
@@ -141,46 +99,6 @@ func TestGlob_Local(t *testing.T) {
 	trial.New(fn, cases).SubTest(t)
 }
 
-func TestGlob_Minio(t *testing.T) {
-	path := "mcs://" + testEndpoint + "/" + testBucket
-	fn := func(input string) ([]string, error) {
-		sts, err := Glob(input, &opts)
-		files := make([]string, len(sts))
-		for i := 0; i < len(sts); i++ {
-			files[i] = sts[i].Path
-		}
-		return files, err
-	}
-	cases := trial.Cases[string, []string]{
-		"star.txt": {
-			Input:    path + "/test/*.txt",
-			Expected: []string{path + "/test/file-1.txt", path + "/test/file2.txt"},
-		},
-		"file?.txt": {
-			Input:    path + "/test/file?.txt",
-			Expected: []string{path + "/test/file2.txt"},
-		},
-		"file?.star": {
-			Input:    path + "/test/file?.*",
-			Expected: []string{path + "/test/file2.txt", path + "/test/file3.gz"},
-		},
-		"folders": {
-			Input:    path + "/test/*/*",
-			Expected: []string{path + "/test/f1/file4.gz", path + "/test/f3/file5.txt", path + "/test/f5/file-6.txt"},
-		},
-		"range": {
-			Input:    path + "/test/f[1-3]/*",
-			Expected: []string{path + "/test/f1/file4.gz", path + "/test/f3/file5.txt"},
-		},
-		"folder/star.txt": {
-			Input:    path + "/test/*/*.txt",
-			Expected: []string{path + "/test/f3/file5.txt", path + "/test/f5/file-6.txt"},
-		},
-	}
-	// TODO: test minio glob m3:localhost:9000/*/*.txt
-	trial.New(fn, cases).SubTest(t)
-}
-
 func TestIterStruct(t *testing.T) {
 	fn := func(path string) (stat.Stats, error) {
 		it := NewIterator(path, nil)
@@ -228,16 +146,3 @@ func createFile(pth string, opt *Options) error {
 	w.Close()
 	return nil
 }
-
-func createBucket(bckt string) error {
-	exists, err := testClient.BucketExists(context.Background(), bckt)
-	if err != nil || exists {
-		return err
-	}
-
-	return testClient.MakeBucket(context.Background(), bckt, minio.MakeBucketOptions{})
-}
-
-func rmBucket(bckt string) error {
-	return testClient.RemoveBucket(context.Background(), bckt)
-}

file/minio/client_test.go

@@ -12,56 +12,79 @@ import (
 )
 
 var (
-	// minio test server credentials
-	//
-	// see:
-	// https://docs.minio.io/docs/golang-client-api-reference
 	testBucket = "task-tools-test"
 	testClient *minio.Client
-	testOption = Option{
-		Host:      "play.minio.io:9000",
-		AccessKey: "Q3AM3UQ867SPQQA43P2F",
-		SecretKey: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
-		Secure:    true,
-	}
 )
 
+var testOption = Option{
+	Host:      getEnv("MINIO_ENDPOINT", "localhost:9000"),
+	AccessKey: getEnv("MINIO_ACCESS_KEY", "minioadmin"),
+	SecretKey: getEnv("MINIO_SECRET_KEY", "minioadmin"),
+	Secure:    false, // local Docker instance uses HTTP
+}
+
+func getEnv(key, fallback string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return fallback
+}
+
 func TestMain(m *testing.M) {
 	var err error
 
 	log.SetFlags(log.Llongfile)
 	// test client
 	testClient, err = newTestClient()
 	if err != nil {
-		log.Printf("SKIP: error with test minio client %v %v", testClient, err)
+		fmt.Println("\033[34mSKIP: Minio server not available - run 'docker run -p 9000:9000 minio/minio server /data' for local testing\033[0m")
 		return
 	}
 
 	// make test bucket
 	if err := createBucket(testBucket); err != nil {
-		log.Printf("SKIP: error with create bucket %v", err)
+		fmt.Printf("\033[34mSKIP: error creating bucket: %v\033[0m\n", err)
+		return
 	}
 
-	// create two test files for reading
-	pth := fmt.Sprintf("mc://%v/read/test.txt", testBucket)
-	if err := createTestFile(pth); err != nil {
-		log.Printf("SKIP: error with create test.txt: %v", err)
-		return
+	// create test files for reading
+	readFiles := []string{
+		fmt.Sprintf("mc://%v/read/test.txt", testBucket),
+		fmt.Sprintf("mc://%v/read/test.gz", testBucket),
+	}
+	for _, pth := range readFiles {
+		if err := createTestFile(pth); err != nil {
+			fmt.Printf("\033[34mSKIP: error creating %s: %v\033[0m\n", pth, err)
+			return
+		}
 	}
 
-	// compressed read test file
-	gzPth := fmt.Sprintf("mc://%v/read/test.gz", testBucket)
-	if err := createTestFile(gzPth); err != nil {
-		log.Printf("SKIP: error with test.gz %v", err)
-		return
+	// create test files for glob testing
+	globFiles := []string{
+		fmt.Sprintf("mc://%v/glob/file-1.txt", testBucket),
+		fmt.Sprintf("mc://%v/glob/file2.txt", testBucket),
+		fmt.Sprintf("mc://%v/glob/file3.gz", testBucket),
+		fmt.Sprintf("mc://%v/glob/f1/file4.gz", testBucket),
+		fmt.Sprintf("mc://%v/glob/f3/file5.txt", testBucket),
+		fmt.Sprintf("mc://%v/glob/f5/file-6.txt", testBucket),
+	}
+	for _, pth := range globFiles {
+		if err := createTestFile(pth); err != nil {
+			fmt.Printf("\033[34mSKIP: error creating %s: %v\033[0m\n", pth, err)
+			return
+		}
 	}
 
 	// run
 	runRslt := m.Run()
 
-	// remove read objects
-	rmTestFile(pth)
-	rmTestFile(gzPth)
+	// remove test objects
+	for _, pth := range readFiles {
+		rmTestFile(pth)
+	}
+	for _, pth := range globFiles {
+		rmTestFile(pth)
+	}
 
 	// remove test bucket
 	rmBucket(testBucket)

file/minio/glob_test.go

@@ -0,0 +1,64 @@
+package minio_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/hydronica/trial"
+	"github.com/pcelvng/task-tools/file"
+)
+
+func getEnv(key, fallback string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return fallback
+}
+
+func TestGlob(t *testing.T) {
+	endpoint := getEnv("MINIO_ENDPOINT", "localhost:9000")
+	accessKey := getEnv("MINIO_ACCESS_KEY", "minioadmin")
+	secretKey := getEnv("MINIO_SECRET_KEY", "minioadmin")
+	testBucket := "task-tools-test"
+
+	path := "mc://" + endpoint + "/" + testBucket
+	opts := &file.Options{
+		AccessKey: accessKey,
+		SecretKey: secretKey,
+	}
+	fn := func(input string) ([]string, error) {
+		sts, err := file.Glob(input, opts)
+		files := make([]string, len(sts))
+		for i := 0; i < len(sts); i++ {
+			files[i] = sts[i].Path
+		}
+		return files, err
+	}
+	cases := trial.Cases[string, []string]{
+		"star.txt": {
+			Input:    path + "/glob/*.txt",
+			Expected: []string{path + "/glob/file-1.txt", path + "/glob/file2.txt"},
+		},
+		"file?.txt": {
+			Input:    path + "/glob/file?.txt",
+			Expected: []string{path + "/glob/file2.txt"},
+		},
+		"file?.star": {
+			Input:    path + "/glob/file?.*",
+			Expected: []string{path + "/glob/file2.txt", path + "/glob/file3.gz"},
+		},
+		"folders": {
+			Input:    path + "/glob/*/*",
+			Expected: []string{path + "/glob/f1/file4.gz", path + "/glob/f3/file5.txt", path + "/glob/f5/file-6.txt"},
+		},
+		"range": {
+			Input:    path + "/glob/f[1-3]/*",
+			Expected: []string{path + "/glob/f1/file4.gz", path + "/glob/f3/file5.txt"},
+		},
+		"folder/star.txt": {
+			Input:    path + "/glob/*/*.txt",
+			Expected: []string{path + "/glob/f3/file5.txt", path + "/glob/f5/file-6.txt"},
+		},
+	}
+	trial.New(fn, cases).SubTest(t)
+}