From b780f0c2c3b12740867e9493d20c05f72ea5bff1 Mon Sep 17 00:00:00 2001 From: Johannes Ziemke Date: Tue, 14 Jan 2025 18:59:44 +0100 Subject: [PATCH 1/8] Update go and golangci-lint-action --- .github/workflows/default.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/default.yaml b/.github/workflows/default.yaml index 8a68638..97cccc9 100644 --- a/.github/workflows/default.yaml +++ b/.github/workflows/default.yaml @@ -14,10 +14,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: - go-version: "1.20" - - uses: golangci/golangci-lint-action@v3 + go-version: "1.22" + - uses: golangci/golangci-lint-action@v6 - run: go test -v ./... build: @@ -35,9 +35,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: - go-version: "1.20" + go-version: "1.22" - run: | ARCH=${{ matrix.arch }} # For aarch64 we need to create a separate python wheel but still use arm64 for the go build From eeb02a42b164200c15022222e8507b15dc3c000d Mon Sep 17 00:00:00 2001 From: Johannes Ziemke Date: Tue, 14 Jan 2025 15:08:27 +0100 Subject: [PATCH 2/8] Add `agent build` function --- go.mod | 31 ++++++++++------- go.sum | 66 +++++++++++++++++++++-------------- pkg/cmd/agent/agent.go | 1 + pkg/cmd/agent/build.go | 69 +++++++++++++++++++++++++++++++++++++ pkg/container/docker.go | 21 +++++++++++ pkg/container/runner.go | 10 ++++++ pkg/diambra/diambra_test.go | 4 +++ 7 files changed, 164 insertions(+), 38 deletions(-) create mode 100644 pkg/cmd/agent/build.go diff --git a/go.mod b/go.mod index 5032dd7..d00d00e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/diambra/cli -go 1.20 +go 1.22 + +toolchain go1.22.4 require ( github.com/diambra/init v0.0.0-20230711105936-6921ee0b2542 @@ -9,32 +11,37 @@ require ( github.com/moby/term v0.5.0 github.com/sergi/go-diff v1.3.1 github.com/spf13/cobra v1.7.0 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.4 ) require ( github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/containerd/containerd v1.7.25 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.3.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/runc v1.2.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/sync v0.10.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect + google.golang.org/protobuf v1.35.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.2.0 // indirect ) require ( - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/containerd/console v1.0.3 + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/containerd/console v1.0.4 github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.5.0 // indirect @@ -43,12 +50,12 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc4 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/spf13/pflag v1.0.5 golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 - google.golang.org/grpc v1.57.1 + google.golang.org/grpc v1.59.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 602c5a0..fa74630 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,18 @@ +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.11.7 h1:vl/nj3Bar/CvJSYo7gIQPyRWc9f3c6IeSNavBTSZNZQ= +github.com/Microsoft/hcsshim v0.11.7/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= +github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= +github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= +github.com/containerd/containerd v1.7.25 h1:khEQOAXOEJalRO228yzVsuASLH42vT7DIo9Ss+9SMFQ= +github.com/containerd/containerd v1.7.25/go.mod h1:tWfHzVI0azhw4CT2vaIjsb2CoV4LJ9PrMPaULAr21Ok= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -27,15 +34,17 @@ github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -48,14 +57,24 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= -github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/runc v1.2.4 h1:yWFgLkghp71D76Fa0l349yAl5g4Gse7DPYNlvkQ9Eiw= +github.com/opencontainers/runc v1.2.4/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -63,7 +82,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -71,8 +91,9 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -80,8 +101,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -91,15 +110,14 @@ golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -116,20 +134,16 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf h1:guOdSPaeFgN+jEJwTo1dQ71hdBm+yKSCCKuTRkJzcVo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= -google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= -google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/cmd/agent/agent.go b/pkg/cmd/agent/agent.go index f338676..16fb8d1 100644 --- a/pkg/cmd/agent/agent.go +++ b/pkg/cmd/agent/agent.go @@ -30,5 +30,6 @@ func NewCommand(logger *log.Logger) *cobra.Command { cmd.AddCommand(NewInitCmd(logger)) cmd.AddCommand(NewSubmitCmd(logger)) cmd.AddCommand(NewTestCmd(logger)) + cmd.AddCommand(NewBuildCmd(logger)) return cmd } diff --git a/pkg/cmd/agent/build.go b/pkg/cmd/agent/build.go new file mode 100644 index 0000000..e58d052 --- /dev/null +++ b/pkg/cmd/agent/build.go @@ -0,0 +1,69 @@ +/* + * Copyright 2025 The DIAMBRA Authors + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package agent + +import ( + "os" + + "github.com/diambra/cli/pkg/container" + "github.com/diambra/cli/pkg/log" + dclient "github.com/docker/docker/client" + "github.com/go-kit/log/level" + "github.com/spf13/cobra" +) + +func NewBuildCmd(logger *log.Logger) *cobra.Command { + tag := "" + cmd := &cobra.Command{ + Use: "build [path/to/agent]", + Short: "Build a container image for submission", + Long: `This builds a container image from the given path.`, + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + args = []string{"."} + } + client, err := dclient.NewClientWithOpts(dclient.FromEnv, dclient.WithAPIVersionNegotiation()) + if err != nil { + level.Error(logger).Log("msg", "failed to create docker client", "err", err) + os.Exit(1) + } + + runner, err := container.NewDockerRunner(logger, client, false) + if err != nil { + level.Error(logger).Log("msg", "failed to create docker runner", "err", err) + os.Exit(1) + } + + if tag == "" { + var err error + tag, err = container.TagFromDir(args[0]) + if err != nil { + level.Error(logger).Log("msg", "failed to get tag from dir", "err", err) + os.Exit(1) + } + } + + if err := runner.Build(args[0], tag); err != nil { + level.Error(logger).Log("msg", "failed to build agent", "err", err) + os.Exit(1) + } + }, + Args: cobra.MaximumNArgs(1), + } + cmd.Flags().StringVarP(&tag, "tag", "t", tag, "Tag for the image") + + return cmd +} diff --git a/pkg/container/docker.go b/pkg/container/docker.go index a1ba0f8..76d9fb5 100644 --- a/pkg/container/docker.go +++ b/pkg/container/docker.go @@ -30,6 +30,7 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/client" + "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/go-connections/nat" "github.com/go-kit/log" @@ -285,3 +286,23 @@ func (r *DockerRunner) StopAll() error { } return nil } + +func (r *DockerRunner) Build(path string, tag string) error { + ctx := context.Background() + + context, err := archive.TarWithOptions(path, &archive.TarOptions{}) + if err != nil { + return err + } + defer context.Close() + resp, err := r.Client.ImageBuild(ctx, context, types.ImageBuildOptions{ + Tags: []string{tag}, + }) + if err != nil { + return err + } + defer resp.Body.Close() + + termFd, isTerm := term.GetFdInfo(os.Stdout) + return jsonmessage.DisplayJSONMessagesStream(resp.Body, io.Writer(os.Stderr), termFd, isTerm, nil) +} diff --git a/pkg/container/runner.go b/pkg/container/runner.go index e66cc10..f9c5947 100644 --- a/pkg/container/runner.go +++ b/pkg/container/runner.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strconv" "strings" @@ -118,4 +119,13 @@ type Runner interface { StopAll() error Attach(id string) (io.WriteCloser, io.ReadCloser, error) Wait(id string) (int, error) + Build(path, tag string) error +} + +func TagFromDir(dir string) (string, error) { + wd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current directory: %w", err) + } + return filepath.Base(filepath.Join(wd, dir)), nil } diff --git a/pkg/diambra/diambra_test.go b/pkg/diambra/diambra_test.go index bc014f6..f80ea2d 100644 --- a/pkg/diambra/diambra_test.go +++ b/pkg/diambra/diambra_test.go @@ -58,6 +58,10 @@ func (r *mockRunner) StopAll() error { panic("not implemented") // TODO: Implement } +func (r *mockRunner) Build(path, tag string) error { + panic("not implemented") // TODO: Implement +} + func TestDiambra(t *testing.T) { var ( logger = log.NewLogfmtLogger(os.Stderr) From e63ade37abf20ebc01349db3e5976a68d38f8731 Mon Sep 17 00:00:00 2001 From: Johannes Ziemke Date: Tue, 14 Jan 2025 16:40:38 +0100 Subject: [PATCH 3/8] Add agent build-and-push --- pkg/cmd/agent/agent.go | 1 + pkg/cmd/agent/build_and_push.go | 115 ++++++++++++++++++++++++++++++++ pkg/container/docker.go | 32 +++++++++ pkg/container/runner.go | 1 + pkg/diambra/client/registry.go | 44 ++++++++++++ pkg/diambra/diambra_test.go | 4 ++ 6 files changed, 197 insertions(+) create mode 100644 pkg/cmd/agent/build_and_push.go create mode 100644 pkg/diambra/client/registry.go diff --git a/pkg/cmd/agent/agent.go b/pkg/cmd/agent/agent.go index 16fb8d1..cba467f 100644 --- a/pkg/cmd/agent/agent.go +++ b/pkg/cmd/agent/agent.go @@ -31,5 +31,6 @@ func NewCommand(logger *log.Logger) *cobra.Command { cmd.AddCommand(NewSubmitCmd(logger)) cmd.AddCommand(NewTestCmd(logger)) cmd.AddCommand(NewBuildCmd(logger)) + cmd.AddCommand(NewBuildAndPushCmd(logger)) return cmd } diff --git a/pkg/cmd/agent/build_and_push.go b/pkg/cmd/agent/build_and_push.go new file mode 100644 index 0000000..ae63db6 --- /dev/null +++ b/pkg/cmd/agent/build_and_push.go @@ -0,0 +1,115 @@ +/* + * Copyright 2024 The DIAMBRA Authors + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package agent + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + + "github.com/diambra/cli/pkg/container" + "github.com/diambra/cli/pkg/diambra/client" + "github.com/diambra/cli/pkg/log" + dclient "github.com/docker/docker/client" + "github.com/go-kit/log/level" + "github.com/spf13/cobra" +) + +const defaultCredPath = "~/.diambra/credentials" + +func NewBuildAndPushCmd(logger *log.Logger) *cobra.Command { + var ( + tag = "" + credPath = defaultCredPath + ) + + cmd := &cobra.Command{ + Use: "build-and-push [path/to/agent]", + Short: "Build a container image and push it to the DIAMBRA registry", + Long: `This builds a container image from the given path, then pushes it to the DIAMBRA registry.`, + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + args = []string{"."} + } + dc, err := dclient.NewClientWithOpts(dclient.FromEnv, dclient.WithAPIVersionNegotiation()) + if err != nil { + level.Error(logger).Log("msg", "failed to create docker client", "err", err) + os.Exit(1) + } + + runner, err := container.NewDockerRunner(logger, dc, false) + if err != nil { + level.Error(logger).Log("msg", "failed to create docker runner", "err", err) + os.Exit(1) + } + + if credPath == defaultCredPath { + homedir, err := os.UserHomeDir() + if err != nil { + level.Error(logger).Log("msg", "couldn't get homedir", "err", err.Error()) + os.Exit(1) + } + credPath = filepath.Join(homedir, ".diambra", "credentials") + } + + cl, err := client.NewClient(logger, credPath) + if err != nil { + level.Error(logger).Log("msg", "failed to create client", "err", err.Error()) + os.Exit(1) + } + + if tag == "" { + var err error + tag, err = container.TagFromDir(args[0]) + if err != nil { + level.Error(logger).Log("msg", "failed to get tag from dir", "err", err) + os.Exit(1) + } + } + + credentials, err := cl.Credentials() + if err != nil { + level.Error(logger).Log("msg", "failed to get credentials", "err", err.Error()) + os.Exit(1) + } + + repositoryURL, err := url.Parse(credentials.Repository) + if err != nil { + level.Error(logger).Log("msg", "failed to parse repository URL", "err", err) + os.Exit(1) + } + + tag = fmt.Sprintf("%s%s:%s", repositoryURL.Host, repositoryURL.Path, tag) + level.Info(logger).Log("msg", "Building agent", "tag", tag) + + if err := runner.Build(args[0], tag); err != nil { + level.Error(logger).Log("msg", "failed to build agent", "err", err) + os.Exit(1) + } + + if err := runner.Push(tag, credentials.Username, credentials.Password, repositoryURL.Host); err != nil { + level.Error(logger).Log("msg", "failed to push agent", "err", err) + os.Exit(1) + } + }, + Args: cobra.MaximumNArgs(1), + } + cmd.Flags().StringVarP(&tag, "tag", "t", tag, "Tag for the image") + cmd.Flags().StringVar(&credPath, "path.credentials", defaultCredPath, "Path to credentials file") + + return cmd +} diff --git a/pkg/container/docker.go b/pkg/container/docker.go index 76d9fb5..bc13494 100644 --- a/pkg/container/docker.go +++ b/pkg/container/docker.go @@ -17,6 +17,8 @@ package container import ( "context" + "encoding/base64" + "encoding/json" "fmt" "io" "os" @@ -306,3 +308,33 @@ func (r *DockerRunner) Build(path string, tag string) error { termFd, isTerm := term.GetFdInfo(os.Stdout) return jsonmessage.DisplayJSONMessagesStream(resp.Body, io.Writer(os.Stderr), termFd, isTerm, nil) } + +type DockerAuth struct { + Username string `json:"username"` + Password string `json:"password"` + ServerAddress string `json:"serveraddress"` +} + +func (r *DockerRunner) Push(tag, username, password, host string) error { + ctx := context.Background() + auth := DockerAuth{ + Username: username, + Password: password, + ServerAddress: host, + } + authStr, err := json.Marshal(auth) + if err != nil { + return err + } + + resp, err := r.Client.ImagePush(ctx, tag, types.ImagePushOptions{ + RegistryAuth: base64.URLEncoding.EncodeToString(authStr), + }) + if err != nil { + return err + } + defer resp.Close() + + termFd, isTerm := term.GetFdInfo(os.Stdout) + return jsonmessage.DisplayJSONMessagesStream(resp, io.Writer(os.Stderr), termFd, isTerm, nil) +} diff --git a/pkg/container/runner.go b/pkg/container/runner.go index f9c5947..79802aa 100644 --- a/pkg/container/runner.go +++ b/pkg/container/runner.go @@ -120,6 +120,7 @@ type Runner interface { Attach(id string) (io.WriteCloser, io.ReadCloser, error) Wait(id string) (int, error) Build(path, tag string) error + Push(tag, username, password, host string) error } func TagFromDir(dir string) (string, error) { diff --git a/pkg/diambra/client/registry.go b/pkg/diambra/client/registry.go new file mode 100644 index 0000000..0991b11 --- /dev/null +++ b/pkg/diambra/client/registry.go @@ -0,0 +1,44 @@ +/* + * Copyright 2024 The DIAMBRA Authors + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "encoding/json" + "fmt" + "net/http" +) + +type CredentialsResponse struct { + Username string `json:"username"` + Password string `json:"password"` + Repository string `json:"repository"` +} + +func (c *Client) Credentials() (*CredentialsResponse, error) { + resp, err := c.Request("POST", "registry/credentials", nil, true) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get credentials: %s", resp.Status) + } + var u CredentialsResponse + if err := json.NewDecoder(resp.Body).Decode(&u); err != nil { + return nil, err + } + return &u, nil +} diff --git a/pkg/diambra/diambra_test.go b/pkg/diambra/diambra_test.go index f80ea2d..dc507e5 100644 --- a/pkg/diambra/diambra_test.go +++ b/pkg/diambra/diambra_test.go @@ -62,6 +62,10 @@ func (r *mockRunner) Build(path, tag string) error { panic("not implemented") // TODO: Implement } +func (r *mockRunner) Push(tag, username, password, host string) error { + panic("not implemented") // TODO: Implement +} + func TestDiambra(t *testing.T) { var ( logger = log.NewLogfmtLogger(os.Stderr) From 1a8e2ff6b6762df7d2c50fee779a95246833b873 Mon Sep 17 00:00:00 2001 From: Johannes Ziemke Date: Tue, 14 Jan 2025 18:45:40 +0100 Subject: [PATCH 4/8] Build-and-push in submit when arg is dir --- pkg/cmd/agent/build.go | 8 +--- pkg/cmd/agent/build_and_push.go | 8 +--- pkg/cmd/agent/submit.go | 74 +++++++++++++++++++++++++++++++-- pkg/cmd/agent/test.go | 7 +--- pkg/cmd/arena/down.go | 8 +--- pkg/cmd/arena/up.go | 7 +--- pkg/cmd/run.go | 7 +--- pkg/container/docker.go | 11 +++-- pkg/diambra/config.go | 4 +- pkg/git/git.go | 66 +++++++++++++++++++++++++++++ 10 files changed, 152 insertions(+), 48 deletions(-) create mode 100644 pkg/git/git.go diff --git a/pkg/cmd/agent/build.go b/pkg/cmd/agent/build.go index e58d052..b6410b5 100644 --- a/pkg/cmd/agent/build.go +++ b/pkg/cmd/agent/build.go @@ -20,7 +20,6 @@ import ( "github.com/diambra/cli/pkg/container" "github.com/diambra/cli/pkg/log" - dclient "github.com/docker/docker/client" "github.com/go-kit/log/level" "github.com/spf13/cobra" ) @@ -35,13 +34,8 @@ func NewBuildCmd(logger *log.Logger) *cobra.Command { if len(args) == 0 { args = []string{"."} } - client, err := dclient.NewClientWithOpts(dclient.FromEnv, dclient.WithAPIVersionNegotiation()) - if err != nil { - level.Error(logger).Log("msg", "failed to create docker client", "err", err) - os.Exit(1) - } - runner, err := container.NewDockerRunner(logger, client, false) + runner, err := container.NewDockerRunner(logger, false) if err != nil { level.Error(logger).Log("msg", "failed to create docker runner", "err", err) os.Exit(1) diff --git a/pkg/cmd/agent/build_and_push.go b/pkg/cmd/agent/build_and_push.go index ae63db6..3fa4827 100644 --- a/pkg/cmd/agent/build_and_push.go +++ b/pkg/cmd/agent/build_and_push.go @@ -24,7 +24,6 @@ import ( "github.com/diambra/cli/pkg/container" "github.com/diambra/cli/pkg/diambra/client" "github.com/diambra/cli/pkg/log" - dclient "github.com/docker/docker/client" "github.com/go-kit/log/level" "github.com/spf13/cobra" ) @@ -45,13 +44,8 @@ func NewBuildAndPushCmd(logger *log.Logger) *cobra.Command { if len(args) == 0 { args = []string{"."} } - dc, err := dclient.NewClientWithOpts(dclient.FromEnv, dclient.WithAPIVersionNegotiation()) - if err != nil { - level.Error(logger).Log("msg", "failed to create docker client", "err", err) - os.Exit(1) - } - runner, err := container.NewDockerRunner(logger, dc, false) + runner, err := container.NewDockerRunner(logger, false) if err != nil { level.Error(logger).Log("msg", "failed to create docker runner", "err", err) os.Exit(1) diff --git a/pkg/cmd/agent/submit.go b/pkg/cmd/agent/submit.go index 05fccac..97bb60e 100644 --- a/pkg/cmd/agent/submit.go +++ b/pkg/cmd/agent/submit.go @@ -17,11 +17,15 @@ package agent import ( "fmt" + "net/url" "os" "path/filepath" + "time" + "github.com/diambra/cli/pkg/container" "github.com/diambra/cli/pkg/diambra" "github.com/diambra/cli/pkg/diambra/client" + "github.com/diambra/cli/pkg/git" "github.com/diambra/cli/pkg/log" "github.com/go-kit/log/level" "github.com/spf13/cobra" @@ -29,9 +33,14 @@ import ( ) func NewSubmitCmd(logger *log.Logger) *cobra.Command { - dump := false - submissionConfig := diambra.SubmissionConfig{} + var ( + dump = false + submissionConfig = diambra.SubmissionConfig{} + name = "" + version = "" + ) submissionConfig.RegisterCredentialsProviders() + c, err := diambra.NewConfig(logger) if err != nil { level.Error(logger).Log("msg", err.Error()) @@ -39,9 +48,9 @@ func NewSubmitCmd(logger *log.Logger) *cobra.Command { } cmd := &cobra.Command{ - Use: "submit [flags] {--submission.manifest submission-manifest.yaml | docker-image} [args/command(s) ...]", + Use: "submit [flags] (directory | --submission.manifest=submission-manifest.yaml | docker-image) [args/command(s) ...]", Short: "Submits an agent for evaluation", - Long: `This takes a docker image or submission manifest and submits it for evaluation.`, + Long: `This takes a directory, existing docker image or submission manifest and submits it for evaluation.`, Run: func(cmd *cobra.Command, args []string) { if err := diambra.EnsureCredentials(logger, c.CredPath); err != nil { level.Error(logger).Log("msg", err.Error()) @@ -61,11 +70,66 @@ func NewSubmitCmd(logger *log.Logger) *cobra.Command { fmt.Println(string(b)) return } + cl, err := client.NewClient(logger, c.CredPath) if err != nil { level.Error(logger).Log("msg", "failed to create client", "err", err.Error()) os.Exit(1) } + // If submission.Image is a directory, we build and push it, then update the name to the resulting image + if stat, err := os.Stat(submission.Manifest.Image); err == nil && stat.IsDir() { + context := submission.Manifest.Image + level.Info(logger).Log("msg", "Building and pushing image", "context", context) + + if name == "" { + name, err = container.TagFromDir(context) + if err != nil { + level.Error(logger).Log("msg", "failed to get tag from dir", "err", err) + os.Exit(1) + } + } + + if version == "" { + version, err = git.GitHeadSHAShort(context, 0) + if err != nil { + level.Warn(logger).Log("msg", "failed to get git head sha", "err", err) + version = time.Now().Format("20060102-150405") + } + } + level.Info(logger).Log("msg", "Building agent", "name", name, "version", version) + runner, err := container.NewDockerRunner(logger, false) + if err != nil { + level.Error(logger).Log("msg", "failed to create docker runner", "err", err) + os.Exit(1) + } + credentials, err := cl.Credentials() + if err != nil { + level.Error(logger).Log("msg", "failed to get credentials", "err", err.Error()) + os.Exit(1) + } + + repositoryURL, err := url.Parse(credentials.Repository) + if err != nil { + level.Error(logger).Log("msg", "failed to parse repository URL", "err", err) + os.Exit(1) + } + + tag := fmt.Sprintf("%s%s:%s-%s", repositoryURL.Host, repositoryURL.Path, name, version) + + if err := runner.Build(context, tag); err != nil { + level.Error(logger).Log("msg", "failed to build and push image", "err", err.Error()) + os.Exit(1) + } + if err := runner.Push(tag, credentials.Username, credentials.Password, repositoryURL.Host); err != nil { + level.Error(logger).Log("msg", "failed to push agent", "err", err) + os.Exit(1) + } + + submission.Manifest.Image = tag + } else { + level.Warn(logger).Log("msg", "Using existing images or submission manifest is not recommended and might get deprecated in the future") + } + id, err := cl.Submit(submission) if err != nil { level.Error(logger).Log("msg", "failed to submit agent", "err", err.Error()) @@ -79,5 +143,7 @@ func NewSubmitCmd(logger *log.Logger) *cobra.Command { cmd.Flags().StringVar(&c.CredPath, "path.credentials", filepath.Join(c.Home, ".diambra/credentials"), "Path to credentials file") cmd.Flags().BoolVar(&dump, "dump", false, "Dump the manifest to stdout instead of submitting") cmd.Flags().SetInterspersed(false) + cmd.Flags().StringVar(&name, "name", name, "Name of the agent image (only used when giving a directory)") + cmd.Flags().StringVar(&version, "version", version, "Version of the agent image (only used when giving a directory)") return cmd } diff --git a/pkg/cmd/agent/test.go b/pkg/cmd/agent/test.go index 01839bb..4130106 100644 --- a/pkg/cmd/agent/test.go +++ b/pkg/cmd/agent/test.go @@ -13,7 +13,6 @@ import ( "github.com/diambra/cli/pkg/diambra" "github.com/diambra/cli/pkg/diambra/client" "github.com/diambra/cli/pkg/log" - dclient "github.com/docker/docker/client" "github.com/go-kit/log/level" "github.com/spf13/cobra" ) @@ -58,11 +57,7 @@ func NewTestCmd(logger *log.Logger) *cobra.Command { func TestFn(logger *log.Logger, c *diambra.EnvConfig, submission *client.Submission) error { level.Debug(logger).Log("manifest", fmt.Sprintf("%#v", submission.Manifest), "config", fmt.Sprintf("%#v", c)) - client, err := dclient.NewClientWithOpts(dclient.FromEnv, dclient.WithAPIVersionNegotiation()) - if err != nil { - return err - } - runner, err := container.NewDockerRunner(logger, client, c.AutoRemove) + runner, err := container.NewDockerRunner(logger, c.AutoRemove) if err != nil { return err } diff --git a/pkg/cmd/arena/down.go b/pkg/cmd/arena/down.go index 7e57c37..1f5969c 100644 --- a/pkg/cmd/arena/down.go +++ b/pkg/cmd/arena/down.go @@ -20,7 +20,6 @@ import ( "github.com/diambra/cli/pkg/container" "github.com/diambra/cli/pkg/log" - "github.com/docker/docker/client" "github.com/go-kit/log/level" "github.com/spf13/cobra" ) @@ -31,12 +30,7 @@ func NewDownCmd(logger *log.Logger) *cobra.Command { Short: "Stop DIAMBRA Arena", Long: `This stops a DIAMBRA Arena running in the background.`, Run: func(_ *cobra.Command, _ []string) { - client, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - level.Error(logger).Log("msg", "failed to create docker client", "err", err.Error()) - os.Exit(1) - } - runner, err := container.NewDockerRunner(logger, client, true) + runner, err := container.NewDockerRunner(logger, true) if err != nil { level.Error(logger).Log("msg", "msg", "failed to create runner", "err", err.Error()) os.Exit(1) diff --git a/pkg/cmd/arena/up.go b/pkg/cmd/arena/up.go index 635abfa..fc1e4d6 100644 --- a/pkg/cmd/arena/up.go +++ b/pkg/cmd/arena/up.go @@ -24,7 +24,6 @@ import ( "github.com/diambra/cli/pkg/container" "github.com/diambra/cli/pkg/diambra" "github.com/diambra/cli/pkg/log" - "github.com/docker/docker/client" "github.com/go-kit/log/level" "github.com/spf13/cobra" ) @@ -68,11 +67,7 @@ func NewUpCmd(logger *log.Logger) *cobra.Command { func RunFn(logger *log.Logger, c *diambra.EnvConfig, args []string) error { level.Debug(logger).Log("config", fmt.Sprintf("%#v", c)) - client, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - return err - } - runner, err := container.NewDockerRunner(logger, client, c.AutoRemove) + runner, err := container.NewDockerRunner(logger, c.AutoRemove) if err != nil { return err } diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 947224c..6f282e3 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -28,7 +28,6 @@ import ( "github.com/diambra/cli/pkg/diambra" "github.com/diambra/cli/pkg/log" - "github.com/docker/docker/client" "github.com/go-kit/log/level" "github.com/spf13/cobra" ) @@ -76,11 +75,7 @@ The flag --agent-image can be used to run the commands in the given image.`, func RunFn(logger *log.Logger, c *diambra.EnvConfig, args []string) error { level.Debug(logger).Log("config", fmt.Sprintf("%#v", c)) - client, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - if err != nil { - return err - } - runner, err := container.NewDockerRunner(logger, client, c.AutoRemove) + runner, err := container.NewDockerRunner(logger, c.AutoRemove) if err != nil { return err } diff --git a/pkg/container/docker.go b/pkg/container/docker.go index bc13494..a8079d3 100644 --- a/pkg/container/docker.go +++ b/pkg/container/docker.go @@ -48,14 +48,19 @@ type DockerRunner struct { AutoRemove bool } -func NewDockerRunner(logger log.Logger, client *client.Client, autoRemove bool) (*DockerRunner, error) { - _, err := client.Ping(context.TODO()) +func NewDockerRunner(logger log.Logger, autoRemove bool) (*DockerRunner, error) { + cl, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + level.Error(logger).Log("msg", "failed to create docker client", "err", err.Error()) + os.Exit(1) + } + _, err = cl.Ping(context.TODO()) if err != nil { return nil, fmt.Errorf("couldn't connect to docker. Make sure your user has docker access: %w", err) } return &DockerRunner{ Logger: logger, - Client: client, + Client: cl, TimeoutStop: 10 * time.Second, AutoRemove: autoRemove, }, nil diff --git a/pkg/diambra/config.go b/pkg/diambra/config.go index 2519e57..cf5e54a 100644 --- a/pkg/diambra/config.go +++ b/pkg/diambra/config.go @@ -232,7 +232,7 @@ const ( DifficultyHard Difficulty = "hard" ) -var ErrInvalidArgs = errors.New("either image, manifest path or submission id must be provided") +var ErrInvalidArgs = errors.New("either directory, image, manifest path or submission id must be provided") type SubmissionConfig struct { Mode string @@ -296,7 +296,7 @@ func (c *SubmissionConfig) Submission(config *EnvConfig, args []string) (*client } default: if nargs == 0 { - return nil, fmt.Errorf("either image, manifest path or submission id must be provided") + return nil, fmt.Errorf("either directory, image, manifest path or submission id must be provided") } // If we don't have a manifest, args are image and args manifest = &client.Manifest{} diff --git a/pkg/git/git.go b/pkg/git/git.go new file mode 100644 index 0000000..a8d443a --- /dev/null +++ b/pkg/git/git.go @@ -0,0 +1,66 @@ +package git + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +const RefPrefix = "ref: " + +func findGitDir(startDir string) (string, error) { + dir, err := filepath.Abs(startDir) + if err != nil { + return "", fmt.Errorf("unable to resolve absolute path: %w", err) + } + + for { + gitPath := filepath.Join(dir, ".git") + if info, err := os.Stat(gitPath); err == nil && info.IsDir() { + return gitPath, nil + } + + parent := filepath.Dir(dir) + // root + if parent == dir { + break + } + dir = parent + } + return "", fmt.Errorf("no .git directory found in parents of %s", dir) +} + +func GitHeadSHA(dir string) (string, error) { + dir, err := findGitDir(dir) + if err != nil { + return "", err + } + file, err := os.ReadFile(filepath.Join(dir, "HEAD")) + if err != nil { + return "", err + } + if !strings.HasPrefix(string(file), RefPrefix) { + return string(file), nil + } + + refFile, err := os.ReadFile(filepath.Join(dir, strings.TrimSpace(string(file)[len(RefPrefix):]))) + if err != nil { + return "", err + } + return strings.TrimSpace(string(refFile)), nil +} + +func GitHeadSHAShort(dir string, n int) (string, error) { + if n <= 0 { + n = 7 + } + sha, err := GitHeadSHA(dir) + if err != nil { + return "", err + } + if len(sha) < n { + return sha, nil + } + return sha[:n], nil +} From 65cdefa6c4a90bbb25ddf4647b6531804021eef9 Mon Sep 17 00:00:00 2001 From: Johannes Ziemke Date: Mon, 20 Jan 2025 17:26:40 +0100 Subject: [PATCH 5/8] Refactor runner auth --- pkg/cmd/agent/build_and_push.go | 3 ++- pkg/cmd/agent/submit.go | 15 ++++++++------- pkg/container/docker.go | 34 +++++++++++++++++++-------------- pkg/container/runner.go | 3 ++- pkg/diambra/diambra_test.go | 6 +++++- 5 files changed, 37 insertions(+), 24 deletions(-) diff --git a/pkg/cmd/agent/build_and_push.go b/pkg/cmd/agent/build_and_push.go index 3fa4827..621fcbd 100644 --- a/pkg/cmd/agent/build_and_push.go +++ b/pkg/cmd/agent/build_and_push.go @@ -87,6 +87,7 @@ func NewBuildAndPushCmd(logger *log.Logger) *cobra.Command { os.Exit(1) } + runner.Login(credentials.Username, credentials.Password, repositoryURL.Host) tag = fmt.Sprintf("%s%s:%s", repositoryURL.Host, repositoryURL.Path, tag) level.Info(logger).Log("msg", "Building agent", "tag", tag) @@ -95,7 +96,7 @@ func NewBuildAndPushCmd(logger *log.Logger) *cobra.Command { os.Exit(1) } - if err := runner.Push(tag, credentials.Username, credentials.Password, repositoryURL.Host); err != nil { + if err := runner.Push(tag); err != nil { level.Error(logger).Log("msg", "failed to push agent", "err", err) os.Exit(1) } diff --git a/pkg/cmd/agent/submit.go b/pkg/cmd/agent/submit.go index 97bb60e..96236bb 100644 --- a/pkg/cmd/agent/submit.go +++ b/pkg/cmd/agent/submit.go @@ -96,31 +96,32 @@ func NewSubmitCmd(logger *log.Logger) *cobra.Command { version = time.Now().Format("20060102-150405") } } - level.Info(logger).Log("msg", "Building agent", "name", name, "version", version) - runner, err := container.NewDockerRunner(logger, false) - if err != nil { - level.Error(logger).Log("msg", "failed to create docker runner", "err", err) - os.Exit(1) - } credentials, err := cl.Credentials() if err != nil { level.Error(logger).Log("msg", "failed to get credentials", "err", err.Error()) os.Exit(1) } + level.Info(logger).Log("msg", "Building agent", "name", name, "version", version) + runner, err := container.NewDockerRunner(logger, false) + if err != nil { + level.Error(logger).Log("msg", "failed to create docker runner", "err", err) + os.Exit(1) + } repositoryURL, err := url.Parse(credentials.Repository) if err != nil { level.Error(logger).Log("msg", "failed to parse repository URL", "err", err) os.Exit(1) } + runner.Login(credentials.Username, credentials.Password, repositoryURL.Host) tag := fmt.Sprintf("%s%s:%s-%s", repositoryURL.Host, repositoryURL.Path, name, version) if err := runner.Build(context, tag); err != nil { level.Error(logger).Log("msg", "failed to build and push image", "err", err.Error()) os.Exit(1) } - if err := runner.Push(tag, credentials.Username, credentials.Password, repositoryURL.Host); err != nil { + if err := runner.Push(tag); err != nil { level.Error(logger).Log("msg", "failed to push agent", "err", err) os.Exit(1) } diff --git a/pkg/container/docker.go b/pkg/container/docker.go index a8079d3..a596128 100644 --- a/pkg/container/docker.go +++ b/pkg/container/docker.go @@ -1,5 +1,5 @@ /* - * Copyright 2022 The DIAMBRA Authors + * Copyright 2022-2025 The DIAMBRA Authors * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -44,8 +44,9 @@ import ( type DockerRunner struct { log.Logger *client.Client - TimeoutStop time.Duration - AutoRemove bool + TimeoutStop time.Duration + AutoRemove bool + registryAuth string } func NewDockerRunner(logger log.Logger, autoRemove bool) (*DockerRunner, error) { @@ -314,26 +315,31 @@ func (r *DockerRunner) Build(path string, tag string) error { return jsonmessage.DisplayJSONMessagesStream(resp.Body, io.Writer(os.Stderr), termFd, isTerm, nil) } -type DockerAuth struct { - Username string `json:"username"` - Password string `json:"password"` - ServerAddress string `json:"serveraddress"` -} +func (r *DockerRunner) Login(username, password, serverAddress string) { + type DockerAuth struct { + Username string `json:"username"` + Password string `json:"password"` + ServerAddress string `json:"serveraddress"` + } -func (r *DockerRunner) Push(tag, username, password, host string) error { - ctx := context.Background() - auth := DockerAuth{ + auth := &DockerAuth{ Username: username, Password: password, - ServerAddress: host, + ServerAddress: serverAddress, } + authStr, err := json.Marshal(auth) if err != nil { - return err + panic(err) } + r.registryAuth = base64.URLEncoding.EncodeToString(authStr) +} + +func (r *DockerRunner) Push(tag string) error { + ctx := context.Background() resp, err := r.Client.ImagePush(ctx, tag, types.ImagePushOptions{ - RegistryAuth: base64.URLEncoding.EncodeToString(authStr), + RegistryAuth: r.registryAuth, }) if err != nil { return err diff --git a/pkg/container/runner.go b/pkg/container/runner.go index 79802aa..f663102 100644 --- a/pkg/container/runner.go +++ b/pkg/container/runner.go @@ -120,7 +120,8 @@ type Runner interface { Attach(id string) (io.WriteCloser, io.ReadCloser, error) Wait(id string) (int, error) Build(path, tag string) error - Push(tag, username, password, host string) error + Login(username, password, registry string) + Push(tag string) error } func TagFromDir(dir string) (string, error) { diff --git a/pkg/diambra/diambra_test.go b/pkg/diambra/diambra_test.go index dc507e5..3bbafc7 100644 --- a/pkg/diambra/diambra_test.go +++ b/pkg/diambra/diambra_test.go @@ -62,7 +62,11 @@ func (r *mockRunner) Build(path, tag string) error { panic("not implemented") // TODO: Implement } -func (r *mockRunner) Push(tag, username, password, host string) error { +func (r *mockRunner) Push(tag string) error { + panic("not implemented") // TODO: Implement +} + +func (r *mockRunner) Login(server, username, password string) { panic("not implemented") // TODO: Implement } From a92039ea1233838929abf952f1cd3cd51051b66c Mon Sep 17 00:00:00 2001 From: Johannes Ziemke Date: Mon, 20 Jan 2025 18:17:49 +0100 Subject: [PATCH 6/8] Check for tag before push --- pkg/cmd/agent/build_and_push.go | 8 ++++++++ pkg/cmd/agent/submit.go | 9 +++++++++ pkg/container/docker.go | 15 +++++++++++++++ pkg/diambra/diambra_test.go | 4 ++++ 4 files changed, 36 insertions(+) diff --git a/pkg/cmd/agent/build_and_push.go b/pkg/cmd/agent/build_and_push.go index 621fcbd..9402f47 100644 --- a/pkg/cmd/agent/build_and_push.go +++ b/pkg/cmd/agent/build_and_push.go @@ -89,6 +89,14 @@ func NewBuildAndPushCmd(logger *log.Logger) *cobra.Command { runner.Login(credentials.Username, credentials.Password, repositoryURL.Host) tag = fmt.Sprintf("%s%s:%s", repositoryURL.Host, repositoryURL.Path, tag) + if exists, err := runner.TagExists(tag); err != nil { + level.Error(logger).Log("msg", "failed to check if tag exists", "err", err) + os.Exit(1) + } else if exists { + level.Error(logger).Log("msg", "tag already exists, use --name or --version to specify unused tag", "tag", tag) + os.Exit(1) + } + level.Info(logger).Log("msg", "Building agent", "tag", tag) if err := runner.Build(args[0], tag); err != nil { diff --git a/pkg/cmd/agent/submit.go b/pkg/cmd/agent/submit.go index 96236bb..04f2a7a 100644 --- a/pkg/cmd/agent/submit.go +++ b/pkg/cmd/agent/submit.go @@ -96,6 +96,7 @@ func NewSubmitCmd(logger *log.Logger) *cobra.Command { version = time.Now().Format("20060102-150405") } } + credentials, err := cl.Credentials() if err != nil { level.Error(logger).Log("msg", "failed to get credentials", "err", err.Error()) @@ -117,6 +118,14 @@ func NewSubmitCmd(logger *log.Logger) *cobra.Command { runner.Login(credentials.Username, credentials.Password, repositoryURL.Host) tag := fmt.Sprintf("%s%s:%s-%s", repositoryURL.Host, repositoryURL.Path, name, version) + if exists, err := runner.TagExists(tag); err != nil { + level.Error(logger).Log("msg", "failed to check if tag exists", "err", err) + os.Exit(1) + } else if exists { + level.Error(logger).Log("msg", fmt.Sprintf("tag %s already exists, use --name or --version to specify unused tag", tag), "tag", tag) + os.Exit(1) + } + if err := runner.Build(context, tag); err != nil { level.Error(logger).Log("msg", "failed to build and push image", "err", err.Error()) os.Exit(1) diff --git a/pkg/container/docker.go b/pkg/container/docker.go index a596128..d9539ac 100644 --- a/pkg/container/docker.go +++ b/pkg/container/docker.go @@ -32,6 +32,7 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/client" + "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/go-connections/nat" @@ -349,3 +350,17 @@ func (r *DockerRunner) Push(tag string) error { termFd, isTerm := term.GetFdInfo(os.Stdout) return jsonmessage.DisplayJSONMessagesStream(resp, io.Writer(os.Stderr), termFd, isTerm, nil) } + +func (r *DockerRunner) TagExists(tag string) (bool, error) { + ctx := context.Background() + _, err := r.Client.DistributionInspect(ctx, tag, r.registryAuth) + if err == nil { + return true, nil + } + + if errdefs.IsNotFound(err) { + return false, nil + } + + return false, err +} diff --git a/pkg/diambra/diambra_test.go b/pkg/diambra/diambra_test.go index 3bbafc7..3d09b53 100644 --- a/pkg/diambra/diambra_test.go +++ b/pkg/diambra/diambra_test.go @@ -70,6 +70,10 @@ func (r *mockRunner) Login(server, username, password string) { panic("not implemented") // TODO: Implement } +func (r *mockRunner) TagExists(tag string) (bool, error) { + panic("not implemented") // TODO: Implement +} + func TestDiambra(t *testing.T) { var ( logger = log.NewLogfmtLogger(os.Stderr) From 708b11347303d4462f39f6a19cfabddfb19f6eff Mon Sep 17 00:00:00 2001 From: Johannes Ziemke Date: Fri, 24 Jan 2025 12:31:23 +0100 Subject: [PATCH 7/8] Reuse build-and-push logic --- pkg/cmd/agent/agent.go | 60 +++++++++++++++++++++++++++++++++ pkg/cmd/agent/build_and_push.go | 56 ++++-------------------------- pkg/cmd/agent/submit.go | 57 ++----------------------------- 3 files changed, 69 insertions(+), 104 deletions(-) diff --git a/pkg/cmd/agent/agent.go b/pkg/cmd/agent/agent.go index cba467f..5dd68be 100644 --- a/pkg/cmd/agent/agent.go +++ b/pkg/cmd/agent/agent.go @@ -16,7 +16,15 @@ package agent import ( + "fmt" + "net/url" + "time" + + "github.com/diambra/cli/pkg/container" + "github.com/diambra/cli/pkg/diambra/client" + "github.com/diambra/cli/pkg/git" "github.com/diambra/cli/pkg/log" + "github.com/go-kit/log/level" "github.com/spf13/cobra" ) @@ -34,3 +42,55 @@ func NewCommand(logger *log.Logger) *cobra.Command { cmd.AddCommand(NewBuildAndPushCmd(logger)) return cmd } + +func buildAndPush(logger *log.Logger, client *client.Client, context, name, version string) (string, error) { + var err error + if name == "" { + name, err = container.TagFromDir(context) + if err != nil { + return "", fmt.Errorf("failed to get tag from dir: %w", err) + } + } + + if version == "" { + version, err = git.GitHeadSHAShort(context, 0) + if err != nil { + version = time.Now().Format("20060102-150405") + level.Warn(logger).Log("msg", fmt.Sprintf("failed to get git head sha, using timestamp %s", version), "err", err) + } + } + + credentials, err := client.Credentials() + if err != nil { + return "", fmt.Errorf("failed to get credentials: %w", err) + } + + level.Info(logger).Log("msg", "Building agent", "name", name, "version", version) + runner, err := container.NewDockerRunner(logger, false) + if err != nil { + return "", fmt.Errorf("failed to create docker runner: %w", err) + } + + repositoryURL, err := url.Parse(credentials.Repository) + if err != nil { + return "", fmt.Errorf("failed to parse repository URL: %w", err) + } + + runner.Login(credentials.Username, credentials.Password, repositoryURL.Host) + + tag := fmt.Sprintf("%s%s:%s-%s", repositoryURL.Host, repositoryURL.Path, name, version) + + if exists, err := runner.TagExists(tag); err != nil { + return "", fmt.Errorf("failed to check if tag exists: %w", err) + } else if exists { + return "", fmt.Errorf("tag %s already exists, use --name or --version to specify unused tag", tag) + } + + if err := runner.Build(context, tag); err != nil { + return "", fmt.Errorf("failed to build agent: %w", err) + } + if err := runner.Push(tag); err != nil { + return "", fmt.Errorf("failed to push agent: %w", err) + } + return tag, nil +} diff --git a/pkg/cmd/agent/build_and_push.go b/pkg/cmd/agent/build_and_push.go index 9402f47..c69521b 100644 --- a/pkg/cmd/agent/build_and_push.go +++ b/pkg/cmd/agent/build_and_push.go @@ -17,11 +17,9 @@ package agent import ( "fmt" - "net/url" "os" "path/filepath" - "github.com/diambra/cli/pkg/container" "github.com/diambra/cli/pkg/diambra/client" "github.com/diambra/cli/pkg/log" "github.com/go-kit/log/level" @@ -32,7 +30,8 @@ const defaultCredPath = "~/.diambra/credentials" func NewBuildAndPushCmd(logger *log.Logger) *cobra.Command { var ( - tag = "" + name = "" + version = "" credPath = defaultCredPath ) @@ -45,12 +44,6 @@ func NewBuildAndPushCmd(logger *log.Logger) *cobra.Command { args = []string{"."} } - runner, err := container.NewDockerRunner(logger, false) - if err != nil { - level.Error(logger).Log("msg", "failed to create docker runner", "err", err) - os.Exit(1) - } - if credPath == defaultCredPath { homedir, err := os.UserHomeDir() if err != nil { @@ -66,53 +59,18 @@ func NewBuildAndPushCmd(logger *log.Logger) *cobra.Command { os.Exit(1) } - if tag == "" { - var err error - tag, err = container.TagFromDir(args[0]) - if err != nil { - level.Error(logger).Log("msg", "failed to get tag from dir", "err", err) - os.Exit(1) - } - } - - credentials, err := cl.Credentials() - if err != nil { - level.Error(logger).Log("msg", "failed to get credentials", "err", err.Error()) - os.Exit(1) - } - - repositoryURL, err := url.Parse(credentials.Repository) + tag, err := buildAndPush(logger, cl, args[0], name, version) if err != nil { - level.Error(logger).Log("msg", "failed to parse repository URL", "err", err) - os.Exit(1) - } - - runner.Login(credentials.Username, credentials.Password, repositoryURL.Host) - tag = fmt.Sprintf("%s%s:%s", repositoryURL.Host, repositoryURL.Path, tag) - if exists, err := runner.TagExists(tag); err != nil { - level.Error(logger).Log("msg", "failed to check if tag exists", "err", err) - os.Exit(1) - } else if exists { - level.Error(logger).Log("msg", "tag already exists, use --name or --version to specify unused tag", "tag", tag) - os.Exit(1) - } - - level.Info(logger).Log("msg", "Building agent", "tag", tag) - - if err := runner.Build(args[0], tag); err != nil { - level.Error(logger).Log("msg", "failed to build agent", "err", err) - os.Exit(1) - } - - if err := runner.Push(tag); err != nil { - level.Error(logger).Log("msg", "failed to push agent", "err", err) + level.Error(logger).Log("msg", "failed to build and push agent", "err", err) os.Exit(1) } + level.Info(logger).Log("msg", fmt.Sprintf("Agent built and pushed: %s", tag), "tag", tag) }, Args: cobra.MaximumNArgs(1), } - cmd.Flags().StringVarP(&tag, "tag", "t", tag, "Tag for the image") cmd.Flags().StringVar(&credPath, "path.credentials", defaultCredPath, "Path to credentials file") + cmd.Flags().StringVar(&name, "name", name, "Name of the agent image (only used when giving a directory)") + cmd.Flags().StringVar(&version, "version", version, "Version of the agent image (only used when giving a directory)") return cmd } diff --git a/pkg/cmd/agent/submit.go b/pkg/cmd/agent/submit.go index 04f2a7a..f7a2592 100644 --- a/pkg/cmd/agent/submit.go +++ b/pkg/cmd/agent/submit.go @@ -17,15 +17,11 @@ package agent import ( "fmt" - "net/url" "os" "path/filepath" - "time" - "github.com/diambra/cli/pkg/container" "github.com/diambra/cli/pkg/diambra" "github.com/diambra/cli/pkg/diambra/client" - "github.com/diambra/cli/pkg/git" "github.com/diambra/cli/pkg/log" "github.com/go-kit/log/level" "github.com/spf13/cobra" @@ -80,58 +76,9 @@ func NewSubmitCmd(logger *log.Logger) *cobra.Command { if stat, err := os.Stat(submission.Manifest.Image); err == nil && stat.IsDir() { context := submission.Manifest.Image level.Info(logger).Log("msg", "Building and pushing image", "context", context) - - if name == "" { - name, err = container.TagFromDir(context) - if err != nil { - level.Error(logger).Log("msg", "failed to get tag from dir", "err", err) - os.Exit(1) - } - } - - if version == "" { - version, err = git.GitHeadSHAShort(context, 0) - if err != nil { - level.Warn(logger).Log("msg", "failed to get git head sha", "err", err) - version = time.Now().Format("20060102-150405") - } - } - - credentials, err := cl.Credentials() - if err != nil { - level.Error(logger).Log("msg", "failed to get credentials", "err", err.Error()) - os.Exit(1) - } - - level.Info(logger).Log("msg", "Building agent", "name", name, "version", version) - runner, err := container.NewDockerRunner(logger, false) - if err != nil { - level.Error(logger).Log("msg", "failed to create docker runner", "err", err) - os.Exit(1) - } - repositoryURL, err := url.Parse(credentials.Repository) + tag, err := buildAndPush(logger, cl, context, name, version) if err != nil { - level.Error(logger).Log("msg", "failed to parse repository URL", "err", err) - os.Exit(1) - } - - runner.Login(credentials.Username, credentials.Password, repositoryURL.Host) - tag := fmt.Sprintf("%s%s:%s-%s", repositoryURL.Host, repositoryURL.Path, name, version) - - if exists, err := runner.TagExists(tag); err != nil { - level.Error(logger).Log("msg", "failed to check if tag exists", "err", err) - os.Exit(1) - } else if exists { - level.Error(logger).Log("msg", fmt.Sprintf("tag %s already exists, use --name or --version to specify unused tag", tag), "tag", tag) - os.Exit(1) - } - - if err := runner.Build(context, tag); err != nil { - level.Error(logger).Log("msg", "failed to build and push image", "err", err.Error()) - os.Exit(1) - } - if err := runner.Push(tag); err != nil { - level.Error(logger).Log("msg", "failed to push agent", "err", err) + level.Error(logger).Log("msg", "failed to build and push agent", "err", err.Error()) os.Exit(1) } From 3245ba5f0bafd7c944144af56f9655fe7af94609 Mon Sep 17 00:00:00 2001 From: Johannes Ziemke Date: Fri, 24 Jan 2025 13:31:45 +0100 Subject: [PATCH 8/8] Update and fix generated example agent --- pkg/cmd/agent/init.go | 8 ++- pkg/diambra/agents/agent.py.tmpl | 13 ++--- pkg/diambra/agents/dockerfile.tmpl | 2 +- pkg/diambra/agents/generator.go | 67 +++++++++++++++--------- pkg/diambra/agents/requirements.txt.tmpl | 2 +- 5 files changed, 55 insertions(+), 37 deletions(-) diff --git a/pkg/cmd/agent/init.go b/pkg/cmd/agent/init.go index 23ae1a0..d72e487 100644 --- a/pkg/cmd/agent/init.go +++ b/pkg/cmd/agent/init.go @@ -38,6 +38,10 @@ func NewInitCmd(logger *log.Logger) *cobra.Command { Short: "Prepares local directory as agent for submission", Long: `This creates all files needed to submit an agent.`, Run: func(cmd *cobra.Command, args []string) { + if err := config.Validate(); err != nil { + level.Error(logger).Log("msg", err.Error()) + os.Exit(1) + } parts, err := diambra.GetInstalledPackageVersion("diambra-arena") if err != nil || len(parts) != 3 || (parts[0] == "0" && parts[1] == "0" && parts[2] == "0") { level.Info(logger).Log("msg", "can't find installed diambra-arena version, using latest", "err", err) @@ -47,7 +51,7 @@ func NewInitCmd(logger *log.Logger) *cobra.Command { os.Exit(1) } } - config.Arena.Version = strings.Join(parts, ".") + config.ArenaVersion = strings.Join(parts, ".") if err := agents.Generate(logger, args[0], config); err != nil { level.Error(logger).Log("msg", "failed to initialize agent", "err", err.Error()) os.Exit(1) @@ -56,7 +60,7 @@ func NewInitCmd(logger *log.Logger) *cobra.Command { }, Args: cobra.ExactArgs(1), } - cmd.Flags().StringVar(&config.Python.Version, "python.version", config.Python.Version, "Python version to use") + cmd.Flags().StringVar(&config.PythonVersion, "python.version", config.PythonVersion, "Python version to use") cmd.Flags().BoolVar(&config.Secret, "secret", config.Secret, "Include secret in agent") return cmd diff --git a/pkg/diambra/agents/agent.py.tmpl b/pkg/diambra/agents/agent.py.tmpl index f556c1a..b0c6c15 100644 --- a/pkg/diambra/agents/agent.py.tmpl +++ b/pkg/diambra/agents/agent.py.tmpl @@ -5,15 +5,12 @@ env = diambra.arena.make("doapp") observation = env.reset() while True: - env.render() + action = env.action_space.sample() + observation, reward, terminated, truncated, info = env.step(action) - actions = env.action_space.sample() - - observation, reward, done, info = env.step(actions) - - if done: - observation = env.reset() - if info["env_done"]: + if terminated or truncated: + observation, info = env.reset() + if info["env_done"] or test is True: break env.close() diff --git a/pkg/diambra/agents/dockerfile.tmpl b/pkg/diambra/agents/dockerfile.tmpl index c9db104..879284e 100644 --- a/pkg/diambra/agents/dockerfile.tmpl +++ b/pkg/diambra/agents/dockerfile.tmpl @@ -1,4 +1,4 @@ -FROM {{ .BaseImage.Registry }}/{{ .BaseImage.Image }}:{{ .Python.Version }} +FROM {{ .Registry }}/{{ .Image }} RUN apt-get -qy update && \ apt-get -qy install libgl1 && \ diff --git a/pkg/diambra/agents/generator.go b/pkg/diambra/agents/generator.go index e84bfd7..e448225 100644 --- a/pkg/diambra/agents/generator.go +++ b/pkg/diambra/agents/generator.go @@ -31,39 +31,49 @@ var ReadmeTemplate string var differ = diffmatchpatch.New() -type PythonConfig struct { - Version string +type TemplateConfig struct { + Registry string + Image string + Secret bool + ArenaVersion string } -type BaseImageConfig struct { - Registry string - Image string +type Config struct { + PythonVersion string + ArenaVersion string + Secret bool } -type ArenaConfig struct { - Version string -} +const ( + OSVersion = "bullseye" +) -type Config struct { - Python PythonConfig - BaseImage BaseImageConfig - Arena ArenaConfig - Secret bool -} +// FIXME: Read from https://github.com/orgs/diambra/packages?repo_name=arena +var ( + PythonVersions = []string{"3.10", "3.9", "3.8", "3.7"} +) func NewConfig() (*Config, error) { return &Config{ - Arena: ArenaConfig{}, - Python: PythonConfig{ - Version: "3.7", // FIXME: Detect version - }, - BaseImage: BaseImageConfig{ - Registry: "docker.io", - Image: "python", - }, + ArenaVersion: "", + PythonVersion: PythonVersions[0], // FIXME: Detect version }, nil } +func (c *Config) Validate() error { + found := false + for _, v := range PythonVersions { + if v == c.PythonVersion { + found = true + break + } + } + if !found { + return fmt.Errorf("python version %s not supported. Available: %s", c.PythonVersion, PythonVersions) + } + return nil +} + func WriteFile(logger log.Logger, path, name, tmpl string, config *Config) error { exists := true if _, err := os.Stat(filepath.Join(path, name)); err != nil { @@ -73,6 +83,13 @@ func WriteFile(logger log.Logger, path, name, tmpl string, config *Config) error exists = false } + templateConfig := TemplateConfig{ + Registry: "ghcr.io/diambra", + Image: fmt.Sprintf("arena-base-on%s-%s:main", config.PythonVersion, OSVersion), + Secret: config.Secret, + ArenaVersion: config.ArenaVersion, + } + if exists { fh, err := os.Open(filepath.Join(path, name)) if err != nil { @@ -85,10 +102,10 @@ func WriteFile(logger log.Logger, path, name, tmpl string, config *Config) error return fmt.Errorf("couldn't read existing file %s: %w", name, err) } new := bytes.Buffer{} - if err := template.Must(template.New(name).Parse(tmpl)).Execute(&new, config); err != nil { + if err := template.Must(template.New(name).Parse(tmpl)).Execute(&new, templateConfig); err != nil { return err } - diffs := differ.DiffMain(string(old), new.String(), true) + diffs := differ.DiffMain(string(old), new.String(), false) if len(diffs) < 2 { level.Info(logger).Log("msg", "Skipping "+name+", content identical", "file", name) return nil @@ -113,7 +130,7 @@ func WriteFile(logger log.Logger, path, name, tmpl string, config *Config) error return err } level.Info(logger).Log("msg", "Creating "+name, "file", name) - return template.Must(template.New(name).Parse(tmpl)).Execute(fh, config) + return template.Must(template.New(name).Parse(tmpl)).Execute(fh, templateConfig) } func Generate(logger log.Logger, path string, config *Config) error { diff --git a/pkg/diambra/agents/requirements.txt.tmpl b/pkg/diambra/agents/requirements.txt.tmpl index da6a1fb..72a845d 100644 --- a/pkg/diambra/agents/requirements.txt.tmpl +++ b/pkg/diambra/agents/requirements.txt.tmpl @@ -1 +1 @@ -diambra-arena=={{ .Arena.Version }} +diambra-arena=={{ .ArenaVersion }}