From fe42d9ec4076b6a7e5572749d48bda04f21ee7d0 Mon Sep 17 00:00:00 2001 From: Oshank Kumar Date: Fri, 30 Nov 2018 15:15:34 +0530 Subject: [PATCH 1/7] gd2 plugin: added a plugin for block volume management - added APIs for creation,deleting and listing block volumes. - added pluggable interface for block volume providers. Refer Design Doc: #1319 Signed-off-by: Oshank Kumar --- Gopkg.lock | 2 +- Gopkg.toml | 2 +- glusterd2/commands/volumes/volume-create.go | 94 +++-- glusterd2/commands/volumes/volume-start.go | 45 +-- glusterd2/plugin/plugins.go | 2 + glusterd2/volume/filters.go | 51 +++ pkg/size/size.go | 187 ++++++++++ pkg/size/size_test.go | 166 +++++++++ plugins/blockvolume/api/types.go | 39 ++ .../blockprovider/blockvolume_provider.go | 61 ++++ .../blockprovider/gluster-block/config.go | 65 ++++ .../gluster-block/glusterblock.go | 337 ++++++++++++++++++ .../blockprovider/gluster-block/volume.go | 108 ++++++ plugins/blockvolume/blockprovider/options.go | 71 ++++ plugins/blockvolume/handlers.go | 107 ++++++ plugins/blockvolume/init.go | 12 + plugins/blockvolume/routes.go | 82 +++++ 17 files changed, 1360 insertions(+), 71 deletions(-) create mode 100644 glusterd2/volume/filters.go create mode 100644 pkg/size/size.go create mode 100644 pkg/size/size_test.go create mode 100644 plugins/blockvolume/api/types.go create mode 100644 plugins/blockvolume/blockprovider/blockvolume_provider.go create mode 100644 plugins/blockvolume/blockprovider/gluster-block/config.go create mode 100644 plugins/blockvolume/blockprovider/gluster-block/glusterblock.go create mode 100644 plugins/blockvolume/blockprovider/gluster-block/volume.go create mode 100644 plugins/blockvolume/blockprovider/options.go create mode 100644 plugins/blockvolume/handlers.go create mode 100644 plugins/blockvolume/init.go create mode 100644 plugins/blockvolume/routes.go diff --git a/Gopkg.lock b/Gopkg.lock index 72d3d5575..fd1686094 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -588,4 +588,4 @@ analyzer-version = 1 inputs-digest = "1a1bf3511263a7300c00a98d7b55df50c80fec36ba01c25fbddde4a85dee99f5" solver-name = "gps-cdcl" - solver-version = 1 + solver-version = 1 \ No newline at end of file diff --git a/Gopkg.toml b/Gopkg.toml index f5c1133ce..ee2fe1b74 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -48,7 +48,7 @@ [[constraint]] name = "github.com/sirupsen/logrus" - version = "~1.0.3" + version = "~1.2.0" [[constraint]] name = "github.com/cockroachdb/cmux" diff --git a/glusterd2/commands/volumes/volume-create.go b/glusterd2/commands/volumes/volume-create.go index 716175e81..3190e3f91 100644 --- a/glusterd2/commands/volumes/volume-create.go +++ b/glusterd2/commands/volumes/volume-create.go @@ -1,6 +1,7 @@ package volumecommands import ( + "context" "errors" "net/http" "path/filepath" @@ -102,9 +103,6 @@ func registerVolCreateStepFuncs() { func volumeCreateHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - ctx, span := trace.StartSpan(ctx, "/volumeCreateHandler") - defer span.End() - logger := gdctx.GetReqLogger(ctx) var err error @@ -114,45 +112,68 @@ func volumeCreateHandler(w http.ResponseWriter, r *http.Request) { return } - if err := validateVolCreateReq(&req); err != nil { - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, err) + if status, err := CreateVolume(ctx, req); err != nil { + restutils.SendHTTPError(ctx, w, status, err) return } - if containsReservedGroupProfile(req.Options) { - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, gderrors.ErrReservedGroupProfile) + volinfo, err := volume.GetVolume(req.Name) + if err != nil { + // FIXME: If volume was created successfully in the txn above and + // then the store goes down by the time we reach here, what do + // we return to the client ? + restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) return } + logger.WithField("volume-name", volinfo.Name).Info("new volume created") + events.Broadcast(volume.NewEvent(volume.EventVolumeCreated, volinfo)) + + resp := createVolumeCreateResp(volinfo) + restutils.SetLocationHeader(r, w, volinfo.Name) + restutils.SendHTTPResponse(ctx, w, http.StatusCreated, resp) +} + +func createVolumeCreateResp(v *volume.Volinfo) *api.VolumeCreateResp { + return (*api.VolumeCreateResp)(volume.CreateVolumeInfoResp(v)) +} + +// CreateVolume creates a volume +func CreateVolume(ctx context.Context, req api.VolCreateReq) (status int, err error) { + ctx, span := trace.StartSpan(ctx, "/volumeCreateHandler") + defer span.End() + + if err := validateVolCreateReq(&req); err != nil { + return http.StatusBadRequest, err + } + + if containsReservedGroupProfile(req.Options) { + return http.StatusBadRequest, gderrors.ErrReservedGroupProfile + } + if req.Size > 0 { applyDefaults(&req) if req.SnapshotReserveFactor < 1 { - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, - errors.New("invalid snapshot reserve factor")) - return + return http.StatusBadRequest, errors.New("invalid snapshot reserve factor") } if err := bricksplanner.PlanBricks(&req); err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return + return http.StatusInternalServerError, err } } else { if err := checkDupBrickEntryVolCreate(req); err != nil { - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, err) - return + return http.StatusBadRequest, err } } req.Options, err = expandGroupOptions(req.Options) if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return + return http.StatusInternalServerError, err } if err := validateOptions(req.Options, req.VolOptionFlags); err != nil { - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, err) - return + return http.StatusBadRequest, err } // Include default Volume Options profile @@ -171,21 +192,17 @@ func volumeCreateHandler(w http.ResponseWriter, r *http.Request) { nodes, err := req.Nodes() if err != nil { - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, err) - return + return http.StatusBadRequest, err } txn, err := transactionv2.NewTxnWithLocks(ctx, req.Name) if err != nil { - status, err := restutils.ErrToStatusCode(err) - restutils.SendHTTPError(ctx, w, status, err) - return + return restutils.ErrToStatusCode(err) } defer txn.Done() if volume.Exists(req.Name) { - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, gderrors.ErrVolExists) - return + return http.StatusBadRequest, gderrors.ErrVolExists } txn.Steps = []*transaction.Step{ @@ -219,8 +236,7 @@ func volumeCreateHandler(w http.ResponseWriter, r *http.Request) { } if err := txn.Ctx.Set("req", &req); err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return + return http.StatusInternalServerError, err } // Add attributes to the span with info that can be viewed along with traces. @@ -231,28 +247,8 @@ func volumeCreateHandler(w http.ResponseWriter, r *http.Request) { ) if err := txn.Do(); err != nil { - status, err := restutils.ErrToStatusCode(err) - restutils.SendHTTPError(ctx, w, status, err) - return + return restutils.ErrToStatusCode(err) } - volinfo, err := volume.GetVolume(req.Name) - if err != nil { - // FIXME: If volume was created successfully in the txn above and - // then the store goes down by the time we reach here, what do - // we return to the client ? - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return - } - - logger.WithField("volume-name", volinfo.Name).Info("new volume created") - events.Broadcast(volume.NewEvent(volume.EventVolumeCreated, volinfo)) - - resp := createVolumeCreateResp(volinfo) - restutils.SetLocationHeader(r, w, volinfo.Name) - restutils.SendHTTPResponse(ctx, w, http.StatusCreated, resp) -} - -func createVolumeCreateResp(v *volume.Volinfo) *api.VolumeCreateResp { - return (*api.VolumeCreateResp)(volume.CreateVolumeInfoResp(v)) + return http.StatusCreated, nil } diff --git a/glusterd2/commands/volumes/volume-start.go b/glusterd2/commands/volumes/volume-start.go index fe8253437..a195bb2da 100644 --- a/glusterd2/commands/volumes/volume-start.go +++ b/glusterd2/commands/volumes/volume-start.go @@ -129,10 +129,6 @@ func registerVolStartStepFuncs() { func volumeStartHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - ctx, span := trace.StartSpan(ctx, "/volumeStartHandler") - defer span.End() - - logger := gdctx.GetReqLogger(ctx) volname := mux.Vars(r)["volname"] var req api.VolumeStartReq @@ -142,24 +138,39 @@ func volumeStartHandler(w http.ResponseWriter, r *http.Request) { return } - txn, err := transactionv2.NewTxnWithLocks(ctx, volname) + volinfo, status, err := StartVolume(ctx, volname, req) if err != nil { - status, err := restutils.ErrToStatusCode(err) restutils.SendHTTPError(ctx, w, status, err) return } + + events.Broadcast(volume.NewEvent(volume.EventVolumeStarted, volinfo)) + + resp := createVolumeStartResp(volinfo) + restutils.SendHTTPResponse(ctx, w, http.StatusOK, resp) +} + +// StartVolume starts a volume +func StartVolume(ctx context.Context, volname string, req api.VolumeStartReq) (volInfo *volume.Volinfo, status int, err error) { + logger := gdctx.GetReqLogger(ctx) + ctx, span := trace.StartSpan(ctx, "/volumeStartHandler") + defer span.End() + + txn, err := transactionv2.NewTxnWithLocks(ctx, volname) + if err != nil { + status, err := restutils.ErrToStatusCode(err) + return nil, status, err + } defer txn.Done() volinfo, err := volume.GetVolume(volname) if err != nil { status, err := restutils.ErrToStatusCode(err) - restutils.SendHTTPError(ctx, w, status, err) - return + return nil, status, err } if volinfo.State == volume.VolStarted && !req.ForceStartBricks { - restutils.SendHTTPError(ctx, w, http.StatusBadRequest, errors.ErrVolAlreadyStarted) - return + return nil, http.StatusBadRequest, errors.ErrVolAlreadyStarted } txn.Steps = []*transaction.Step{ @@ -182,15 +193,13 @@ func volumeStartHandler(w http.ResponseWriter, r *http.Request) { } if err := txn.Ctx.Set("oldvolinfo", volinfo); err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return + return nil, http.StatusInternalServerError, err } volinfo.State = volume.VolStarted if err := txn.Ctx.Set("volinfo", volinfo); err != nil { - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return + return nil, http.StatusInternalServerError, err } span.AddAttributes( @@ -201,14 +210,10 @@ func volumeStartHandler(w http.ResponseWriter, r *http.Request) { if err := txn.Do(); err != nil { logger.WithError(err).WithField( "volume", volname).Error("transaction to start volume failed") - restutils.SendHTTPError(ctx, w, http.StatusInternalServerError, err) - return + return nil, http.StatusInternalServerError, err } - events.Broadcast(volume.NewEvent(volume.EventVolumeStarted, volinfo)) - - resp := createVolumeStartResp(volinfo) - restutils.SendHTTPResponse(ctx, w, http.StatusOK, resp) + return volinfo, http.StatusOK, nil } func createVolumeStartResp(v *volume.Volinfo) *api.VolumeStartResp { diff --git a/glusterd2/plugin/plugins.go b/glusterd2/plugin/plugins.go index 90afd966a..03d46e1bc 100644 --- a/glusterd2/plugin/plugins.go +++ b/glusterd2/plugin/plugins.go @@ -4,6 +4,7 @@ package plugin import ( "github.com/gluster/glusterd2/plugins/bitrot" + "github.com/gluster/glusterd2/plugins/blockvolume" "github.com/gluster/glusterd2/plugins/device" "github.com/gluster/glusterd2/plugins/events" "github.com/gluster/glusterd2/plugins/georeplication" @@ -25,4 +26,5 @@ var PluginsList = []GlusterdPlugin{ &glustershd.Plugin{}, &device.Plugin{}, &rebalance.Plugin{}, + &blockvolume.BlockVolume{}, } diff --git a/glusterd2/volume/filters.go b/glusterd2/volume/filters.go new file mode 100644 index 000000000..30d141120 --- /dev/null +++ b/glusterd2/volume/filters.go @@ -0,0 +1,51 @@ +package volume + +const ( + // BlockHosted is plugin name for FilterBlockHostedVolumes + BlockHosted = "block-hosted" +) + +// Filter will receive a slice of *Volinfo and filters out the undesired one and return slice of desired one only +type Filter func([]*Volinfo) []*Volinfo + +var filters = make(map[string]Filter) + +// InstallFilter will register a custom Filter +func InstallFilter(name string, f Filter) { + filters[name] = f +} + +// ApplyFilters applies all registered filters passed in the args to a slice of *Volinfo +func ApplyFilters(volumes []*Volinfo, names ...string) []*Volinfo { + for _, name := range names { + if filter, found := filters[name]; found { + volumes = filter(volumes) + } + } + return volumes +} + +// ApplyCustomFilters applies all custom filter to a slice of *Volinfo +func ApplyCustomFilters(volumes []*Volinfo, filters ...Filter) []*Volinfo { + for _, filter := range filters { + volumes = filter(volumes) + } + + return volumes +} + +// FilterBlockHostedVolumes filters out volume which are suitable for hosting block volume +func FilterBlockHostedVolumes(volumes []*Volinfo) []*Volinfo { + var volInfos []*Volinfo + for _, volume := range volumes { + val, found := volume.Metadata["block-hosting"] + if found && val == "yes" { + volInfos = append(volInfos, volume) + } + } + return volInfos +} + +func init() { + InstallFilter(BlockHosted, FilterBlockHostedVolumes) +} diff --git a/pkg/size/size.go b/pkg/size/size.go new file mode 100644 index 000000000..98acf8a5d --- /dev/null +++ b/pkg/size/size.go @@ -0,0 +1,187 @@ +package size + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +// Size represents unit to measure information size +type Size uint64 + +// Byte represents one byte of information +const Byte Size = 1 + +const ( + // KiB is multiple of unite Byte the binary prefix Ki represents 2^10 + KiB = 1024 * Byte + // MiB is multiple of unite Byte the binary prefix Mi represents 2^20 + MiB = 1024 * KiB + // GiB is multiple of unite Byte the binary prefix Gi represents 2^30 + GiB = 1024 * MiB + // TiB is multiple of unite Byte the binary prefix Ti represents 2^40 + TiB = 1024 * GiB + // PiB is multiple of unite Byte the binary prefix Pi represents 2^50 + PiB = 1024 * TiB +) + +const ( + // KB is a multiple of the unit byte the prefix K represents 10^3 + KB = 1e3 * Byte + // MB is a multiple of the unit byte the prefix M represents 10^6 + MB = 1e3 * KB + // GB is a multiple of the unit byte the prefix G represents 10^9 + GB = 1e3 * MB + // TB is a multiple of the unit byte the prefix T represents 10^12 + TB = 1e3 * GB + // PB is a multiple of the unit byte the prefix T represents 10^15 + PB = 1e3 * TB +) + +// Bytes returns number of bytes +func (s Size) Bytes() int64 { return int64(s) } + +// KiloBytes returns numbers of KiloBytes in floating point +func (s Size) KiloBytes() float64 { + kb := s / KB + bytes := s % KB + + return float64(kb) + float64(bytes)/1e3 +} + +// MegaBytes returns numbers of MegaBytes in floating point +func (s Size) MegaBytes() float64 { + mb := s / MB + bytes := s % MB + + return float64(mb) + float64(bytes)/(1e6) +} + +// GigaBytes returns number of GigaBytes in floating point +func (s Size) GigaBytes() float64 { + gb := s / GB + bytes := s % GB + + return float64(gb) + float64(bytes)/(1e9) +} + +// TeraBytes returns number of TeraBytes in floating point +func (s Size) TeraBytes() float64 { + tb := s / TB + bytes := s % TB + + return float64(tb) + float64(bytes)/(1e12) +} + +// KibiBytes returns number of KiB in floating point +func (s Size) KibiBytes() float64 { + kib := s / KiB + bytes := s % KiB + + return float64(kib) + float64(bytes)/1024 +} + +// MebiBytes returns number of MiB in floating point +func (s Size) MebiBytes() float64 { + mib := s / MiB + bytes := s % MiB + + return float64(mib) + float64(bytes)/(1024*1024) +} + +// GibiBytes returns number of GiB in floating point +func (s Size) GibiBytes() float64 { + gib := s / GiB + bytes := s % GiB + + return float64(gib) + float64(bytes)/(1024*1024*1024) +} + +// TebiBytes returns number of TiB in floating point +func (s Size) TebiBytes() float64 { + tib := s / TiB + bytes := s % TiB + + return float64(tib) + float64(bytes)/(1024*1024*1024*1024) +} + +// String string representation of Size in form XXKB/MB/TB/GB/Bytes +// TODO: support for string representation in XiB format +func (s Size) String() string { + + if s >= TB { + return fmt.Sprintf("%.2fTB", s.TeraBytes()) + } + + if s >= GB { + return fmt.Sprintf("%.2fGB", s.GigaBytes()) + } + + if s >= MB { + return fmt.Sprintf("%.2fMB", s.MegaBytes()) + } + + if s >= KB { + return fmt.Sprintf("%.2fKB", s.KiloBytes()) + } + + return fmt.Sprintf("%d Bytes", s) +} + +// Parse parses a string representation of size and returns the Size value it represents. +// Supported formats are {TiB,GiB,MiB,KiB,TB,GB,MB,KB} +func Parse(s string) (Size, error) { + var ( + count float64 + size Size + err error + regex = regexp.MustCompile(`^([\d.]+)([KMGT]i?B)$`) + ) + + s = strings.Replace(s, " ", "", -1) + matches := regex.FindStringSubmatch(s) + + if len(matches) != 3 { + return size, errors.New("invalid size format") + } + + switch matches[2] { + case "GiB": + _, err = fmt.Sscanf(s, "%fGiB", &count) + size = Size(count * float64(1*GiB)) + + case "MiB": + _, err = fmt.Sscanf(s, "%fMiB", &count) + size = Size(count * float64(1*MiB)) + + case "KiB": + _, err = fmt.Sscanf(s, "%fKiB", &count) + size = Size(count * float64(1*KiB)) + + case "TiB": + _, err = fmt.Sscanf(s, "%fTiB", &count) + size = Size(count * float64(1*TiB)) + + case "KB": + _, err = fmt.Sscanf(s, "%fKB", &count) + size = Size(count * float64(1*KB)) + + case "MB": + _, err = fmt.Sscanf(s, "%fMB", &count) + size = Size(count * float64(1*MB)) + + case "GB": + _, err = fmt.Sscanf(s, "%fGB", &count) + size = Size(count * float64(1*GB)) + + case "TB": + _, err = fmt.Sscanf(s, "%fTB", &count) + size = Size(count * float64(1*TB)) + + default: + return 0, errors.New("can not parse to size") + } + + return size, err +} diff --git a/pkg/size/size_test.go b/pkg/size/size_test.go new file mode 100644 index 000000000..01b9c620c --- /dev/null +++ b/pkg/size/size_test.go @@ -0,0 +1,166 @@ +package size + +import "testing" + +var ( + kbSizeTests = []struct { + s Size + want float64 + }{ + {Size(1000), 1}, + {Size(2000), 2}, + {Size(2500), 2.5}, + {Size(8750), 8.75}, + } + + mbSizeTests = []struct { + s Size + want float64 + }{ + {Size(1e6), 1}, + {Size(2.5e6), 2.5}, + {Size(8.75e6), 8.75}, + {Size(2047e6), 2047}, + } + + gbSizeTests = []struct { + s Size + want float64 + }{ + {Size(1e9), 1}, + {Size(2.5e9), 2.5}, + {Size(8.75e9), 8.75}, + {Size(0.25e9), 0.25}, + } + + tbSizeTests = []struct { + s Size + want float64 + }{ + {Size(1e12), 1}, + {Size(2.5e12), 2.5}, + {Size(8.75e12), 8.75}, + {Size(0.75e12), 0.75}, + } + + stringSizeTests = []struct { + s Size + want string + }{ + {Size(1e12), "1.00TB"}, + {Size(2.5e9), "2.50GB"}, + {Size(8.75e6), "8.75MB"}, + {Size(768e3), "768.00KB"}, + {Size(500), "500 Bytes"}, + } + + parseSizeTests = []struct { + s string + want Size + }{ + // Binary format + {"1GiB", Size(1 * float64(GiB))}, + {"2.5GiB", Size(2.5 * float64(GiB))}, + {"1MiB", Size(1 * float64(MiB))}, + {"100.5MiB", Size(100.5 * float64(MiB))}, + {"50KiB", Size(50 * float64(KiB))}, + {"0050KiB", Size(50 * float64(KiB))}, + {"2.50KiB", Size(2.5 * float64(KiB))}, + {"2.50TiB", Size(2.5 * float64(TiB))}, + // Decimal format + {"2.50TB", Size(2.5 * float64(TB))}, + {"2.50MB", Size(2.5 * float64(MB))}, + {"0.5KB", Size(0.5 * float64(KB))}, + {"052GB", Size(52 * float64(GB))}, + // having space in between + {"0.5 KB", Size(0.5 * float64(KB))}, + {"052 GB", Size(52 * float64(GB))}, + {"0050 KiB", Size(50 * float64(KiB))}, + {"2.5 KiB", Size(2.5 * float64(KiB))}, + {"2.50 TiB", Size(2.5 * float64(TiB))}, + } + + parseSizeFailureTest = []string{ + "1xGiB", + "x1TiB", + "5kiB", + "7.4xKiB", + "7.4KKiB", + "7.4KMiB", + "7.4MiBT", + // + "5KBM", + "x5KB", + "05xMB", + "5.5.5MB", + } +) + +func TestSizeBytes(t *testing.T) { + var s Size = 2048 + bytes := s.Bytes() + if bytes != 2048 { + t.Errorf("s.Bytes() = %v; want: %v", bytes, 2048) + } +} + +func TestSizeKiloBytes(t *testing.T) { + for _, tt := range kbSizeTests { + if got := tt.s.KiloBytes(); got != tt.want { + t.Errorf("s.KiloBytes() = %v; want: %v", got, tt.want) + } + } +} + +func TestSizeMegaBytes(t *testing.T) { + for _, tt := range mbSizeTests { + if got := tt.s.MegaBytes(); got != tt.want { + t.Errorf("s.MegaBytes() = %v; want: %v", got, tt.want) + } + } +} + +func TestSizeGigaBytes(t *testing.T) { + for _, tt := range gbSizeTests { + if got := tt.s.GigaBytes(); got != tt.want { + t.Errorf("s.GigaBytes() = %v; want: %v", got, tt.want) + } + } +} + +func TestSizeTeraBytes(t *testing.T) { + for _, tt := range tbSizeTests { + if got := tt.s.TeraBytes(); got != tt.want { + t.Errorf("s.TeraBytes() = %v; want: %v", got, tt.want) + } + } +} + +func TestSizeString(t *testing.T) { + for _, tt := range stringSizeTests { + if got := tt.s.String(); got != tt.want { + t.Errorf("s.String() = %v; want: %v", got, tt.want) + } + } +} + +func TestParse(t *testing.T) { + for _, tt := range parseSizeTests { + got, err := Parse(tt.s) + if err != nil { + t.Error("error in s.Parse() :", err) + } else { + if got != tt.want { + t.Errorf("s.Parse() = %v; want: %v", got, tt.want) + } + } + } +} + +func TestParseFailure(t *testing.T) { + for _, s := range parseSizeFailureTest { + if sz, err := Parse(s); err == nil { + t.Errorf("s.Parse() = %v; wanted error", sz) + } + } +} diff --git a/plugins/blockvolume/api/types.go b/plugins/blockvolume/api/types.go new file mode 100644 index 000000000..1a6814685 --- /dev/null +++ b/plugins/blockvolume/api/types.go @@ -0,0 +1,39 @@ +package api + +// BlockVolumeInfo represents block volume info +type BlockVolumeInfo struct { + // HostingVolume name is optional + HostingVolume string `json:"hostingvolume"` + // Name represents block Volume name + Name string `json:"name"` + // Size represents Block Volume size in bytes + Size int64 `json:"size,omitempty"` + HaCount int `json:"hacount,omitempty"` +} + +// BlockVolumeCreateRequest represents req Body for Block vol create req +type BlockVolumeCreateRequest struct { + *BlockVolumeInfo + Clusters []string `json:"clusters,omitempty"` + Auth bool `json:"auth,omitempty"` +} + +// BlockVolumeCreateResp represents resp body for a Block Vol Create req +type BlockVolumeCreateResp struct { + *BlockVolumeInfo + Hosts []string `json:"hosts"` + Iqn string `json:"iqn"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` +} + +// BlockVolumeListResp represents resp body for a Block List req +type BlockVolumeListResp []BlockVolumeInfo + +// BlockVolumeGetResp represents a resp body for Block Vol Get req +type BlockVolumeGetResp struct { + *BlockVolumeInfo + Hosts []string `json:"hosts"` + GBID string `json:"gbid"` + Password string `json:"password,omitempty"` +} diff --git a/plugins/blockvolume/blockprovider/blockvolume_provider.go b/plugins/blockvolume/blockprovider/blockvolume_provider.go new file mode 100644 index 000000000..c7a89edca --- /dev/null +++ b/plugins/blockvolume/blockprovider/blockvolume_provider.go @@ -0,0 +1,61 @@ +package blockprovider + +import ( + "fmt" + "sync" + + log "github.com/sirupsen/logrus" +) + +// ProviderFunc returns a block Provider instance. It also returns an error +// If occurred any while creating a Provider instance +type ProviderFunc func() (Provider, error) + +var ( + providersMutex sync.Mutex + providerFactory = make(map[string]ProviderFunc) +) + +// Provider is an abstract, pluggable interface for block volume providers +type Provider interface { + CreateBlockVolume(name string, size int64, hosts []string, options ...BlockVolOption) (BlockVolume, error) + DeleteBlockVolume(name string, options ...BlockVolOption) error + GetBlockVolume(id string) (BlockVolume, error) + BlockVolumes() []BlockVolume + ProviderName() string +} + +// BlockVolume is an interface which provides information about a block volume +type BlockVolume interface { + Name() string + ID() string + HostVolume() string + HostAddresses() []string + IQN() string + Username() string + Password() string + Size() uint64 + HaCount() int +} + +// RegisterBlockProvider will register a block provider +func RegisterBlockProvider(name string, f ProviderFunc) { + providersMutex.Lock() + defer providersMutex.Unlock() + if _, found := providerFactory[name]; found { + log.WithField("name", name).Error("block provider was registered twice") + } + log.WithField("name", name).Infof("Registered block provider") + providerFactory[name] = f +} + +// GetBlockProvider will return a block Provider instance if it has been registered. +func GetBlockProvider(name string) (Provider, error) { + providersMutex.Lock() + defer providersMutex.Unlock() + f, found := providerFactory[name] + if !found { + return nil, fmt.Errorf("%s block provider does not exist", name) + } + return f() +} diff --git a/plugins/blockvolume/blockprovider/gluster-block/config.go b/plugins/blockvolume/blockprovider/gluster-block/config.go new file mode 100644 index 000000000..2ec621d02 --- /dev/null +++ b/plugins/blockvolume/blockprovider/gluster-block/config.go @@ -0,0 +1,65 @@ +package glusterblock + +import ( + "github.com/gluster/glusterd2/pkg/api" + + "github.com/pborman/uuid" + "github.com/spf13/viper" +) + +// VolumeType represents a volume type +type VolumeType string + +const ( + // Replica represents a replica volume type + Replica VolumeType = "Replica" +) + +// ClientConfig holds various config information needed to create a gluster-block rest client +type ClientConfig struct { + HostAddress string + User string + Secret string + CaCertFile string + Insecure bool +} + +// ApplyFromConfig sets the ClientConfig options from various config sources +func (c *ClientConfig) ApplyFromConfig(conf *viper.Viper) { + c.CaCertFile = conf.GetString("gluster-block-cacert") + c.HostAddress = conf.GetString("gluster-block-hostaddr") + c.User = conf.GetString("gluster-block-user") + c.Secret = conf.GetString("gluster-block-secret") + c.Insecure = conf.GetBool("gluster-block-insecure") +} + +// HostingVolumeOptions holds various information which will be used in creating hosting volume +type HostingVolumeOptions struct { + Size int64 + Type VolumeType + ReplicaCount int + AutoCreate bool +} + +// ApplyFromConfig sets HostingVolumeOptions member values from given config source +func (h *HostingVolumeOptions) ApplyFromConfig(conf *viper.Viper) { + h.Size = conf.GetInt64("block-hosting-volume-size") + h.Type = VolumeType(conf.GetString("block-hosting-volume-type")) + h.ReplicaCount = conf.GetInt("block-hosting-volume-replica-count") + h.AutoCreate = conf.GetBool("auto-create-block-hosting-volumes") +} + +// PrepareVolumeCreateReq will create a request body to be use for creating a gluster volume +func (h *HostingVolumeOptions) PrepareVolumeCreateReq() *api.VolCreateReq { + name := "block_hosting_volume_" + uuid.NewRandom().String() + + req := &api.VolCreateReq{ + Name: name, + Transport: "tcp", + Size: uint64(h.Size), + ReplicaCount: h.ReplicaCount, + SubvolType: string(h.Type), + } + + return req +} diff --git a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go new file mode 100644 index 000000000..7df086785 --- /dev/null +++ b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go @@ -0,0 +1,337 @@ +package glusterblock + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/gluster/gluster-block-restapi/client" + "github.com/gluster/gluster-block-restapi/pkg/api" + "github.com/gluster/glusterd2/glusterd2/commands/volumes" + "github.com/gluster/glusterd2/glusterd2/gdctx" + "github.com/gluster/glusterd2/glusterd2/volume" + gd2api "github.com/gluster/glusterd2/pkg/api" + "github.com/gluster/glusterd2/pkg/size" + "github.com/gluster/glusterd2/plugins/blockvolume/blockprovider" + + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" +) + +const providerName = "gluster-block" + +func init() { + log.WithField("name", providerName).Infof("Registering block provider") + blockprovider.RegisterBlockProvider(providerName, newGlusterBlock) +} + +// GlusterBlock implements block Provider interface. It represents a gluster-block +type GlusterBlock struct { + client client.GlusterBlockClient + ClientConf *ClientConfig + HostingVolumeOptions *HostingVolumeOptions +} + +func newGlusterBlock() (blockprovider.Provider, error) { + var ( + gb = &GlusterBlock{} + clientConf = &ClientConfig{} + hostVolOpts = &HostingVolumeOptions{} + opts = []client.OptFuncs{} + ) + + clientConf.ApplyFromConfig(viper.GetViper()) + hostVolOpts.ApplyFromConfig(viper.GetViper()) + + gb.ClientConf = clientConf + gb.HostingVolumeOptions = hostVolOpts + + opts = append(opts, + client.WithAuth(clientConf.User, clientConf.Secret), + client.WithTLSConfig(&client.TLSOptions{CaCertFile: clientConf.CaCertFile, InsecureSkipVerify: clientConf.Insecure}), + ) + + gbClient, err := client.NewClientWithOpts(clientConf.HostAddress, opts...) + if err != nil { + return nil, err + } + gb.client = gbClient + + return gb, nil +} + +// CreateBlockVolume will create a gluster block volume with given name and size. If hosting volume is not provide then it will +// create a gluster volume for hosting gluster block. +func (g *GlusterBlock) CreateBlockVolume(name string, size int64, hosts []string, options ...blockprovider.BlockVolOption) (blockprovider.BlockVolume, error) { + var ( + blockVolOpts = &blockprovider.BlockVolumeOptions{} + volCreateReq = g.HostingVolumeOptions.PrepareVolumeCreateReq() + volInfo *volume.Volinfo + ctx = gdctx.WithReqLogger(context.Background(), log.StandardLogger()) + ) + + blockVolOpts.ApplyOpts(options...) + + // ERROR if If HostingVolume is not specified and auto-create-block-hosting-volumes is false + if blockVolOpts.HostVol == "" && !g.HostingVolumeOptions.AutoCreate { + err := errors.New("host volume is not provided and auto creation is not enabled") + log.WithError(err).Error("failed in creating block volume") + return nil, err + } + + // If HostingVolume name is not empty, then create block volume with requested size. + // If available size is less than requested size then ERROR. Set block related + // metadata and volume options if not exists. + if blockVolOpts.HostVol != "" { + vInfo, err := volume.GetVolume(blockVolOpts.HostVol) + if err != nil { + log.WithError(err).Error("error in fetching volume info") + return nil, err + } + volInfo = vInfo + } + + // If HostingVolume is not specified. List all available volumes and see if any volume is + // available with Metadata:block-hosting=yes + if blockVolOpts.HostVol == "" { + vInfo, err := GetExistingBlockHostingVolume(size) + if err != nil { + log.WithError(err).Debug("no block hosting volumes present") + } + volInfo = vInfo + } + + // If No volumes are available with Metadata:block-hosting=yes or if no space available to create block + // volumes(Metadata:block-hosting-available-size is less than request size), then try to create a new + // block hosting Volume with generated name with default size and volume type configured + if blockVolOpts.HostVol == "" && volInfo == nil { + vInfo, err := CreateBlockHostingVolume(volCreateReq) + if err != nil { + log.WithError(err).Error("error in auto creating block hosting volume") + return nil, err + } + + vInfo, _, err = volumecommands.StartVolume(ctx, vInfo.Name, gd2api.VolumeStartReq{}) + if err != nil { + log.WithError(err).Error("error in starting auto created block hosting volume") + return nil, err + } + + volInfo = vInfo + } + + if _, found := volInfo.Metadata["block-hosting"]; !found { + volInfo.Metadata["block-hosting"] = "yes" + } + + blockHosting := volInfo.Metadata["block-hosting"] + + if strings.ToLower(blockHosting) == "no" { + return nil, errors.New("not a block hosting volume") + } + + if _, found := volInfo.Metadata["block-hosting-available-size"]; !found { + volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", g.HostingVolumeOptions.Size) + } + + availableSizeInBytes, err := strconv.Atoi(volInfo.Metadata["block-hosting-available-size"]) + + if err != nil { + return nil, err + } + + if int64(availableSizeInBytes) < size { + return nil, fmt.Errorf("available size is less than requested size,request size: %d, available size: %d", size, availableSizeInBytes) + } + + if volInfo.State != volume.VolStarted { + return nil, errors.New("volume has not been started") + } + + req := &api.BlockVolumeCreateReq{ + HaCount: blockVolOpts.Ha, + AuthEnabled: blockVolOpts.Auth, + FullPrealloc: blockVolOpts.FullPrealloc, + Size: uint64(size), + Storage: blockVolOpts.Storage, + RingBufferSizeInMB: blockVolOpts.RingBufferSizeInMB, + Hosts: hosts, + } + + resp, err := g.client.CreateBlockVolume(volInfo.Name, name, req) + if err != nil { + return nil, err + } + + volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", int64(availableSizeInBytes)-size) + + if err := volume.AddOrUpdateVolume(volInfo); err != nil { + log.WithError(err).Error("failed in updating volume info to store") + } + + return &BlockVolume{ + hostVolume: volInfo.Name, + name: name, + hosts: resp.Portals, + iqn: resp.IQN, + username: resp.Username, + password: resp.Password, + size: int64(size), + ha: blockVolOpts.Ha, + }, nil +} + +// DeleteBlockVolume deletes a gluster block volume of give name +func (g *GlusterBlock) DeleteBlockVolume(name string, options ...blockprovider.BlockVolOption) error { + var ( + blockVolOpts = &blockprovider.BlockVolumeOptions{} + hostVol string + ) + + blockVolOpts.ApplyOpts(options...) + + blockVols := g.BlockVolumes() + + for _, blockVol := range blockVols { + if blockVol.Name() == name { + hostVol = blockVol.HostVolume() + break + } + } + + if hostVol == "" { + return errors.New("block volume not found") + } + + blockInfo, err := g.client.BlockVolumeInfo(hostVol, name) + if err != nil { + return err + } + + req := &api.BlockVolumeDeleteReq{ + UnlinkStorage: blockVolOpts.UnlinkStorage, + Force: blockVolOpts.ForceDelete, + } + + if err := g.client.DeleteBlockVolume(hostVol, name, req); err != nil { + return err + } + + if err := ResizeBlockHostingVolume(hostVol, blockInfo.Size); err != nil { + log.WithError(err).Error("error in resizing the block hosting volume") + } + + return nil +} + +// GetBlockVolume gives info about a gluster block volume +func (g *GlusterBlock) GetBlockVolume(name string) (blockprovider.BlockVolume, error) { + var ( + blockVolume blockprovider.BlockVolume + availableBlockVolumes = g.BlockVolumes() + ) + + for _, blockVol := range availableBlockVolumes { + if blockVol.Name() == name { + blockVolume = blockVol + break + } + } + + if blockVolume == nil { + return nil, errors.New("block volume not found") + } + + blockInfo, err := g.client.BlockVolumeInfo(blockVolume.HostVolume(), blockVolume.Name()) + if err != nil { + return nil, err + } + + glusterBlockVol := &BlockVolume{ + name: blockInfo.Name, + hostVolume: blockInfo.Volume, + password: blockInfo.Password, + hosts: blockInfo.ExportedOn, + gbID: blockInfo.GBID, + ha: blockInfo.Ha, + } + + if blockSize, err := size.Parse(blockInfo.Size); err == nil { + glusterBlockVol.size = int64(blockSize) + } + + return glusterBlockVol, nil +} + +// BlockVolumes returns all available gluster block volume +func (g *GlusterBlock) BlockVolumes() []blockprovider.BlockVolume { + var glusterBlockVolumes = []blockprovider.BlockVolume{} + + volumes, err := volume.GetVolumes(context.Background()) + if err != nil { + return glusterBlockVolumes + } + + volumes = volume.ApplyFilters(volumes, volume.BlockHosted) + + for _, vol := range volumes { + blockList, err := g.client.ListBlockVolumes(vol.Name) + if err != nil { + continue + } + + for _, block := range blockList.Blocks { + glusterBlockVolumes = append(glusterBlockVolumes, &BlockVolume{name: block, hostVolume: vol.Name}) + } + } + + return glusterBlockVolumes +} + +// ProviderName returns name of block provider +func (g *GlusterBlock) ProviderName() string { + return providerName +} + +// BlockVolume implements blockprovider.BlockVolume interface. +// It holds information about a gluster-block volume +type BlockVolume struct { + hosts []string + iqn string + username string + password string + hostVolume string + name string + size int64 + gbID string + ha int +} + +// HostAddresses returns host addresses of a gluster block vol +func (gv *BlockVolume) HostAddresses() []string { return gv.hosts } + +// IQN returns IQN of a gluster block vol +func (gv *BlockVolume) IQN() string { return gv.iqn } + +// Username returns username of a gluster-block vol. +func (gv *BlockVolume) Username() string { return gv.username } + +// Password returns password for a gluster block vol +func (gv *BlockVolume) Password() string { return gv.password } + +// HostVolume returns host vol name of gluster block +func (gv *BlockVolume) HostVolume() string { return gv.hostVolume } + +// Name returns name of gluster block vol +func (gv *BlockVolume) Name() string { return gv.name } + +// Size returns size of a gluster block vol in bytes +func (gv *BlockVolume) Size() uint64 { return uint64(gv.size) } + +// ID returns Gluster Block ID +func (gv *BlockVolume) ID() string { return gv.gbID } + +// HaCount returns high availability count +func (gv *BlockVolume) HaCount() int { return gv.ha } diff --git a/plugins/blockvolume/blockprovider/gluster-block/volume.go b/plugins/blockvolume/blockprovider/gluster-block/volume.go new file mode 100644 index 000000000..632d8a044 --- /dev/null +++ b/plugins/blockvolume/blockprovider/gluster-block/volume.go @@ -0,0 +1,108 @@ +package glusterblock + +import ( + "context" + "errors" + "fmt" + "math/rand" + "net/http" + "strconv" + "time" + + "github.com/gluster/glusterd2/glusterd2/commands/volumes" + "github.com/gluster/glusterd2/glusterd2/volume" + "github.com/gluster/glusterd2/pkg/api" + "github.com/gluster/glusterd2/pkg/size" +) + +// BlockSizeFilter returns a volume Filter, which will filter out volumes +// haing block-hosting-available-size greater than give size. +func BlockSizeFilter(size int64) volume.Filter { + return func(volinfos []*volume.Volinfo) []*volume.Volinfo { + var volumes []*volume.Volinfo + + for _, volinfo := range volinfos { + availableSize, found := volinfo.Metadata["block-hosting-available-size"] + if !found { + continue + } + + if availableSizeInBytes, err := strconv.Atoi(availableSize); err == nil && int64(availableSizeInBytes) > size { + volumes = append(volumes, volinfo) + } + } + return volumes + } +} + +// GetExistingBlockHostingVolume returns a existing volume which is suitable for hosting a gluster-block +func GetExistingBlockHostingVolume(size int64) (*volume.Volinfo, error) { + var ( + filters = []volume.Filter{volume.FilterBlockHostedVolumes, BlockSizeFilter(size)} + ctx, cancel = context.WithTimeout(context.Background(), time.Second*5) + ) + + defer cancel() + + volumes, err := volume.GetVolumes(ctx) + if err != nil || len(volumes) == 0 { + return nil, fmt.Errorf("%v/no volumes found", err) + } + + volumes = volume.ApplyCustomFilters(volumes, filters...) + + return SelectRandomVolume(volumes) +} + +// CreateBlockHostingVolume will create a gluster volume with metadata block-hosting-volume-auto-created=yes +func CreateBlockHostingVolume(req *api.VolCreateReq) (*volume.Volinfo, error) { + status, err := volumecommands.CreateVolume(context.Background(), *req) + if err != nil || status != http.StatusCreated { + return nil, err + } + + vInfo, err := volume.GetVolume(req.Name) + if err != nil { + return nil, err + } + + vInfo.Metadata["block-hosting-volume-auto-created"] = "yes" + return vInfo, nil +} + +// ResizeBlockHostingVolume will adds deletedBlockSize to block-hosting-available-size +// in metadata and update the new vol info to store. +func ResizeBlockHostingVolume(volname string, deletedBlockSize string) error { + volInfo, err := volume.GetVolume(volname) + if err != nil { + return err + } + + deletedSizeInBytes, err := size.Parse(deletedBlockSize) + if err != nil { + return err + } + + if _, found := volInfo.Metadata["block-hosting-available-size"]; !found { + return errors.New("block-hosting-available-size metadata not found for volume") + } + + availableSizeInBytes, err := strconv.Atoi(volInfo.Metadata["block-hosting-available-size"]) + if err != nil { + return err + } + + volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", size.Size(availableSizeInBytes)+deletedSizeInBytes) + + return volume.AddOrUpdateVolume(volInfo) +} + +// SelectRandomVolume will select a random volume from a given slice of volumes +func SelectRandomVolume(volumes []*volume.Volinfo) (*volume.Volinfo, error) { + if len(volumes) == 0 { + return nil, errors.New("no available volumes") + } + + i := rand.Int() % len(volumes) + return volumes[i], nil +} diff --git a/plugins/blockvolume/blockprovider/options.go b/plugins/blockvolume/blockprovider/options.go new file mode 100644 index 000000000..08990acfc --- /dev/null +++ b/plugins/blockvolume/blockprovider/options.go @@ -0,0 +1,71 @@ +package blockprovider + +// BlockVolOption configures various optional parameters for a block operation +type BlockVolOption func(*BlockVolumeOptions) + +// BlockVolumeOptions represents various optional params to be used for a block volume operation +type BlockVolumeOptions struct { + HostVol string + Auth bool + FullPrealloc bool + Storage string + Ha int + RingBufferSizeInMB uint64 + ForceDelete bool + UnlinkStorage bool +} + +// ApplyOpts applies configured optional parameters on BlockVolumeOptions +func (op *BlockVolumeOptions) ApplyOpts(optFuncs ...BlockVolOption) { + for _, optFunc := range optFuncs { + optFunc(op) + } +} + +// WithHostVolume configures a HostVolume param +func WithHostVolume(name string) BlockVolOption { + return func(options *BlockVolumeOptions) { + options.HostVol = name + } +} + +// WithHaCount configures haCount for block creation +func WithHaCount(count int) BlockVolOption { + return func(options *BlockVolumeOptions) { + options.Ha = count + } +} + +// WithStorage configures storage param for block-vol creation +func WithStorage(storage string) BlockVolOption { + return func(options *BlockVolumeOptions) { + options.Storage = storage + } +} + +// WithRingBufferSizeInMB configures ring-buffer param (size should in MB units) +func WithRingBufferSizeInMB(size uint64) BlockVolOption { + return func(options *BlockVolumeOptions) { + options.RingBufferSizeInMB = size + } +} + +// WithForceDelete configures force param in a block delete req +func WithForceDelete(options *BlockVolumeOptions) { + options.ForceDelete = true +} + +// WithUnlinkStorage configures unlink-storage param in block delete req +func WithUnlinkStorage(options *BlockVolumeOptions) { + options.UnlinkStorage = true +} + +// WithAuthEnabled enables auth for block creation +func WithAuthEnabled(options *BlockVolumeOptions) { + options.Auth = true +} + +// WithFullPrealloc configures "prealloc" param +func WithFullPrealloc(options *BlockVolumeOptions) { + options.FullPrealloc = true +} diff --git a/plugins/blockvolume/handlers.go b/plugins/blockvolume/handlers.go new file mode 100644 index 000000000..71a77736f --- /dev/null +++ b/plugins/blockvolume/handlers.go @@ -0,0 +1,107 @@ +package blockvolume + +import ( + "github.com/gorilla/mux" + "net/http" + + "github.com/gluster/glusterd2/glusterd2/servers/rest/utils" + "github.com/gluster/glusterd2/plugins/blockvolume/api" + "github.com/gluster/glusterd2/plugins/blockvolume/blockprovider" +) + +// CreateVolume is a http Handler for creating a block volume +func (b *BlockVolume) CreateVolume(w http.ResponseWriter, r *http.Request) { + var ( + req = &api.BlockVolumeCreateRequest{} + resp = &api.BlockVolumeCreateResp{} + opts = []blockprovider.BlockVolOption{} + ) + + if err := utils.UnmarshalRequest(r, req); err != nil { + utils.SendHTTPError(r.Context(), w, http.StatusBadRequest, err) + return + } + + opts = append(opts, + blockprovider.WithHostVolume(req.HostingVolume), + blockprovider.WithHaCount(req.HaCount), + ) + + if req.Auth { + opts = append(opts, blockprovider.WithAuthEnabled) + } + + blockVol, err := b.blockProvider.CreateBlockVolume(req.Name, req.Size, req.Clusters, opts...) + if err != nil { + utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) + return + } + + { + resp.BlockVolumeInfo = req.BlockVolumeInfo + resp.HostingVolume = blockVol.HostVolume() + resp.Name = blockVol.Name() + resp.Iqn = blockVol.IQN() + resp.Username = blockVol.Username() + resp.Password = blockVol.Password() + resp.Hosts = blockVol.HostAddresses() + } + + utils.SendHTTPResponse(r.Context(), w, http.StatusCreated, resp) +} + +// DeleteVolume is a http Handler for deleting a specific block-volume +func (b *BlockVolume) DeleteVolume(w http.ResponseWriter, r *http.Request) { + var ( + pathParams = mux.Vars(r) + ) + + if err := b.blockProvider.DeleteBlockVolume(pathParams["name"]); err != nil { + utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) + return + } + + utils.SendHTTPResponse(r.Context(), w, http.StatusNoContent, nil) +} + +// ListBlockVolumes is a http handler for listing all available block volumes +func (b *BlockVolume) ListBlockVolumes(w http.ResponseWriter, r *http.Request) { + var ( + resp = api.BlockVolumeListResp{} + ) + + blockVols := b.blockProvider.BlockVolumes() + + for _, blockVol := range blockVols { + resp = append(resp, api.BlockVolumeInfo{Name: blockVol.Name(), HostingVolume: blockVol.HostVolume()}) + } + + utils.SendHTTPResponse(r.Context(), w, http.StatusOK, resp) +} + +// GetBlockVolume is a http Handler for getting info about a block volume. +func (b *BlockVolume) GetBlockVolume(w http.ResponseWriter, r *http.Request) { + var ( + pathParams = mux.Vars(r) + resp = &api.BlockVolumeGetResp{} + ) + + blockVol, err := b.blockProvider.GetBlockVolume(pathParams["name"]) + if err != nil { + utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) + return + } + + { + resp.BlockVolumeInfo = &api.BlockVolumeInfo{} + resp.Name = blockVol.Name() + resp.HostingVolume = blockVol.HostVolume() + resp.Size = int64(blockVol.Size()) + resp.Hosts = blockVol.HostAddresses() + resp.Password = blockVol.Password() + resp.GBID = blockVol.ID() + resp.HaCount = blockVol.HaCount() + } + + utils.SendHTTPResponse(r.Context(), w, http.StatusOK, resp) +} diff --git a/plugins/blockvolume/init.go b/plugins/blockvolume/init.go new file mode 100644 index 000000000..4a1291199 --- /dev/null +++ b/plugins/blockvolume/init.go @@ -0,0 +1,12 @@ +package blockvolume + +import ( + // initialise all block providers + _ "github.com/gluster/glusterd2/plugins/blockvolume/blockprovider/gluster-block" + + config "github.com/spf13/viper" +) + +func init() { + config.SetDefault("block-provider", "gluster-block") +} diff --git a/plugins/blockvolume/routes.go b/plugins/blockvolume/routes.go new file mode 100644 index 000000000..f56948bcf --- /dev/null +++ b/plugins/blockvolume/routes.go @@ -0,0 +1,82 @@ +package blockvolume + +import ( + "net/http" + "sync" + + "github.com/gluster/glusterd2/glusterd2/servers/rest/route" + "github.com/gluster/glusterd2/pkg/utils" + "github.com/gluster/glusterd2/plugins/blockvolume/api" + "github.com/gluster/glusterd2/plugins/blockvolume/blockprovider" + + log "github.com/sirupsen/logrus" + config "github.com/spf13/viper" +) + +// BlockVolume represents BlockVolume plugin +type BlockVolume struct { + blockProvider blockprovider.Provider + initOnce sync.Once +} + +// Name returns underlying block provider name +func (b *BlockVolume) Name() string { + b.mustInitBlockProvider() + return b.blockProvider.ProviderName() +} + +// RestRoutes returns list of REST API routes of BlockVolume to register with Glusterd. +func (b *BlockVolume) RestRoutes() route.Routes { + b.mustInitBlockProvider() + return route.Routes{ + { + Name: "BlockCreate", + Method: http.MethodPost, + Pattern: "/blockvolumes", + Version: 1, + RequestType: utils.GetTypeString((*api.BlockVolumeCreateRequest)(nil)), + ResponseType: utils.GetTypeString((*api.BlockVolumeCreateResp)(nil)), + HandlerFunc: b.CreateVolume, + }, + { + Name: "BlockDelete", + Method: http.MethodDelete, + Pattern: "/blockvolumes/{name}", + Version: 1, + HandlerFunc: b.DeleteVolume, + }, + { + Name: "BlockList", + Method: http.MethodGet, + Pattern: "/blockvolumes", + Version: 1, + HandlerFunc: b.ListBlockVolumes, + }, + { + Name: "BlockGet", + Method: http.MethodGet, + Pattern: "/blockvolumes/{name}", + Version: 1, + HandlerFunc: b.GetBlockVolume, + }, + } +} + +// RegisterStepFuncs registers all step functions +// Here it is a no-op func +func (*BlockVolume) RegisterStepFuncs() { + +} + +// mustInitBlockProvider will initialize the underlying block provider only once. +// calling it multiple times will do nothing +func (b *BlockVolume) mustInitBlockProvider() { + b.initOnce.Do(func() { + providerName := config.GetString("block-provider") + provider, err := blockprovider.GetBlockProvider(providerName) + if err != nil { + log.WithError(err).Panic("failed in initializing block-volume provider") + } + b.blockProvider = provider + }) +} From 4dabeb102b89bfd10b760eb823ec364793d2509a Mon Sep 17 00:00:00 2001 From: Oshank Kumar Date: Tue, 18 Dec 2018 14:43:36 +0530 Subject: [PATCH 2/7] block-volume: moved common code to a utils package Signed-off-by: Oshank Kumar --- .../blockprovider/gluster-block/glusterblock.go | 7 ++++--- .../{blockprovider/gluster-block => utils}/volume.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) rename plugins/blockvolume/{blockprovider/gluster-block => utils}/volume.go (99%) diff --git a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go index 7df086785..2e0e46c6f 100644 --- a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go +++ b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/gluster/glusterd2/plugins/blockvolume/utils" "strconv" "strings" @@ -96,7 +97,7 @@ func (g *GlusterBlock) CreateBlockVolume(name string, size int64, hosts []string // If HostingVolume is not specified. List all available volumes and see if any volume is // available with Metadata:block-hosting=yes if blockVolOpts.HostVol == "" { - vInfo, err := GetExistingBlockHostingVolume(size) + vInfo, err := utils.GetExistingBlockHostingVolume(size) if err != nil { log.WithError(err).Debug("no block hosting volumes present") } @@ -107,7 +108,7 @@ func (g *GlusterBlock) CreateBlockVolume(name string, size int64, hosts []string // volumes(Metadata:block-hosting-available-size is less than request size), then try to create a new // block hosting Volume with generated name with default size and volume type configured if blockVolOpts.HostVol == "" && volInfo == nil { - vInfo, err := CreateBlockHostingVolume(volCreateReq) + vInfo, err := utils.CreateBlockHostingVolume(volCreateReq) if err != nil { log.WithError(err).Error("error in auto creating block hosting volume") return nil, err @@ -219,7 +220,7 @@ func (g *GlusterBlock) DeleteBlockVolume(name string, options ...blockprovider.B return err } - if err := ResizeBlockHostingVolume(hostVol, blockInfo.Size); err != nil { + if err := utils.ResizeBlockHostingVolume(hostVol, blockInfo.Size); err != nil { log.WithError(err).Error("error in resizing the block hosting volume") } diff --git a/plugins/blockvolume/blockprovider/gluster-block/volume.go b/plugins/blockvolume/utils/volume.go similarity index 99% rename from plugins/blockvolume/blockprovider/gluster-block/volume.go rename to plugins/blockvolume/utils/volume.go index 632d8a044..70fdace18 100644 --- a/plugins/blockvolume/blockprovider/gluster-block/volume.go +++ b/plugins/blockvolume/utils/volume.go @@ -1,4 +1,4 @@ -package glusterblock +package utils import ( "context" From 9391a6c597f08fd6b1ef513da0273e75589e4f69 Mon Sep 17 00:00:00 2001 From: Oshank Kumar Date: Tue, 18 Dec 2018 18:07:42 +0530 Subject: [PATCH 3/7] block-volume: added a interface for host volume management. - added block volume provider name in path parameter of url - block provider will not be responsible for managing host volumes. Signed-off-by: Oshank Kumar --- .../blockprovider/blockvolume_provider.go | 2 +- .../blockprovider/gluster-block/config.go | 42 ----- .../gluster-block/glusterblock.go | 122 +++---------- plugins/blockvolume/blockprovider/options.go | 8 - plugins/blockvolume/handlers.go | 59 +++++-- plugins/blockvolume/hostvol_manager.go | 163 ++++++++++++++++++ plugins/blockvolume/hostvol_opts.go | 47 +++++ plugins/blockvolume/init.go | 6 - plugins/blockvolume/routes.go | 37 ++-- plugins/blockvolume/utils/volume.go | 12 ++ 10 files changed, 308 insertions(+), 190 deletions(-) create mode 100644 plugins/blockvolume/hostvol_manager.go create mode 100644 plugins/blockvolume/hostvol_opts.go diff --git a/plugins/blockvolume/blockprovider/blockvolume_provider.go b/plugins/blockvolume/blockprovider/blockvolume_provider.go index c7a89edca..91f932821 100644 --- a/plugins/blockvolume/blockprovider/blockvolume_provider.go +++ b/plugins/blockvolume/blockprovider/blockvolume_provider.go @@ -18,7 +18,7 @@ var ( // Provider is an abstract, pluggable interface for block volume providers type Provider interface { - CreateBlockVolume(name string, size int64, hosts []string, options ...BlockVolOption) (BlockVolume, error) + CreateBlockVolume(name string, size int64, hosts []string, hostVolume string, options ...BlockVolOption) (BlockVolume, error) DeleteBlockVolume(name string, options ...BlockVolOption) error GetBlockVolume(id string) (BlockVolume, error) BlockVolumes() []BlockVolume diff --git a/plugins/blockvolume/blockprovider/gluster-block/config.go b/plugins/blockvolume/blockprovider/gluster-block/config.go index 2ec621d02..9980e1655 100644 --- a/plugins/blockvolume/blockprovider/gluster-block/config.go +++ b/plugins/blockvolume/blockprovider/gluster-block/config.go @@ -1,20 +1,9 @@ package glusterblock import ( - "github.com/gluster/glusterd2/pkg/api" - - "github.com/pborman/uuid" "github.com/spf13/viper" ) -// VolumeType represents a volume type -type VolumeType string - -const ( - // Replica represents a replica volume type - Replica VolumeType = "Replica" -) - // ClientConfig holds various config information needed to create a gluster-block rest client type ClientConfig struct { HostAddress string @@ -32,34 +21,3 @@ func (c *ClientConfig) ApplyFromConfig(conf *viper.Viper) { c.Secret = conf.GetString("gluster-block-secret") c.Insecure = conf.GetBool("gluster-block-insecure") } - -// HostingVolumeOptions holds various information which will be used in creating hosting volume -type HostingVolumeOptions struct { - Size int64 - Type VolumeType - ReplicaCount int - AutoCreate bool -} - -// ApplyFromConfig sets HostingVolumeOptions member values from given config source -func (h *HostingVolumeOptions) ApplyFromConfig(conf *viper.Viper) { - h.Size = conf.GetInt64("block-hosting-volume-size") - h.Type = VolumeType(conf.GetString("block-hosting-volume-type")) - h.ReplicaCount = conf.GetInt("block-hosting-volume-replica-count") - h.AutoCreate = conf.GetBool("auto-create-block-hosting-volumes") -} - -// PrepareVolumeCreateReq will create a request body to be use for creating a gluster volume -func (h *HostingVolumeOptions) PrepareVolumeCreateReq() *api.VolCreateReq { - name := "block_hosting_volume_" + uuid.NewRandom().String() - - req := &api.VolCreateReq{ - Name: name, - Transport: "tcp", - Size: uint64(h.Size), - ReplicaCount: h.ReplicaCount, - SubvolType: string(h.Type), - } - - return req -} diff --git a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go index 2e0e46c6f..a85354471 100644 --- a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go +++ b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go @@ -4,19 +4,16 @@ import ( "context" "errors" "fmt" - "github.com/gluster/glusterd2/plugins/blockvolume/utils" "strconv" - "strings" - "github.com/gluster/gluster-block-restapi/client" - "github.com/gluster/gluster-block-restapi/pkg/api" - "github.com/gluster/glusterd2/glusterd2/commands/volumes" - "github.com/gluster/glusterd2/glusterd2/gdctx" + "github.com/gluster/glusterd2/glusterd2/transaction" "github.com/gluster/glusterd2/glusterd2/volume" - gd2api "github.com/gluster/glusterd2/pkg/api" "github.com/gluster/glusterd2/pkg/size" "github.com/gluster/glusterd2/plugins/blockvolume/blockprovider" + "github.com/gluster/glusterd2/plugins/blockvolume/utils" + "github.com/gluster/gluster-block-restapi/client" + "github.com/gluster/gluster-block-restapi/pkg/api" log "github.com/sirupsen/logrus" "github.com/spf13/viper" ) @@ -30,24 +27,19 @@ func init() { // GlusterBlock implements block Provider interface. It represents a gluster-block type GlusterBlock struct { - client client.GlusterBlockClient - ClientConf *ClientConfig - HostingVolumeOptions *HostingVolumeOptions + client client.GlusterBlockClient + ClientConf *ClientConfig } func newGlusterBlock() (blockprovider.Provider, error) { var ( - gb = &GlusterBlock{} - clientConf = &ClientConfig{} - hostVolOpts = &HostingVolumeOptions{} - opts = []client.OptFuncs{} + gb = &GlusterBlock{} + clientConf = &ClientConfig{} + opts = []client.OptFuncs{} ) clientConf.ApplyFromConfig(viper.GetViper()) - hostVolOpts.ApplyFromConfig(viper.GetViper()) - gb.ClientConf = clientConf - gb.HostingVolumeOptions = hostVolOpts opts = append(opts, client.WithAuth(clientConf.User, clientConf.Secret), @@ -63,82 +55,38 @@ func newGlusterBlock() (blockprovider.Provider, error) { return gb, nil } -// CreateBlockVolume will create a gluster block volume with given name and size. If hosting volume is not provide then it will -// create a gluster volume for hosting gluster block. -func (g *GlusterBlock) CreateBlockVolume(name string, size int64, hosts []string, options ...blockprovider.BlockVolOption) (blockprovider.BlockVolume, error) { +// CreateBlockVolume will create a gluster block volume with given name and size having `hostVolume` as hosting volume +func (g *GlusterBlock) CreateBlockVolume(name string, size int64, hosts []string, hostVolume string, options ...blockprovider.BlockVolOption) (blockprovider.BlockVolume, error) { var ( blockVolOpts = &blockprovider.BlockVolumeOptions{} - volCreateReq = g.HostingVolumeOptions.PrepareVolumeCreateReq() - volInfo *volume.Volinfo - ctx = gdctx.WithReqLogger(context.Background(), log.StandardLogger()) + clusterLocks = transaction.Locks{} ) blockVolOpts.ApplyOpts(options...) - // ERROR if If HostingVolume is not specified and auto-create-block-hosting-volumes is false - if blockVolOpts.HostVol == "" && !g.HostingVolumeOptions.AutoCreate { - err := errors.New("host volume is not provided and auto creation is not enabled") - log.WithError(err).Error("failed in creating block volume") - return nil, err - } - - // If HostingVolume name is not empty, then create block volume with requested size. - // If available size is less than requested size then ERROR. Set block related - // metadata and volume options if not exists. - if blockVolOpts.HostVol != "" { - vInfo, err := volume.GetVolume(blockVolOpts.HostVol) - if err != nil { - log.WithError(err).Error("error in fetching volume info") - return nil, err - } - volInfo = vInfo - } - - // If HostingVolume is not specified. List all available volumes and see if any volume is - // available with Metadata:block-hosting=yes - if blockVolOpts.HostVol == "" { - vInfo, err := utils.GetExistingBlockHostingVolume(size) - if err != nil { - log.WithError(err).Debug("no block hosting volumes present") - } - volInfo = vInfo - } - - // If No volumes are available with Metadata:block-hosting=yes or if no space available to create block - // volumes(Metadata:block-hosting-available-size is less than request size), then try to create a new - // block hosting Volume with generated name with default size and volume type configured - if blockVolOpts.HostVol == "" && volInfo == nil { - vInfo, err := utils.CreateBlockHostingVolume(volCreateReq) - if err != nil { - log.WithError(err).Error("error in auto creating block hosting volume") - return nil, err - } - - vInfo, _, err = volumecommands.StartVolume(ctx, vInfo.Name, gd2api.VolumeStartReq{}) - if err != nil { - log.WithError(err).Error("error in starting auto created block hosting volume") - return nil, err - } - - volInfo = vInfo + req := &api.BlockVolumeCreateReq{ + HaCount: blockVolOpts.Ha, + AuthEnabled: blockVolOpts.Auth, + FullPrealloc: blockVolOpts.FullPrealloc, + Size: uint64(size), + Storage: blockVolOpts.Storage, + RingBufferSizeInMB: blockVolOpts.RingBufferSizeInMB, + Hosts: hosts, } - if _, found := volInfo.Metadata["block-hosting"]; !found { - volInfo.Metadata["block-hosting"] = "yes" + volInfo, err := volume.GetVolume(hostVolume) + if err != nil { + return nil, fmt.Errorf("error in getting host vol details: %s", err) } - blockHosting := volInfo.Metadata["block-hosting"] - - if strings.ToLower(blockHosting) == "no" { - return nil, errors.New("not a block hosting volume") + if err := clusterLocks.Lock(volInfo.Name); err != nil { + log.WithError(err).Error("error in acquiring cluster lock") + return nil, err } - if _, found := volInfo.Metadata["block-hosting-available-size"]; !found { - volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", g.HostingVolumeOptions.Size) - } + defer clusterLocks.UnLock(context.Background()) availableSizeInBytes, err := strconv.Atoi(volInfo.Metadata["block-hosting-available-size"]) - if err != nil { return nil, err } @@ -147,21 +95,7 @@ func (g *GlusterBlock) CreateBlockVolume(name string, size int64, hosts []string return nil, fmt.Errorf("available size is less than requested size,request size: %d, available size: %d", size, availableSizeInBytes) } - if volInfo.State != volume.VolStarted { - return nil, errors.New("volume has not been started") - } - - req := &api.BlockVolumeCreateReq{ - HaCount: blockVolOpts.Ha, - AuthEnabled: blockVolOpts.Auth, - FullPrealloc: blockVolOpts.FullPrealloc, - Size: uint64(size), - Storage: blockVolOpts.Storage, - RingBufferSizeInMB: blockVolOpts.RingBufferSizeInMB, - Hosts: hosts, - } - - resp, err := g.client.CreateBlockVolume(volInfo.Name, name, req) + resp, err := g.client.CreateBlockVolume(hostVolume, name, req) if err != nil { return nil, err } diff --git a/plugins/blockvolume/blockprovider/options.go b/plugins/blockvolume/blockprovider/options.go index 08990acfc..f19d8e958 100644 --- a/plugins/blockvolume/blockprovider/options.go +++ b/plugins/blockvolume/blockprovider/options.go @@ -5,7 +5,6 @@ type BlockVolOption func(*BlockVolumeOptions) // BlockVolumeOptions represents various optional params to be used for a block volume operation type BlockVolumeOptions struct { - HostVol string Auth bool FullPrealloc bool Storage string @@ -22,13 +21,6 @@ func (op *BlockVolumeOptions) ApplyOpts(optFuncs ...BlockVolOption) { } } -// WithHostVolume configures a HostVolume param -func WithHostVolume(name string) BlockVolOption { - return func(options *BlockVolumeOptions) { - options.HostVol = name - } -} - // WithHaCount configures haCount for block creation func WithHaCount(count int) BlockVolOption { return func(options *BlockVolumeOptions) { diff --git a/plugins/blockvolume/handlers.go b/plugins/blockvolume/handlers.go index 71a77736f..b3fb3193b 100644 --- a/plugins/blockvolume/handlers.go +++ b/plugins/blockvolume/handlers.go @@ -1,20 +1,22 @@ package blockvolume import ( - "github.com/gorilla/mux" "net/http" "github.com/gluster/glusterd2/glusterd2/servers/rest/utils" "github.com/gluster/glusterd2/plugins/blockvolume/api" "github.com/gluster/glusterd2/plugins/blockvolume/blockprovider" + + "github.com/gorilla/mux" ) // CreateVolume is a http Handler for creating a block volume func (b *BlockVolume) CreateVolume(w http.ResponseWriter, r *http.Request) { var ( - req = &api.BlockVolumeCreateRequest{} - resp = &api.BlockVolumeCreateResp{} - opts = []blockprovider.BlockVolOption{} + req = &api.BlockVolumeCreateRequest{} + resp = &api.BlockVolumeCreateResp{} + opts = []blockprovider.BlockVolOption{} + pathParams = mux.Vars(r) ) if err := utils.UnmarshalRequest(r, req); err != nil { @@ -22,16 +24,25 @@ func (b *BlockVolume) CreateVolume(w http.ResponseWriter, r *http.Request) { return } - opts = append(opts, - blockprovider.WithHostVolume(req.HostingVolume), - blockprovider.WithHaCount(req.HaCount), - ) + opts = append(opts, blockprovider.WithHaCount(req.HaCount)) if req.Auth { opts = append(opts, blockprovider.WithAuthEnabled) } - blockVol, err := b.blockProvider.CreateBlockVolume(req.Name, req.Size, req.Clusters, opts...) + blockProvider, err := blockprovider.GetBlockProvider(pathParams["provider"]) + if err != nil { + utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) + return + } + + hostVolInfo, err := b.hostVolManager.GetOrCreateHostingVolume(req.HostingVolume, req.Size) + if err != nil { + utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) + return + } + + blockVol, err := blockProvider.CreateBlockVolume(req.Name, req.Size, req.Clusters, hostVolInfo.Name, opts...) if err != nil { utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) return @@ -52,11 +63,14 @@ func (b *BlockVolume) CreateVolume(w http.ResponseWriter, r *http.Request) { // DeleteVolume is a http Handler for deleting a specific block-volume func (b *BlockVolume) DeleteVolume(w http.ResponseWriter, r *http.Request) { - var ( - pathParams = mux.Vars(r) - ) + pathParams := mux.Vars(r) + blockProvider, err := blockprovider.GetBlockProvider(pathParams["provider"]) + if err != nil { + utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) + return + } - if err := b.blockProvider.DeleteBlockVolume(pathParams["name"]); err != nil { + if err := blockProvider.DeleteBlockVolume(pathParams["name"]); err != nil { utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) return } @@ -67,10 +81,17 @@ func (b *BlockVolume) DeleteVolume(w http.ResponseWriter, r *http.Request) { // ListBlockVolumes is a http handler for listing all available block volumes func (b *BlockVolume) ListBlockVolumes(w http.ResponseWriter, r *http.Request) { var ( - resp = api.BlockVolumeListResp{} + resp = api.BlockVolumeListResp{} + pathParams = mux.Vars(r) ) - blockVols := b.blockProvider.BlockVolumes() + blockProvider, err := blockprovider.GetBlockProvider(pathParams["provider"]) + if err != nil { + utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) + return + } + + blockVols := blockProvider.BlockVolumes() for _, blockVol := range blockVols { resp = append(resp, api.BlockVolumeInfo{Name: blockVol.Name(), HostingVolume: blockVol.HostVolume()}) @@ -86,7 +107,13 @@ func (b *BlockVolume) GetBlockVolume(w http.ResponseWriter, r *http.Request) { resp = &api.BlockVolumeGetResp{} ) - blockVol, err := b.blockProvider.GetBlockVolume(pathParams["name"]) + blockProvider, err := blockprovider.GetBlockProvider(pathParams["provider"]) + if err != nil { + utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) + return + } + + blockVol, err := blockProvider.GetBlockVolume(pathParams["name"]) if err != nil { utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) return diff --git a/plugins/blockvolume/hostvol_manager.go b/plugins/blockvolume/hostvol_manager.go new file mode 100644 index 000000000..65e915f90 --- /dev/null +++ b/plugins/blockvolume/hostvol_manager.go @@ -0,0 +1,163 @@ +package blockvolume + +import ( + "context" + "errors" + "fmt" + "path" + "strconv" + "strings" + "time" + + "github.com/gluster/glusterd2/glusterd2/commands/volumes" + "github.com/gluster/glusterd2/glusterd2/gdctx" + "github.com/gluster/glusterd2/glusterd2/transaction" + "github.com/gluster/glusterd2/glusterd2/volume" + "github.com/gluster/glusterd2/pkg/api" + "github.com/gluster/glusterd2/plugins/blockvolume/utils" + + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" +) + +const ( + globalLockID = "host-vol-lock" +) + +// HostingVolumeManager provides methods for host volume management +type HostingVolumeManager interface { + GetHostingVolumesInUse() []*volume.Volinfo + GetOrCreateHostingVolume(name string, minSizeLimit int64) (*volume.Volinfo, error) +} + +// glusterVolManager is a concrete implementation of HostingVolumeManager +type glusterVolManager struct { + hostVolOpts *HostingVolumeOptions +} + +// newGlusterVolManager returns a glusterVolManager instance +func newGlusterVolManager() *glusterVolManager { + var ( + g = &glusterVolManager{} + hostVolOpts = &HostingVolumeOptions{} + ) + + hostVolOpts.ApplyFromConfig(viper.GetViper()) + g.hostVolOpts = hostVolOpts + return g +} + +// GetHostingVolumesInUse lists all volumes which used in hosting block-vols +func (g *glusterVolManager) GetHostingVolumesInUse() []*volume.Volinfo { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + volumes, err := volume.GetVolumes(ctx) + if err != nil || len(volumes) == 0 { + return nil + } + + return volume.ApplyFilters(volumes, volume.BlockHosted) +} + +// GetOrCreateHostingVolume will returns volume details for a given volume name and having a minimum size of `minSizeLimit`. +// If volume name is not provided then it will create a gluster volume with default size for hosting gluster block. +func (g *glusterVolManager) GetOrCreateHostingVolume(name string, minSizeLimit int64) (*volume.Volinfo, error) { + var ( + volInfo *volume.Volinfo + volCreateReq = g.hostVolOpts.PrepareVolumeCreateReq() + ctx = gdctx.WithReqLogger(context.Background(), log.StandardLogger()) + clusterLocks = transaction.Locks{} + ) + + if err := clusterLocks.Lock(path.Join(globalLockID, name)); err != nil { + return nil, err + } + defer clusterLocks.UnLock(context.Background()) + + // ERROR if If HostingVolume is not specified and auto-create-block-hosting-volumes is false + if name == "" && !g.hostVolOpts.AutoCreate { + err := errors.New("host volume is not provided and auto creation is not enabled") + log.WithError(err).Error("failed in creating block volume") + return nil, err + } + + // If HostingVolume name is not empty, then create block volume with requested size. + // If available size is less than requested size then ERROR. Set block related + // metadata and volume options if not exists. + if name != "" { + vInfo, err := volume.GetVolume(name) + if err != nil { + log.WithError(err).Error("error in fetching volume info") + return nil, err + } + volInfo = vInfo + } + + // If HostingVolume is not specified. List all available volumes and see if any volume is + // available with Metadata:block-hosting=yes + if name == "" { + vInfo, err := utils.GetExistingBlockHostingVolume(minSizeLimit) + if err != nil { + log.WithError(err).Debug("no block hosting volumes present") + } + volInfo = vInfo + } + + // If No volumes are available with Metadata:block-hosting=yes or if no space available to create block + // volumes(Metadata:block-hosting-available-size is less than request size), then try to create a new + // block hosting Volume with generated name with default size and volume type configured + if name == "" && volInfo == nil { + vInfo, err := utils.CreateBlockHostingVolume(volCreateReq) + if err != nil { + log.WithError(err).Error("error in auto creating block hosting volume") + return nil, err + } + + log.WithField("name", vInfo.Name).Debug("auto creation of hosting volume succeeded") + + vInfo, _, err = volumecommands.StartVolume(ctx, vInfo.Name, api.VolumeStartReq{}) + if err != nil { + log.WithError(err).Error("error in starting auto created block hosting volume") + return nil, err + } + + log.WithField("name", vInfo.Name).Debug("host volume started successfully") + + volInfo = vInfo + } + + if _, found := volInfo.Metadata["block-hosting"]; !found { + volInfo.Metadata["block-hosting"] = "yes" + } + + blockHosting := volInfo.Metadata["block-hosting"] + + if strings.ToLower(blockHosting) != "yes" { + return nil, errors.New("not a block hosting volume") + } + + if _, found := volInfo.Metadata["block-hosting-available-size"]; !found { + volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", g.hostVolOpts.Size) + } + + availableSizeInBytes, err := strconv.Atoi(volInfo.Metadata["block-hosting-available-size"]) + + if err != nil { + return nil, err + } + + if int64(availableSizeInBytes) < minSizeLimit { + return nil, fmt.Errorf("available size is less than requested size,request size: %d, available size: %d", minSizeLimit, availableSizeInBytes) + } + + if volInfo.State != volume.VolStarted { + return nil, errors.New("volume has not been started") + } + + if err := volume.AddOrUpdateVolume(volInfo); err != nil { + log.WithError(err).Error("failed in updating volume info to store") + } + + return volInfo, nil +} diff --git a/plugins/blockvolume/hostvol_opts.go b/plugins/blockvolume/hostvol_opts.go new file mode 100644 index 000000000..063034836 --- /dev/null +++ b/plugins/blockvolume/hostvol_opts.go @@ -0,0 +1,47 @@ +package blockvolume + +import ( + "github.com/gluster/glusterd2/pkg/api" + + "github.com/pborman/uuid" + "github.com/spf13/viper" +) + +// VolumeType represents a volume type +type VolumeType string + +const ( + // Replica represents a replica volume type + Replica VolumeType = "Replica" +) + +// HostingVolumeOptions holds various information which will be used in creating hosting volume +type HostingVolumeOptions struct { + Size int64 + Type VolumeType + ReplicaCount int + AutoCreate bool +} + +// ApplyFromConfig sets HostingVolumeOptions member values from given config source +func (h *HostingVolumeOptions) ApplyFromConfig(conf *viper.Viper) { + h.Size = conf.GetInt64("block-hosting-volume-size") + h.Type = VolumeType(conf.GetString("block-hosting-volume-type")) + h.ReplicaCount = conf.GetInt("block-hosting-volume-replica-count") + h.AutoCreate = conf.GetBool("auto-create-block-hosting-volumes") +} + +// PrepareVolumeCreateReq will create a request body to be use for creating a gluster volume +func (h *HostingVolumeOptions) PrepareVolumeCreateReq() *api.VolCreateReq { + name := "block_hosting_volume_" + uuid.NewRandom().String() + + req := &api.VolCreateReq{ + Name: name, + Transport: "tcp", + Size: uint64(h.Size), + ReplicaCount: h.ReplicaCount, + SubvolType: string(h.Type), + } + + return req +} diff --git a/plugins/blockvolume/init.go b/plugins/blockvolume/init.go index 4a1291199..3bbd19897 100644 --- a/plugins/blockvolume/init.go +++ b/plugins/blockvolume/init.go @@ -3,10 +3,4 @@ package blockvolume import ( // initialise all block providers _ "github.com/gluster/glusterd2/plugins/blockvolume/blockprovider/gluster-block" - - config "github.com/spf13/viper" ) - -func init() { - config.SetDefault("block-provider", "gluster-block") -} diff --git a/plugins/blockvolume/routes.go b/plugins/blockvolume/routes.go index f56948bcf..1eafdb8ff 100644 --- a/plugins/blockvolume/routes.go +++ b/plugins/blockvolume/routes.go @@ -7,32 +7,28 @@ import ( "github.com/gluster/glusterd2/glusterd2/servers/rest/route" "github.com/gluster/glusterd2/pkg/utils" "github.com/gluster/glusterd2/plugins/blockvolume/api" - "github.com/gluster/glusterd2/plugins/blockvolume/blockprovider" - - log "github.com/sirupsen/logrus" - config "github.com/spf13/viper" ) // BlockVolume represents BlockVolume plugin type BlockVolume struct { - blockProvider blockprovider.Provider - initOnce sync.Once + hostVolManager HostingVolumeManager + initOnce sync.Once } -// Name returns underlying block provider name +// Name returns plugin name func (b *BlockVolume) Name() string { - b.mustInitBlockProvider() - return b.blockProvider.ProviderName() + b.Init() + return "block-volume" } // RestRoutes returns list of REST API routes of BlockVolume to register with Glusterd. func (b *BlockVolume) RestRoutes() route.Routes { - b.mustInitBlockProvider() + b.Init() return route.Routes{ { Name: "BlockCreate", Method: http.MethodPost, - Pattern: "/blockvolumes", + Pattern: "/blockvolumes/{provider}", Version: 1, RequestType: utils.GetTypeString((*api.BlockVolumeCreateRequest)(nil)), ResponseType: utils.GetTypeString((*api.BlockVolumeCreateResp)(nil)), @@ -41,21 +37,21 @@ func (b *BlockVolume) RestRoutes() route.Routes { { Name: "BlockDelete", Method: http.MethodDelete, - Pattern: "/blockvolumes/{name}", + Pattern: "/blockvolumes/{provider}/{name}", Version: 1, HandlerFunc: b.DeleteVolume, }, { Name: "BlockList", Method: http.MethodGet, - Pattern: "/blockvolumes", + Pattern: "/blockvolumes/{provider}", Version: 1, HandlerFunc: b.ListBlockVolumes, }, { Name: "BlockGet", Method: http.MethodGet, - Pattern: "/blockvolumes/{name}", + Pattern: "/blockvolumes/{provider}/{name}", Version: 1, HandlerFunc: b.GetBlockVolume, }, @@ -63,20 +59,15 @@ func (b *BlockVolume) RestRoutes() route.Routes { } // RegisterStepFuncs registers all step functions -// Here it is a no-op func +// Here it is a no-op method func (*BlockVolume) RegisterStepFuncs() { } -// mustInitBlockProvider will initialize the underlying block provider only once. +// Init will initialize the underlying HostVolume manager only once. // calling it multiple times will do nothing -func (b *BlockVolume) mustInitBlockProvider() { +func (b *BlockVolume) Init() { b.initOnce.Do(func() { - providerName := config.GetString("block-provider") - provider, err := blockprovider.GetBlockProvider(providerName) - if err != nil { - log.WithError(err).Panic("failed in initializing block-volume provider") - } - b.blockProvider = provider + b.hostVolManager = newGlusterVolManager() }) } diff --git a/plugins/blockvolume/utils/volume.go b/plugins/blockvolume/utils/volume.go index 70fdace18..ab3c0d5c7 100644 --- a/plugins/blockvolume/utils/volume.go +++ b/plugins/blockvolume/utils/volume.go @@ -10,9 +10,12 @@ import ( "time" "github.com/gluster/glusterd2/glusterd2/commands/volumes" + "github.com/gluster/glusterd2/glusterd2/transaction" "github.com/gluster/glusterd2/glusterd2/volume" "github.com/gluster/glusterd2/pkg/api" "github.com/gluster/glusterd2/pkg/size" + + log "github.com/sirupsen/logrus" ) // BlockSizeFilter returns a volume Filter, which will filter out volumes @@ -73,6 +76,8 @@ func CreateBlockHostingVolume(req *api.VolCreateReq) (*volume.Volinfo, error) { // ResizeBlockHostingVolume will adds deletedBlockSize to block-hosting-available-size // in metadata and update the new vol info to store. func ResizeBlockHostingVolume(volname string, deletedBlockSize string) error { + clusterLocks := transaction.Locks{} + volInfo, err := volume.GetVolume(volname) if err != nil { return err @@ -94,6 +99,13 @@ func ResizeBlockHostingVolume(volname string, deletedBlockSize string) error { volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", size.Size(availableSizeInBytes)+deletedSizeInBytes) + if err := clusterLocks.Lock(volInfo.Name); err != nil { + log.WithError(err).Error("error in acquiring cluster lock") + return err + } + + defer clusterLocks.UnLock(context.Background()) + return volume.AddOrUpdateVolume(volInfo) } From ffadf9eb47de692c4faae4fdcd13e7588726840f Mon Sep 17 00:00:00 2001 From: Oshank Kumar Date: Fri, 21 Dec 2018 11:31:58 +0530 Subject: [PATCH 4/7] block-volume: change size type from int64 to uint64 Signed-off-by: Oshank Kumar --- plugins/blockvolume/api/types.go | 4 +- .../blockprovider/blockvolume_provider.go | 2 +- .../gluster-block/glusterblock.go | 44 ++++++++++--------- plugins/blockvolume/handlers.go | 2 +- plugins/blockvolume/hostvol_manager.go | 8 ++-- plugins/blockvolume/utils/volume.go | 25 +++++------ 6 files changed, 43 insertions(+), 42 deletions(-) diff --git a/plugins/blockvolume/api/types.go b/plugins/blockvolume/api/types.go index 1a6814685..907b58a37 100644 --- a/plugins/blockvolume/api/types.go +++ b/plugins/blockvolume/api/types.go @@ -7,8 +7,8 @@ type BlockVolumeInfo struct { // Name represents block Volume name Name string `json:"name"` // Size represents Block Volume size in bytes - Size int64 `json:"size,omitempty"` - HaCount int `json:"hacount,omitempty"` + Size uint64 `json:"size,omitempty"` + HaCount int `json:"hacount,omitempty"` } // BlockVolumeCreateRequest represents req Body for Block vol create req diff --git a/plugins/blockvolume/blockprovider/blockvolume_provider.go b/plugins/blockvolume/blockprovider/blockvolume_provider.go index 91f932821..4b5cfbebd 100644 --- a/plugins/blockvolume/blockprovider/blockvolume_provider.go +++ b/plugins/blockvolume/blockprovider/blockvolume_provider.go @@ -18,7 +18,7 @@ var ( // Provider is an abstract, pluggable interface for block volume providers type Provider interface { - CreateBlockVolume(name string, size int64, hosts []string, hostVolume string, options ...BlockVolOption) (BlockVolume, error) + CreateBlockVolume(name string, size uint64, hosts []string, hostVolume string, options ...BlockVolOption) (BlockVolume, error) DeleteBlockVolume(name string, options ...BlockVolOption) error GetBlockVolume(id string) (BlockVolume, error) BlockVolumes() []BlockVolume diff --git a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go index a85354471..bdd7f8ea4 100644 --- a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go +++ b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go @@ -56,7 +56,7 @@ func newGlusterBlock() (blockprovider.Provider, error) { } // CreateBlockVolume will create a gluster block volume with given name and size having `hostVolume` as hosting volume -func (g *GlusterBlock) CreateBlockVolume(name string, size int64, hosts []string, hostVolume string, options ...blockprovider.BlockVolOption) (blockprovider.BlockVolume, error) { +func (g *GlusterBlock) CreateBlockVolume(name string, size uint64, hosts []string, hostVolume string, options ...blockprovider.BlockVolOption) (blockprovider.BlockVolume, error) { var ( blockVolOpts = &blockprovider.BlockVolumeOptions{} clusterLocks = transaction.Locks{} @@ -68,30 +68,29 @@ func (g *GlusterBlock) CreateBlockVolume(name string, size int64, hosts []string HaCount: blockVolOpts.Ha, AuthEnabled: blockVolOpts.Auth, FullPrealloc: blockVolOpts.FullPrealloc, - Size: uint64(size), + Size: size, Storage: blockVolOpts.Storage, RingBufferSizeInMB: blockVolOpts.RingBufferSizeInMB, Hosts: hosts, } - volInfo, err := volume.GetVolume(hostVolume) - if err != nil { - return nil, fmt.Errorf("error in getting host vol details: %s", err) - } - - if err := clusterLocks.Lock(volInfo.Name); err != nil { + if err := clusterLocks.Lock(hostVolume); err != nil { log.WithError(err).Error("error in acquiring cluster lock") return nil, err } - defer clusterLocks.UnLock(context.Background()) - availableSizeInBytes, err := strconv.Atoi(volInfo.Metadata["block-hosting-available-size"]) + volInfo, err := volume.GetVolume(hostVolume) + if err != nil { + return nil, fmt.Errorf("error in getting host vol details: %s", err) + } + + availableSizeInBytes, err := strconv.ParseUint(volInfo.Metadata["block-hosting-available-size"], 10, 64) if err != nil { return nil, err } - if int64(availableSizeInBytes) < size { + if availableSizeInBytes < size { return nil, fmt.Errorf("available size is less than requested size,request size: %d, available size: %d", size, availableSizeInBytes) } @@ -100,9 +99,9 @@ func (g *GlusterBlock) CreateBlockVolume(name string, size int64, hosts []string return nil, err } - volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", int64(availableSizeInBytes)-size) + volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", availableSizeInBytes-size) - if err := volume.AddOrUpdateVolume(volInfo); err != nil { + if err = volume.AddOrUpdateVolume(volInfo); err != nil { log.WithError(err).Error("failed in updating volume info to store") } @@ -113,9 +112,9 @@ func (g *GlusterBlock) CreateBlockVolume(name string, size int64, hosts []string iqn: resp.IQN, username: resp.Username, password: resp.Password, - size: int64(size), + size: size, ha: blockVolOpts.Ha, - }, nil + }, err } // DeleteBlockVolume deletes a gluster block volume of give name @@ -154,11 +153,14 @@ func (g *GlusterBlock) DeleteBlockVolume(name string, options ...blockprovider.B return err } - if err := utils.ResizeBlockHostingVolume(hostVol, blockInfo.Size); err != nil { - log.WithError(err).Error("error in resizing the block hosting volume") + if err = utils.ResizeBlockHostingVolume(hostVol, blockInfo.Size); err != nil { + log.WithFields(log.Fields{ + "error": err, + "size": blockInfo.Size, + }).Error("error in resizing the block hosting volume") } - return nil + return err } // GetBlockVolume gives info about a gluster block volume @@ -194,7 +196,7 @@ func (g *GlusterBlock) GetBlockVolume(name string) (blockprovider.BlockVolume, e } if blockSize, err := size.Parse(blockInfo.Size); err == nil { - glusterBlockVol.size = int64(blockSize) + glusterBlockVol.size = uint64(blockSize) } return glusterBlockVol, nil @@ -239,7 +241,7 @@ type BlockVolume struct { password string hostVolume string name string - size int64 + size uint64 gbID string ha int } @@ -263,7 +265,7 @@ func (gv *BlockVolume) HostVolume() string { return gv.hostVolume } func (gv *BlockVolume) Name() string { return gv.name } // Size returns size of a gluster block vol in bytes -func (gv *BlockVolume) Size() uint64 { return uint64(gv.size) } +func (gv *BlockVolume) Size() uint64 { return gv.size } // ID returns Gluster Block ID func (gv *BlockVolume) ID() string { return gv.gbID } diff --git a/plugins/blockvolume/handlers.go b/plugins/blockvolume/handlers.go index b3fb3193b..b4edd9a7d 100644 --- a/plugins/blockvolume/handlers.go +++ b/plugins/blockvolume/handlers.go @@ -123,7 +123,7 @@ func (b *BlockVolume) GetBlockVolume(w http.ResponseWriter, r *http.Request) { resp.BlockVolumeInfo = &api.BlockVolumeInfo{} resp.Name = blockVol.Name() resp.HostingVolume = blockVol.HostVolume() - resp.Size = int64(blockVol.Size()) + resp.Size = blockVol.Size() resp.Hosts = blockVol.HostAddresses() resp.Password = blockVol.Password() resp.GBID = blockVol.ID() diff --git a/plugins/blockvolume/hostvol_manager.go b/plugins/blockvolume/hostvol_manager.go index 65e915f90..82713e0ec 100644 --- a/plugins/blockvolume/hostvol_manager.go +++ b/plugins/blockvolume/hostvol_manager.go @@ -27,7 +27,7 @@ const ( // HostingVolumeManager provides methods for host volume management type HostingVolumeManager interface { GetHostingVolumesInUse() []*volume.Volinfo - GetOrCreateHostingVolume(name string, minSizeLimit int64) (*volume.Volinfo, error) + GetOrCreateHostingVolume(name string, minSizeLimit uint64) (*volume.Volinfo, error) } // glusterVolManager is a concrete implementation of HostingVolumeManager @@ -62,7 +62,7 @@ func (g *glusterVolManager) GetHostingVolumesInUse() []*volume.Volinfo { // GetOrCreateHostingVolume will returns volume details for a given volume name and having a minimum size of `minSizeLimit`. // If volume name is not provided then it will create a gluster volume with default size for hosting gluster block. -func (g *glusterVolManager) GetOrCreateHostingVolume(name string, minSizeLimit int64) (*volume.Volinfo, error) { +func (g *glusterVolManager) GetOrCreateHostingVolume(name string, minSizeLimit uint64) (*volume.Volinfo, error) { var ( volInfo *volume.Volinfo volCreateReq = g.hostVolOpts.PrepareVolumeCreateReq() @@ -141,13 +141,13 @@ func (g *glusterVolManager) GetOrCreateHostingVolume(name string, minSizeLimit i volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", g.hostVolOpts.Size) } - availableSizeInBytes, err := strconv.Atoi(volInfo.Metadata["block-hosting-available-size"]) + availableSizeInBytes, err := strconv.ParseUint(volInfo.Metadata["block-hosting-available-size"], 10, 64) if err != nil { return nil, err } - if int64(availableSizeInBytes) < minSizeLimit { + if availableSizeInBytes < minSizeLimit { return nil, fmt.Errorf("available size is less than requested size,request size: %d, available size: %d", minSizeLimit, availableSizeInBytes) } diff --git a/plugins/blockvolume/utils/volume.go b/plugins/blockvolume/utils/volume.go index ab3c0d5c7..ba5a35719 100644 --- a/plugins/blockvolume/utils/volume.go +++ b/plugins/blockvolume/utils/volume.go @@ -20,7 +20,7 @@ import ( // BlockSizeFilter returns a volume Filter, which will filter out volumes // haing block-hosting-available-size greater than give size. -func BlockSizeFilter(size int64) volume.Filter { +func BlockSizeFilter(size uint64) volume.Filter { return func(volinfos []*volume.Volinfo) []*volume.Volinfo { var volumes []*volume.Volinfo @@ -30,7 +30,7 @@ func BlockSizeFilter(size int64) volume.Filter { continue } - if availableSizeInBytes, err := strconv.Atoi(availableSize); err == nil && int64(availableSizeInBytes) > size { + if availableSizeInBytes, err := strconv.ParseUint(availableSize, 10, 64); err == nil && availableSizeInBytes > size { volumes = append(volumes, volinfo) } } @@ -39,7 +39,7 @@ func BlockSizeFilter(size int64) volume.Filter { } // GetExistingBlockHostingVolume returns a existing volume which is suitable for hosting a gluster-block -func GetExistingBlockHostingVolume(size int64) (*volume.Volinfo, error) { +func GetExistingBlockHostingVolume(size uint64) (*volume.Volinfo, error) { var ( filters = []volume.Filter{volume.FilterBlockHostedVolumes, BlockSizeFilter(size)} ctx, cancel = context.WithTimeout(context.Background(), time.Second*5) @@ -78,12 +78,18 @@ func CreateBlockHostingVolume(req *api.VolCreateReq) (*volume.Volinfo, error) { func ResizeBlockHostingVolume(volname string, deletedBlockSize string) error { clusterLocks := transaction.Locks{} + if err := clusterLocks.Lock(volname); err != nil { + log.WithError(err).Error("error in acquiring cluster lock") + return err + } + defer clusterLocks.UnLock(context.Background()) + volInfo, err := volume.GetVolume(volname) if err != nil { return err } - deletedSizeInBytes, err := size.Parse(deletedBlockSize) + deletedSize, err := size.Parse(deletedBlockSize) if err != nil { return err } @@ -92,19 +98,12 @@ func ResizeBlockHostingVolume(volname string, deletedBlockSize string) error { return errors.New("block-hosting-available-size metadata not found for volume") } - availableSizeInBytes, err := strconv.Atoi(volInfo.Metadata["block-hosting-available-size"]) + availableSizeInBytes, err := strconv.ParseUint(volInfo.Metadata["block-hosting-available-size"], 10, 64) if err != nil { return err } - volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", size.Size(availableSizeInBytes)+deletedSizeInBytes) - - if err := clusterLocks.Lock(volInfo.Name); err != nil { - log.WithError(err).Error("error in acquiring cluster lock") - return err - } - - defer clusterLocks.UnLock(context.Background()) + volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", availableSizeInBytes+uint64(deletedSize)) return volume.AddOrUpdateVolume(volInfo) } From c60180ac0a8083a4f5949f7948d35fa383e81a9d Mon Sep 17 00:00:00 2001 From: Oshank Kumar Date: Fri, 21 Dec 2018 18:22:03 +0530 Subject: [PATCH 5/7] update dependencies using dep ensure Signed-off-by: Oshank Kumar --- Gopkg.lock | 233 ++++++++++++++++-- glusterd2.toml.example | 12 + glusterd2/volume/filters.go | 2 +- glusterd2/volume/metadata.go | 10 + pkg/size/size.go | 104 ++++---- pkg/size/size_test.go | 26 +- .../blockprovider/blockvolume_provider.go | 3 +- .../gluster-block/glusterblock.go | 4 +- plugins/blockvolume/hostvol_manager.go | 33 +-- plugins/blockvolume/utils/volume.go | 25 +- 10 files changed, 347 insertions(+), 105 deletions(-) create mode 100644 glusterd2/volume/metadata.go diff --git a/Gopkg.lock b/Gopkg.lock index fd1686094..9f776149b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,41 +3,54 @@ [[projects]] branch = "master" + digest = "1:eee9386329f4fcdf8d6c0def0c9771b634bdd5ba460d888aa98c17d59b37a76c" name = "git.apache.org/thrift.git" packages = ["lib/go/thrift"] + pruneopts = "NUT" revision = "f12cacf56145e2c8f0d4429694fedf5453648089" source = "github.com/apache/thrift" [[projects]] + digest = "1:2ca151c1203d07f3fca238e42bffbbf7525089585365070c4eff5bcbeaa951e1" name = "github.com/asaskevich/govalidator" packages = ["."] + pruneopts = "NUT" revision = "521b25f4b05fd26bec69d9dedeb8f9c9a83939a8" version = "v8" [[projects]] branch = "master" + digest = "1:707ebe952a8b3d00b343c01536c79c73771d100f63ec6babeaed5c79e2b8a8dd" name = "github.com/beorn7/perks" packages = ["quantile"] + pruneopts = "NUT" revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] + digest = "1:ac52ddd069a36a6545ba4869aa169de501b2d9711437688a632cf9f8d8f5053c" name = "github.com/cespare/xxhash" packages = ["."] + pruneopts = "NUT" revision = "5c37fe3735342a2e0d01c87a907579987c8936cc" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:afea4ad0d6b3bf7efe0c248049c3efd427136bdb6e10985dd956fde5c57010a3" name = "github.com/cockroachdb/cmux" packages = ["."] + pruneopts = "NUT" revision = "30d10be492927e2dcae0089c374c455d42414fcb" [[projects]] + digest = "1:9610254926720daa703fae889010d45e190e8c264c6da76354f0ac8540ca3271" name = "github.com/coreos/bbolt" packages = ["."] + pruneopts = "NUT" revision = "48ea1b39c25fc1bab3506fbc712ecbaa842c4d2d" [[projects]] + digest = "1:e4da502dee22f817d7060ece41e2024c6a662011fd8cd410cfc30dc3ccff9820" name = "github.com/coreos/etcd" packages = [ "alarm", @@ -107,70 +120,102 @@ "store", "version", "wal", - "wal/walpb" + "wal/walpb", ] + pruneopts = "NUT" revision = "33245c6b5b49130ca99280408fadfab01aac0e48" version = "v3.3.8" [[projects]] + digest = "1:0ef770954bca104ee99b3b6b7f9b240605ac03517d9f98cbc1893daa03f3c038" name = "github.com/coreos/go-semver" packages = ["semver"] + pruneopts = "NUT" revision = "8ab6407b697782a06568d4b7f1db25550ec2e4c6" version = "v0.2.0" [[projects]] + digest = "1:e3b55611cb86797357cc1cd2b3d38faf07ae3c4079f04b32e84d497422ab0d60" name = "github.com/coreos/go-systemd" packages = ["journal"] + pruneopts = "NUT" revision = "39ca1b05acc7ad1220e09f133283b8859a8b71ab" version = "v17" [[projects]] + digest = "1:9a89d8c37f5fadb9651b180fc74fc6ceb3e6abb74cbf8e783bbfba05bd8f98c4" name = "github.com/coreos/pkg" packages = ["capnslog"] + pruneopts = "NUT" revision = "3ac0863d7acf3bc44daf49afef8919af12f704ef" version = "v3" [[projects]] + digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "NUT" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] + digest = "1:01ced7908dbaae42990af9521328922b8948bdcb174c23bba6db572381515716" name = "github.com/dgrijalva/jwt-go" packages = ["."] + pruneopts = "NUT" revision = "dbeaa9332f19a944acb5736b4456cfcc02140e29" version = "v3.1.0" [[projects]] + digest = "1:1b91ae0dc69a41d4c2ed23ea5cffb721ea63f5037ca4b81e6d6771fbb8f45129" name = "github.com/fsnotify/fsnotify" packages = ["."] + pruneopts = "NUT" revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" version = "v1.4.7" [[projects]] + digest = "1:81466b4218bf6adddac2572a30ac733a9255919bc2f470b4827a317bd4ee1756" name = "github.com/ghodss/yaml" packages = ["."] + pruneopts = "NUT" revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" version = "v1.0.0" [[projects]] + digest = "1:e81535d65eb6c17070d6b2318fdf47ff8bb62fe28be69c0817abb5cf3bd05b19" + name = "github.com/gluster/gluster-block-restapi" + packages = [ + "client", + "pkg/api", + "pkg/utils", + ] + pruneopts = "NUT" + revision = "c1abbfc09bd1e22e3cc8c8df2eb0b8d2d110651c" + version = "v0.2" + +[[projects]] + digest = "1:ca3369c0fc8d471d8698f85a37a4f8c98a847402681a31431fb87a84fa2e5373" name = "github.com/godbus/dbus" packages = ["."] + pruneopts = "NUT" revision = "a389bdde4dd695d414e47b755e95e72b7826432c" version = "v4.1.0" [[projects]] + digest = "1:2e15dd2dcea234f9f10e96041a3862830d10feadf8fd9c3795ca71fcb4cf2a3d" name = "github.com/gogo/protobuf" packages = [ "gogoproto", "proto", - "protoc-gen-gogo/descriptor" + "protoc-gen-gogo/descriptor", ] + pruneopts = "NUT" revision = "1adfc126b41513cc696b209667c8656ea7aac67c" version = "v1.0.0" [[projects]] + digest = "1:9aa21412e5a184d06dfcdf5cc7babdf5b29da82727744a2056b9fab91f3f8491" name = "github.com/golang/protobuf" packages = [ "jsonpb", @@ -179,58 +224,74 @@ "ptypes/any", "ptypes/duration", "ptypes/struct", - "ptypes/timestamp" + "ptypes/timestamp", ] + pruneopts = "NUT" revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:245bd4eb633039cd66106a5d340ae826d87f4e36a8602fcc940e14176fd26ea7" name = "github.com/google/btree" packages = ["."] + pruneopts = "NUT" revision = "e89373fe6b4a7413d7acd6da1725b83ef713e6e4" [[projects]] + digest = "1:c01767916c59f084bb7c41a7d5877c0f3099b1595cfa066e84ec6ad6b084dd89" name = "github.com/gorilla/context" packages = ["."] + pruneopts = "NUT" revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" version = "v1.1.1" [[projects]] + digest = "1:fe47be2bd5d0196679d314fcc6e02f40a51e641c09e1541528a05cccefee9b0e" name = "github.com/gorilla/handlers" packages = ["."] + pruneopts = "NUT" revision = "90663712d74cb411cbef281bc1e08c19d1a76145" version = "v1.3.0" [[projects]] + digest = "1:bf5cf1d53d703332e9bd8984c69784645b73a938317bf5ace9aadf20ac49379a" name = "github.com/gorilla/mux" packages = ["."] + pruneopts = "NUT" revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" version = "v1.6.2" [[projects]] + digest = "1:3b708ebf63bfa9ba3313bedb8526bc0bb284e51474e65e958481476a9d4a12aa" name = "github.com/gorilla/websocket" packages = ["."] + pruneopts = "NUT" revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" version = "v1.2.0" [[projects]] + digest = "1:5872c7f130f62fc34bfda20babad36be6309c00b5c9207717f7cd2a51536fff4" name = "github.com/grpc-ecosystem/go-grpc-prometheus" packages = ["."] + pruneopts = "NUT" revision = "c225b8c3b01faf2899099b768856a9e916e5087b" version = "v1.2.0" [[projects]] + digest = "1:0b200ca72bccd47ffca4d57f63168cf77d712e547849460e7e6f8efa622472a8" name = "github.com/grpc-ecosystem/grpc-gateway" packages = [ "runtime", "runtime/internal", - "utilities" + "utilities", ] + pruneopts = "NUT" revision = "8cc3a55af3bcf171a1c23a90c4df9cf591706104" [[projects]] branch = "master" + digest = "1:11c6c696067d3127ecf332b10f89394d386d9083f82baf71f40f2da31841a009" name = "github.com/hashicorp/hcl" packages = [ ".", @@ -242,209 +303,279 @@ "hcl/token", "json/parser", "json/scanner", - "json/token" + "json/token", ] + pruneopts = "NUT" revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" [[projects]] + digest = "1:406338ad39ab2e37b7f4452906442a3dbf0eb3379dd1f06aafb5c07e769a5fbb" name = "github.com/inconshreveable/mousetrap" packages = ["."] + pruneopts = "NUT" revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" version = "v1.0" [[projects]] + digest = "1:6b1eae4bb93e5ccd23cb09d1e005ecb391316d27701b7a5264f8555a6e2f3d87" name = "github.com/jonboulle/clockwork" packages = ["."] + pruneopts = "NUT" revision = "2eee05ed794112d45db504eb05aa693efd2b8b09" version = "v0.1.0" [[projects]] branch = "master" + digest = "1:01144ccbbd97fe4d710b96c1db19e631ea3b491fffc8b4fb53274cdd2be1061e" name = "github.com/justinas/alice" packages = ["."] + pruneopts = "NUT" revision = "03f45bd4b7dad4734bc4620e46a35789349abb20" [[projects]] + digest = "1:4059c14e87a2de3a434430340521b5feece186c1469eff0834c29a63870de3ed" + name = "github.com/konsorten/go-windows-terminal-sequences" + packages = ["."] + pruneopts = "NUT" + revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242" + version = "v1.0.1" + +[[projects]] + digest = "1:d244f8666a838fe6ad70ec8fe77f50ebc29fdc3331a2729ba5886bef8435d10d" name = "github.com/magiconair/properties" packages = ["."] + pruneopts = "NUT" revision = "c2353362d570a7bfa228149c62842019201cfb71" version = "v1.8.0" [[projects]] + digest = "1:cb591533458f6eb6e2c1065ff3eac6b50263d7847deb23fc9f79b25bc608970e" name = "github.com/mattn/go-runewidth" packages = ["."] + pruneopts = "NUT" revision = "9e777a8366cce605130a531d2cd6363d07ad7317" version = "v0.0.2" [[projects]] + digest = "1:5985ef4caf91ece5d54817c11ea25f182697534f8ae6521eadcd628c142ac4b6" name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] + pruneopts = "NUT" revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" version = "v1.0.1" [[projects]] branch = "master" + digest = "1:ffbb5e45e7e69a915767cc2fcff99525645afdfefffee72c551940e4b055e538" name = "github.com/mitchellh/mapstructure" packages = ["."] + pruneopts = "NUT" revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b" [[projects]] branch = "master" + digest = "1:e6baebf0bd20950e285254902a8f2241b4870528fb63fdc10460c67b1f31b1a3" name = "github.com/olekukonko/tablewriter" packages = ["."] + pruneopts = "NUT" revision = "d4647c9c7a84d847478d890b816b7d8b62b0b279" [[projects]] + digest = "1:cce3a18fb0b96b5015cd8ca03a57d20a662679de03c4dc4b6ff5f17ea2050fa6" name = "github.com/pborman/uuid" packages = ["."] + pruneopts = "NUT" revision = "e790cca94e6cc75c7064b1332e63811d4aae1a53" version = "v1.1" [[projects]] + digest = "1:84b6c3d03118e04267f3c57d34f0ae8a885e5486b127935b87e3499cea3f180a" name = "github.com/pelletier/go-toml" packages = ["."] + pruneopts = "NUT" revision = "16398bac157da96aa88f98a2df640c7f32af1da2" version = "v1.0.1" [[projects]] branch = "master" + digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" name = "github.com/petar/GoLLRB" packages = ["llrb"] + pruneopts = "NUT" revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" [[projects]] + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" name = "github.com/pmezard/go-difflib" packages = ["difflib"] + pruneopts = "NUT" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] + digest = "1:03bca087b180bf24c4f9060775f137775550a0834e18f0bca0520a868679dbd7" name = "github.com/prometheus/client_golang" packages = [ "prometheus", - "prometheus/promhttp" + "prometheus/promhttp", ] + pruneopts = "NUT" revision = "c5b7fccd204277076155f10851dad72b76a49317" version = "v0.8.0" [[projects]] branch = "master" + digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4" name = "github.com/prometheus/client_model" packages = ["go"] + pruneopts = "NUT" revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f" [[projects]] branch = "master" + digest = "1:768b555b86742de2f28beb37f1dedce9a75f91f871d75b5717c96399c1a78c08" name = "github.com/prometheus/common" packages = [ "expfmt", "internal/bitbucket.org/ww/goautoneg", - "model" + "model", ] + pruneopts = "NUT" revision = "7600349dcfe1abd18d72d3a1770870d9800a7801" [[projects]] branch = "master" + digest = "1:6621142cd60b7150ab66f38ff36303ca55843dc5a635c1f9a28f95ecddab72b4" name = "github.com/prometheus/procfs" packages = [ ".", "internal/util", "nfs", - "xfs" + "xfs", ] + pruneopts = "NUT" revision = "ae68e2d4c00fed4943b5f6698d504a5fe083da8a" [[projects]] branch = "master" + digest = "1:a3e3c31d1d2b080cd85940a1c06e51e1c2e9dafa684d4ec736375c0e53038dd9" name = "github.com/rasky/go-xdr" packages = ["xdr2"] + pruneopts = "NUT" revision = "1a41d1a06c93bf8a0e0385be3847a277bb793187" [[projects]] + digest = "1:d848e2bdc690ea54c4b49894b67a05db318a97ee6561879b814c2c1f82f61406" name = "github.com/sirupsen/logrus" packages = ["."] - revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc" - version = "v1.0.5" + pruneopts = "NUT" + revision = "bcd833dfe83d3cebad139e4a29ed79cb2318bf95" + version = "v1.2.0" [[projects]] + digest = "1:2f31ab8c0f8b4fad1cd69cf250605009461bd31092a2daf80988fa1fbd7abfd8" name = "github.com/soheilhy/cmux" packages = ["."] + pruneopts = "NUT" revision = "e09e9389d85d8492d313d73d1469c029e710623f" version = "v0.1.4" [[projects]] + digest = "1:330e9062b308ac597e28485699c02223bd052437a6eed32a173c9227dcb9d95a" name = "github.com/spf13/afero" packages = [ ".", - "mem" + "mem", ] + pruneopts = "NUT" revision = "787d034dfe70e44075ccc060d346146ef53270ad" version = "v1.1.1" [[projects]] + digest = "1:3fa7947ca83b98ae553590d993886e845a4bff19b7b007e869c6e0dd3b9da9cd" name = "github.com/spf13/cast" packages = ["."] + pruneopts = "NUT" revision = "8965335b8c7107321228e3e3702cab9832751bac" version = "v1.2.0" [[projects]] + digest = "1:343d44e06621142ab09ae0c76c1799104cdfddd3ffb445d78b1adf8dc3ffaf3d" name = "github.com/spf13/cobra" packages = ["."] + pruneopts = "NUT" revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" version = "v0.0.3" [[projects]] branch = "master" + digest = "1:f29f83301ed096daed24a90f4af591b7560cb14b9cc3e1827abbf04db7269ab5" name = "github.com/spf13/jwalterweatherman" packages = ["."] + pruneopts = "NUT" revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" [[projects]] + digest = "1:15e5c398fbd9d2c439b635a08ac161b13d04f0c2aa587fe256b65dc0c3efe8b7" name = "github.com/spf13/pflag" packages = ["."] + pruneopts = "NUT" revision = "583c0c0531f06d5278b7d917446061adc344b5cd" version = "v1.0.1" [[projects]] + digest = "1:ea67fb4941c0a1a92f828e73cf426533c71db02df45a2cdf55a14c3e7b74c07a" name = "github.com/spf13/viper" packages = ["."] + pruneopts = "NUT" revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736" version = "v1.0.2" [[projects]] + digest = "1:0331452965d8695c0a5633e0f509012987681a654f388f2ad0c3b2d7f7004b1c" name = "github.com/stretchr/testify" packages = [ "assert", - "require" + "require", ] + pruneopts = "NUT" revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" version = "v1.2.2" [[projects]] + digest = "1:f7a4645103dec85e565bd3ea0d9c6bb9612dd38ddc20e81ff4a63e1b61856280" name = "github.com/thejerf/suture" packages = ["."] + pruneopts = "NUT" revision = "f44dbcdd98cf42f65b78faac17288ad1ba2c041b" version = "v2.0.3" [[projects]] branch = "master" + digest = "1:9fbba9f266710aae2dfe8bcdb24703b25ada0512623cabc96efb99bc7d579aef" name = "github.com/tmc/grpc-websocket-proxy" packages = ["wsproxy"] + pruneopts = "NUT" revision = "830351dc03c6f07d625727d5f993a463babb20e1" [[projects]] + digest = "1:919fc2a81add8ac4a7a236dceaf3e4c29ba4df96d13c3f2ce13a4d0132ebefb8" name = "github.com/ugorji/go" packages = ["codec"] + pruneopts = "NUT" revision = "b4c50a2b199d93b13dc15e78929cfb23bfdf21ab" version = "v1.1.1" [[projects]] + digest = "1:e4ffb3b860a4c71f7805415b4ba1e9a27e04eb9c4c03ddd943e9ff1790eda506" name = "github.com/xiang90/probing" packages = ["."] + pruneopts = "NUT" revision = "07dd2e8dfe18522e9c447ba95f2fe95262f63bb2" version = "0.0.1" [[projects]] + digest = "1:d08d0e06026f77a198f980be4b7f3c36cda3640d750fe360830bbaef67bb9739" name = "go.opencensus.io" packages = [ "exporter/jaeger", @@ -460,23 +591,27 @@ "tag", "trace", "trace/internal", - "trace/propagation" + "trace/propagation", ] + pruneopts = "NUT" revision = "c3ed530f775d85e577ca652cb052a52c078aad26" version = "v0.11.0" [[projects]] branch = "master" + digest = "1:001a4e7a40e50ff2ef32e2556bca50c4f77daa457db3ac6afc8bea9bb2122cfb" name = "golang.org/x/crypto" packages = [ "bcrypt", "blowfish", - "ssh/terminal" + "ssh/terminal", ] + pruneopts = "NUT" revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" [[projects]] branch = "master" + digest = "1:d39e451e542c7c064d7e29050eb50993a7484d4a3b538387c0f2ac32b5b9cdd8" name = "golang.org/x/net" packages = [ "context", @@ -485,26 +620,32 @@ "http2/hpack", "idna", "internal/timeseries", - "trace" + "trace", ] + pruneopts = "NUT" revision = "49bb7cea24b1df9410e1712aa6433dae904ff66a" [[projects]] branch = "master" + digest = "1:e0140c0c868c6e0f01c0380865194592c011fe521d6e12d78bfd33e756fe018a" name = "golang.org/x/sync" packages = ["semaphore"] + pruneopts = "NUT" revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" [[projects]] branch = "master" + digest = "1:5420733d35e5e562fffc7c5b99660f3587af042b5420f19b17fe0426e6a5259d" name = "golang.org/x/sys" packages = [ "unix", - "windows" + "windows", ] + pruneopts = "NUT" revision = "1b2967e3c290b7c545b3db0deeda16e9be4f98a2" [[projects]] + digest = "1:e7071ed636b5422cc51c0e3a6cebc229d6c9fffc528814b519a980641422d619" name = "golang.org/x/text" packages = [ "collate", @@ -520,30 +661,38 @@ "unicode/bidi", "unicode/cldr", "unicode/norm", - "unicode/rangetable" + "unicode/rangetable", ] + pruneopts = "NUT" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] branch = "master" + digest = "1:c9e7a4b4d47c0ed205d257648b0e5b0440880cb728506e318f8ac7cd36270bc4" name = "golang.org/x/time" packages = ["rate"] + pruneopts = "NUT" revision = "fbb02b2291d28baffd63558aa44b4b56f178d650" [[projects]] branch = "master" + digest = "1:c0c17c94fe8bc1ab34e7f586a4a8b788c5e1f4f9f750ff23395b8b2f5a523530" name = "google.golang.org/api" packages = ["support/bundler"] + pruneopts = "NUT" revision = "781db45e5b94469718b0c615037e76333cd050e3" [[projects]] branch = "master" + digest = "1:601e63e7d4577f907118bec825902505291918859d223bce015539e79f1160e3" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] + pruneopts = "NUT" revision = "e92b116572682a5b432ddd840aeaba2a559eeff1" [[projects]] + digest = "1:04403ed28896279b683ae57464170afd8e196c273feeb853f0347fe0015b525c" name = "google.golang.org/grpc" packages = [ ".", @@ -572,20 +721,64 @@ "stats", "status", "tap", - "transport" + "transport", ] + pruneopts = "NUT" revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" version = "v1.13.0" [[projects]] + digest = "1:7c95b35057a0ff2e19f707173cc1a947fa43a6eb5c4d300d196ece0334046082" name = "gopkg.in/yaml.v2" packages = ["."] + pruneopts = "NUT" revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" version = "v2.2.1" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "1a1bf3511263a7300c00a98d7b55df50c80fec36ba01c25fbddde4a85dee99f5" + input-imports = [ + "github.com/asaskevich/govalidator", + "github.com/cespare/xxhash", + "github.com/cockroachdb/cmux", + "github.com/coreos/etcd/clientv3", + "github.com/coreos/etcd/clientv3/concurrency", + "github.com/coreos/etcd/clientv3/namespace", + "github.com/coreos/etcd/embed", + "github.com/coreos/etcd/etcdserver/etcdserverpb", + "github.com/coreos/etcd/mvcc/mvccpb", + "github.com/coreos/etcd/pkg/transport", + "github.com/coreos/etcd/pkg/types", + "github.com/coreos/pkg/capnslog", + "github.com/dgrijalva/jwt-go", + "github.com/ghodss/yaml", + "github.com/gluster/gluster-block-restapi/client", + "github.com/gluster/gluster-block-restapi/pkg/api", + "github.com/godbus/dbus", + "github.com/golang/protobuf/proto", + "github.com/gorilla/handlers", + "github.com/gorilla/mux", + "github.com/justinas/alice", + "github.com/olekukonko/tablewriter", + "github.com/pborman/uuid", + "github.com/pelletier/go-toml", + "github.com/rasky/go-xdr/xdr2", + "github.com/sirupsen/logrus", + "github.com/spf13/cobra", + "github.com/spf13/pflag", + "github.com/spf13/viper", + "github.com/stretchr/testify/assert", + "github.com/stretchr/testify/require", + "github.com/thejerf/suture", + "go.opencensus.io/exporter/jaeger", + "go.opencensus.io/plugin/ocgrpc", + "go.opencensus.io/plugin/ochttp", + "go.opencensus.io/trace", + "golang.org/x/net/context", + "golang.org/x/sys/unix", + "google.golang.org/grpc", + "google.golang.org/grpc/codes", + ] solver-name = "gps-cdcl" - solver-version = 1 \ No newline at end of file + solver-version = 1 diff --git a/glusterd2.toml.example b/glusterd2.toml.example index caefc3ad6..98cb7b6cc 100644 --- a/glusterd2.toml.example +++ b/glusterd2.toml.example @@ -3,3 +3,15 @@ peeraddress = ":24008" clientaddress = ":24007" #restauth enables/disables REST authentication in glusterd2 #restauth = true + +#[gluster-block-hosting-vol-options] +block-hosting-volume-size = 20971520 +auto-create-block-hosting-volumes = true +block-hosting-volume-replica-count = 1 +#block-hosting-volume-type = "Distribute" + +#[gluster-block-client-config] +gluster-block-hostaddr = "192.168.122.16:8081" +#gluster-block-cacert = "/path/to/ca.crt" +#gluster-block-user = "username_for_rest_authentication" +#gluster-block-secret = "secret_for_rest_authentication" \ No newline at end of file diff --git a/glusterd2/volume/filters.go b/glusterd2/volume/filters.go index 30d141120..1d6883ef7 100644 --- a/glusterd2/volume/filters.go +++ b/glusterd2/volume/filters.go @@ -38,7 +38,7 @@ func ApplyCustomFilters(volumes []*Volinfo, filters ...Filter) []*Volinfo { func FilterBlockHostedVolumes(volumes []*Volinfo) []*Volinfo { var volInfos []*Volinfo for _, volume := range volumes { - val, found := volume.Metadata["block-hosting"] + val, found := volume.Metadata[BlockHosting] if found && val == "yes" { volInfos = append(volInfos, volume) } diff --git a/glusterd2/volume/metadata.go b/glusterd2/volume/metadata.go new file mode 100644 index 000000000..b009e3432 --- /dev/null +++ b/glusterd2/volume/metadata.go @@ -0,0 +1,10 @@ +package volume + +const ( + // BlockHostingAvailableSize is a volume metadata to store available size of hosting volume to create block devices. + BlockHostingAvailableSize = "_block-hosting-available-size" + // BlockHostingVolumeAutoCreated is volume metadata which will be set as `yes` if gd2 auto create the hosting volume + BlockHostingVolumeAutoCreated = "block-hosting-volume-auto-created" + // BlockHosting is a volume metadata which will be set as `yes' for volumes which are able to host block devices. + BlockHosting = "block-hosting" +) diff --git a/pkg/size/size.go b/pkg/size/size.go index 98acf8a5d..f9be3ee79 100644 --- a/pkg/size/size.go +++ b/pkg/size/size.go @@ -1,9 +1,9 @@ package size import ( - "errors" "fmt" "regexp" + "strconv" "strings" ) @@ -39,6 +39,34 @@ const ( PB = 1e3 * TB ) +var sizeMultiple = map[string]Size{ + "B": Byte, + + "KB": KB, + "MB": MB, + "GB": GB, + "TB": TB, + "PB": PB, + + "K": KiB, + "M": MiB, + "G": GiB, + "T": TiB, + "P": PiB, + + "KiB": KiB, + "MiB": MiB, + "GiB": GiB, + "TiB": TiB, + "PiB": PiB, + + "Ki": KiB, + "Mi": MiB, + "Gi": GiB, + "Ti": TiB, + "Pi": PiB, +} + // Bytes returns number of bytes func (s Size) Bytes() int64 { return int64(s) } @@ -126,62 +154,44 @@ func (s Size) String() string { return fmt.Sprintf("%.2fKB", s.KiloBytes()) } - return fmt.Sprintf("%d Bytes", s) + return fmt.Sprintf("%d B", s) } -// Parse parses a string representation of size and returns the Size value it represents. -// Supported formats are {TiB,GiB,MiB,KiB,TB,GB,MB,KB} -func Parse(s string) (Size, error) { - var ( - count float64 - size Size - err error - regex = regexp.MustCompile(`^([\d.]+)([KMGT]i?B)$`) - ) - - s = strings.Replace(s, " ", "", -1) - matches := regex.FindStringSubmatch(s) - - if len(matches) != 3 { - return size, errors.New("invalid size format") - } - - switch matches[2] { - case "GiB": - _, err = fmt.Sscanf(s, "%fGiB", &count) - size = Size(count * float64(1*GiB)) +var validSizePattern = regexp.MustCompile( + `(\d+(\.\d+)?)([KMGTP]?i?B?)`, +) - case "MiB": - _, err = fmt.Sscanf(s, "%fMiB", &count) - size = Size(count * float64(1*MiB)) +// Parse parses a string representation of size and returns the Size value it represents. +// Supported formats are {PiB,TiB,GiB,MiB,KiB,PB,TB,GB,MB,KB,B,Pi,Ti,Gi,Mi,Ki} +func Parse(sizeStr string) (Size, error) { + sizeStr = strings.Replace(sizeStr, " ", "", -1) - case "KiB": - _, err = fmt.Sscanf(s, "%fKiB", &count) - size = Size(count * float64(1*KiB)) + if !validSizePattern.MatchString(sizeStr) { + return 0, fmt.Errorf("size parse error: %s", sizeStr) + } - case "TiB": - _, err = fmt.Sscanf(s, "%fTiB", &count) - size = Size(count * float64(1*TiB)) + matches := validSizePattern.FindStringSubmatch(sizeStr) - case "KB": - _, err = fmt.Sscanf(s, "%fKB", &count) - size = Size(count * float64(1*KB)) + if len(matches) != 4 { + return 0, fmt.Errorf("size parse error: %s invalid fields (%v)", sizeStr, matches) + } - case "MB": - _, err = fmt.Sscanf(s, "%fMB", &count) - size = Size(count * float64(1*MB)) + if matches[0] != sizeStr { + return 0, fmt.Errorf("size parse error: %s invalid fields (%v)", sizeStr, matches) + } - case "GB": - _, err = fmt.Sscanf(s, "%fGB", &count) - size = Size(count * float64(1*GB)) + unit := matches[3] + size := matches[1] - case "TB": - _, err = fmt.Sscanf(s, "%fTB", &count) - size = Size(count * float64(1*TB)) + multiplier, ok := sizeMultiple[unit] + if !ok { + return 0, fmt.Errorf("multiplier not found for unit: %s", unit) + } - default: - return 0, errors.New("can not parse to size") + val, err := strconv.ParseFloat(size, 64) + if err != nil { + return 0, err } - return size, err + return Size(float64(multiplier) * val), nil } diff --git a/pkg/size/size_test.go b/pkg/size/size_test.go index 01b9c620c..53d1a798f 100644 --- a/pkg/size/size_test.go +++ b/pkg/size/size_test.go @@ -51,13 +51,18 @@ var ( {Size(2.5e9), "2.50GB"}, {Size(8.75e6), "8.75MB"}, {Size(768e3), "768.00KB"}, - {Size(500), "500 Bytes"}, + {Size(500), "500 B"}, } parseSizeTests = []struct { s string want Size }{ + // Byte Format + {"1B", Byte}, + {"512.0 B", 512 * Byte}, + {"125.0B", 125 * Byte}, + // Binary format {"1GiB", Size(1 * float64(GiB))}, {"2.5GiB", Size(2.5 * float64(GiB))}, @@ -67,11 +72,28 @@ var ( {"0050KiB", Size(50 * float64(KiB))}, {"2.50KiB", Size(2.5 * float64(KiB))}, {"2.50TiB", Size(2.5 * float64(TiB))}, + + {"1Gi", Size(1 * float64(GiB))}, + {"2.5Gi", Size(2.5 * float64(GiB))}, + {"1Mi", Size(1 * float64(MiB))}, + {"100.5Mi", Size(100.5 * float64(MiB))}, + {"50Ki", Size(50 * float64(KiB))}, + {"0050Ki", Size(50 * float64(KiB))}, + {"2.50Ki", Size(2.5 * float64(KiB))}, + {"2.50Ti", Size(2.5 * float64(TiB))}, + + {"100.5M", Size(100.5 * float64(MiB))}, + {"50K", Size(50 * float64(KiB))}, + {"0050K", Size(50 * float64(KiB))}, + {"2.50K", Size(2.5 * float64(KiB))}, + {"2.50T", Size(2.5 * float64(TiB))}, + // Decimal format {"2.50TB", Size(2.5 * float64(TB))}, {"2.50MB", Size(2.5 * float64(MB))}, {"0.5KB", Size(0.5 * float64(KB))}, {"052GB", Size(52 * float64(GB))}, + // having space in between {"0.5 KB", Size(0.5 * float64(KB))}, {"052 GB", Size(52 * float64(GB))}, @@ -93,6 +115,8 @@ var ( "x5KB", "05xMB", "5.5.5MB", + "5BB", + "4.5.5B", } ) diff --git a/plugins/blockvolume/blockprovider/blockvolume_provider.go b/plugins/blockvolume/blockprovider/blockvolume_provider.go index 4b5cfbebd..9c25f42f3 100644 --- a/plugins/blockvolume/blockprovider/blockvolume_provider.go +++ b/plugins/blockvolume/blockprovider/blockvolume_provider.go @@ -43,7 +43,8 @@ func RegisterBlockProvider(name string, f ProviderFunc) { providersMutex.Lock() defer providersMutex.Unlock() if _, found := providerFactory[name]; found { - log.WithField("name", name).Error("block provider was registered twice") + log.WithField("name", name).Error("failed to register block provider, provider already exist") + return } log.WithField("name", name).Infof("Registered block provider") providerFactory[name] = f diff --git a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go index bdd7f8ea4..0856237d4 100644 --- a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go +++ b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go @@ -85,7 +85,7 @@ func (g *GlusterBlock) CreateBlockVolume(name string, size uint64, hosts []strin return nil, fmt.Errorf("error in getting host vol details: %s", err) } - availableSizeInBytes, err := strconv.ParseUint(volInfo.Metadata["block-hosting-available-size"], 10, 64) + availableSizeInBytes, err := strconv.ParseUint(volInfo.Metadata[volume.BlockHostingAvailableSize], 10, 64) if err != nil { return nil, err } @@ -99,7 +99,7 @@ func (g *GlusterBlock) CreateBlockVolume(name string, size uint64, hosts []strin return nil, err } - volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", availableSizeInBytes-size) + volInfo.Metadata[volume.BlockHostingAvailableSize] = fmt.Sprintf("%d", availableSizeInBytes-size) if err = volume.AddOrUpdateVolume(volInfo); err != nil { log.WithError(err).Error("failed in updating volume info to store") diff --git a/plugins/blockvolume/hostvol_manager.go b/plugins/blockvolume/hostvol_manager.go index 82713e0ec..0f898a9c9 100644 --- a/plugins/blockvolume/hostvol_manager.go +++ b/plugins/blockvolume/hostvol_manager.go @@ -9,11 +9,8 @@ import ( "strings" "time" - "github.com/gluster/glusterd2/glusterd2/commands/volumes" - "github.com/gluster/glusterd2/glusterd2/gdctx" "github.com/gluster/glusterd2/glusterd2/transaction" "github.com/gluster/glusterd2/glusterd2/volume" - "github.com/gluster/glusterd2/pkg/api" "github.com/gluster/glusterd2/plugins/blockvolume/utils" log "github.com/sirupsen/logrus" @@ -66,7 +63,6 @@ func (g *glusterVolManager) GetOrCreateHostingVolume(name string, minSizeLimit u var ( volInfo *volume.Volinfo volCreateReq = g.hostVolOpts.PrepareVolumeCreateReq() - ctx = gdctx.WithReqLogger(context.Background(), log.StandardLogger()) clusterLocks = transaction.Locks{} ) @@ -106,42 +102,31 @@ func (g *glusterVolManager) GetOrCreateHostingVolume(name string, minSizeLimit u // If No volumes are available with Metadata:block-hosting=yes or if no space available to create block // volumes(Metadata:block-hosting-available-size is less than request size), then try to create a new - // block hosting Volume with generated name with default size and volume type configured + // block hosting Volume with generated name with default size and volume type configured. if name == "" && volInfo == nil { - vInfo, err := utils.CreateBlockHostingVolume(volCreateReq) + vInfo, err := utils.CreateAndStartHostingVolume(volCreateReq) if err != nil { - log.WithError(err).Error("error in auto creating block hosting volume") + log.WithError(err).Error("error in auto creation of block hosting volume") return nil, err } - - log.WithField("name", vInfo.Name).Debug("auto creation of hosting volume succeeded") - - vInfo, _, err = volumecommands.StartVolume(ctx, vInfo.Name, api.VolumeStartReq{}) - if err != nil { - log.WithError(err).Error("error in starting auto created block hosting volume") - return nil, err - } - - log.WithField("name", vInfo.Name).Debug("host volume started successfully") - volInfo = vInfo } - if _, found := volInfo.Metadata["block-hosting"]; !found { - volInfo.Metadata["block-hosting"] = "yes" + if _, found := volInfo.Metadata[volume.BlockHosting]; !found { + volInfo.Metadata[volume.BlockHosting] = "yes" } - blockHosting := volInfo.Metadata["block-hosting"] + blockHosting := volInfo.Metadata[volume.BlockHosting] if strings.ToLower(blockHosting) != "yes" { return nil, errors.New("not a block hosting volume") } - if _, found := volInfo.Metadata["block-hosting-available-size"]; !found { - volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", g.hostVolOpts.Size) + if _, found := volInfo.Metadata[volume.BlockHostingAvailableSize]; !found { + volInfo.Metadata[volume.BlockHostingAvailableSize] = fmt.Sprintf("%d", g.hostVolOpts.Size) } - availableSizeInBytes, err := strconv.ParseUint(volInfo.Metadata["block-hosting-available-size"], 10, 64) + availableSizeInBytes, err := strconv.ParseUint(volInfo.Metadata[volume.BlockHostingAvailableSize], 10, 64) if err != nil { return nil, err diff --git a/plugins/blockvolume/utils/volume.go b/plugins/blockvolume/utils/volume.go index ba5a35719..557643be9 100644 --- a/plugins/blockvolume/utils/volume.go +++ b/plugins/blockvolume/utils/volume.go @@ -10,6 +10,7 @@ import ( "time" "github.com/gluster/glusterd2/glusterd2/commands/volumes" + "github.com/gluster/glusterd2/glusterd2/gdctx" "github.com/gluster/glusterd2/glusterd2/transaction" "github.com/gluster/glusterd2/glusterd2/volume" "github.com/gluster/glusterd2/pkg/api" @@ -25,7 +26,7 @@ func BlockSizeFilter(size uint64) volume.Filter { var volumes []*volume.Volinfo for _, volinfo := range volinfos { - availableSize, found := volinfo.Metadata["block-hosting-available-size"] + availableSize, found := volinfo.Metadata[volume.BlockHostingAvailableSize] if !found { continue } @@ -57,19 +58,25 @@ func GetExistingBlockHostingVolume(size uint64) (*volume.Volinfo, error) { return SelectRandomVolume(volumes) } -// CreateBlockHostingVolume will create a gluster volume with metadata block-hosting-volume-auto-created=yes -func CreateBlockHostingVolume(req *api.VolCreateReq) (*volume.Volinfo, error) { - status, err := volumecommands.CreateVolume(context.Background(), *req) +// CreateAndStartHostingVolume creates and starts a gluster volume and returns volume Info on success. +// Set Metadata:block-hosting-volume-auto-created=yes if Block hosting volume is created and started successfully. +func CreateAndStartHostingVolume(req *api.VolCreateReq) (*volume.Volinfo, error) { + ctx := gdctx.WithReqLogger(context.Background(), log.StandardLogger()) + + status, err := volumecommands.CreateVolume(ctx, *req) if err != nil || status != http.StatusCreated { + log.WithError(err).Error("error in auto creating block hosting volume") return nil, err } - vInfo, err := volume.GetVolume(req.Name) + vInfo, _, err := volumecommands.StartVolume(ctx, req.Name, api.VolumeStartReq{}) if err != nil { + log.WithError(err).Error("error in starting auto created block hosting volume") return nil, err } - vInfo.Metadata["block-hosting-volume-auto-created"] = "yes" + vInfo.Metadata[volume.BlockHostingVolumeAutoCreated] = "yes" + log.WithField("name", vInfo.Name).Debug("host volume created and started successfully") return vInfo, nil } @@ -94,16 +101,16 @@ func ResizeBlockHostingVolume(volname string, deletedBlockSize string) error { return err } - if _, found := volInfo.Metadata["block-hosting-available-size"]; !found { + if _, found := volInfo.Metadata[volume.BlockHostingAvailableSize]; !found { return errors.New("block-hosting-available-size metadata not found for volume") } - availableSizeInBytes, err := strconv.ParseUint(volInfo.Metadata["block-hosting-available-size"], 10, 64) + availableSizeInBytes, err := strconv.ParseUint(volInfo.Metadata[volume.BlockHostingAvailableSize], 10, 64) if err != nil { return err } - volInfo.Metadata["block-hosting-available-size"] = fmt.Sprintf("%d", availableSizeInBytes+uint64(deletedSize)) + volInfo.Metadata[volume.BlockHostingAvailableSize] = fmt.Sprintf("%d", availableSizeInBytes+uint64(deletedSize)) return volume.AddOrUpdateVolume(volInfo) } From 827f2f09df06bbb4bcd0aa8298d67660951315d1 Mon Sep 17 00:00:00 2001 From: Oshank Kumar Date: Thu, 3 Jan 2019 13:57:08 +0530 Subject: [PATCH 6/7] block volume: Fix Merge conflicts in volume-start.go Signed-off-by: Oshank Kumar --- glusterd2.toml.example | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/glusterd2.toml.example b/glusterd2.toml.example index 98cb7b6cc..af19b2486 100644 --- a/glusterd2.toml.example +++ b/glusterd2.toml.example @@ -4,14 +4,14 @@ clientaddress = ":24007" #restauth enables/disables REST authentication in glusterd2 #restauth = true -#[gluster-block-hosting-vol-options] -block-hosting-volume-size = 20971520 +#[block-hosting-vol-options] +block-hosting-volume-size = 5368709120 #5GiB auto-create-block-hosting-volumes = true -block-hosting-volume-replica-count = 1 -#block-hosting-volume-type = "Distribute" +block-hosting-volume-replica-count = 3 +#block-hosting-volume-type = "Replicate" #[gluster-block-client-config] gluster-block-hostaddr = "192.168.122.16:8081" #gluster-block-cacert = "/path/to/ca.crt" #gluster-block-user = "username_for_rest_authentication" -#gluster-block-secret = "secret_for_rest_authentication" \ No newline at end of file +#gluster-block-secret = "secret_for_rest_authentication" From 0561c0ee7d4fecf426ef78df7375f5334d2d3427 Mon Sep 17 00:00:00 2001 From: Oshank Kumar Date: Thu, 3 Jan 2019 17:17:12 +0530 Subject: [PATCH 7/7] block volume: added a common function for updating available hosting volume size - moved hosts parameter from mandatory to optional field in CreateBlockVolume method of BlockProvider interface,since hosts field may not required for other block providers like loopback. - a common function for updating available hosting volume size will prevent from duplicate code Signed-off-by: Oshank Kumar --- doc/endpoints.md | 4 ++ pkg/size/size.go | 20 +++---- pkg/size/size_test.go | 8 +-- .../blockprovider/blockvolume_provider.go | 2 +- .../gluster-block/glusterblock.go | 55 ++++++------------- plugins/blockvolume/blockprovider/options.go | 8 +++ plugins/blockvolume/handlers.go | 7 ++- plugins/blockvolume/utils/volume.go | 39 +++++++++---- 8 files changed, 79 insertions(+), 64 deletions(-) diff --git a/doc/endpoints.md b/doc/endpoints.md index 4153cea6f..0c36469c5 100644 --- a/doc/endpoints.md +++ b/doc/endpoints.md @@ -91,6 +91,10 @@ DevicesList | GET | /devices | [](https://godoc.org/github.com/gluster/glusterd2 RebalanceStart | POST | /volumes/{volname}/rebalance/start | [StartReq](https://godoc.org/github.com/gluster/glusterd2/plugins/rebalance/api#StartReq) | [](https://godoc.org/github.com/gluster/glusterd2/plugins/rebalance/api#) RebalanceStop | POST | /volumes/{volname}/rebalance/stop | [](https://godoc.org/github.com/gluster/glusterd2/plugins/rebalance/api#) | [](https://godoc.org/github.com/gluster/glusterd2/plugins/rebalance/api#) RebalanceStatus | GET | /volumes/{volname}/rebalance | [](https://godoc.org/github.com/gluster/glusterd2/plugins/rebalance/api#) | [](https://godoc.org/github.com/gluster/glusterd2/plugins/rebalance/api#) +BlockCreate | POST | /blockvolumes/{provider} | [BlockVolumeCreateRequest](https://godoc.org/github.com/gluster/glusterd2/pkg/api#BlockVolumeCreateRequest) | [BlockVolumeCreateResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#BlockVolumeCreateResp) +BlockDelete | DELETE | /blockvolumes/{provider}/{name} | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) +BlockList | GET | /blockvolumes/{provider} | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) +BlockGet | GET | /blockvolumes/{provider}/{name} | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) Statedump | GET | /statedump | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) List Endpoints | GET | /endpoints | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [ListEndpointsResp](https://godoc.org/github.com/gluster/glusterd2/pkg/api#ListEndpointsResp) Glusterd2 service status | GET | /ping | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) | [](https://godoc.org/github.com/gluster/glusterd2/pkg/api#) diff --git a/pkg/size/size.go b/pkg/size/size.go index f9be3ee79..acaa3a570 100644 --- a/pkg/size/size.go +++ b/pkg/size/size.go @@ -134,24 +134,24 @@ func (s Size) TebiBytes() float64 { return float64(tib) + float64(bytes)/(1024*1024*1024*1024) } -// String string representation of Size in form XXKB/MB/TB/GB/Bytes -// TODO: support for string representation in XiB format +// String string representation of Size in form XXKiB/MiB/TiB/GiB/B +// TODO: support for string representation in other formats func (s Size) String() string { - if s >= TB { - return fmt.Sprintf("%.2fTB", s.TeraBytes()) + if s >= TiB { + return fmt.Sprintf("%.2fTiB", s.TebiBytes()) } - if s >= GB { - return fmt.Sprintf("%.2fGB", s.GigaBytes()) + if s >= GiB { + return fmt.Sprintf("%.2fGiB", s.GibiBytes()) } - if s >= MB { - return fmt.Sprintf("%.2fMB", s.MegaBytes()) + if s >= MiB { + return fmt.Sprintf("%.2fMiB", s.MebiBytes()) } - if s >= KB { - return fmt.Sprintf("%.2fKB", s.KiloBytes()) + if s >= KiB { + return fmt.Sprintf("%.2fKiB", s.KibiBytes()) } return fmt.Sprintf("%d B", s) diff --git a/pkg/size/size_test.go b/pkg/size/size_test.go index 53d1a798f..4e7c0302a 100644 --- a/pkg/size/size_test.go +++ b/pkg/size/size_test.go @@ -47,10 +47,10 @@ var ( s Size want string }{ - {Size(1e12), "1.00TB"}, - {Size(2.5e9), "2.50GB"}, - {Size(8.75e6), "8.75MB"}, - {Size(768e3), "768.00KB"}, + {Size(1099511627776), "1.00TiB"}, + {Size(2684354560), "2.50GiB"}, + {Size(9175040), "8.75MiB"}, + {Size(786432), "768.00KiB"}, {Size(500), "500 B"}, } diff --git a/plugins/blockvolume/blockprovider/blockvolume_provider.go b/plugins/blockvolume/blockprovider/blockvolume_provider.go index 9c25f42f3..8a7241a73 100644 --- a/plugins/blockvolume/blockprovider/blockvolume_provider.go +++ b/plugins/blockvolume/blockprovider/blockvolume_provider.go @@ -18,7 +18,7 @@ var ( // Provider is an abstract, pluggable interface for block volume providers type Provider interface { - CreateBlockVolume(name string, size uint64, hosts []string, hostVolume string, options ...BlockVolOption) (BlockVolume, error) + CreateBlockVolume(name string, size uint64, hostVolume string, options ...BlockVolOption) (BlockVolume, error) DeleteBlockVolume(name string, options ...BlockVolOption) error GetBlockVolume(id string) (BlockVolume, error) BlockVolumes() []BlockVolume diff --git a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go index 0856237d4..c368f4bde 100644 --- a/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go +++ b/plugins/blockvolume/blockprovider/gluster-block/glusterblock.go @@ -3,14 +3,11 @@ package glusterblock import ( "context" "errors" - "fmt" - "strconv" - - "github.com/gluster/glusterd2/glusterd2/transaction" "github.com/gluster/glusterd2/glusterd2/volume" "github.com/gluster/glusterd2/pkg/size" "github.com/gluster/glusterd2/plugins/blockvolume/blockprovider" "github.com/gluster/glusterd2/plugins/blockvolume/utils" + "time" "github.com/gluster/gluster-block-restapi/client" "github.com/gluster/gluster-block-restapi/pkg/api" @@ -44,6 +41,7 @@ func newGlusterBlock() (blockprovider.Provider, error) { opts = append(opts, client.WithAuth(clientConf.User, clientConf.Secret), client.WithTLSConfig(&client.TLSOptions{CaCertFile: clientConf.CaCertFile, InsecureSkipVerify: clientConf.Insecure}), + client.WithTimeOut(time.Minute), ) gbClient, err := client.NewClientWithOpts(clientConf.HostAddress, opts...) @@ -56,13 +54,14 @@ func newGlusterBlock() (blockprovider.Provider, error) { } // CreateBlockVolume will create a gluster block volume with given name and size having `hostVolume` as hosting volume -func (g *GlusterBlock) CreateBlockVolume(name string, size uint64, hosts []string, hostVolume string, options ...blockprovider.BlockVolOption) (blockprovider.BlockVolume, error) { - var ( - blockVolOpts = &blockprovider.BlockVolumeOptions{} - clusterLocks = transaction.Locks{} - ) - +func (g *GlusterBlock) CreateBlockVolume(name string, size uint64, hostVolume string, options ...blockprovider.BlockVolOption) (blockprovider.BlockVolume, error) { + blockVolOpts := &blockprovider.BlockVolumeOptions{} blockVolOpts.ApplyOpts(options...) + logger := log.WithFields(log.Fields{ + "block_name": name, + "hostvol": hostVolume, + "requested_block_size": size, + }) req := &api.BlockVolumeCreateReq{ HaCount: blockVolOpts.Ha, @@ -71,42 +70,22 @@ func (g *GlusterBlock) CreateBlockVolume(name string, size uint64, hosts []strin Size: size, Storage: blockVolOpts.Storage, RingBufferSizeInMB: blockVolOpts.RingBufferSizeInMB, - Hosts: hosts, - } - - if err := clusterLocks.Lock(hostVolume); err != nil { - log.WithError(err).Error("error in acquiring cluster lock") - return nil, err - } - defer clusterLocks.UnLock(context.Background()) - - volInfo, err := volume.GetVolume(hostVolume) - if err != nil { - return nil, fmt.Errorf("error in getting host vol details: %s", err) - } - - availableSizeInBytes, err := strconv.ParseUint(volInfo.Metadata[volume.BlockHostingAvailableSize], 10, 64) - if err != nil { - return nil, err - } - - if availableSizeInBytes < size { - return nil, fmt.Errorf("available size is less than requested size,request size: %d, available size: %d", size, availableSizeInBytes) + Hosts: blockVolOpts.Hosts, } resp, err := g.client.CreateBlockVolume(hostVolume, name, req) if err != nil { + logger.WithError(err).Error("failed in creating gluster block volume") return nil, err } - volInfo.Metadata[volume.BlockHostingAvailableSize] = fmt.Sprintf("%d", availableSizeInBytes-size) - - if err = volume.AddOrUpdateVolume(volInfo); err != nil { - log.WithError(err).Error("failed in updating volume info to store") + resizeFunc := func(blockHostingAvailableSize, blockSize uint64) uint64 { return blockHostingAvailableSize - blockSize } + if err = utils.ResizeBlockHostingVolume(hostVolume, size, resizeFunc); err != nil { + logger.WithError(err).Error("failed in updating hostvolume _block-hosting-available-size metadata") } return &BlockVolume{ - hostVolume: volInfo.Name, + hostVolume: hostVolume, name: name, hosts: resp.Portals, iqn: resp.IQN, @@ -153,7 +132,9 @@ func (g *GlusterBlock) DeleteBlockVolume(name string, options ...blockprovider.B return err } - if err = utils.ResizeBlockHostingVolume(hostVol, blockInfo.Size); err != nil { + resizeFunc := func(blockHostingAvailableSize, blockSize uint64) uint64 { return blockHostingAvailableSize + blockSize } + + if err = utils.ResizeBlockHostingVolume(hostVol, blockInfo.Size, resizeFunc); err != nil { log.WithFields(log.Fields{ "error": err, "size": blockInfo.Size, diff --git a/plugins/blockvolume/blockprovider/options.go b/plugins/blockvolume/blockprovider/options.go index f19d8e958..7f33fff1b 100644 --- a/plugins/blockvolume/blockprovider/options.go +++ b/plugins/blockvolume/blockprovider/options.go @@ -12,6 +12,7 @@ type BlockVolumeOptions struct { RingBufferSizeInMB uint64 ForceDelete bool UnlinkStorage bool + Hosts []string } // ApplyOpts applies configured optional parameters on BlockVolumeOptions @@ -61,3 +62,10 @@ func WithAuthEnabled(options *BlockVolumeOptions) { func WithFullPrealloc(options *BlockVolumeOptions) { options.FullPrealloc = true } + +// WithHosts configures required hosts for block creation +func WithHosts(hosts []string) BlockVolOption { + return func(options *BlockVolumeOptions) { + options.Hosts = hosts + } +} diff --git a/plugins/blockvolume/handlers.go b/plugins/blockvolume/handlers.go index b4edd9a7d..aec8266ec 100644 --- a/plugins/blockvolume/handlers.go +++ b/plugins/blockvolume/handlers.go @@ -24,7 +24,10 @@ func (b *BlockVolume) CreateVolume(w http.ResponseWriter, r *http.Request) { return } - opts = append(opts, blockprovider.WithHaCount(req.HaCount)) + opts = append(opts, + blockprovider.WithHaCount(req.HaCount), + blockprovider.WithHosts(req.Clusters), + ) if req.Auth { opts = append(opts, blockprovider.WithAuthEnabled) @@ -42,7 +45,7 @@ func (b *BlockVolume) CreateVolume(w http.ResponseWriter, r *http.Request) { return } - blockVol, err := blockProvider.CreateBlockVolume(req.Name, req.Size, req.Clusters, hostVolInfo.Name, opts...) + blockVol, err := blockProvider.CreateBlockVolume(req.Name, req.Size, hostVolInfo.Name, opts...) if err != nil { utils.SendHTTPError(r.Context(), w, http.StatusInternalServerError, err) return diff --git a/plugins/blockvolume/utils/volume.go b/plugins/blockvolume/utils/volume.go index 557643be9..38d690cc8 100644 --- a/plugins/blockvolume/utils/volume.go +++ b/plugins/blockvolume/utils/volume.go @@ -80,25 +80,39 @@ func CreateAndStartHostingVolume(req *api.VolCreateReq) (*volume.Volinfo, error) return vInfo, nil } -// ResizeBlockHostingVolume will adds deletedBlockSize to block-hosting-available-size -// in metadata and update the new vol info to store. -func ResizeBlockHostingVolume(volname string, deletedBlockSize string) error { - clusterLocks := transaction.Locks{} +// ResizeBlockHostingVolume will resize the _block-hosting-available-size metadata and update the new vol info to store. +// resizeFunc is use to update the new new value to the _block-hosting-available-size metadata +// e.g for adding the value to the _block-hosting-available-size metadata we can use `resizeFunc` as +// f := func(a, b uint64) uint64{return a +b } +func ResizeBlockHostingVolume(volName string, blockSize interface{}, resizeFunc func(blockHostingAvailableSize, blockSize uint64) uint64) error { + var ( + blkSize size.Size + clusterLocks = transaction.Locks{} + ) - if err := clusterLocks.Lock(volname); err != nil { + if err := clusterLocks.Lock(volName); err != nil { log.WithError(err).Error("error in acquiring cluster lock") return err } defer clusterLocks.UnLock(context.Background()) - volInfo, err := volume.GetVolume(volname) + volInfo, err := volume.GetVolume(volName) if err != nil { return err } - deletedSize, err := size.Parse(deletedBlockSize) - if err != nil { - return err + switch sz := blockSize.(type) { + case string: + blkSize, err = size.Parse(sz) + if err != nil { + return err + } + case uint64: + blkSize = size.Size(sz) + case int64: + blkSize = size.Size(sz) + default: + return fmt.Errorf("blocksize is not a supported type(%T)", blockSize) } if _, found := volInfo.Metadata[volume.BlockHostingAvailableSize]; !found { @@ -110,7 +124,12 @@ func ResizeBlockHostingVolume(volname string, deletedBlockSize string) error { return err } - volInfo.Metadata[volume.BlockHostingAvailableSize] = fmt.Sprintf("%d", availableSizeInBytes+uint64(deletedSize)) + log.WithFields(log.Fields{ + "blockHostingAvailableSize": size.Size(availableSizeInBytes), + "blockSize": blkSize, + }).Debug("resizing hosting volume") + + volInfo.Metadata[volume.BlockHostingAvailableSize] = fmt.Sprintf("%d", resizeFunc(availableSizeInBytes, uint64(blkSize))) return volume.AddOrUpdateVolume(volInfo) }