From 013c161bde7386647590b84afb1e586f306ec78d Mon Sep 17 00:00:00 2001 From: Cory Snyder Date: Thu, 14 Feb 2019 14:44:25 -0500 Subject: [PATCH 1/2] refactor project to use go modules and to conform to conventional layout --- .gitignore | 5 +- README.md | 5 + {src/exporter => exporter}/environment.go | 0 .../exporter => exporter}/environment_test.go | 0 {src/exporter => exporter}/hardwareprofile.go | 0 {src/exporter => exporter}/swiftmetrics.go | 0 .../systemperformance.go | 0 go.mod | 9 + go.sum | 25 ++ src/swift_exporter.go | 317 ------------------ swift_exporter.go | 285 ++++++++++++++++ swift_exporter_config.yaml | 69 ++-- 12 files changed, 361 insertions(+), 354 deletions(-) rename {src/exporter => exporter}/environment.go (100%) rename {src/exporter => exporter}/environment_test.go (100%) rename {src/exporter => exporter}/hardwareprofile.go (100%) rename {src/exporter => exporter}/swiftmetrics.go (100%) rename {src/exporter => exporter}/systemperformance.go (100%) create mode 100644 go.mod create mode 100644 go.sum delete mode 100644 src/swift_exporter.go create mode 100644 swift_exporter.go diff --git a/.gitignore b/.gitignore index 0c08b9f..8b83cb0 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,7 @@ pkg/* bin/* #ignore the generated binary file as well... -src/swift_exporter +swift_exporter + +# ignore idea configuration +.idea diff --git a/README.md b/README.md index df61562..4846d42 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,11 @@ To get the **swift_exporter** plugin install, you will need the following: 1. swift_exporter (the binary, not the .go file) 2. swift_exporter.service +To generate the swift_exporter binary, do the following: +1. navigate to the root of this project +2. run `go get` to retrieve project dependencies +3. run `go build -o swift_exporter` to generate the executable in the project folder + After getting both files, place them at the following locations: 1. swift_exporter = `/opt/ss/support/bin/` diff --git a/src/exporter/environment.go b/exporter/environment.go similarity index 100% rename from src/exporter/environment.go rename to exporter/environment.go diff --git a/src/exporter/environment_test.go b/exporter/environment_test.go similarity index 100% rename from src/exporter/environment_test.go rename to exporter/environment_test.go diff --git a/src/exporter/hardwareprofile.go b/exporter/hardwareprofile.go similarity index 100% rename from src/exporter/hardwareprofile.go rename to exporter/hardwareprofile.go diff --git a/src/exporter/swiftmetrics.go b/exporter/swiftmetrics.go similarity index 100% rename from src/exporter/swiftmetrics.go rename to exporter/swiftmetrics.go diff --git a/src/exporter/systemperformance.go b/exporter/systemperformance.go similarity index 100% rename from src/exporter/systemperformance.go rename to exporter/systemperformance.go diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a120129 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/ilanddev/swift-exporter + +require ( + github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 + github.com/prometheus/client_golang v0.9.2 + github.com/shirou/gopsutil v2.18.12+incompatible + golang.org/x/sys v0.0.0-20190213121743-983097b1a8a3 // indirect + gopkg.in/yaml.v2 v2.2.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..59ca021 --- /dev/null +++ b/go.sum @@ -0,0 +1,25 @@ +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= +github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190213121743-983097b1a8a3 h1:+KlxhGbYkFs8lMfwKn+2ojry1ID5eBSMXprS2u/wqCE= +golang.org/x/sys v0.0.0-20190213121743-983097b1a8a3/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/src/swift_exporter.go b/src/swift_exporter.go deleted file mode 100644 index 7b6a241..0000000 --- a/src/swift_exporter.go +++ /dev/null @@ -1,317 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "time" - - "exporter" - - "github.com/docopt/docopt-go" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "gopkg.in/yaml.v2" -) - -// Location of the .recon files in Swift nodes -const accountReconFile = "/var/cache/swift/account.recon" -const containerReconFile = "/var/cache/swift/container.recon" -const objectReconFile = "/var/cache/swift/object.recon" -const replicationProgressFile = "/opt/ss/var/lib/replication_progress.json" -const swiftConfig = "/etc/swift/swift.conf" -const swiftLog = "/var/log/swift/all.log" - -// ModulesOnOff holds the on / off settings from the swift_exporter.yml file. -type ModulesOnOff struct { - CheckObjectServerConnectionEnable bool - GrabSwiftPartitionEnable bool - GatherReplicationEstimateEnable bool - GatherStoragePolicyUtilizationEnable bool - ExposePerCPUUsageEnable bool - ExposePerNICMetricEnable bool - ReadReconFileEnable bool - SwiftDiskUsageEnable bool - SwiftDriveIOEnable bool - SwiftLogSizeEnable bool -} - -/* -This var() section sets the port which promohttp (Promethes HTTP server) uses. -In addition, accountServer, containerServer, and objectServer initializes gauge-type prometheus -metrics data. -*/ -var ( - scriptVersion = "0.8.5" - timeLastRun = "00:00:00" - swiftExporterLog, swiftExporterLogError = os.OpenFile("/var/log/swift_exporter.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) - addr = flag.String("listen-address", ":53167", "The addres to listen on for HTTP requests.") - abScriptVersionPara = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "ac_script_version", - Help: "swift_exporter version 0.8.5", - }, []string{"script_version"}) - - defaultConfig = map[string]bool{ - "ReadReconFile": true, - "GrabSwiftPartition": true, - "SwiftDiskUsage": true, - "SwiftDriveIO": true, - "GatherReplicationEstimate": true, - "GatherStoragePolicyUtilization": true, - "CheckObjectServerConnection": true, - "ExposePerCPUUsage": true, - "ExposePerNICMetric": true, - } - - SelectedModule ModulesOnOff - argv []string - Usage = `Usage: - /opt/ss/bin/swift_exporter - /opt/ss/bin/swift_exporter [] - /opt/ss/bin/swift_exporter --help | --version ` -) - -// Metrics have to be registeered to be expose, so this is done below. -func init() { - prometheus.MustRegister(abScriptVersionPara) - - if swiftExporterLogError != nil { - fmt.Println("Error Opening File: %", swiftExporterLog) - } -} - -// SanityCheckOnFiles checks is a function being called in -func SanityCheckOnFiles(SelectedModule ModulesOnOff) { - - writeLogFile := log.New(swiftExporterLog, "SanityCheckOnFiles: ", log.Ldate|log.Ltime|log.Lshortfile) - - if _, swiftConfigErr := os.Stat(swiftConfig); os.IsNotExist(swiftConfigErr) { - writeLogFile.Println("swift.conf does not exist! Exiting this script!") - os.Exit(1) - } else { - writeLogFile.Println("Swift config file (swift.conf) exist. Continue checking other files") - writeLogFile.Println("Checking if *.recon (/var/cache/swift/*recon) file exist...") - if SelectedModule.ReadReconFileEnable { - writeLogFile.Println("Script is set to expose data collected from /var/cache/swift/*.recon files (ReadReconFile module enable). Check to see if those file exist") - if _, err := os.Stat(accountReconFile); err == nil { - writeLogFile.Println(" ===> account.recon file exists. Moving on to check if container.recon file exists...") - } else { - writeLogFile.Println(" ===> account.recon file does not exist. We will need all 3 (account, container, object) recon files for this module to work, but you have enable the ReadReconFile module. Turning it off...") - SelectedModule.ReadReconFileEnable = false - } - if _, err := os.Stat(containerReconFile); err == nil { - writeLogFile.Println(" ===> container.recon file exists. Moving on to check if container.recon file exists...") - } else { - writeLogFile.Println(" ===> container.recon file does not exist. We will need all 3 (account, container, object) recon files for this module to work, but you have enable the ReadReconFile module. Turning it off...") - SelectedModule.ReadReconFileEnable = false - } - if _, err := os.Stat(objectReconFile); err == nil { - writeLogFile.Println(" ===> object.recon file exists. Moving on to check if object.recon file exists") - } else { - writeLogFile.Println(" ===> object.recon file does not exist. We will need all 3 (account, container, object) recon files for this module to work, but you have enable the ReadReconFile module. Turning it off...") - SelectedModule.ReadReconFileEnable = false - } - writeLogFile.Println("===> account.recon, container.recon, and object.recon file exist. Check for this module has completed. Enable this module...") - SelectedModule.ReadReconFileEnable = true - writeLogFile.Println() - } else { - writeLogFile.Println("ReadReconFile module is disabled. Skip this check.") - writeLogFile.Println() - } - if SelectedModule.GrabSwiftPartitionEnable { - writeLogFile.Println("Script is set to expose data collected from /opt/ss/var/lib/replication_progress.json (GrabSwiftPartition module enable). Check to see if that file exist...") - if _, err := os.Stat(replicationProgressFile); err == nil { - log.Println("===> /opt/ss/var/lib/replication_progress.json exists. Check for this module has completed. Enable the module...") - SelectedModule.GrabSwiftPartitionEnable = true - writeLogFile.Println() - } else { - writeLogFile.Println("===> /opt/ss/var/lib/replication_progress.json does not exists, but you have enabled it. Disable the module...") - SelectedModule.GrabSwiftPartitionEnable = false - writeLogFile.Println() - } - } else { - writeLogFile.Println("GrabSwiftPartition module is disabled. Skip this check.") - } - if SelectedModule.GatherReplicationEstimateEnable { - writeLogFile.Println("Script is set to expose data collected from /var/log/swift/all.log (GatherReplicationEstimate module enable). Check to see if that file exist...") - if _, err := os.Stat(swiftLog); err == nil { - writeLogFile.Println("===> /var/log/swift/all.log exists. Check for this module has completed. Enable the module...") - SelectedModule.GatherReplicationEstimateEnable = true - writeLogFile.Println() - } else { - writeLogFile.Println("===> /var/log/swift/all.log does not exists, but you have enabled it. Disable the module...") - SelectedModule.GatherReplicationEstimateEnable = false - writeLogFile.Println() - } - } else { - writeLogFile.Println("GatherReplicationEstimate module is disabled. Skip this check.") - writeLogFile.Println() - } - if SelectedModule.GatherStoragePolicyUtilizationEnable { - writeLogFile.Println("GatherStoragePolicyUtilization module is enabled. Since there is no config, there is nothing to check.") - writeLogFile.Println() - } else { - writeLogFile.Println("GatherStoragePolicyUtilization module is disabled. Skip this check.") - writeLogFile.Println() - } - if SelectedModule.ExposePerCPUUsageEnable { - writeLogFile.Println("ExposePerCPUUsage module is enabled. Since there is no config, there is nothing to check.") - writeLogFile.Println() - } else { - writeLogFile.Println("ExposePerCPUUsage module is disabled. Skip this check.") - writeLogFile.Println() - } - if SelectedModule.ExposePerNICMetricEnable { - writeLogFile.Println("ExposePerNICMetric module is enabled. Since there is no config, there is nothing to check.") - writeLogFile.Println() - } else { - writeLogFile.Println("ExposePerNICMetric module is disabled. Skip this check.") - writeLogFile.Println() - } - writeLogFile.Println("All checks complete. Proceed on turning modules on / off.") - writeLogFile.Println() - } -} - -//TurnOnModules reads through the yaml file and turns on the modules available in this script. -//Input Argument: Location of the yaml file. -//Output Argument: 3 boolean values that will enable func GrabSwiftPartition, -//func SwiftDiskUsage, and func GatherReplicationEstimate -func TurnOnModules(configFileLocation string) (SelectedModule ModulesOnOff) { - - writeLogFile := log.New(swiftExporterLog, "TurnOnModules: ", log.Ldate|log.Ltime|log.Lshortfile) - - // To parse the data correctly, we need the following. - // Reference: http://squarism.com/2014/10/13/yaml-go/ - var config = make(map[string][]bool) - - filename, _ := os.Open(configFileLocation) - yamlFile, _ := ioutil.ReadAll(filename) - - err := yaml.Unmarshal(yamlFile, &config) - - // If yaml.Unmarshal cannot extra data and put into the map data structure, do the following: - if err != nil { - writeLogFile.Fatalf("cannot unmarshal %v", err) - writeLogFile.Println(err) - } - - // If yaml.Unmarshal can output the data into map data structure, do the following: - SelectedModule.ReadReconFileEnable = config["ReadReconFile"][0] - SelectedModule.GrabSwiftPartitionEnable = config["GrabSwiftPartition"][0] - SelectedModule.SwiftDiskUsageEnable = config["SwiftDiskUsage"][0] - SelectedModule.SwiftDriveIOEnable = config["SwiftDriveIO"][0] - SelectedModule.GatherStoragePolicyUtilizationEnable = config["GatherStoragePolicyUtilization"][0] - SelectedModule.CheckObjectServerConnectionEnable = config["CheckObjectServerConnection"][0] - SelectedModule.ExposePerCPUUsageEnable = config["ExposePerCPUUsage"][0] - SelectedModule.ExposePerNICMetricEnable = config["ExposePerNICMetric"][0] - - return SelectedModule - //return ReadReconFileEnable, GrabSwiftPartitionEnable, SwiftDiskUsageEnable, SwiftDriveIOEnable, GatherReplicationEstimateEnable, GatherStoragePolicyUtilizationEnable, CheckObjectServerConnectionEnable, ExposePerCPUUsageEnable, ExposePerNICMetricEnable -} - -func main() { - - writeLogFile := log.New(swiftExporterLog, "main: ", log.Ldate|log.Ltime|log.Lshortfile) - - // If user pass an empty argument to the script, use the default value. Assign dummy variable "all" - // that turns on ALL modules in this script. - if len(os.Args) < 2 { - argv = []string{"all"} - } else if _, err := os.Stat(os.Args[1]); err == nil { // To check if a file exists, equivalent to Python's if os.path.exists(filename): - argv = []string{os.Args[1]} - } - - // user docopt to show menu, forward argument, and display version number. - opts, _ := docopt.ParseArgs(Usage, argv, scriptVersion) - - // extra config file entry. - ConfigFileExist, _ := opts.String("") - - abScriptVersionPara.WithLabelValues(scriptVersion).Set(0.00) - - // If no agrument is presented when the code is run. - if ConfigFileExist == "all" { - writeLogFile.Println("swift_export_config.yaml is NOT detected") - SelectedModule.ReadReconFileEnable = defaultConfig["ReadReconFile"] - SelectedModule.GrabSwiftPartitionEnable = defaultConfig["GrabSwiftPartition"] - SelectedModule.SwiftDiskUsageEnable = defaultConfig["SwiftDiskUsage"] - SelectedModule.SwiftDriveIOEnable = defaultConfig["SwiftDriveIO"] - SelectedModule.GatherReplicationEstimateEnable = defaultConfig["GatherReplicationEstimate"] - SelectedModule.GatherStoragePolicyUtilizationEnable = defaultConfig["GatherStoragePolicyUtilization"] - SelectedModule.CheckObjectServerConnectionEnable = defaultConfig["CheckObjectServerConnection"] - SelectedModule.ExposePerCPUUsageEnable = defaultConfig["ExposePerCPUUsage"] - SelectedModule.ExposePerNICMetricEnable = defaultConfig["ExposePerNICMetric"] - SanityCheckOnFiles(SelectedModule) - } else if _, err := os.Stat(ConfigFileExist); err == nil { // To check if a file exists, equivalent to Python's if os.path.exists(filename): - writeLogFile.Println("swift_export_config.yaml is detected") - SelectedModule = TurnOnModules(ConfigFileExist) - SanityCheckOnFiles(SelectedModule) - } - - // Declare Go routines below so that we can grab the metrics and expose them to the - // prometheus HTTP server periodically. - // Fixed issue #6 in gitlab - // Reference: https://gobyexample.com/goroutines - // Reference2: https://github.com/prometheus/client_golang/blob/master/examples/random/main.go - go func() { - for { - exporter.ReadReconFile(accountReconFile, "account", SelectedModule.ReadReconFileEnable) - exporter.ReadReconFile(containerReconFile, "container", SelectedModule.ReadReconFileEnable) - exporter.ReadReconFile(objectReconFile, "object", SelectedModule.ReadReconFileEnable) - exporter.GrabSwiftPartition(replicationProgressFile, SelectedModule.GrabSwiftPartitionEnable) - exporter.SwiftDiskUsage(SelectedModule.SwiftDiskUsageEnable) - exporter.SwiftDriveIO(SelectedModule.SwiftDriveIOEnable) - exporter.CheckObjectServerConnection(SelectedModule.CheckObjectServerConnectionEnable) - exporter.ExposePerCPUUsage(SelectedModule.ExposePerCPUUsageEnable) - exporter.ExposePerNICMetric(SelectedModule.ExposePerNICMetricEnable) - exporter.GrabNICMTU() - - // Setting time to sleep for 1 Minute. If you need to set it to milliseconds, change the - // "time.Minute" to "time.Millisecond" - // Reference: https://golang.org/pkg/time/#Sleep - time.Sleep(1 * time.Minute) - } - }() - - // the following go routine will be run every 5 minutes - go func() { - for { - //GatherReplicationEstimate(swiftLog, timeLastRun, SelectedModule.GatherReplicationEstimateEnable) - exporter.CheckSwiftService() - time.Sleep(5 * time.Minute) - } - }() - - // the following go routine will be run every hour. - go func() { - for { - exporter.RunSMARTCTL() - time.Sleep(1 * time.Hour) - } - }() - - // the following go routine will be run every 3 hours. - go func() { - for { - exporter.CheckSwiftLogSize(swiftLog) - exporter.CountFilesPerSwiftDrive() - time.Sleep(3 * time.Hour) - } - }() - - // the following go routine will be run every 6 hours. - go func() { - for { - exporter.GatherStoragePolicyUtilization(SelectedModule.GatherStoragePolicyUtilizationEnable) - time.Sleep(6 * time.Hour) - } - }() - // Call the promhttp method in Prometheus to expose the data for Prometheus to grab. - flag.Parse() - http.Handle("/metrics", promhttp.Handler()) - writeLogFile.Fatal(http.ListenAndServe(*addr, nil)) -} diff --git a/swift_exporter.go b/swift_exporter.go new file mode 100644 index 0000000..19e41db --- /dev/null +++ b/swift_exporter.go @@ -0,0 +1,285 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "time" + + "github.com/ilanddev/swift-exporter/exporter" + + "github.com/docopt/docopt-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "gopkg.in/yaml.v2" +) + +// Config holds the configuration settings from the swift_exporter.yml file. +type Config struct { + CheckObjectServerConnectionEnable bool `yaml:"CheckObjectServerConnection"` + GrabSwiftPartitionEnable bool `yaml:"GrabSwiftPartition"` + GatherReplicationEstimateEnable bool `yaml:"GatherReplicationEstimate"` + GatherStoragePolicyUtilizationEnable bool `yaml:"GatherStoragePolicyUtilization"` + ExposePerCPUUsageEnable bool `yaml:"ExposePerCPUUsage"` + ExposePerNICMetricEnable bool `yaml:"ExposePerNICMetric"` + ReadReconFileEnable bool `yaml:"ReadReconFile"` + SwiftDiskUsageEnable bool `yaml:"SwiftDiskUsage"` + SwiftDriveIOEnable bool `yaml:"SwiftDriveIO"` + SwiftLogFile string `yaml:"SwiftLogFile"` + SwiftConfigFile string `yaml:"SwiftConfigFile"` + ReplicationProgressFile string `yaml:"ReplicationProgressFile"` + ObjectReconFile string `yaml:"ObjectReconFile"` + ContainerReconFile string `yaml:"ContainerReconFile"` + AccountReconFile string `yaml:"AccountReconFile"` +} + +/* +This var() section sets the port which promohttp (Promethes HTTP server) uses. +In addition, accountServer, containerServer, and objectServer initializes gauge-type prometheus +metrics data. +*/ +var ( + scriptVersion = "0.8.5" + timeLastRun = "00:00:00" + swiftExporterLogFile = "/var/log/swift_exporter.log" + swiftExporterLog, swiftExporterLogError = os.OpenFile(swiftExporterLogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + addr = flag.String("listen-address", ":53167", "The addres to listen on for HTTP requests.") + abScriptVersionPara = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ac_script_version", + Help: "swift_exporter version 0.8.5", + }, []string{"script_version"}) + + config = Config{ + ReadReconFileEnable: true, + GrabSwiftPartitionEnable: true, + SwiftDiskUsageEnable: true, + SwiftDriveIOEnable: true, + GatherReplicationEstimateEnable: true, + GatherStoragePolicyUtilizationEnable: true, + CheckObjectServerConnectionEnable: true, + ExposePerCPUUsageEnable: true, + ExposePerNICMetricEnable: true, + SwiftLogFile: "/var/log/swift/all.log", + SwiftConfigFile: "/etc/swift/swift.conf", + ReplicationProgressFile: "/opt/ss/var/lib/replication_progress.json", + ObjectReconFile: "/var/cache/swift/object.recon", + ContainerReconFile: "/var/cache/swift/container.recon", + AccountReconFile: "/var/cache/swift/account.recon", + } + argv []string + Usage = `Usage: + /opt/ss/bin/swift_exporter + /opt/ss/bin/swift_exporter [] + /opt/ss/bin/swift_exporter --help | --version ` +) + +// Metrics have to be registeered to be expose, so this is done below. +func init() { + prometheus.MustRegister(abScriptVersionPara) + if swiftExporterLogError != nil { + fmt.Printf("Error Opening File '%s': %v\n", swiftExporterLogFile, swiftExporterLogError) + } +} + +// SanityCheckOnFiles checks is a function being called in +func SanityCheckOnFiles() { + + writeLogFile := log.New(swiftExporterLog, "SanityCheckOnFiles: ", log.Ldate|log.Ltime|log.Lshortfile) + + if _, swiftConfigErr := os.Stat(config.SwiftConfigFile); os.IsNotExist(swiftConfigErr) { + writeLogFile.Printf("%s does not exist! Exiting this script!\n", config.SwiftConfigFile) + os.Exit(1) + } else { + writeLogFile.Println("Swift config file (swift.conf) exist. Continue checking other files") + writeLogFile.Println("Checking if *.recon (default /var/cache/swift/*recon) file exist...") + if config.ReadReconFileEnable { + writeLogFile.Println("Script is set to expose data collected from /var/cache/swift/*.recon files (ReadReconFile module enable). Check to see if those file exist") + if _, err := os.Stat(config.AccountReconFile); err == nil { + writeLogFile.Println(" ===> account.recon file exists. Moving on to check if container.recon file exists...") + } else { + writeLogFile.Printf(" ===> %s file does not exist. We will need all 3 (account, container, object) recon files for this module to work, but you have enable the ReadReconFile module. Turning it off...\n", config.AccountReconFile) + config.ReadReconFileEnable = false + } + if _, err := os.Stat(config.ContainerReconFile); err == nil { + writeLogFile.Println(" ===> container.recon file exists. Moving on to check if container.recon file exists...") + } else { + writeLogFile.Printf(" ===> %s file does not exist. We will need all 3 (account, container, object) recon files for this module to work, but you have enable the ReadReconFile module. Turning it off...\n", config.ContainerReconFile) + config.ReadReconFileEnable = false + } + if _, err := os.Stat(config.ObjectReconFile); err == nil { + writeLogFile.Println(" ===> object.recon file exists. Moving on to check if object.recon file exists") + } else { + writeLogFile.Printf(" ===> %s file does not exist. We will need all 3 (account, container, object) recon files for this module to work, but you have enable the ReadReconFile module. Turning it off...\n", config.ObjectReconFile) + config.ReadReconFileEnable = false + } + writeLogFile.Println("===> account.recon, container.recon, and object.recon file exist. Check for this module has completed. Enable this module...") + config.ReadReconFileEnable = true + writeLogFile.Println() + } else { + writeLogFile.Println("ReadReconFile module is disabled. Skip this check.") + writeLogFile.Println() + } + if config.GrabSwiftPartitionEnable { + writeLogFile.Printf("Script is set to expose data collected from %s (GrabSwiftPartition module enable). Check to see if that file exist...\n", config.ReplicationProgressFile) + if _, err := os.Stat(config.ReplicationProgressFile); err == nil { + log.Printf("===> %s exists. Check for this module has completed. Enable the module...\n", config.ReplicationProgressFile) + config.GrabSwiftPartitionEnable = true + writeLogFile.Println() + } else { + writeLogFile.Printf("===> %s does not exists, but you have enabled it. Disable the module...\n", config.ReplicationProgressFile) + config.GrabSwiftPartitionEnable = false + writeLogFile.Println() + } + } else { + writeLogFile.Println("GrabSwiftPartition module is disabled. Skip this check.") + } + if config.GatherReplicationEstimateEnable { + writeLogFile.Printf("Script is set to expose data collected from %s (GatherReplicationEstimate module enable). Check to see if that file exist...\n", config.SwiftLogFile) + if _, err := os.Stat(config.SwiftLogFile); err == nil { + writeLogFile.Printf("===> %s exists. Check for this module has completed. Enable the module...\n", config.SwiftLogFile) + config.GatherReplicationEstimateEnable = true + writeLogFile.Println() + } else { + writeLogFile.Printf("===> %s does not exists, but you have enabled it. Disable the module...\n", config.SwiftLogFile) + config.GatherReplicationEstimateEnable = false + writeLogFile.Println() + } + } else { + writeLogFile.Println("GatherReplicationEstimate module is disabled. Skip this check.") + writeLogFile.Println() + } + if config.GatherStoragePolicyUtilizationEnable { + writeLogFile.Println("GatherStoragePolicyUtilization module is enabled. Since there is no config, there is nothing to check.") + writeLogFile.Println() + } else { + writeLogFile.Println("GatherStoragePolicyUtilization module is disabled. Skip this check.") + writeLogFile.Println() + } + if config.ExposePerCPUUsageEnable { + writeLogFile.Println("ExposePerCPUUsage module is enabled. Since there is no config, there is nothing to check.") + writeLogFile.Println() + } else { + writeLogFile.Println("ExposePerCPUUsage module is disabled. Skip this check.") + writeLogFile.Println() + } + if config.ExposePerNICMetricEnable { + writeLogFile.Println("ExposePerNICMetric module is enabled. Since there is no config, there is nothing to check.") + writeLogFile.Println() + } else { + writeLogFile.Println("ExposePerNICMetric module is disabled. Skip this check.") + writeLogFile.Println() + } + writeLogFile.Println("All checks complete. Proceed on turning modules on / off.") + writeLogFile.Println() + } +} + +//ParseConfigFile reads through the yaml file, turns on the modules available in this script, and parses other config options. +func ParseConfigFile(configFileLocation string) () { + writeLogFile := log.New(swiftExporterLog, "TurnOnModules: ", log.Ldate|log.Ltime|log.Lshortfile) + filename, _ := os.Open(configFileLocation) + yamlFile, _ := ioutil.ReadAll(filename) + err := yaml.Unmarshal(yamlFile, &config) + // If yaml.Unmarshal cannot extra data and put into the map data structure, do the following: + if err != nil { + writeLogFile.Fatalf("cannot unmarshal %v", err) + writeLogFile.Println(err) + } +} + +func main() { + + writeLogFile := log.New(swiftExporterLog, "main: ", log.Ldate|log.Ltime|log.Lshortfile) + + // If user pass an empty argument to the script, use the default value. Assign dummy variable "all" + // that turns on ALL modules in this script. + if len(os.Args) < 2 { + argv = []string{"all"} + } else if _, err := os.Stat(os.Args[1]); err == nil { // To check if a file exists, equivalent to Python's if os.path.exists(filename): + argv = []string{os.Args[1]} + } + + // user docopt to show menu, forward argument, and display version number. + opts, _ := docopt.ParseArgs(Usage, argv, scriptVersion) + + // extra config file entry. + ConfigFileExist, _ := opts.String("") + + abScriptVersionPara.WithLabelValues(scriptVersion).Set(0.00) + + // If no argument is presented when the code is run. + if ConfigFileExist == "all" { + writeLogFile.Println("swift_export_config.yaml is NOT detected") + SanityCheckOnFiles() + } else if _, err := os.Stat(ConfigFileExist); err == nil { // To check if a file exists, equivalent to Python's if os.path.exists(filename): + writeLogFile.Println("swift_export_config.yaml is detected") + ParseConfigFile(ConfigFileExist) + SanityCheckOnFiles() + } + + // Declare Go routines below so that we can grab the metrics and expose them to the + // prometheus HTTP server periodically. + // Fixed issue #6 in gitlab + // Reference: https://gobyexample.com/goroutines + // Reference2: https://github.com/prometheus/client_golang/blob/master/examples/random/main.go + go func() { + for { + exporter.ReadReconFile(config.AccountReconFile, "account", config.ReadReconFileEnable) + exporter.ReadReconFile(config.ContainerReconFile, "container", config.ReadReconFileEnable) + exporter.ReadReconFile(config.ObjectReconFile, "object", config.ReadReconFileEnable) + exporter.GrabSwiftPartition(config.ReplicationProgressFile, config.GrabSwiftPartitionEnable) + exporter.SwiftDiskUsage(config.SwiftDiskUsageEnable) + exporter.SwiftDriveIO(config.SwiftDriveIOEnable) + exporter.CheckObjectServerConnection(config.CheckObjectServerConnectionEnable) + exporter.ExposePerCPUUsage(config.ExposePerCPUUsageEnable) + exporter.ExposePerNICMetric(config.ExposePerNICMetricEnable) + exporter.GrabNICMTU() + + // Setting time to sleep for 1 Minute. If you need to set it to milliseconds, change the + // "time.Minute" to "time.Millisecond" + // Reference: https://golang.org/pkg/time/#Sleep + time.Sleep(1 * time.Minute) + } + }() + + // the following go routine will be run every 5 minutes + go func() { + for { + //GatherReplicationEstimate(swiftLog, timeLastRun, SelectedModule.GatherReplicationEstimateEnable) + exporter.CheckSwiftService() + time.Sleep(5 * time.Minute) + } + }() + + // the following go routine will be run every hour. + go func() { + for { + exporter.RunSMARTCTL() + time.Sleep(1 * time.Hour) + } + }() + + // the following go routine will be run every 3 hours. + go func() { + for { + exporter.CheckSwiftLogSize(config.SwiftLogFile) + exporter.CountFilesPerSwiftDrive() + time.Sleep(3 * time.Hour) + } + }() + + // the following go routine will be run every 6 hours. + go func() { + for { + exporter.GatherStoragePolicyUtilization(config.GatherStoragePolicyUtilizationEnable) + time.Sleep(6 * time.Hour) + } + }() + // Call the promhttp method in Prometheus to expose the data for Prometheus to grab. + flag.Parse() + http.Handle("/metrics", promhttp.Handler()) + writeLogFile.Fatal(http.ListenAndServe(*addr, nil)) +} diff --git a/swift_exporter_config.yaml b/swift_exporter_config.yaml index 43d480d..d77c107 100644 --- a/swift_exporter_config.yaml +++ b/swift_exporter_config.yaml @@ -1,36 +1,33 @@ -ReadReconFile: - # module_description: this moudule reads the *.recon file from /var/cache/swift directory, and parse - # data such as replication time, object audit time...etc. Generally speaking, this is the module that - # contains data most users will find useful. - - yes -GrabSwiftPartition: - # module_description: this module grab the primary and handoff partitions count in a Swift node. - # Enter "yes" to enable, and "no" to disable. - - yes -SwiftDiskUsage: - # module_description: this module checks Swift nodes and grab information about disk. - # Enter "yes" to enable, and "no" to disable. - - yes -SwiftDriveIO: - # module_description: this module pull disk IO stats from all Swift drives (those that are mounted in "/srv/node/d") - # and expose them via Prometheus. - - yes -GatherReplicationEstimate: - # module_description: this module will read the latest entry in all.log and search for keyword "replicated", - # then extract the data out and parse for the estimated time complete, and part/sec. Enter "yes" to enable, - # and "no" to disable. - - yes -GatherStoragePolicyUtilization: - # module_description: this module grabs the size of each drive that has a storage policy assigned to it, - # then expose it to Prometheus. - - yes -CheckObjectServerConnection: - # module_description: this module checks the number of object server spinned. This is useful for replica based storage policy. - # The number of object server = number of drives * object server per port setting. - - yes -ExposePerCPUUsage: - # module_description: this module expose cpu usage and expose them in prometheus. - - yes -ExposePerNICMetric: - # module_description: this module expose cpu usage and expose them in prometheus. - - yes \ No newline at end of file +# module_description: this moudule reads the *.recon file from /var/cache/swift directory, and parse +# data such as replication time, object audit time...etc. Generally speaking, this is the module that +# contains data most users will find useful. +ReadReconFile: yes +# module_description: this module grab the primary and handoff partitions count in a Swift node. +# Enter "yes" to enable, and "no" to disable. +GrabSwiftPartition: yes +# module_description: this module checks Swift nodes and grab information about disk. +# Enter "yes" to enable, and "no" to disable. +SwiftDiskUsage: yes +# module_description: this module pull disk IO stats from all Swift drives (those that are mounted in "/srv/node/d") +# and expose them via Prometheus. +SwiftDriveIO: yes +# module_description: this module will read the latest entry in all.log and search for keyword "replicated", +# then extract the data out and parse for the estimated time complete, and part/sec. Enter "yes" to enable, +# and "no" to disable. +GatherReplicationEstimate: yes +# module_description: this module grabs the size of each drive that has a storage policy assigned to it, +# then expose it to Prometheus. +GatherStoragePolicyUtilization: yes +# module_description: this module checks the number of object server spinned. This is useful for replica based storage policy. +# The number of object server = number of drives * object server per port setting. +CheckObjectServerConnection: yes +# module_description: this module expose cpu usage and expose them in prometheus. +ExposePerCPUUsage: yes +# module_description: this module expose cpu usage and expose them in prometheus. +ExposePerNICMetric: yes +SwiftLogFile: "/var/log/swift/all.log" +SwiftConfigFile: "/etc/swift/swift.conf" +ReplicationProgressFile: "/opt/ss/var/lib/replication_progress.json" +ObjectReconFile: "/var/cache/swift/object.recon" +ContainerReconFile: "/var/cache/swift/container.recon" +AccountReconFile: "/var/cache/swift/account.recon" From be68d853af0f301aa60d33ac7b116491ca7f5c63 Mon Sep 17 00:00:00 2001 From: David Orman Date: Tue, 12 Mar 2019 22:05:39 +0000 Subject: [PATCH 2/2] Removed external dependencies on swiftstack specific configuration --- exporter/environment.go | 95 +------ exporter/hardwareprofile.go | 106 ++++--- exporter/swiftmetrics.go | 515 +++++++++++++--------------------- exporter/systemperformance.go | 33 ++- swift_exporter.go | 35 +-- swift_exporter_config.yaml | 18 +- 6 files changed, 274 insertions(+), 528 deletions(-) diff --git a/exporter/environment.go b/exporter/environment.go index 7313ce3..0f6cca5 100644 --- a/exporter/environment.go +++ b/exporter/environment.go @@ -1,18 +1,10 @@ package exporter import ( - "bufio" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "os" "os/exec" "strings" ) -const ssnodeConfFile = "/etc/ssnode.conf" - type formpostParameter struct { } @@ -77,98 +69,15 @@ type NodeSwiftSetting struct { TempURL tempURLParameter `json:"tempurl"` } -// GetSwiftEnvironmentParameters - this function runs a curl call to http:///info to get the -// node parameter of the system. Environment variables like Swift version, S3 version...etc will be expose and -// reference in other modules in the script. -func GetSwiftEnvironmentParameters() (swiftEnvironmentParameters NodeSwiftSetting) { - apiIP, apiPort, apiHostname, _ := GetAPIAddress(ssnodeConfFile) - var targetEndpoint string - var read NodeSwiftSetting - var target []string - - if len(apiHostname) != 0 { - target = []string{"http://", apiHostname} - } else if len(apiIP) != 0 { - target = []string{"http://", apiIP} - } - - if apiPort == "443" { - target[0] = "https://" - } - - target[0] = strings.Join(target, "") - target[1] = "/info" - targetEndpoint = strings.Join(target, "") - - resp, err := http.Get(targetEndpoint) - if err != nil { - fmt.Println(err) - fmt.Println("Error here!!") - } - - defer resp.Body.Close() - - body, _ := ioutil.ReadAll(resp.Body) - err2 := json.Unmarshal(body, &read) - - if err2 == nil { - fmt.Println(err2) - fmt.Println("ERROR") - } - - swiftEnvironmentParameters = read - return swiftEnvironmentParameters -} - -// GetAPIAddress reads the ssnode.conf file and get the API IP, API Port, and API Hostname inside the file. -func GetAPIAddress(ssnodeConfig string) (apiAddress string, apiPort string, apiHostname string, err error) { - - openFile, err := os.Open(ssnodeConfig) - if err != nil { - fmt.Println("I cannot read this file") - } - defer openFile.Close() - - readFile := bufio.NewScanner(openFile) - for readFile.Scan() { - if strings.Contains(readFile.Text(), "api_ip") { - apiAddress = strings.TrimSpace(strings.Split(readFile.Text(), "=")[1]) - } - if strings.Contains(readFile.Text(), "api_port") { - apiPort = strings.TrimSpace(strings.Split(readFile.Text(), "=")[1]) - } - if strings.Contains(readFile.Text(), "api_hostname") { - apiHostname = strings.TrimSpace(strings.Split(readFile.Text(), "=")[1]) - } - } - - return apiAddress, apiPort, apiHostname, err -} - // GetUUIDAndFQDN runs "hostname -f" and reads the ssnode.conf to get the full FQDN and the UUID of a Swift node. -func GetUUIDAndFQDN(ssnodeConfig string) (UUID string, FQDN string, err error) { +func GetUUIDAndFQDN() (FQDN string, err error) { // to get this module to run, please do he following: // read /etc/ssnode.conf to get the UUID of the node // run hostnamectl to get the FQDN of the node - var nodeUUID string var hostName string - openFile, err := os.Open(ssnodeConfig) - - if err != nil { - fmt.Println("I cannot read this file") - } - defer openFile.Close() - - readFile := bufio.NewScanner(openFile) - for readFile.Scan() { - if strings.Contains(readFile.Text(), "node_uuid") { - nodeUUID = strings.TrimSpace(strings.Split(readFile.Text(), "=")[1]) - } - } - runCommand, _ := exec.Command("hostname", "-f").Output() hostName = strings.TrimRight(string(runCommand), "\n") - return hostName, nodeUUID, err + return hostName, err } diff --git a/exporter/hardwareprofile.go b/exporter/hardwareprofile.go index 00129df..01c3d66 100644 --- a/exporter/hardwareprofile.go +++ b/exporter/hardwareprofile.go @@ -1,9 +1,9 @@ package exporter import ( - "fmt" +// "fmt" "io/ioutil" - "log" +// "log" "os" "os/exec" "strconv" @@ -20,15 +20,15 @@ var ( individualCPUStatValue = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "cpu_stat", Help: "CPU Stat - the data exposed here is in percentage (with 1 = 100%).", - }, []string{"cpu_name", "metrics_name", "FQDN", "UUID"}) + }, []string{"cpu_name", "metrics_name"}) nicMetric = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "nic_stat", Help: "NIC Stat - 'byte_*' metrics is measured in bytes. While 'pckt_*' and 'err_*' are measured in packet counts.", - }, []string{"nic_name", "mac_address", "metrics_name", "FQDN", "UUID"}) + }, []string{"nic_name", "mac_address", "metrics_name"}) nicMTU = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "nic_mtu", Help: "NIC MTU Reading", - }, []string{"nic_name", "FQDN", "UUID"}) + }, []string{"nic_name"}) swiftDriveUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "swift_drive_usage", Help: "Swift Drive Usage in bytes (B)", @@ -44,11 +44,11 @@ var ( swiftDriveIOStat = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "swift_drive_io_stat", Help: "swift_drive_io_stat expose drive io-related data to prometheus measures in Bytes (B).", - }, []string{"swift_drive", "metric_name", "drive_type", "FQDN", "UUID"}) + }, []string{"swift_drive", "metric_name", "drive_type"}) swiftStoragePolicyUsage = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "swift_storage_policy_usage", Help: "Utilization Per Storage Policy. This metrics will be fetched every 6 hours instead of minutes", - }, []string{"swift_drive_mountpoint", "swift_drive_label", "storage_policy_name", "FQDN", "UUID"}) + }, []string{"swift_drive_mountpoint", "swift_drive_label", "storage_policy_name"}) ) func init() { @@ -65,11 +65,10 @@ func init() { // and convert them into percentage (out of 1), and expose them out to prometheus. func ExposePerCPUUsage(ExposePerCPUUsageEnable bool) { - writeLogFile := log.New(swiftExporterLog, "ExposePerCPUUsage: ", log.Ldate|log.Ltime|log.Lshortfile) + //writeLogFile := log.New(swiftExporterLog, "ExposePerCPUUsage: ", log.Ldate|log.Ltime|log.Lshortfile) if ExposePerCPUUsageEnable { - writeLogFile.Println("ExposeCPUUsage ENABLED") - hostFQDN, hostUUID, _ := GetUUIDAndFQDN(ssnodeConfFile) + //writeLogFile.Println("ExposeCPUUsage ENABLED") PerCPUUsageTime, _ := cpu.Times(true) for i := 0; i < len(PerCPUUsageTime); i++ { @@ -105,23 +104,23 @@ func ExposePerCPUUsage(ExposePerCPUUsageEnable bool) { stolenValue := cpuStolenTime / totalTimeUsed //expose the metrics out to prometheus - individualCPUStatValue.WithLabelValues(cpuName, "usr", hostFQDN, hostUUID).Set(usrValue) - individualCPUStatValue.WithLabelValues(cpuName, "sys", hostFQDN, hostUUID).Set(sysValue) - individualCPUStatValue.WithLabelValues(cpuName, "idle", hostFQDN, hostUUID).Set(idleValue) - individualCPUStatValue.WithLabelValues(cpuName, "nice", hostFQDN, hostUUID).Set(niceValue) - individualCPUStatValue.WithLabelValues(cpuName, "iowait", hostFQDN, hostUUID).Set(iowaitValue) - individualCPUStatValue.WithLabelValues(cpuName, "irq", hostFQDN, hostUUID).Set(irqValue) - individualCPUStatValue.WithLabelValues(cpuName, "softirq", hostFQDN, hostUUID).Set(softirqValue) - individualCPUStatValue.WithLabelValues(cpuName, "steal", hostFQDN, hostUUID).Set(stealValue) - individualCPUStatValue.WithLabelValues(cpuName, "guest", hostFQDN, hostUUID).Set(guestValue) - individualCPUStatValue.WithLabelValues(cpuName, "guestnice", hostFQDN, hostUUID).Set(guestniceValue) - individualCPUStatValue.WithLabelValues(cpuName, "stolen", hostFQDN, hostUUID).Set(stolenValue) + individualCPUStatValue.WithLabelValues(cpuName, "usr").Set(usrValue) + individualCPUStatValue.WithLabelValues(cpuName, "sys").Set(sysValue) + individualCPUStatValue.WithLabelValues(cpuName, "idle").Set(idleValue) + individualCPUStatValue.WithLabelValues(cpuName, "nice").Set(niceValue) + individualCPUStatValue.WithLabelValues(cpuName, "iowait").Set(iowaitValue) + individualCPUStatValue.WithLabelValues(cpuName, "irq").Set(irqValue) + individualCPUStatValue.WithLabelValues(cpuName, "softirq").Set(softirqValue) + individualCPUStatValue.WithLabelValues(cpuName, "steal").Set(stealValue) + individualCPUStatValue.WithLabelValues(cpuName, "guest").Set(guestValue) + individualCPUStatValue.WithLabelValues(cpuName, "guestnice").Set(guestniceValue) + individualCPUStatValue.WithLabelValues(cpuName, "stolen").Set(stolenValue) //for each metrcis, do the percentage calculate. } } else { - writeLogFile.Println("ExposeCPUUsage Disabled") - writeLogFile.Println() + //writeLogFile.Println("ExposeCPUUsage Disabled") + //writeLogFile.Println() } } @@ -131,16 +130,15 @@ func ExposePerCPUUsage(ExposePerCPUUsageEnable bool) { // prometheus. func ExposePerNICMetric(ExposePerNICMetricEnable bool) { - writeLogFile := log.New(swiftExporterLog, "ExposePerNICMetric: ", log.Ldate|log.Ltime|log.Lshortfile) + //writeLogFile := log.New(swiftExporterLog, "ExposePerNICMetric: ", log.Ldate|log.Ltime|log.Lshortfile) if ExposePerNICMetricEnable { // perNicMetric get the IO counts of each interface available in the node. // nicInfo gets the MAC and IP address of each interface available in the node. perNicMetric, _ := net.IOCounters(true) nicInfo, _ := net.Interfaces() - hostFQDN, hostUUID, _ := GetUUIDAndFQDN(ssnodeConfFile) - writeLogFile.Println("ExposePerNICMetric Module ENABLED") + //writeLogFile.Println("ExposePerNICMetric Module ENABLED") for i := 0; i < len(perNicMetric); i++ { var nicName string @@ -161,17 +159,17 @@ func ExposePerNICMetric(ExposePerNICMetricEnable bool) { nicErrIn := perNicMetric[i].Errin nicErrOut := perNicMetric[i].Errout - nicMetric.WithLabelValues(nicName, nicMACAddr, "byte_sent", hostFQDN, hostUUID).Set(float64(nicByteSent)) - nicMetric.WithLabelValues(nicName, nicMACAddr, "byte_recv", hostFQDN, hostUUID).Set(float64(nicByteRecv)) - nicMetric.WithLabelValues(nicName, nicMACAddr, "pckt_sent", hostFQDN, hostUUID).Set(float64(nicPcktSent)) - nicMetric.WithLabelValues(nicName, nicMACAddr, "pckt_recv", hostFQDN, hostUUID).Set(float64(nicPcktRecv)) - nicMetric.WithLabelValues(nicName, nicMACAddr, "err_in", hostFQDN, hostUUID).Set(float64(nicErrIn)) - nicMetric.WithLabelValues(nicName, nicMACAddr, "err_out", hostFQDN, hostUUID).Set(float64(nicErrOut)) + nicMetric.WithLabelValues(nicName, nicMACAddr, "byte_sent").Set(float64(nicByteSent)) + nicMetric.WithLabelValues(nicName, nicMACAddr, "byte_recv").Set(float64(nicByteRecv)) + nicMetric.WithLabelValues(nicName, nicMACAddr, "pckt_sent").Set(float64(nicPcktSent)) + nicMetric.WithLabelValues(nicName, nicMACAddr, "pckt_recv").Set(float64(nicPcktRecv)) + nicMetric.WithLabelValues(nicName, nicMACAddr, "err_in").Set(float64(nicErrIn)) + nicMetric.WithLabelValues(nicName, nicMACAddr, "err_out").Set(float64(nicErrOut)) } } else { - writeLogFile.Println("ExposePerNICMetric Disabled") - writeLogFile.Println() + //writeLogFile.Println("ExposePerNICMetric Disabled") + //writeLogFile.Println() } } @@ -179,7 +177,6 @@ func ExposePerNICMetric(ExposePerNICMetricEnable bool) { // GrabNICMTU grabs the MTU setting of a NIC card by reading the /sys/class/net file. func GrabNICMTU() { - hostFQDN, hostUUID, _ := GetUUIDAndFQDN(ssnodeConfFile) cmd, _ := exec.Command("ls", "/sys/class/net").Output() outputString := strings.Split(string(cmd), "\n") outputString = outputString[:len(outputString)-1] @@ -199,7 +196,7 @@ func GrabNICMTU() { getMTU, _ := exec.Command("cat", targetLocation).Output() mtu, _ := strconv.ParseFloat(string(getMTU[:len(getMTU)-1]), 64) - nicMTU.WithLabelValues(outputString[i], hostFQDN, hostUUID).Set(mtu) + nicMTU.WithLabelValues(outputString[i]).Set(mtu) } } @@ -209,10 +206,10 @@ func GrabNICMTU() { // them via Prometheus. func SwiftDiskUsage(SwiftDiskUsageEnable bool) { - writeLogFile := log.New(swiftExporterLog, "SwiftDiskUsage: ", log.Ldate|log.Ltime|log.Lshortfile) + //writeLogFile := log.New(swiftExporterLog, "SwiftDiskUsage: ", log.Ldate|log.Ltime|log.Lshortfile) if SwiftDiskUsageEnable { - writeLogFile.Println("SwiftDiskUsage Module ENABLED") + //writeLogFile.Println("SwiftDiskUsage Module ENABLED") swiftDrive, _ := disk.Partitions(false) for i := 0; i < len(swiftDrive); i++ { swiftDriveLabel := swiftDrive[i].Mountpoint @@ -234,7 +231,7 @@ func SwiftDiskUsage(SwiftDiskUsageEnable bool) { } } else { - writeLogFile.Println("SwiftDiskUsage Module DISABLED") + //writeLogFile.Println("SwiftDiskUsage Module DISABLED") } } @@ -242,36 +239,35 @@ func SwiftDiskUsage(SwiftDiskUsageEnable bool) { // related metrics and expose them via Prometheus. func SwiftDriveIO(SwiftDriveIOEnable bool) { - writeLogFile := log.New(swiftExporterLog, "SwiftDriveIO: ", log.Ldate|log.Ltime|log.Lshortfile) + //writeLogFile := log.New(swiftExporterLog, "SwiftDriveIO: ", log.Ldate|log.Ltime|log.Lshortfile) if SwiftDriveIOEnable { - writeLogFile.Println("SwiftDriveIO Module ENABLED") + //writeLogFile.Println("SwiftDriveIO Module ENABLED") swiftDrive, _ := disk.Partitions(false) swiftDiskIO, _ := disk.IOCounters() - nodeHostname, nodeUUID, _ := GetUUIDAndFQDN(ssnodeConfFile) for i := 0; i < len(swiftDrive); i++ { if strings.Contains(swiftDrive[i].Mountpoint, "/srv/node") { deviceName := swiftDrive[i].Device deviceName = strings.Split(deviceName, "/")[2] deviceType := HddOrSSD(swiftDrive[i].Device) - swiftDriveIOStat.WithLabelValues(deviceName, "readCount", deviceType, nodeHostname, nodeUUID).Set(float64(swiftDiskIO[deviceName].ReadCount)) - swiftDriveIOStat.WithLabelValues(deviceName, "mergedReadCount", deviceType, nodeHostname, nodeUUID).Set(float64(swiftDiskIO[deviceName].MergedReadCount)) - swiftDriveIOStat.WithLabelValues(deviceName, "writeCount", deviceType, nodeHostname, nodeUUID).Set(float64(swiftDiskIO[deviceName].WriteCount)) - swiftDriveIOStat.WithLabelValues(deviceName, "mergedWriteCount", deviceType, nodeHostname, nodeUUID).Set(float64(swiftDiskIO[deviceName].MergedWriteCount)) - swiftDriveIOStat.WithLabelValues(deviceName, "readBytes", deviceType, nodeHostname, nodeUUID).Set(float64(swiftDiskIO[deviceName].ReadBytes)) - swiftDriveIOStat.WithLabelValues(deviceName, "writeBytes", deviceType, nodeHostname, nodeUUID).Set(float64(swiftDiskIO[deviceName].WriteBytes)) - swiftDriveIOStat.WithLabelValues(deviceName, "readTime", deviceType, nodeHostname, nodeUUID).Set(float64(swiftDiskIO[deviceName].ReadTime)) - swiftDriveIOStat.WithLabelValues(deviceName, "writeTime", deviceType, nodeHostname, nodeUUID).Set(float64(swiftDiskIO[deviceName].WriteTime)) - swiftDriveIOStat.WithLabelValues(deviceName, "iopsInProgress", deviceType, nodeHostname, nodeUUID).Set(float64(swiftDiskIO[deviceName].IopsInProgress)) - swiftDriveIOStat.WithLabelValues(deviceName, "ioTime", deviceType, nodeHostname, nodeUUID).Set(float64(swiftDiskIO[deviceName].IoTime)) - swiftDriveIOStat.WithLabelValues(deviceName, "weightedIO", deviceType, nodeHostname, nodeUUID).Set(float64(swiftDiskIO[deviceName].WeightedIO)) + swiftDriveIOStat.WithLabelValues(deviceName, "readCount", deviceType).Set(float64(swiftDiskIO[deviceName].ReadCount)) + swiftDriveIOStat.WithLabelValues(deviceName, "mergedReadCount", deviceType).Set(float64(swiftDiskIO[deviceName].MergedReadCount)) + swiftDriveIOStat.WithLabelValues(deviceName, "writeCount", deviceType).Set(float64(swiftDiskIO[deviceName].WriteCount)) + swiftDriveIOStat.WithLabelValues(deviceName, "mergedWriteCount", deviceType).Set(float64(swiftDiskIO[deviceName].MergedWriteCount)) + swiftDriveIOStat.WithLabelValues(deviceName, "readBytes", deviceType).Set(float64(swiftDiskIO[deviceName].ReadBytes)) + swiftDriveIOStat.WithLabelValues(deviceName, "writeBytes", deviceType).Set(float64(swiftDiskIO[deviceName].WriteBytes)) + swiftDriveIOStat.WithLabelValues(deviceName, "readTime", deviceType).Set(float64(swiftDiskIO[deviceName].ReadTime)) + swiftDriveIOStat.WithLabelValues(deviceName, "writeTime", deviceType).Set(float64(swiftDiskIO[deviceName].WriteTime)) + swiftDriveIOStat.WithLabelValues(deviceName, "iopsInProgress", deviceType).Set(float64(swiftDiskIO[deviceName].IopsInProgress)) + swiftDriveIOStat.WithLabelValues(deviceName, "ioTime", deviceType).Set(float64(swiftDiskIO[deviceName].IoTime)) + swiftDriveIOStat.WithLabelValues(deviceName, "weightedIO", deviceType).Set(float64(swiftDiskIO[deviceName].WeightedIO)) } } } else { - writeLogFile.Println("SwiftDriveIO Module DISABLED") + //writeLogFile.Println("SwiftDriveIO Module DISABLED") } } @@ -295,7 +291,7 @@ func HddOrSSD(deviceName string) (driveType string) { } else if strings.Compare(strings.TrimSuffix(string(data), "\n"), "0") == 0 { typeOfDrive = "SSD" } else { - fmt.Println(data) + //fmt.Println(data) } return typeOfDrive diff --git a/exporter/swiftmetrics.go b/exporter/swiftmetrics.go index b35cd3f..a35cd0d 100644 --- a/exporter/swiftmetrics.go +++ b/exporter/swiftmetrics.go @@ -101,7 +101,7 @@ type ObjectSwiftRole struct { ObjectReplicationPerDisk map[string]ReplicationPerDisk `json:"object_replication_per_disk"` ObjectUpdaterSweep float64 `json:"object_updater_sweep"` ObjectReplicationLast float64 `json:"replication_last"` - ObjectReplicationTime float64 `json:"object_replication_time"` + ObjectReplicationTime float64 `json:"replication_time"` } // ObjectAuditorStats contains parts of the sub-list of account/container metrics that you can find in a *.recon file. @@ -165,88 +165,80 @@ type PartCounts struct { var ( //swiftExporterLog, swiftExporterLogError = os.OpenFile("/var/log/swift_exporter.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) - accountServer = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "account_server", + swiftAccountServer = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "swift_account_server", Help: "Account Server Metrics", - }, []string{"service_name", "metrics_name", "FQDN", "UUID"}) - accountDBCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "account_db", + }, []string{"service_name", "metrics_name"}) + swiftAccountDBCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "swift_account_db", Help: "Number of Account DBs", - }, []string{"FQDN", "UUID"}) - accountDBPendingCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "account_db_pending", + }) + swiftAccountDBPendingCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "swift_account_db_pending", Help: "Number of Pending Account DBs", - }, []string{"FQDN", "UUID"}) - containerServer = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "container_server", + }) + swiftContainerServer = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "swift_container_server", Help: "Container Server Metrics", - }, []string{"service_name", "metrics_name", "FQDN", "UUID"}) - containerDBCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "container_db", + }, []string{"service_name", "metrics_name"}) + swiftContainerDBCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "swift_container_db", Help: "Number of Container DBs", - }, []string{"FQDN", "UUID"}) - containerDBPendingCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "container_db_pending", + }) + swiftContainerDBPendingCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "swift_container_db_pending", Help: "Number of Pending Container DBs", - }, []string{"FQDN", "UUID"}) - objectServer = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "object_server", + }) + swiftObjectServer = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "swift_object_server", Help: "Object Server Metrics", - }, []string{"service_name", "metrics_name", "FQDN", "UUID"}) - objectFileCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "object_file_count", + }, []string{"service_name", "metrics_name"}) + swiftObjectFileCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "swift_object_file_count", Help: "Number of Object Files", - }, []string{"FQDN", "UUID"}) + }) swiftObjectReplicationPerDisk = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "swift_object_replication_per_disk", Help: "Swift Object Replication Per Disk Metrics", - }, []string{"service_name", "metrics_type", "swift_disk", "FQDN", "UUID"}) + }, []string{"service_name", "metrics_type", "swift_disk"}) swiftObjectReplicationEstimate = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "swift_object_replication_estimate", Help: "Swift Object Server - Replication Estimate in seconds (s) and parts/second (/sec)", - }, []string{"metrics_type", "FQDN", "UUID"}) + }, []string{"metrics_type"}) swiftObjectReplicationPerDiskEstimate = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "swift_object_replication_per_disk_estimate", Help: "Swift Object Server - Replication Per Disk Estimate in seconds(s) and parts/second (/sec)", - }, []string{"metrics_type", "swift_disk", "FQDN", "UUID"}) + }, []string{"metrics_type", "swift_disk"}) swiftContainerReplicationEstimate = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "swift_container_replication_estimate", Help: "Swift Container Server - Replicator Estimate in parts/second (/sec)", - }, []string{"metrics_type", "FQDN", "UUID"}) + }, []string{"metrics_type"}) swiftContainerSharding = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "swift_container_sharding", Help: "Swift Container Sharding", - }, []string{"metric_name", "parameter", "FQDN", "UUID"}) + }, []string{"metric_name", "parameter"}) swiftAccountReplicationEstimate = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "swift_account_replication_estimage", Help: "Swift Account Server - Replicatin Estimage in parts/second (/sec)", - }, []string{"metric_type", "FQDN", "UUID"}) + }, []string{"metric_type"}) swiftDrivePrimaryParitions = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "swift_drive_primary_partitions", Help: "Swift Drive Primary Partitions - the number of primary partition, no specific unit.", - }, []string{"FQDN", "UUID", "swift_drive_label", "storage_policy", "swift_role", "drive_type"}) + }, []string{"swift_drive_label", "storage_policy", "swift_role", "drive_type"}) swiftDriveHandoffPartitions = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "swift_drive_handoff_partitions", Help: "Swift Drive Handoff Partitions - the number of handoff partition, no specific unit.", - }, []string{"FQDN", "UUID", "swift_drive_label", "storage_policy", "swift_role", "drive_type"}) + }, []string{"swift_drive_label", "storage_policy", "swift_role", "drive_type"}) swiftLogFileSize = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "swift_log_file_size", Help: "Size of swift all.log", }) - swiftServiceStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "swift_service_status", - Help: "Swift Main Service Status - Services like 'Accouts', 'Containers', and 'Objects' are recorded here", - }, []string{"FQDN", "UUID", "SwiftServiceName"}) - swiftSubServiceStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "swift_sub_service_status", - Help: "Swift Sub Service Status - Services like 'Auditors', 'Replicator', and 'Expirer'...etc are recorded here", - }, []string{"FQDN", "UUID", "SwiftSubServiceName"}) ) func init() { - prometheus.MustRegister(accountServer) - prometheus.MustRegister(containerServer) - prometheus.MustRegister(objectServer) + prometheus.MustRegister(swiftAccountServer) + prometheus.MustRegister(swiftContainerServer) + prometheus.MustRegister(swiftObjectServer) prometheus.MustRegister(swiftObjectReplicationPerDisk) prometheus.MustRegister(swiftObjectReplicationEstimate) prometheus.MustRegister(swiftObjectReplicationPerDiskEstimate) @@ -257,13 +249,11 @@ func init() { prometheus.MustRegister(swiftStoragePolicyUsage) prometheus.MustRegister(swiftContainerReplicationEstimate) prometheus.MustRegister(swiftAccountReplicationEstimate) - prometheus.MustRegister(accountDBCount) - prometheus.MustRegister(accountDBPendingCount) - prometheus.MustRegister(containerDBCount) - prometheus.MustRegister(containerDBPendingCount) - prometheus.MustRegister(objectFileCount) - prometheus.MustRegister(swiftServiceStatus) - prometheus.MustRegister(swiftSubServiceStatus) + prometheus.MustRegister(swiftAccountDBCount) + prometheus.MustRegister(swiftAccountDBPendingCount) + prometheus.MustRegister(swiftContainerDBCount) + prometheus.MustRegister(swiftContainerDBPendingCount) + prometheus.MustRegister(swiftObjectFileCount) } // GatherStoragePolicyCommonName reads throught /etc/swift/swift.conf file to get the storage policy name. @@ -271,13 +261,14 @@ func init() { // relate the storage policy name with the policy number. func GatherStoragePolicyCommonName() map[string]string { - writeLogFile := log.New(swiftExporterLog, "GatherStoragePolicyCommonName: ", log.Ldate|log.Ltime|log.Lshortfile) +// TODO: fix logging +// writeLogFile := log.New(swiftExporterLog, "GatherStoragePolicyCommonName: ", log.Ldate|log.Ltime|log.Lshortfile) openFile, err := os.Open(swiftConfig) StoragePolicyName := make(map[string]string) if err != nil { - writeLogFile.Fatal(err) +// writeLogFile.Fatal(err) } defer openFile.Close() @@ -289,7 +280,6 @@ func GatherStoragePolicyCommonName() map[string]string { readStoragePolicyNameBuffer := readFile.Text() readStoragePolicyName := strings.Split(readStoragePolicyNameBuffer, "=") StoragePolicyName[storagePolicyIndex] = strings.TrimLeft(readStoragePolicyName[1], " ") - //StoragePolicyName = append(StoragePolicyName, strings.TrimLeft(readStoragePolicyName[1], " ")) } } return StoragePolicyName @@ -298,17 +288,16 @@ func GatherStoragePolicyCommonName() map[string]string { // ReadReconFile parses the .recon files, put them into the struct defined above and expose them out //in prometheus. This function takes in 2 argument - ReconFile is the const configured above that reflects //the exact location of the .recon file in Swift nodes. -func ReadReconFile(ReconFile string, SwiftRole string, ReadReconFileEnable bool) { +func ReadReconFile(ReconFile string, SwiftRole string, ReadReconFileEnable bool, SwiftVersion string) { - writeLogFile := log.New(swiftExporterLog, "ReadReconFile: ", log.Ldate|log.Ltime|log.Lshortfile) - hostFQDN, hostUUID, _ := GetUUIDAndFQDN(ssnodeConfFile) - swiftParameter := GetSwiftEnvironmentParameters() - swiftVersion := strings.Split(swiftParameter.Swift.Version, ".") +// TODO: fix logging, this should be enabled via flag +// writeLogFile := log.New(swiftExporterLog, "ReadReconFile: ", log.Ldate|log.Ltime|log.Lshortfile) + swiftVersion := strings.Split(SwiftVersion, ".") swiftMajorVersion, _ := strconv.ParseInt(swiftVersion[0], 10, 64) swiftMinorVersion, _ := strconv.ParseInt(swiftVersion[1], 10, 64) if ReadReconFileEnable { - writeLogFile.Println("ReadReconFile Module ENABLED") +// writeLogFile.Println("ReadReconFile Module ENABLED") jsonFile, err := os.Open(ReconFile) if err != nil { log.Println(err) @@ -321,273 +310,196 @@ func ReadReconFile(ReconFile string, SwiftRole string, ReadReconFileEnable bool) if SwiftRole == "account" { var account AccountSwiftRole json.Unmarshal(byteValue, &account) - writeLogFile.Println(string(byteValue)) +// writeLogFile.Println(string(byteValue)) //fmt.Println(&account) - accountServer.WithLabelValues("auditor", "passed", hostFQDN, hostUUID).Set(account.AccountAuditsPassed) - accountServer.WithLabelValues("auditor", "failed", hostFQDN, hostUUID).Set(account.AccountAuditsFailed) - accountServer.WithLabelValues("auditor", "passed_completed", hostFQDN, hostUUID).Set(account.PassCompleted) - accountServer.WithLabelValues("replicator", "remote_merge", hostFQDN, hostUUID).Set(account.AccountReplicator.RemoteMerge) - accountServer.WithLabelValues("replicator", "diff", hostFQDN, hostUUID).Set(account.AccountReplicator.Diff) - accountServer.WithLabelValues("replicator", "diff_capped", hostFQDN, hostUUID).Set(account.AccountReplicator.DiffCapped) - accountServer.WithLabelValues("replicator", "no_change", hostFQDN, hostUUID).Set(account.AccountReplicator.NoChange) - accountServer.WithLabelValues("replicator", "ts_repl", hostFQDN, hostUUID).Set(account.AccountReplicator.TsRepl) - accountServer.WithLabelValues("replicator", "replication_time", hostFQDN, hostUUID).Set(account.ReplicationTime) - - accountServer.WithLabelValues("replicator", "rsync", hostFQDN, hostUUID).Set(account.AccountReplicator.Rsync) - accountServer.WithLabelValues("replicator", "success", hostFQDN, hostUUID).Set(account.AccountReplicator.Success) - accountServer.WithLabelValues("replicator", "failure", hostFQDN, hostUUID).Set(account.AccountReplicator.Failure) - accountServer.WithLabelValues("replicator", "attempted", hostFQDN, hostUUID).Set(account.AccountReplicator.Attempted) - accountServer.WithLabelValues("replicator", "hashmatch", hostFQDN, hostUUID).Set(account.AccountReplicator.Hashmatch) - - accountReplicationPartsPerSecond := account.AccountReplicator.Attempted / account.ReplicationTime - swiftAccountReplicationEstimate.WithLabelValues("parts_per_second", hostFQDN, hostUUID).Set(accountReplicationPartsPerSecond) + swiftAccountServer.WithLabelValues("auditor", "passed").Set(account.AccountAuditsPassed) + swiftAccountServer.WithLabelValues("auditor", "failed").Set(account.AccountAuditsFailed) + swiftAccountServer.WithLabelValues("auditor", "passed_completed").Set(account.PassCompleted) + swiftAccountServer.WithLabelValues("replication", "remote_merge").Set(account.AccountReplicator.RemoteMerge) + swiftAccountServer.WithLabelValues("replication", "diff").Set(account.AccountReplicator.Diff) + swiftAccountServer.WithLabelValues("replication", "diff_capped").Set(account.AccountReplicator.DiffCapped) + swiftAccountServer.WithLabelValues("replication", "no_change").Set(account.AccountReplicator.NoChange) + swiftAccountServer.WithLabelValues("replication", "ts_repl").Set(account.AccountReplicator.TsRepl) + swiftAccountServer.WithLabelValues("replication", "replication_time").Set(account.ReplicationTime) + + swiftAccountServer.WithLabelValues("replication", "rsync").Set(account.AccountReplicator.Rsync) + swiftAccountServer.WithLabelValues("replication", "success").Set(account.AccountReplicator.Success) + swiftAccountServer.WithLabelValues("replication", "failure").Set(account.AccountReplicator.Failure) + swiftAccountServer.WithLabelValues("replication", "attempted").Set(account.AccountReplicator.Attempted) + swiftAccountServer.WithLabelValues("replication", "hashmatch").Set(account.AccountReplicator.Hashmatch) + } if SwiftRole == "container" { var container ContainerSwiftRole json.Unmarshal(byteValue, &container) - writeLogFile.Println(string(byteValue)) - - containerServer.WithLabelValues("auditor", "passed", hostFQDN, hostUUID).Set(container.ContainerAuditsPassed) - containerServer.WithLabelValues("auditor", "failed", hostFQDN, hostUUID).Set(container.ContainerAuditsFailed) - containerServer.WithLabelValues("auditor", "passed_completed", hostFQDN, hostUUID).Set(container.ContainerAuditorPassCompleted) - containerServer.WithLabelValues("replicator", "remote_merge", hostFQDN, hostUUID).Set(container.ContainerReplicator.RemoteMerge) - containerServer.WithLabelValues("replicator", "diff", hostFQDN, hostUUID).Set(container.ContainerReplicator.Diff) - containerServer.WithLabelValues("replicator", "diff_capped", hostFQDN, hostUUID).Set(container.ContainerReplicator.DiffCapped) - containerServer.WithLabelValues("replicator", "no_change", hostFQDN, hostUUID).Set(container.ContainerReplicator.NoChange) - containerServer.WithLabelValues("replicator", "ts_repl", hostFQDN, hostUUID).Set(container.ContainerReplicator.TsRepl) - containerServer.WithLabelValues("replicator", "replication_time", hostFQDN, hostUUID).Set(container.ReplicationTime) - - containerServer.WithLabelValues("replicator", "rsync", hostFQDN, hostUUID).Set(container.ContainerReplicator.Rsync) - containerServer.WithLabelValues("replicator", "success", hostFQDN, hostUUID).Set(container.ContainerReplicator.Success) - containerServer.WithLabelValues("replicator", "failure", hostFQDN, hostUUID).Set(container.ContainerReplicator.Failure) - containerServer.WithLabelValues("replicator", "attempted", hostFQDN, hostUUID).Set(container.ContainerReplicator.Attempted) - containerServer.WithLabelValues("replicator", "hashmatch", hostFQDN, hostUUID).Set(container.ContainerReplicator.Hashmatch) +// writeLogFile.Println(string(byteValue)) + + swiftContainerServer.WithLabelValues("auditor", "passed").Set(container.ContainerAuditsPassed) + swiftContainerServer.WithLabelValues("auditor", "failed").Set(container.ContainerAuditsFailed) + swiftContainerServer.WithLabelValues("auditor", "passed_completed").Set(container.ContainerAuditorPassCompleted) + swiftContainerServer.WithLabelValues("replication", "remote_merge").Set(container.ContainerReplicator.RemoteMerge) + swiftContainerServer.WithLabelValues("replication", "diff").Set(container.ContainerReplicator.Diff) + swiftContainerServer.WithLabelValues("replication", "diff_capped").Set(container.ContainerReplicator.DiffCapped) + swiftContainerServer.WithLabelValues("replication", "no_change").Set(container.ContainerReplicator.NoChange) + swiftContainerServer.WithLabelValues("replication", "ts_repl").Set(container.ContainerReplicator.TsRepl) + swiftContainerServer.WithLabelValues("replication", "replication_time").Set(container.ReplicationTime) + + swiftContainerServer.WithLabelValues("replication", "rsync").Set(container.ContainerReplicator.Rsync) + swiftContainerServer.WithLabelValues("replication", "success").Set(container.ContainerReplicator.Success) + swiftContainerServer.WithLabelValues("replication", "failure").Set(container.ContainerReplicator.Failure) + swiftContainerServer.WithLabelValues("replication", "attempted").Set(container.ContainerReplicator.Attempted) + swiftContainerServer.WithLabelValues("replication", "hashmatch").Set(container.ContainerReplicator.Hashmatch) if swiftMajorVersion >= 2 { - if swiftMinorVersion >= 15 { - swiftContainerSharding.WithLabelValues("sharding_stats", "attempted", hostFQDN, hostUUID).Set(container.ShardingStats.Attempted) - swiftContainerSharding.WithLabelValues("sharding_stats", "deffered", hostFQDN, hostUUID).Set(container.ShardingStats.Deferred) - swiftContainerSharding.WithLabelValues("sharding_stats", "diff", hostFQDN, hostUUID).Set(container.ShardingStats.Diff) - swiftContainerSharding.WithLabelValues("sharding_stats", "diff_capped", hostFQDN, hostUUID).Set(container.ShardingStats.DiffCapped) - swiftContainerSharding.WithLabelValues("sharding_stats", "empty", hostFQDN, hostUUID).Set(container.ShardingStats.Empty) - swiftContainerSharding.WithLabelValues("sharding_stats", "failure", hostFQDN, hostUUID).Set(container.ShardingStats.Failure) - swiftContainerSharding.WithLabelValues("sharding_stats", "hashmatch", hostFQDN, hostUUID).Set(container.ShardingStats.Hashmatch) - swiftContainerSharding.WithLabelValues("sharding_stats", "no_change", hostFQDN, hostUUID).Set(container.ShardingStats.NoChange) - swiftContainerSharding.WithLabelValues("sharding_stats", "remote_merge", hostFQDN, hostUUID).Set(container.ShardingStats.RemoteMerge) - swiftContainerSharding.WithLabelValues("sharding_stats", "remove", hostFQDN, hostUUID).Set(container.ShardingStats.Remove) - swiftContainerSharding.WithLabelValues("sharding_stats", "rsync", hostFQDN, hostUUID).Set(container.ShardingStats.Rsync) - - swiftContainerSharding.WithLabelValues("audit_root", "attempted", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.AuditRoot.Attempted) - swiftContainerSharding.WithLabelValues("audit_root", "failure", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.AuditRoot.Failure) - swiftContainerSharding.WithLabelValues("audit_root", "success", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.AuditRoot.Success) - swiftContainerSharding.WithLabelValues("audit_shard", "attempted", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.AuditShard.Attempted) - swiftContainerSharding.WithLabelValues("audit_shard", "failure", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.AuditShard.Failure) - swiftContainerSharding.WithLabelValues("audit_shard", "attempted", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.AuditShard.Attempted) - - swiftContainerSharding.WithLabelValues("cleaved", "attempted", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Cleaved.Attempted) - swiftContainerSharding.WithLabelValues("cleaved", "failure", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Cleaved.Failure) - swiftContainerSharding.WithLabelValues("cleaved", "max_time", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Cleaved.MaxTime) - swiftContainerSharding.WithLabelValues("cleaved", "min_time", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Cleaved.MinTime) - swiftContainerSharding.WithLabelValues("cleaved", "success", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Cleaved.Success) - - swiftContainerSharding.WithLabelValues("created", "attempted", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Created.Attempted) - swiftContainerSharding.WithLabelValues("created", "failure", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Created.Failure) - swiftContainerSharding.WithLabelValues("created", "success", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Created.Success) - - swiftContainerSharding.WithLabelValues("misplaced", "attempted", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Misplaced.Attempted) - swiftContainerSharding.WithLabelValues("misplaced", "failure", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Misplaced.Failure) - swiftContainerSharding.WithLabelValues("misplaced", "found", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Misplaced.Found) - swiftContainerSharding.WithLabelValues("misplaced", "max_time", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Misplaced.MaxTime) - swiftContainerSharding.WithLabelValues("misplaced", "min_time", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Misplaced.MinTime) - swiftContainerSharding.WithLabelValues("misplaced", "success", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Misplaced.Success) - - swiftContainerSharding.WithLabelValues("scanned", "attempted", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Scanned.Attempted) - swiftContainerSharding.WithLabelValues("scanned", "failure", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Scanned.Failure) - swiftContainerSharding.WithLabelValues("scanned", "found", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Scanned.Found) - swiftContainerSharding.WithLabelValues("scanned", "max_time", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Scanned.MaxTime) - swiftContainerSharding.WithLabelValues("scanned", "min_time", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Scanned.MinTime) - swiftContainerSharding.WithLabelValues("scanned", "success", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Scanned.Success) - - swiftContainerSharding.WithLabelValues("sharding_candidates", "found", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.ShardingCandidates.Found) - - swiftContainerSharding.WithLabelValues("visited", "attempted", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Visited.Attempted) - swiftContainerSharding.WithLabelValues("visited", "completed", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Visited.Completed) - swiftContainerSharding.WithLabelValues("visited", "failure", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Visited.Failure) - swiftContainerSharding.WithLabelValues("visited", "skipped", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Visited.Skipped) - swiftContainerSharding.WithLabelValues("visited", "success", hostFQDN, hostUUID).Set(container.ShardingStats.Sharding.Visited.Success) + if swiftMinorVersion >= 18 { + swiftContainerSharding.WithLabelValues("sharding_stats", "attempted").Set(container.ShardingStats.Attempted) + swiftContainerSharding.WithLabelValues("sharding_stats", "deffered").Set(container.ShardingStats.Deferred) + swiftContainerSharding.WithLabelValues("sharding_stats", "diff").Set(container.ShardingStats.Diff) + swiftContainerSharding.WithLabelValues("sharding_stats", "diff_capped").Set(container.ShardingStats.DiffCapped) + swiftContainerSharding.WithLabelValues("sharding_stats", "empty").Set(container.ShardingStats.Empty) + swiftContainerSharding.WithLabelValues("sharding_stats", "failure").Set(container.ShardingStats.Failure) + swiftContainerSharding.WithLabelValues("sharding_stats", "hashmatch").Set(container.ShardingStats.Hashmatch) + swiftContainerSharding.WithLabelValues("sharding_stats", "no_change").Set(container.ShardingStats.NoChange) + swiftContainerSharding.WithLabelValues("sharding_stats", "remote_merge").Set(container.ShardingStats.RemoteMerge) + swiftContainerSharding.WithLabelValues("sharding_stats", "remove").Set(container.ShardingStats.Remove) + swiftContainerSharding.WithLabelValues("sharding_stats", "rsync").Set(container.ShardingStats.Rsync) + + swiftContainerSharding.WithLabelValues("audit_root", "attempted").Set(container.ShardingStats.Sharding.AuditRoot.Attempted) + swiftContainerSharding.WithLabelValues("audit_root", "failure").Set(container.ShardingStats.Sharding.AuditRoot.Failure) + swiftContainerSharding.WithLabelValues("audit_root", "success").Set(container.ShardingStats.Sharding.AuditRoot.Success) + swiftContainerSharding.WithLabelValues("audit_shard", "attempted").Set(container.ShardingStats.Sharding.AuditShard.Attempted) + swiftContainerSharding.WithLabelValues("audit_shard", "failure").Set(container.ShardingStats.Sharding.AuditShard.Failure) + swiftContainerSharding.WithLabelValues("audit_shard", "attempted").Set(container.ShardingStats.Sharding.AuditShard.Attempted) + + swiftContainerSharding.WithLabelValues("cleaved", "attempted").Set(container.ShardingStats.Sharding.Cleaved.Attempted) + swiftContainerSharding.WithLabelValues("cleaved", "failure").Set(container.ShardingStats.Sharding.Cleaved.Failure) + swiftContainerSharding.WithLabelValues("cleaved", "max_time").Set(container.ShardingStats.Sharding.Cleaved.MaxTime) + swiftContainerSharding.WithLabelValues("cleaved", "min_time").Set(container.ShardingStats.Sharding.Cleaved.MinTime) + swiftContainerSharding.WithLabelValues("cleaved", "success").Set(container.ShardingStats.Sharding.Cleaved.Success) + + swiftContainerSharding.WithLabelValues("created", "attempted").Set(container.ShardingStats.Sharding.Created.Attempted) + swiftContainerSharding.WithLabelValues("created", "failure").Set(container.ShardingStats.Sharding.Created.Failure) + swiftContainerSharding.WithLabelValues("created", "success").Set(container.ShardingStats.Sharding.Created.Success) + + swiftContainerSharding.WithLabelValues("misplaced", "attempted").Set(container.ShardingStats.Sharding.Misplaced.Attempted) + swiftContainerSharding.WithLabelValues("misplaced", "failure").Set(container.ShardingStats.Sharding.Misplaced.Failure) + swiftContainerSharding.WithLabelValues("misplaced", "found").Set(container.ShardingStats.Sharding.Misplaced.Found) + swiftContainerSharding.WithLabelValues("misplaced", "max_time").Set(container.ShardingStats.Sharding.Misplaced.MaxTime) + swiftContainerSharding.WithLabelValues("misplaced", "min_time").Set(container.ShardingStats.Sharding.Misplaced.MinTime) + swiftContainerSharding.WithLabelValues("misplaced", "success").Set(container.ShardingStats.Sharding.Misplaced.Success) + + swiftContainerSharding.WithLabelValues("scanned", "attempted").Set(container.ShardingStats.Sharding.Scanned.Attempted) + swiftContainerSharding.WithLabelValues("scanned", "failure").Set(container.ShardingStats.Sharding.Scanned.Failure) + swiftContainerSharding.WithLabelValues("scanned", "found").Set(container.ShardingStats.Sharding.Scanned.Found) + swiftContainerSharding.WithLabelValues("scanned", "max_time").Set(container.ShardingStats.Sharding.Scanned.MaxTime) + swiftContainerSharding.WithLabelValues("scanned", "min_time").Set(container.ShardingStats.Sharding.Scanned.MinTime) + swiftContainerSharding.WithLabelValues("scanned", "success").Set(container.ShardingStats.Sharding.Scanned.Success) + + swiftContainerSharding.WithLabelValues("sharding_candidates", "found").Set(container.ShardingStats.Sharding.ShardingCandidates.Found) + + swiftContainerSharding.WithLabelValues("visited", "attempted").Set(container.ShardingStats.Sharding.Visited.Attempted) + swiftContainerSharding.WithLabelValues("visited", "completed").Set(container.ShardingStats.Sharding.Visited.Completed) + swiftContainerSharding.WithLabelValues("visited", "failure").Set(container.ShardingStats.Sharding.Visited.Failure) + swiftContainerSharding.WithLabelValues("visited", "skipped").Set(container.ShardingStats.Sharding.Visited.Skipped) + swiftContainerSharding.WithLabelValues("visited", "success").Set(container.ShardingStats.Sharding.Visited.Success) } else { - writeLogFile.Println("You are runnig Swift version that does not have Container Sharding") + //writeLogFile.Println("You are runnig Swift version that does not have Container Sharding") } } else { - writeLogFile.Print("You are running a very old version ") + //writeLogFile.Print("You are running a very old version ") } - containerReplicationPartsPerSecond := container.ContainerReplicator.Attempted / container.ReplicationTime - swiftContainerReplicationEstimate.WithLabelValues("parts_per_second", hostFQDN, hostUUID).Set(containerReplicationPartsPerSecond) } if SwiftRole == "object" { var object ObjectSwiftRole json.Unmarshal(byteValue, &object) - writeLogFile.Println(string(byteValue)) + //writeLogFile.Println(string(byteValue)) //fmt.Println(object) - objectServer.WithLabelValues("server", "async_pending", hostFQDN, hostUUID).Set(object.AsyncPending) - objectServer.WithLabelValues("replicator", "object_replication_time", hostFQDN, hostUUID).Set(object.ObjectReplicationTime) - objectServer.WithLabelValues("reconstructor", "object_reconstruction_time", hostFQDN, hostUUID).Set(object.ObjectReconstructionTime) - objectServer.WithLabelValues("server", "replication_last", hostFQDN, hostUUID).Set(object.ObjectReplicationLast) - - objectServer.WithLabelValues("auditor_ALL", "audit_time", hostFQDN, hostUUID).Set(object.ObjectAuditorStatsALL.AuditTime) - objectServer.WithLabelValues("auditor_ALL", "byte_processed", hostFQDN, hostUUID).Set(object.ObjectAuditorStatsALL.ByteProcessed) - objectServer.WithLabelValues("auditor_ALL", "errors", hostFQDN, hostUUID).Set(object.ObjectAuditorStatsALL.Errors) - objectServer.WithLabelValues("auditor_ALL", "passes", hostFQDN, hostUUID).Set(object.ObjectAuditorStatsALL.Passes) - objectServer.WithLabelValues("auditor_ALL", "quarantined", hostFQDN, hostUUID).Set(object.ObjectAuditorStatsALL.Quarantined) - objectServer.WithLabelValues("auditor_ZBF", "audit_time", hostFQDN, hostUUID).Set(object.ObjectAuditorStatsZBF.AuditTime) - objectServer.WithLabelValues("auditor_ZBF", "byte_processed", hostFQDN, hostUUID).Set(object.ObjectAuditorStatsZBF.ByteProcessed) - objectServer.WithLabelValues("auditor_ZBF", "errors", hostFQDN, hostUUID).Set(object.ObjectAuditorStatsZBF.Errors) - objectServer.WithLabelValues("auditor_ZBF", "passes", hostFQDN, hostUUID).Set(object.ObjectAuditorStatsZBF.Passes) - - // If Swift version running in the node is 2.15 or after, go ahead and parse the replication per disk metrics. + swiftObjectServer.WithLabelValues("server", "async_pending").Set(object.AsyncPending) + swiftObjectServer.WithLabelValues("replication", "replication_time").Set(object.ObjectReplicationTime) + swiftObjectServer.WithLabelValues("reconstructor", "object_reconstruction_time").Set(object.ObjectReconstructionTime) + swiftObjectServer.WithLabelValues("server", "replication_last").Set(object.ObjectReplicationLast) + + swiftObjectServer.WithLabelValues("auditor_ALL", "audit_time").Set(object.ObjectAuditorStatsALL.AuditTime) + swiftObjectServer.WithLabelValues("auditor_ALL", "byte_processed").Set(object.ObjectAuditorStatsALL.ByteProcessed) + swiftObjectServer.WithLabelValues("auditor_ALL", "errors").Set(object.ObjectAuditorStatsALL.Errors) + swiftObjectServer.WithLabelValues("auditor_ALL", "passes").Set(object.ObjectAuditorStatsALL.Passes) + swiftObjectServer.WithLabelValues("auditor_ALL", "quarantined").Set(object.ObjectAuditorStatsALL.Quarantined) + swiftObjectServer.WithLabelValues("auditor_ZBF", "audit_time").Set(object.ObjectAuditorStatsZBF.AuditTime) + swiftObjectServer.WithLabelValues("auditor_ZBF", "byte_processed").Set(object.ObjectAuditorStatsZBF.ByteProcessed) + swiftObjectServer.WithLabelValues("auditor_ZBF", "errors").Set(object.ObjectAuditorStatsZBF.Errors) + swiftObjectServer.WithLabelValues("auditor_ZBF", "passes").Set(object.ObjectAuditorStatsZBF.Passes) + + // If Swift version running in the node is 2.18 or later, parse the replication per disk metrics. if swiftMajorVersion >= 2 { - if swiftMinorVersion >= 15 { + if swiftMinorVersion >= 18 { for swiftDrive := range object.ObjectReplicationPerDisk { - swiftObjectReplicationPerDisk.WithLabelValues("replicator_per_disk", "rsync", swiftDrive, hostFQDN, hostUUID).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Rsync) - swiftObjectReplicationPerDisk.WithLabelValues("replicator_per_disk", "success", swiftDrive, hostFQDN, hostUUID).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Success) - swiftObjectReplicationPerDisk.WithLabelValues("replicator_per_disk", "failure", swiftDrive, hostFQDN, hostUUID).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Failure) - swiftObjectReplicationPerDisk.WithLabelValues("replicator_per_disk", "attempted", swiftDrive, hostFQDN, hostUUID).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Attempted) - swiftObjectReplicationPerDisk.WithLabelValues("replicator_per_disk", "hashmatch", swiftDrive, hostFQDN, hostUUID).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Hashmatch) - swiftObjectReplicationPerDisk.WithLabelValues("replicator_per_disk", "remove", swiftDrive, hostFQDN, hostUUID).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Remove) - swiftObjectReplicationPerDisk.WithLabelValues("replicator_per_disk", "suffix_count", swiftDrive, hostFQDN, hostUUID).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.SuffixCount) - swiftObjectReplicationPerDisk.WithLabelValues("replicator_per_disk", "suffix_hash", swiftDrive, hostFQDN, hostUUID).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.SuffixHash) - swiftObjectReplicationPerDisk.WithLabelValues("replicator_per_disk", "suffix_sync", swiftDrive, hostFQDN, hostUUID).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.SuffixSync) - swiftObjectReplicationPerDisk.WithLabelValues("replicator_per_disk", "replication_last", swiftDrive, hostFQDN, hostUUID).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicationLast) - swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "replication_time", swiftDrive, hostFQDN, hostUUID).Set(object.ObjectReplicationPerDisk[swiftDrive].ReplicationTime) - - partitionReplicatedPerDisk := object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Attempted - replicationTimeUsedPerDisk := object.ObjectReplicationPerDisk[swiftDrive].ReplicationTime * 60 - replicationPartPerSecondPerDisk := partitionReplicatedPerDisk / replicationTimeUsedPerDisk - - swiftObjectReplicationPerDiskEstimate.WithLabelValues("parts_per_second_per_disk", swiftDrive, hostFQDN, hostUUID).Set(replicationPartPerSecondPerDisk) - swiftObjectReplicationPerDiskEstimate.WithLabelValues("time_used_per_disk", swiftDrive, hostFQDN, hostUUID).Set(replicationTimeUsedPerDisk) + swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "rsync", swiftDrive).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Rsync) + swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "success", swiftDrive).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Success) + swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "failure", swiftDrive).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Failure) + swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "attempted", swiftDrive).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Attempted) + swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "hashmatch", swiftDrive).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Hashmatch) + swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "remove", swiftDrive).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.Remove) + swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "suffix_count", swiftDrive).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.SuffixCount) + swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "suffix_hash", swiftDrive).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.SuffixHash) + swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "suffix_sync", swiftDrive).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicatorStats.SuffixSync) + swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "replication_last", swiftDrive).Set(object.ObjectReplicationPerDisk[swiftDrive].ObjectReplicationLast) + swiftObjectReplicationPerDisk.WithLabelValues("replication_per_disk", "replication_time", swiftDrive).Set(object.ObjectReplicationPerDisk[swiftDrive].ReplicationTime) + } } else { - writeLogFile.Println("You are running Swift 2.15 or below. This is not meant for you") + //writeLogFile.Println("You are running a version prior to Swift 2.18. Per disk replication was not enabled at that point. https://github.com/openstack/swift/commit/c28004deb0e938a9a9532c9c2e0f3197b6e572cb") } } else { - writeLogFile.Println("You are running a very very old version of Swift") + //writeLogFile.Println("You are running a very very old version of Swift") } - objectServer.WithLabelValues("replicator", "rsync", hostFQDN, hostUUID).Set(object.ObjectReplicatorStats.Rsync) - objectServer.WithLabelValues("replicator", "success", hostFQDN, hostUUID).Set(object.ObjectReplicatorStats.Success) - objectServer.WithLabelValues("replicator", "failure", hostFQDN, hostUUID).Set(object.ObjectReplicatorStats.Failure) - objectServer.WithLabelValues("replicator", "attempted", hostFQDN, hostUUID).Set(object.ObjectReplicatorStats.Attempted) - objectServer.WithLabelValues("replicator", "suffixes_checked", hostFQDN, hostUUID).Set(object.ObjectReplicatorStats.Hashmatch) - objectServer.WithLabelValues("replicator", "start", hostFQDN, hostUUID).Set(object.ObjectReplicatorStats.StartTime) - objectServer.WithLabelValues("updater", "object_updater_sweep", hostFQDN, hostUUID).Set(object.ObjectUpdaterSweep) - - partitionReplicated := object.ObjectReplicatorStats.Attempted - replicationTimeUsed := object.ObjectReplicationTime * 60 - replicationPartPerSecond := partitionReplicated / replicationTimeUsed - - swiftObjectReplicationEstimate.WithLabelValues("parts_per_second", hostFQDN, hostUUID).Set(replicationPartPerSecond) - swiftObjectReplicationEstimate.WithLabelValues("time_used", hostFQDN, hostUUID).Set(replicationTimeUsed) + swiftObjectServer.WithLabelValues("replication", "rsync").Set(object.ObjectReplicatorStats.Rsync) + swiftObjectServer.WithLabelValues("replication", "success").Set(object.ObjectReplicatorStats.Success) + swiftObjectServer.WithLabelValues("replication", "failure").Set(object.ObjectReplicatorStats.Failure) + swiftObjectServer.WithLabelValues("replication", "attempted").Set(object.ObjectReplicatorStats.Attempted) + swiftObjectServer.WithLabelValues("replication", "suffixes_checked").Set(object.ObjectReplicatorStats.Hashmatch) + swiftObjectServer.WithLabelValues("replication", "start").Set(object.ObjectReplicatorStats.StartTime) + swiftObjectServer.WithLabelValues("updater", "object_updater_sweep").Set(object.ObjectUpdaterSweep) } } else { - writeLogFile.Println("ReadReconFile Module DISABLED") + //writeLogFile.Println("ReadReconFile Module DISABLED") } } -// GrabSwiftPartition reads the /opt/ss/var/lib/replication_progress.json file and gets the -// primary and handoff partition, then expose them to the prometheus. -func GrabSwiftPartition(replicationProgressFile string, GrabSwiftPartitionEnable bool) { - - writeLogFile := log.New(swiftExporterLog, "GrabSwiftPartition: ", log.Ldate|log.Ltime|log.Lshortfile) - nodeHostname, nodeUUID, _ := GetUUIDAndFQDN(ssnodeConfFile) // getting node FQDN and UUID - - if GrabSwiftPartitionEnable { - writeLogFile.Println("GrabSwiftPartition Module ENABLED") - storagePolicyNameList := GatherStoragePolicyCommonName() - var parts = make(map[string]map[string]PartCounts) // do NOT remove!! - drivesAvailable, _ := disk.Partitions(false) // List out all the drives detected in OS. - - jsonFile, err := os.Open(replicationProgressFile) - if err != nil { - writeLogFile.Println(err) - } - defer jsonFile.Close() - byteValue, _ := ioutil.ReadAll(jsonFile) - err = json.Unmarshal(byteValue, &parts) - writeLogFile.Println(string(byteValue)) - - for i := 0; i < len(drivesAvailable); i++ { - - swiftMountPoint := drivesAvailable[i].Mountpoint - swiftDriveLabel := drivesAvailable[i].Device - driveType := HddOrSSD(swiftDriveLabel) - if strings.Contains(swiftMountPoint, "/srv/node") { - swiftMountPoint := strings.Split(swiftMountPoint, "/")[3] - swiftDrivePrimaryParitions.WithLabelValues(nodeHostname, nodeUUID, swiftMountPoint, "Account & Container", "account", driveType).Set(parts[swiftMountPoint]["accounts"].Primary) - swiftDrivePrimaryParitions.WithLabelValues(nodeHostname, nodeUUID, swiftMountPoint, "Account & Container", "container", driveType).Set(parts[swiftMountPoint]["containers"].Primary) - //swiftDrivePrimaryParitions.WithLabelValues(swiftMountPoint, "object").Set(parts[swiftMountPoint].ObjectPartCount.Primary) - swiftDriveHandoffPartitions.WithLabelValues(nodeHostname, nodeUUID, swiftMountPoint, "Account & Container", "account", driveType).Set(parts[swiftMountPoint]["accounts"].Handoff) - swiftDriveHandoffPartitions.WithLabelValues(nodeHostname, nodeUUID, swiftMountPoint, "Account & Container", "container", driveType).Set(parts[swiftMountPoint]["containers"].Handoff) - //swiftDriveHandoffPartitions.WithLabelValues(swiftMountPoint, "object").Set(parts[swiftMountPoint].ObjectPartCount.Handoff) - if len(storagePolicyNameList) > 0 { - for j := 0; j < len(storagePolicyNameList); j++ { - objectDirectoryBody := []string{"objects"} - storagePolicyIndex := strconv.FormatInt(int64(j), 10) - if j == 0 { - swiftDrivePrimaryParitions.WithLabelValues(nodeHostname, nodeUUID, swiftMountPoint, storagePolicyNameList[storagePolicyIndex], "objects", driveType).Set(parts[swiftMountPoint]["objects"].Primary) - swiftDriveHandoffPartitions.WithLabelValues(nodeHostname, nodeUUID, swiftMountPoint, storagePolicyNameList[storagePolicyIndex], "objects", driveType).Set(parts[swiftMountPoint]["objects"].Handoff) - } else { - objectDirectoryBody = append(objectDirectoryBody, storagePolicyIndex) - objectDirectoryComplete := strings.Join(objectDirectoryBody, "-") - swiftDrivePrimaryParitions.WithLabelValues(nodeHostname, nodeUUID, swiftMountPoint, storagePolicyNameList[storagePolicyIndex], objectDirectoryComplete, driveType).Set(parts[swiftMountPoint][objectDirectoryComplete].Primary) - swiftDriveHandoffPartitions.WithLabelValues(nodeHostname, nodeUUID, swiftMountPoint, storagePolicyNameList[storagePolicyIndex], objectDirectoryComplete, driveType).Set(parts[swiftMountPoint][objectDirectoryComplete].Handoff) - objectDirectoryBody = objectDirectoryBody[:len(objectDirectoryBody)-1] - } - } - } - } else { - writeLogFile.Println("Not a Swift Partition") - continue - } - } - } else { - writeLogFile.Println("GrabSwiftPartition Module DISABLED") - } -} - // CheckSwiftLogSize Description: this function checks the size of Swift all.log at /var/log/swift/all.log and returns its size. // Once the data is retrieved, we will put expose it over Prometheus. func CheckSwiftLogSize(swiftLog string) { - writeLogFile := log.New(swiftExporterLog, "CheckSwiftLogSize: ", log.Ldate|log.Ltime|log.Lshortfile) +// TODO: fix logging +// writeLogFile := log.New(swiftExporterLog, "CheckSwiftLogSize: ", log.Ldate|log.Ltime|log.Lshortfile) swiftLogFileHandle, err := os.Open(swiftLog) if err != nil { - writeLogFile.Println("Cannot open this file. Exiting") +// writeLogFile.Println("Cannot open this file. Exiting") os.Exit(1) } defer swiftLogFileHandle.Close() fileInfo, err := swiftLogFileHandle.Stat() swiftLogFileSize.Set(float64(fileInfo.Size())) - writeLogFile.Printf("Swift all.log Size: %f", float64(fileInfo.Size())) +// writeLogFile.Printf("Swift all.log Size: %f", float64(fileInfo.Size())) } // GatherStoragePolicyUtilization do a "du -s" across all Swift nodes ("/srv/node") and expose // actual disk size through the Prometheus. func GatherStoragePolicyUtilization(GatherStoragePolicyUtilizationEnable bool) { - writeLogFile := log.New(swiftExporterLog, "GatherStoragePolicyUtilization: ", log.Ldate|log.Ltime|log.Lshortfile) +// TODO: fix logging +// writeLogFile := log.New(swiftExporterLog, "GatherStoragePolicyUtilization: ", log.Ldate|log.Ltime|log.Lshortfile) if GatherStoragePolicyUtilizationEnable { - writeLogFile.Println("GatherStoragePolicyUtilization Module ENABLED") + //writeLogFile.Println("GatherStoragePolicyUtilization Module ENABLED") storagePolicyNameList := GatherStoragePolicyCommonName() - hostFQDN, hostUUID, _ := GetUUIDAndFQDN(ssnodeConfFile) var storagePolicyName string // disk.Partition is from gopsutil library and it returns a structure of @@ -627,7 +539,7 @@ func GatherStoragePolicyUtilization(GatherStoragePolicyUtilizationEnable bool) { // convert the outut to float64 from string usageFloat, _ := strconv.ParseFloat(usage[0], 64) // expose the data out to Prometheus - swiftStoragePolicyUsage.WithLabelValues(driveLocation, f.Name(), storagePolicyName, hostFQDN, hostUUID).Set(usageFloat) + swiftStoragePolicyUsage.WithLabelValues(driveLocation, f.Name(), storagePolicyName).Set(usageFloat) // Removing the last element from slice "storagePolicyList", to "reset" the slice. Otherwise, // data in this entry will be carried over to the next one. Causing error... storagePolicyList = storagePolicyList[:len(storagePolicyList)-1] @@ -636,7 +548,7 @@ func GatherStoragePolicyUtilization(GatherStoragePolicyUtilizationEnable bool) { } } } else { - writeLogFile.Println("GatherStoragePolicyUtilization Module DISABLED") + //writeLogFile.Println("GatherStoragePolicyUtilization Module DISABLED") } } @@ -650,8 +562,6 @@ func CountFilesPerSwiftDrive() { var objectFiles []string swiftDrivesRoot := "/srv/node/" - nodeHostname, nodeUUID, _ := GetUUIDAndFQDN(ssnodeConfFile) // getting node FQDN and UUID - err := filepath.Walk(swiftDrivesRoot, func(path string, info os.FileInfo, err error) error { if strings.Contains(path, "accounts") { if strings.Contains(path, ".db") { @@ -675,54 +585,9 @@ func CountFilesPerSwiftDrive() { if err != nil { panic(err) } - accountDBCount.WithLabelValues(nodeHostname, nodeUUID).Set(float64(len(accountsDB))) - accountDBPendingCount.WithLabelValues(nodeHostname, nodeUUID).Set(float64(len(accountsPendingDB))) - containerDBCount.WithLabelValues(nodeHostname, nodeUUID).Set(float64(len(containersDB))) - containerDBPendingCount.WithLabelValues(nodeHostname, nodeUUID).Set(float64(len(containersPendingDB))) - objectFileCount.WithLabelValues(nodeHostname, nodeUUID).Set(float64(len(objectFiles))) -} - -// CheckSwiftService is a service check on all Swift / Swift-related services running in a node. -func CheckSwiftService() { - nodeHostname, nodeUUID, _ := GetUUIDAndFQDN(ssnodeConfFile) // getting node FQDN and UUID - swiftServices := [4]string{"ssswift-proxy", "ssswift-account@server", "ssswift-container@server", "ssswift-object@server"} - swiftSubServices := [14]string{"ssswift-object-replication@server", "ssswift-object-replication@reconstructor.service", - "ssswift-object-replication@replicator", "ssswift-object@updater", "ssswift-object@auditor", "ssswift-container-replication@sharder", - "ssswift-container-replication@replicator", "ssswift-container-replication@server", "ssswift-container@updater", "ssswift-container@auditor", - "ssswift-account-replication@replicator", "ssswift-account-replication@server", "ssswift-account@reaper", "ssswift-account@auditor"} - - for i := 0; i < len(swiftServices); i++ { - cmd := exec.Command("systemctl", "check", swiftServices[i]) - out, err := cmd.CombinedOutput() - if err != nil { - fmt.Println("Cannot find main process") - fmt.Println("Process Not Running: ", swiftServices[i]) - fmt.Println("Service Status: ", string(out)) - swiftServiceStatus.WithLabelValues(nodeHostname, nodeUUID, swiftServices[i]).Set(float64(0)) - } else { - if strings.TrimRight(string(out), "\n") == "active" { - fmt.Println("Now Checking: ", swiftServices[i]) - fmt.Println("Service Status: ", string(out)) - swiftServiceStatus.WithLabelValues(nodeHostname, nodeUUID, swiftServices[i]).Set(float64(1)) - } - } - } - - for j := 0; j < len(swiftSubServices); j++ { - cmd := exec.Command("systemctl", "check", swiftSubServices[j]) - out, err := cmd.CombinedOutput() - if err != nil { - fmt.Println("Cannot find process") - fmt.Println("Process Not Running: ", swiftServices[j]) - fmt.Println("Service Status: ", string(out)) - swiftServiceStatus.WithLabelValues(nodeHostname, nodeUUID, swiftServices[j]).Set(float64(0)) - - } else { - if strings.TrimRight(string(out), "\n") == "active" { - fmt.Println("Now Checking: ", swiftServices[j]) - fmt.Println("Service Status: ", string(out)) - swiftServiceStatus.WithLabelValues(nodeHostname, nodeUUID, swiftServices[j]).Set(float64(1)) - } - } - } + swiftAccountDBCount.Set(float64(len(accountsDB))) + swiftAccountDBPendingCount.Set(float64(len(accountsPendingDB))) + swiftContainerDBCount.Set(float64(len(containersDB))) + swiftContainerDBPendingCount.Set(float64(len(containersPendingDB))) + swiftObjectFileCount.Set(float64(len(objectFiles))) } diff --git a/exporter/systemperformance.go b/exporter/systemperformance.go index a58c0f9..23546fd 100644 --- a/exporter/systemperformance.go +++ b/exporter/systemperformance.go @@ -2,7 +2,7 @@ package exporter import ( "fmt" - "log" +// "log" "os/exec" "strconv" "strings" @@ -48,14 +48,14 @@ func init() { // instance that is being run. func CheckObjectServerConnection(CheckObjectServerConnectionEnable bool) { - writeLogFile := log.New(swiftExporterLog, "CheckObjectServerConnection: ", log.Ldate|log.Ltime|log.Lshortfile) + //writeLogFile := log.New(swiftExporterLog, "CheckObjectServerConnection: ", log.Ldate|log.Ltime|log.Lshortfile) if CheckObjectServerConnectionEnable { // Get all running processes in the node runningProcess, _ := process.Pids() counter := 0 - numberOfRunningProcess := len(runningProcess) - writeLogFile.Println(numberOfRunningProcess) + //numberOfRunningProcess := len(runningProcess) + //writeLogFile.Println(numberOfRunningProcess) // For each running process, check to see if there is an opened network connection for i := 0; i < len(runningProcess); i++ { @@ -66,13 +66,13 @@ func CheckObjectServerConnection(CheckObjectServerConnectionEnable bool) { } else { if processConnected[0].Laddr.Port == uint32(6000) { counter++ - writeLogFile.Println(processConnected) + //writeLogFile.Println(processConnected) } } } swiftObjectServerConnection.Set(float64(counter) - 1) } else { - writeLogFile.Println("CheckObjectServerConnection Module DISABLED") + //writeLogFile.Println("CheckObjectServerConnection Module DISABLED") } } @@ -88,8 +88,6 @@ func RunSMARTCTL() { var mediaWearoutIndicator float64 fmt.Println("Staring RunSMARTCTL Module...") - // get the FQDN and UUID of the node as part tag used when exposing the data out to prometheus. - nodeFQDN, nodeUUID, _ := GetUUIDAndFQDN(ssnodeConfFile) // grabbing the device list from the node using the disk library in gopsutil library. grabNodeDeviceList, _ := disk.Partitions(false) @@ -97,11 +95,12 @@ func RunSMARTCTL() { for i := 0; i < len(grabNodeDeviceList); i++ { driveList := grabNodeDeviceList[i].Device // get device list swiftDriveType := HddOrSSD(driveList) // find out whether the drive is a HDD or SSD - fmt.Println(driveList) +// we need to clean up all output in this software, and enable flags for verbosity + //fmt.Println(driveList) smartctlExist, smartctlDoesNotExist := exec.Command("which", "smartctl").Output() smartctlLocation := string(smartctlExist) - fmt.Println("Checking to see if RunSMARTCTL exist...") + //fmt.Println("Checking to see if RunSMARTCTL exist...") if smartctlDoesNotExist != nil { // if exec.Command returns error, that is either binary is not available / there is something wrong with the binary, // print the error message out. @@ -109,7 +108,7 @@ func RunSMARTCTL() { break } - fmt.Println("smartctl exists...Running smartctl command to grab SMART data...") + //fmt.Println("smartctl exists...Running smartctl command to grab SMART data...") runCommand, _ := exec.Command(smartctlLocation, "-A", driveList).Output() // run "smartctl -A " command // if exec.Command returns good result, reformat the output to expose them out in prometheus. @@ -121,14 +120,14 @@ func RunSMARTCTL() { if strings.Contains(output[j], "Reallocated_Sector_Ct") { parseOutput := strings.Split(output[j], " ") reallocationSectorsCount, _ = strconv.ParseFloat(string(parseOutput[len(parseOutput)-1]), 64) - swiftDriveReallocatedSectorCount.WithLabelValues(swiftDriveType, nodeFQDN, nodeUUID).Set(reallocationSectorsCount) + swiftDriveReallocatedSectorCount.WithLabelValues(swiftDriveType).Set(reallocationSectorsCount) } else if strings.Contains(output[j], "Offline_Uncorrectable") { parseOutput := strings.Split(output[j], " ") offlineUncorrectableCount, _ = strconv.ParseFloat(string(parseOutput[len(parseOutput)-1]), 64) - swiftDriveOfflineUncorrectableCount.WithLabelValues(swiftDriveType, nodeFQDN, nodeUUID).Set(offlineUncorrectableCount) + swiftDriveOfflineUncorrectableCount.WithLabelValues(swiftDriveType).Set(offlineUncorrectableCount) } else { - fmt.Println(output) - fmt.Println("False") + //fmt.Println(output) + //fmt.Println("False") } } } else if strings.Compare(swiftDriveType, "SSD") == 0 { @@ -141,7 +140,7 @@ func RunSMARTCTL() { if strings.Contains(output[j], "Wear Leveling Count") { parseOutput := strings.Split(output[j], " ") wearLevelingCount, _ = strconv.ParseFloat(string(parseOutput[len(parseOutput)-1]), 64) - swiftDriveWearLevelingCount.WithLabelValues(driveList, swiftDriveType, nodeFQDN, nodeUUID).Set(wearLevelingCount) + swiftDriveWearLevelingCount.WithLabelValues(driveList, swiftDriveType).Set(wearLevelingCount) } } } else if strings.Contains(manufactureOutput[k], "Intel") { @@ -149,7 +148,7 @@ func RunSMARTCTL() { if strings.Contains(output[j], "Media Wearout Indicator") { parseOutput := strings.Split(output[j], " ") mediaWearoutIndicator, _ = strconv.ParseFloat(string(parseOutput[3]), 64) - swiftDriveMediaWearoutIndicatorCount.WithLabelValues(driveList, swiftDriveType, nodeFQDN, nodeUUID).Set(mediaWearoutIndicator) + swiftDriveMediaWearoutIndicatorCount.WithLabelValues(driveList, swiftDriveType).Set(mediaWearoutIndicator) } } } diff --git a/swift_exporter.go b/swift_exporter.go index 19e41db..8956943 100644 --- a/swift_exporter.go +++ b/swift_exporter.go @@ -20,7 +20,6 @@ import ( // Config holds the configuration settings from the swift_exporter.yml file. type Config struct { CheckObjectServerConnectionEnable bool `yaml:"CheckObjectServerConnection"` - GrabSwiftPartitionEnable bool `yaml:"GrabSwiftPartition"` GatherReplicationEstimateEnable bool `yaml:"GatherReplicationEstimate"` GatherStoragePolicyUtilizationEnable bool `yaml:"GatherStoragePolicyUtilization"` ExposePerCPUUsageEnable bool `yaml:"ExposePerCPUUsage"` @@ -28,6 +27,7 @@ type Config struct { ReadReconFileEnable bool `yaml:"ReadReconFile"` SwiftDiskUsageEnable bool `yaml:"SwiftDiskUsage"` SwiftDriveIOEnable bool `yaml:"SwiftDriveIO"` + SwiftVersion string `yaml:"SwiftVersion"` SwiftLogFile string `yaml:"SwiftLogFile"` SwiftConfigFile string `yaml:"SwiftConfigFile"` ReplicationProgressFile string `yaml:"ReplicationProgressFile"` @@ -54,7 +54,6 @@ var ( config = Config{ ReadReconFileEnable: true, - GrabSwiftPartitionEnable: true, SwiftDiskUsageEnable: true, SwiftDriveIOEnable: true, GatherReplicationEstimateEnable: true, @@ -62,6 +61,7 @@ var ( CheckObjectServerConnectionEnable: true, ExposePerCPUUsageEnable: true, ExposePerNICMetricEnable: true, + SwiftVersion: "0.0", SwiftLogFile: "/var/log/swift/all.log", SwiftConfigFile: "/etc/swift/swift.conf", ReplicationProgressFile: "/opt/ss/var/lib/replication_progress.json", @@ -122,20 +122,6 @@ func SanityCheckOnFiles() { writeLogFile.Println("ReadReconFile module is disabled. Skip this check.") writeLogFile.Println() } - if config.GrabSwiftPartitionEnable { - writeLogFile.Printf("Script is set to expose data collected from %s (GrabSwiftPartition module enable). Check to see if that file exist...\n", config.ReplicationProgressFile) - if _, err := os.Stat(config.ReplicationProgressFile); err == nil { - log.Printf("===> %s exists. Check for this module has completed. Enable the module...\n", config.ReplicationProgressFile) - config.GrabSwiftPartitionEnable = true - writeLogFile.Println() - } else { - writeLogFile.Printf("===> %s does not exists, but you have enabled it. Disable the module...\n", config.ReplicationProgressFile) - config.GrabSwiftPartitionEnable = false - writeLogFile.Println() - } - } else { - writeLogFile.Println("GrabSwiftPartition module is disabled. Skip this check.") - } if config.GatherReplicationEstimateEnable { writeLogFile.Printf("Script is set to expose data collected from %s (GatherReplicationEstimate module enable). Check to see if that file exist...\n", config.SwiftLogFile) if _, err := os.Stat(config.SwiftLogFile); err == nil { @@ -227,16 +213,14 @@ func main() { // Reference2: https://github.com/prometheus/client_golang/blob/master/examples/random/main.go go func() { for { - exporter.ReadReconFile(config.AccountReconFile, "account", config.ReadReconFileEnable) - exporter.ReadReconFile(config.ContainerReconFile, "container", config.ReadReconFileEnable) - exporter.ReadReconFile(config.ObjectReconFile, "object", config.ReadReconFileEnable) - exporter.GrabSwiftPartition(config.ReplicationProgressFile, config.GrabSwiftPartitionEnable) + exporter.ReadReconFile(config.AccountReconFile, "account", config.ReadReconFileEnable, config.SwiftVersion) + exporter.ReadReconFile(config.ContainerReconFile, "container", config.ReadReconFileEnable, config.SwiftVersion) + exporter.ReadReconFile(config.ObjectReconFile, "object", config.ReadReconFileEnable, config.SwiftVersion) exporter.SwiftDiskUsage(config.SwiftDiskUsageEnable) exporter.SwiftDriveIO(config.SwiftDriveIOEnable) exporter.CheckObjectServerConnection(config.CheckObjectServerConnectionEnable) exporter.ExposePerCPUUsage(config.ExposePerCPUUsageEnable) exporter.ExposePerNICMetric(config.ExposePerNICMetricEnable) - exporter.GrabNICMTU() // Setting time to sleep for 1 Minute. If you need to set it to milliseconds, change the // "time.Minute" to "time.Millisecond" @@ -245,15 +229,6 @@ func main() { } }() - // the following go routine will be run every 5 minutes - go func() { - for { - //GatherReplicationEstimate(swiftLog, timeLastRun, SelectedModule.GatherReplicationEstimateEnable) - exporter.CheckSwiftService() - time.Sleep(5 * time.Minute) - } - }() - // the following go routine will be run every hour. go func() { for { diff --git a/swift_exporter_config.yaml b/swift_exporter_config.yaml index d77c107..904462e 100644 --- a/swift_exporter_config.yaml +++ b/swift_exporter_config.yaml @@ -2,9 +2,6 @@ # data such as replication time, object audit time...etc. Generally speaking, this is the module that # contains data most users will find useful. ReadReconFile: yes -# module_description: this module grab the primary and handoff partitions count in a Swift node. -# Enter "yes" to enable, and "no" to disable. -GrabSwiftPartition: yes # module_description: this module checks Swift nodes and grab information about disk. # Enter "yes" to enable, and "no" to disable. SwiftDiskUsage: yes @@ -17,17 +14,22 @@ SwiftDriveIO: yes GatherReplicationEstimate: yes # module_description: this module grabs the size of each drive that has a storage policy assigned to it, # then expose it to Prometheus. -GatherStoragePolicyUtilization: yes +#GatherStoragePolicyUtilization: yes +GatherStoragePolicyUtilization: no # module_description: this module checks the number of object server spinned. This is useful for replica based storage policy. # The number of object server = number of drives * object server per port setting. CheckObjectServerConnection: yes # module_description: this module expose cpu usage and expose them in prometheus. -ExposePerCPUUsage: yes +#ExposePerCPUUsage: yes +ExposePerCPUUsage: no # module_description: this module expose cpu usage and expose them in prometheus. -ExposePerNICMetric: yes -SwiftLogFile: "/var/log/swift/all.log" +#ExposePerNICMetric: yes +ExposePerNICMetric: no +SwiftVersion: "2.15.1" +SwiftLogFile: "/var/log/swift/swift.log" SwiftConfigFile: "/etc/swift/swift.conf" -ReplicationProgressFile: "/opt/ss/var/lib/replication_progress.json" +#ReplicationProgressFile: "/var/log/swift/swift_exporter_replication_progress.json" ObjectReconFile: "/var/cache/swift/object.recon" ContainerReconFile: "/var/cache/swift/container.recon" AccountReconFile: "/var/cache/swift/account.recon" +