diff --git a/README.md b/README.md
index f8e34d3fd25b38b9e95e479643ad5bb6affd6a33..0125ab41112585fae4ba63932d352712b6cc3d57 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
It's a Prometheus exporter that exposes metrics from ArvanCloud products. Currently, it supports the following products:
- [x] CDN
-- [ ] Object Storage (coming soon)
+- [x] Object Storage
- [ ] PaaS/CaaS (coming soon)
- [ ] IaaS (coming soon)
@@ -80,3 +80,38 @@ The CDN collector exposes the following metrics:
>>>
š¤¯ We welcome contributions from the community. Please report any issues you find on the Issues page or send us an email at <cdn@arvancloud.ir>.
+
+#### Object Storage
+
+```yaml
+products:
+ object:
+ enable: true
+ buckets:
+ - "bucket1"
+ - "bucket2"
+```
+
+- **`enable`**: Set the enable field to true to activate the object storage collector.
+
+- **`buckets`**: Specify the buckets that should be included in bucket usage metrics report collection.
+
+##### Metrics
+
+The object storage collector exposes the following metrics:
+
+| Name | Description | Type |
+| -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ----- |
+| `arvancloud_storage_modified_time` | Indicates the most recent update time of the bucket usage in UTC (if it's zero, it means there's no usage for today) | gauge |
+| `arvancloud_storage_total_object_count` | Represents the total count of objects in bucket | gauge |
+| `arvancloud_storage_multi_part_object_count` | Represents the total count of multipart objects | gauge |
+| `arvancloud_storage_bucket_size` | Represents the actual size of the bucket in bytes | gauge |
+| `arvancloud_storage_bucket_download` | Represents bucket downloads in bytes | gauge |
+| `arvancloud_storage_bucket_upload` | Represents bucket uploads in bytes | gauge |
+| `arvancloud_storage_bucket_request_count` | Represents the total count of successful operations per `operation_type` label | gauge |
+
+>>>
+**ā¯•Note**: `operation_type` label in `arvancloud_storage_bucket_request_count` can be one of following types: `put_object`, `get_object`, `delete_object`, `post_head` & `head_bucket`.
+
+**ā¯•Note**: All metrics will be labeled with `bucket` label.
+>>>
diff --git a/collector/collector.go b/collector/collector.go
index 928c8ed383d4821fd1571c04acc06777d8430063..cb1ae0726fb20e8c9d24dbdcd98642cc5856b39b 100644
--- a/collector/collector.go
+++ b/collector/collector.go
@@ -56,7 +56,9 @@ func WithCDN() Option {
// WithObject enables ObjectStorage metrics
func WithObject() Option {
return func(c *collector) {
- c.collectors = append(c.collectors, newObjectCollector())
+ for _, bucket := range c.cfg.Products.OBJECT.Buckets {
+ c.collectors = append(c.collectors, newObjectCollector(bucket))
+ }
}
}
diff --git a/collector/object_collector.go b/collector/object_collector.go
index 299fb7eccb7a348717c55e07f0f3be6cd1586b75..338c397e19c582a7dda6e17d5799fe61e9f5132c 100644
--- a/collector/object_collector.go
+++ b/collector/object_collector.go
@@ -1,27 +1,144 @@
package collector
import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "time"
+
"github.com/prometheus/client_golang/prometheus"
+ log "github.com/sirupsen/logrus"
)
+const objectAPIHost = "https://panel.arvanstorage.ir"
+
type objectCollector struct {
- leasesActiveCountDesc *prometheus.Desc
+ descriptions map[string]*prometheus.Desc
+ bucket string
}
func (c *objectCollector) init() {
- const prefix = "object"
+ const prefix = "storage"
+
+ log.Debugf("Initializing object collector for bucket: %s", c.bucket)
+
+ c.descriptions = make(map[string]*prometheus.Desc)
+ c.descriptions["modified_time"] = description(prefix, "modified_time", "The last timestamp of bucket usage being updated", []string{"bucket"})
+ c.descriptions["total_object_count"] = description(prefix, "total_object_count", "Total count of objects in the bucket", []string{"bucket"})
+ c.descriptions["multi_part_object_count"] = description(prefix, "multi_part_object_count", "Total count of multi-part objects in the bucket", []string{"bucket"})
+ c.descriptions["bucket_size"] = description(prefix, "bucket_size", "Bucket size in bytes", []string{"bucket"})
+ c.descriptions["bucket_download"] = description(prefix, "bucket_download", "Total bucket download of today", []string{"bucket"})
+ c.descriptions["bucket_upload"] = description(prefix, "bucket_upload", "Total bucket upload of today", []string{"bucket"})
+ c.descriptions["request_count"] = description(prefix, "request_count", "Total count of successful operations per type", []string{"bucket", "operation_type"})
}
-func newObjectCollector() ArvancloudCollector {
- c := &objectCollector{}
+func newObjectCollector(bucket string) ArvancloudCollector {
+ c := &objectCollector{
+ bucket: bucket,
+ }
c.init()
return c
}
func (c *objectCollector) describe(ch chan<- *prometheus.Desc) {
- ch <- c.leasesActiveCountDesc
+ for _, d := range c.descriptions {
+ ch <- d
+ }
}
func (c *objectCollector) collect(ctx *collectorContext) error {
+ err := c.GetBucketMetrics(ctx)
+ if err != nil {
+ log.Errorf("Error when getting bucket usage metrics: %v\n", err)
+ return err
+ }
+
+ return nil
+}
+
+// Bucket usage metrics response model
+type bucketMetricsResponse struct {
+ Message string `json:"message"`
+ Data bucketMetrics `json:"data"`
+}
+
+// Bucket metrics
+type bucketMetrics struct {
+ MTime time.Time `json:"arvancloud_aos_modified_time" example:"2023-11-07T06:18:22.860248911Z"` // Indicates the most recent update time of the data
+ TotalObjectCount int64 `json:"arvancloud_aos_total_object_count" example:"100"` // Represents the total count of objects in bucket
+ MultiPartObjectCount int64 `json:"arvancloud_aos_multi_part_object_count" example:"5"` // Represents the total count of multipart objects
+ BucketSize int64 `json:"arvancloud_aos_bucket_size" example:"52428800"` // Represents the actual size of the bucket in bytes
+ BucketDownload int64 `json:"arvancloud_aos_bucket_download" example:"13687300"` // Represents bucket downloads in bytes
+ BucketUpload int64 `json:"arvancloud_aos_bucket_upload" example:"1642400"` // Represents bucket uploads in bytes
+ RequestsByType bucketRequestsByType `json:"arvancloud_aos_requests_by_type"` // Represents the total count of successful operations
+}
+
+// Total count of RGW successful operations per type of operations
+type bucketRequestsByType struct {
+ PutObject int64 `json:"put_object" example:"5"` // Represents the count of successful "put_object" operations
+ GetObject int64 `json:"get_object" example:"43"` // Represents the count of successful "get_object" operations
+ DeleteObject int64 `json:"delete_object" example:"2"` // Represents the count of successful "delete_object"" operations
+ PostObject int64 `json:"post_object" example:"1"` // Represents the count of successful "post_object" operations
+ HeadBucket int64 `json:"head_bucket" example:"7"` // Represents the count of successful "head_bucket" operations
+}
+
+// Gets bucket usage metrics from object API
+func (c *objectCollector) GetBucketMetrics(ctx *collectorContext) error {
+ var bucketMetricsResp bucketMetricsResponse
+ url := fmt.Sprintf("%v/panel/v1/bucket/%v/usage", objectAPIHost, c.bucket)
+
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ log.Errorf("Error creating request: %v\n", err)
+ return err
+ }
+
+ req.Header.Set("Authorization", ctx.cfg.Token)
+
+ client := &http.Client{}
+ resp, err := client.Do(req)
+ if err != nil {
+ log.Errorf("Error sending request: %v\n", err)
+ return err
+ }
+ defer resp.Body.Close()
+
+ err = json.NewDecoder(resp.Body).Decode(&bucketMetricsResp)
+ if err != nil {
+ log.Errorf("Error decoding response: %v\n", err)
+ return err
+ }
+
+ if resp.StatusCode != 200 {
+ return fmt.Errorf("Bucket usage request status code: %v. Message: %v", resp.Status, bucketMetricsResp.Message)
+ }
+
+ bucketMetrics := bucketMetricsResp.Data
+
+ desc := c.descriptions["modified_time"]
+ ctx.ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(bucketMetrics.MTime.Unix()), c.bucket)
+
+ desc = c.descriptions["total_object_count"]
+ ctx.ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(bucketMetrics.TotalObjectCount), c.bucket)
+
+ desc = c.descriptions["multi_part_object_count"]
+ ctx.ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(bucketMetrics.MultiPartObjectCount), c.bucket)
+
+ desc = c.descriptions["bucket_size"]
+ ctx.ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(bucketMetrics.BucketSize), c.bucket)
+
+ desc = c.descriptions["bucket_download"]
+ ctx.ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(bucketMetrics.BucketDownload), c.bucket)
+
+ desc = c.descriptions["bucket_upload"]
+ ctx.ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(bucketMetrics.BucketUpload), c.bucket)
+
+ desc = c.descriptions["request_count"]
+ ctx.ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(bucketMetrics.RequestsByType.PutObject), c.bucket, "put_obj")
+ ctx.ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(bucketMetrics.RequestsByType.GetObject), c.bucket, "get_obj")
+ ctx.ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(bucketMetrics.RequestsByType.DeleteObject), c.bucket, "delete_obj")
+ ctx.ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(bucketMetrics.RequestsByType.PostObject), c.bucket, "post_obj")
+ ctx.ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(bucketMetrics.RequestsByType.HeadBucket), c.bucket, "head_bucket")
+
return nil
}
diff --git a/config/config.go b/config/config.go
index 02af1da55d1ef7454921e5db3bab0ac14dcde879..64760866e7501dc992d5324a16d2e6428fd93b76 100644
--- a/config/config.go
+++ b/config/config.go
@@ -16,8 +16,8 @@ type Config struct {
Period string `yaml:"period,omitempty"`
} `yaml:"cdn,omitempty"`
OBJECT struct {
- Enabled bool `yaml:"enable,omitempty"`
- Period string `yaml:"period,omitempty"`
+ Enabled bool `yaml:"enable,omitempty"`
+ Buckets []string `yaml:"buckets,omitempty"`
} `yaml:"object,omitempty"`
} `yaml:"products,omitempty"`
}
diff --git a/config/config.test.yml b/config/config.test.yml
index 5ac67cf085daae307e2af83cf0eee460c5dbff6a..4ec6e33d7e8ef70b501c1aec93326ab472d1bcc1 100644
--- a/config/config.test.yml
+++ b/config/config.test.yml
@@ -7,5 +7,7 @@ products:
- "example.com"
period: 3h
object:
- enable: false
- period: 3h
+ enable: true
+ buckets:
+ - "bucket1"
+ - "bucket2"
diff --git a/config/config_test.go b/config/config_test.go
index 918fa14d5dacdd0afdaed450c1481a40af3a4695..03271faae937d6e53125e0d513c5dd5eb671634d 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -53,8 +53,8 @@ func TestEnabledProducts(t *testing.T) {
Period string `yaml:"period,omitempty"`
} `yaml:"cdn,omitempty"`
OBJECT struct {
- Enabled bool `yaml:"enable,omitempty"`
- Period string `yaml:"period,omitempty"`
+ Enabled bool `yaml:"enable,omitempty"`
+ Buckets []string `yaml:"buckets,omitempty"`
} `yaml:"object,omitempty"`
}{
CDN: struct {
@@ -67,11 +67,10 @@ func TestEnabledProducts(t *testing.T) {
Period: "daily",
},
OBJECT: struct {
- Enabled bool `yaml:"enable,omitempty"`
- Period string `yaml:"period,omitempty"`
+ Enabled bool `yaml:"enable,omitempty"`
+ Buckets []string `yaml:"buckets,omitempty"`
}{
Enabled: false,
- Period: "weekly",
},
},
}
@@ -92,8 +91,8 @@ func TestEnabledProducts(t *testing.T) {
Period string `yaml:"period,omitempty"`
} `yaml:"cdn,omitempty"`
OBJECT struct {
- Enabled bool `yaml:"enable,omitempty"`
- Period string `yaml:"period,omitempty"`
+ Enabled bool `yaml:"enable,omitempty"`
+ Buckets []string `yaml:"buckets,omitempty"`
} `yaml:"object,omitempty"`
}{
CDN: struct {
@@ -106,11 +105,10 @@ func TestEnabledProducts(t *testing.T) {
Period: "daily",
},
OBJECT: struct {
- Enabled bool `yaml:"enable,omitempty"`
- Period string `yaml:"period,omitempty"`
+ Enabled bool `yaml:"enable,omitempty"`
+ Buckets []string `yaml:"buckets,omitempty"`
}{
Enabled: true,
- Period: "weekly",
},
},
}