Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Add R2 metrics #130

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

feat: Add R2 metrics #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3ed476a
feat: Add R2 metrics
Cyb3r-Jak3 Aug 1, 2024
26f0536
docs: Add new metrics to README
Cyb3r-Jak3 Aug 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ Required authentication scopes:

To authenticate this way, only set `CF_API_TOKEN` (omit `CF_API_EMAIL` and `CF_API_KEY`)

[Shortcut to create the API token](https://dash.cloudflare.com/profile/api-tokens?permissionGroupKeys=%5B%7B%22key%22%3A%22account_analytics%22%2C%22type%22%3A%22read%22%7D%2C%7B%22key%22%3A%22account_settings%22%2C%22type%22%3A%22read%22%7D%2C%7B%22key%22%3A%22analytics%22%2C%22type%22%3A%22read%22%7D%2C%7B%22key%22%3A%22firewall_services%22%2C%22type%22%3A%22read%22%7D%5D&name=Cloudflare+Exporter&accountId=*&zoneId=all)

### User email + API key
To authenticate with user email + API key, use the `Global API Key` from the Cloudflare dashboard.
Beware that this key authenticates with write access to every Cloudflare resource.
Expand Down Expand Up @@ -104,6 +106,9 @@ Note: `ZONE_<name>` configuration is not supported as flag.
# HELP cloudflare_zone_pool_requests_total Requests per pool
# HELP cloudflare_logpush_failed_jobs_account_count Number of failed logpush jobs on the account level
# HELP cloudflare_logpush_failed_jobs_zone_count Number of failed logpush jobs on the zone level
# HELP cloudflare_r2_operation_count Number of operations performed by R2
# HELP cloudflare_r2_storage_bytes Storage used by R2
# HELP cloudflare_r2_storage_total_bytes Total storage used by R2
```

## Helm chart repository
Expand Down
152 changes: 103 additions & 49 deletions cloudflare.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ import (
"strings"
"time"

cloudflare "github.com/cloudflare/cloudflare-go"
"github.com/cloudflare/cloudflare-go"
"github.com/machinebox/graphql"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)

var (
const (
cfGraphQLEndpoint = "https://api.cloudflare.com/client/v4/graphql/"
)

Expand Down Expand Up @@ -45,6 +45,35 @@ type cloudflareResponseLogpushAccount struct {
} `json:"viewer"`
}

type r2AccountResp struct {
R2StorageGroups []struct {
Dimensions struct {
BucketName string `json:"bucketName"`
} `json:"dimensions"`
Max struct {
MetadataSize uint64 `json:"metadataSize"`
PayloadSize uint64 `json:"payloadSize"`
ObjectCount uint64 `json:"objectCount"`
} `json:"max"`
} `json:"r2StorageAdaptiveGroups"`

R2StorageOperations []struct {
Dimensions struct {
Action string `json:"actionType"`
BucketName string `json:"bucketName"`
} `json:"dimensions"`
Sum struct {
Requests uint64 `json:"requests"`
} `json:"sum"`
} `json:"r2OperationsAdaptiveGroups"`
}

type cloudflareResponseR2Account struct {
Viewer struct {
Accounts []r2AccountResp `json:"accounts"`
}
}

type cloudflareResponseLogpushZone struct {
Viewer struct {
Zones []logpushResponse `json:"zones"`
Expand Down Expand Up @@ -251,60 +280,38 @@ type lbResp struct {
}

func fetchZones() []cloudflare.Zone {
var api *cloudflare.API
var err error
if len(viper.GetString("cf_api_token")) > 0 {
api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token"))
} else {
api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email"))
}
if err != nil {
log.Fatal(err)
}

ctx := context.Background()
z, err := api.ListZones(ctx)
z, err := cloudflareAPI.ListZones(ctx)
if err != nil {
log.Fatal(err)
log.Fatalf("Error fetching zones: %s", err)
}

return z
}

func fetchFirewallRules(zoneID string) map[string]string {
var api *cloudflare.API
var err error
if len(viper.GetString("cf_api_token")) > 0 {
api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token"))
} else {
api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email"))
}
if err != nil {
log.Fatal(err)
}

ctx := context.Background()
listOfRules, _, err := api.FirewallRules(ctx,
listOfRules, _, err := cloudflareAPI.FirewallRules(ctx,
cloudflare.ZoneIdentifier(zoneID),
cloudflare.FirewallRuleListParams{})
if err != nil {
log.Fatal(err)
log.Fatalf("Error fetching firewall rules: %s", err)
}
firewallRulesMap := make(map[string]string)

for _, rule := range listOfRules {
firewallRulesMap[rule.ID] = rule.Description
}

listOfRulesets, err := api.ListRulesets(ctx, cloudflare.ZoneIdentifier(zoneID), cloudflare.ListRulesetsParams{})
listOfRulesets, err := cloudflareAPI.ListRulesets(ctx, cloudflare.ZoneIdentifier(zoneID), cloudflare.ListRulesetsParams{})
if err != nil {
log.Fatal(err)
log.Fatalf("Error listing rulesets: %s", err)
}
for _, rulesetDesc := range listOfRulesets {
if rulesetDesc.Phase == "http_request_firewall_managed" {
ruleset, err := api.GetRuleset(ctx, cloudflare.ZoneIdentifier(zoneID), rulesetDesc.ID)
ruleset, err := cloudflareAPI.GetRuleset(ctx, cloudflare.ZoneIdentifier(zoneID), rulesetDesc.ID)
if err != nil {
log.Fatal(err)
log.Fatalf("Error fetching ruleset: %s", err)
}
for _, rule := range ruleset.Rules {
firewallRulesMap[rule.ID] = rule.Description
Expand All @@ -316,21 +323,10 @@ func fetchFirewallRules(zoneID string) map[string]string {
}

func fetchAccounts() []cloudflare.Account {
var api *cloudflare.API
var err error
if len(viper.GetString("cf_api_token")) > 0 {
api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token"))
} else {
api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email"))
}
if err != nil {
log.Fatal(err)
}

ctx := context.Background()
a, _, err := api.Accounts(ctx, cloudflare.AccountsListParams{PaginationOptions: cloudflare.PaginationOptions{PerPage: 100}})
a, _, err := cloudflareAPI.Accounts(ctx, cloudflare.AccountsListParams{PaginationOptions: cloudflare.PaginationOptions{PerPage: 100}})
if err != nil {
log.Fatal(err)
log.Fatalf("Error fetching accounts: %s", err)
}

return a
Expand Down Expand Up @@ -570,7 +566,7 @@ func fetchWorkerTotals(accountID string) (*cloudflareResponseAccts, error) {
graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
var resp cloudflareResponseAccts
if err := graphqlClient.Run(ctx, request, &resp); err != nil {
log.Error(err)
log.Errorf("Error fetching worker totals: %s", err)
return nil, err
}

Expand Down Expand Up @@ -647,7 +643,7 @@ func fetchLoadBalancerTotals(zoneIDs []string) (*cloudflareResponseLb, error) {
graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
var resp cloudflareResponseLb
if err := graphqlClient.Run(ctx, request, &resp); err != nil {
log.Error(err)
log.Errorf("Error fetching load balancer totals: %s", err)
return nil, err
}
return &resp, nil
Expand Down Expand Up @@ -699,7 +695,7 @@ func fetchLogpushAccount(accountID string) (*cloudflareResponseLogpushAccount, e
graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
var resp cloudflareResponseLogpushAccount
if err := graphqlClient.Run(ctx, request, &resp); err != nil {
log.Error(err)
log.Errorf("Error fetching logpush account totals: %s", err)
return nil, err
}
return &resp, nil
Expand Down Expand Up @@ -751,13 +747,71 @@ func fetchLogpushZone(zoneIDs []string) (*cloudflareResponseLogpushZone, error)
graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
var resp cloudflareResponseLogpushZone
if err := graphqlClient.Run(ctx, request, &resp); err != nil {
log.Error(err)
log.Errorf("Error fetching logpush zone totals: %s", err)
return nil, err
}

return &resp, nil
}

func fetchR2Account(accountID string) (*cloudflareResponseR2Account, error) {
now := time.Now().Add(-time.Duration(viper.GetInt("scrape_delay")) * time.Second).UTC()
s := 60 * time.Second
now = now.Truncate(s)

request := graphql.NewRequest(`query($accountID: String!, $limit: Int!, $date: String!) {
viewer {
accounts(filter: {accountTag : $accountID }) {
r2StorageAdaptiveGroups(
filter: {
date: $date
},
limit: $limit
) {
dimensions {
bucketName
}
max {
metadataSize
payloadSize
objectCount
}
}
r2OperationsAdaptiveGroups(filter: { date: $date }, limit: $limit) {
dimensions {
actionType
bucketName
}
sum {
requests
}
}
}
}
}`)

if len(viper.GetString("cf_api_token")) > 0 {
request.Header.Set("Authorization", "Bearer "+viper.GetString("cf_api_token"))
} else {
request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email"))
request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key"))
}

request.Var("accountID", accountID)
request.Var("limit", 9999)
request.Var("date", now.Format("2006-01-02"))

ctx := context.Background()
graphqlClient := graphql.NewClient(cfGraphQLEndpoint)
graphqlClient.Log = func(s string) { log.Debug(s) }
var resp cloudflareResponseR2Account
if err := graphqlClient.Run(ctx, request, &resp); err != nil {
log.Errorf("Error fetching R2 account: %s", err)
return nil, err
}
return &resp, nil
}

func findZoneAccountName(zones []cloudflare.Zone, ID string) (string, string) {
for _, z := range zones {
if z.ID == ID {
Expand Down
Loading