Skip to content

Commit

Permalink
container/libcontainer: Improve limits file parsing perf
Browse files Browse the repository at this point in the history
Only max open files is ever parsed from limits files, therefore this
change optimizes for that case.

Benchmark:

```
$ benchstat old.txt new.txt
goos: linux
goarch: amd64
pkg: github.com/google/cadvisor/container/libcontainer
cpu: AMD Ryzen 5 3400GE with Radeon Vega Graphics
                    │   old.txt    │               new.txt               │
                    │    sec/op    │   sec/op     vs base                │
ProcessLimitsFile-8   85.012µ ± 1%   1.324µ ± 0%  -98.44% (p=0.000 n=10)
```

On a GKE v1.27.4 production cluster, this code path used roughly 1.5% of
the total kubelet CPU usage, and at 98.44% improvement this likely
results in at least a 1.5% CPU reduction (perhaps even more since also
less garbage is produced to be collected by the GC).
  • Loading branch information
brancz committed Sep 15, 2023
1 parent f76ba48 commit 4c90b90
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 35 deletions.
71 changes: 36 additions & 35 deletions container/libcontainer/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ import (
)

var (
whitelistedUlimits = [...]string{"max_open_files"}
referencedResetInterval = flag.Uint64("referenced_reset_interval", 0,
"Reset interval for referenced bytes (container_referenced_bytes metric), number of measurement cycles after which referenced bytes are cleared, if set to 0 referenced bytes are never cleared (default: 0)")

Expand Down Expand Up @@ -205,51 +204,53 @@ func parseUlimit(value string) (int64, error) {
return num, nil
}

func isUlimitWhitelisted(name string) bool {
for _, whitelist := range whitelistedUlimits {
if name == whitelist {
return true
}
}
return false
}

func processLimitsFile(fileData string) []info.UlimitSpec {
const maxOpenFilesLinePrefix = "Max open files"

limits := strings.Split(fileData, "\n")
ulimits := make([]info.UlimitSpec, 0, len(limits))
for _, lim := range limits {
// Skip any headers/footers
if strings.HasPrefix(lim, "Max") {

// Line format: Max open files 16384 16384 files
fields := regexp.MustCompile(`[\s]{2,}`).Split(lim, -1)
name := strings.Replace(strings.ToLower(strings.TrimSpace(fields[0])), " ", "_", -1)

found := isUlimitWhitelisted(name)
if !found {
continue
}

soft := strings.TrimSpace(fields[1])
softNum, softErr := parseUlimit(soft)

hard := strings.TrimSpace(fields[2])
hardNum, hardErr := parseUlimit(hard)

// Omit metric if there were any parsing errors
if softErr == nil && hardErr == nil {
ulimitSpec := info.UlimitSpec{
Name: name,
SoftLimit: int64(softNum),
HardLimit: int64(hardNum),
}
ulimits = append(ulimits, ulimitSpec)
if strings.HasPrefix(lim, "Max open files") {
// Remove line prefix
ulimit, err := processMaxOpenFileLimitLine(
"max_open_files",
lim[len(maxOpenFilesLinePrefix):],
)
if err == nil {
ulimits = append(ulimits, ulimit)
}
}
}
return ulimits
}

// Any caller of processMaxOpenFileLimitLine must ensure that the name prefix is already removed from the limit line.
// with the "Max open files" prefix.
func processMaxOpenFileLimitLine(name, line string) (info.UlimitSpec, error) {
// Remove any leading whitespace
line = strings.TrimSpace(line)
// Split on whitespace
fields := strings.Fields(line)
if len(fields) != 3 {
return info.UlimitSpec{}, fmt.Errorf("unable to parse max open files line: %s", line)
}
// The first field is the soft limit, the second is the hard limit
soft, err := parseUlimit(fields[0])
if err != nil {
return info.UlimitSpec{}, err
}
hard, err := parseUlimit(fields[1])
if err != nil {
return info.UlimitSpec{}, err
}
return info.UlimitSpec{
Name: name,
SoftLimit: soft,
HardLimit: hard,
}, nil
}

func processRootProcUlimits(rootFs string, rootPid int) []info.UlimitSpec {
filePath := path.Join(rootFs, "/proc", strconv.Itoa(rootPid), "limits")
out, err := os.ReadFile(filePath)
Expand Down
10 changes: 10 additions & 0 deletions container/libcontainer/handler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -311,3 +311,13 @@ func BenchmarkProcessLimitsFile(b *testing.B) {
// Ensure the compiler doesn't optimize away the benchmark
_ = ulimits
}

func TestProcessMaxOpenFileLimitLine(t *testing.T) {
line := " 1073741816 1073741816 files "

ulimit, err := processMaxOpenFileLimitLine("max_open_files", line)
assert.Nil(t, err)
assert.Equal(t, "max_open_files", ulimit.Name)
assert.Equal(t, int64(1073741816), ulimit.SoftLimit)
assert.Equal(t, int64(1073741816), ulimit.HardLimit)
}

0 comments on commit 4c90b90

Please sign in to comment.