From 27586424f8283659502234d67e6d36e8fcc1ed11 Mon Sep 17 00:00:00 2001 From: Glenn Horton-Smith Date: Wed, 20 Jun 2018 11:48:37 -0500 Subject: [PATCH] Initial commit of the working code, and expansion of README.md. --- .gitignore | 3 ++ README.md | 26 ++++++++++ linux_memavailable.py | 108 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 .gitignore create mode 100644 linux_memavailable.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9a57116 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.pyc +*.py~ +__pycache__ diff --git a/README.md b/README.md index 7d511df..ad1c91f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,28 @@ # linux-memavailable-python A python backport of the /proc/meminfo "MemAvailable" metric which was introduced in the Linux 3.14 kernel. + +This module provides a single python function that uses information +available to users in pre-3.14 kernels to calculate the "MemAvailable" +metric which was introduced in Linux 3.14 kernels. + +For a simple command line test, you can do this: + python -c 'import linux_memavailable; print(linux_memavailable.linux_memavailable())' + +Linus Torvalds' explanation of the algorithm and the C code can be +found here: + https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773 + +A more conservative estimate was later implemented: + https://github.com/torvalds/linux/commit/84ad5802a33a4964a49b8f7d24d80a214a096b19 + +This implementation implements exactly the algorithm above. + +Note: in the very latest kernel source as of 2018-06-20, the algorithm is +almost unchanged except for addition of some amount of memory +considered releasable by the (latest) kernel: + https://github.com/torvalds/linux/blob/b5d903c2d656e9bc54bc76554a477d796a63120d/mm/page_alloc.c#L4701 + +This implementation omits that extra bit of kernel-releasable memory. + +This python implementation owes much to the Perl implementation by Ivan +Zahariev (famzah): https://github.com/famzah/linux-memavailable-procfs diff --git a/linux_memavailable.py b/linux_memavailable.py new file mode 100644 index 0000000..535223b --- /dev/null +++ b/linux_memavailable.py @@ -0,0 +1,108 @@ +"""A python function that uses statistics available to users in +pre-3.14 kernels to calculate the "MemAvailable" metric which was +introduced in Linux 3.14 kernels. Linus Torvalds' explanation of the +algorithm and the C code can be found here: + + https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773 + +A more conservative estimate was later implemented: + https://github.com/torvalds/linux/commit/84ad5802a33a4964a49b8f7d24d80a214a096b19 + +This implementation implements exactly the algorithm above. + +Note: in the very latest kernel source as of 2018-06-20, the algorithm is +almost unchanged except for addition of some amount of memory +considered releasable by the (latest) kernel: +https://github.com/torvalds/linux/blob/b5d903c2d656e9bc54bc76554a477d796a63120d/mm/page_alloc.c#L4701 + +This implementation omits that extra bit of kernel-releasable memory. + +This python implementation owes much to the Perl implementation by Ivan +Zahariev (famzah): https://github.com/famzah/linux-memavailable-procfs +""" + +__author__ = "Glenn Horton-Smith" + + +from mmap import PAGESIZE + +def linux_memavailable(): + """Estimate the amount of memory available for userspace allocations, + without causing swapping. Pure python implementation of the Linux + algorithm introduced in the 3.14 kernel. + """ + #-- find the sum of all reserved low watermarks for all zones + wmark_low, totalreserve = calc_wmark_low_and_totalreserve_pages() + wmark_low = (wmark_low * PAGESIZE) // 1024 + totalreserve = (totalreserve * PAGESIZE) // 1024 + #-- get the MemFree, Active(file), Inactive(file), and SReclaimable values + meminfo = {"MemFree" : 0, + "Active(file)" : 0, + "Inactive(file)" : 0, + "SReclaimable" : 0} + for line in open("/proc/meminfo"): + info = line.split(None, 2) + if len(info) == 3: + key = info[0][:-1] + if key in meminfo: + meminfo[key] = int(info[1]) + #-- Start with "MemFree" minus low watermark + # "Free memory cannot be taken below the low watermark, before the + # system starts swapping." + available = meminfo["MemFree"] - totalreserve + #-- Add in the reclaimable page cache + # "Not all the page cache can be freed, otherwise the system will + # start swapping. Assume at least half of the page cache, or the + # low watermark worth of cache, needs to stay." + pagecache = meminfo["Active(file)"] + meminfo["Inactive(file)"] + pagecache -= min(pagecache//2, wmark_low) + available += pagecache + #-- Add in reclaimable swap ("SReclaimable") + # "Part of the reclaimable swap consists of items that are in use, + # and cannot be freed. Cap this estimate at the low watermark." + slab_reclaimable = meminfo["SReclaimable"] + available += slab_reclaimable - min(slab_reclaimable//2, wmark_low) + #-- Make non-negative for compatibility + if available < 0: + available = 0 + return available + + +def calc_wmark_low_and_totalreserve_pages(): + """Calculate sum of low watermarks and total reserved space over all + zones, and return those two values. Values are in pages. + """ + fzoneinfo = open("/proc/zoneinfo") + wmark_low = 0 + totalreserve = 0 + keep_reading = True + while keep_reading: + keep_reading = False + managed = 0 + high = 0 + max_lowmem_reserve = 0 + #-- loop over lines in zone + for line in fzoneinfo: + #-- new zone encountered if line starts with "Node" + if line.startswith("Node"): + keep_reading = True + break + info = line.split(None, 1) + if len(info) >= 2: + if info[0] == 'low': + wmark_low += int(info[1]) + elif info[0] == 'high': + high = int(info[1]) + elif info[0] == 'managed': + managed = int(info[1]) + elif info[0] == 'protection:': + lowmem_reserve = list( + int(l) for l in info[1].strip('()\n ').split(',')) + max_lowmem_reserve = max(lowmem_reserve) + #-- calculate reserved pages for this zone according to algorithm + # https://github.com/torvalds/linux/blob/6aa303defb7454a2520c4ddcdf6b081f62a15890/mm/page_alloc.c#L6559 + reserve = max_lowmem_reserve + high + if reserve > managed: + reserve = managed + totalreserve += reserve + return wmark_low, totalreserve