UVM: basic implementation of amap

include/aarch64/vm_param.h

@@ -3,6 +3,7 @@
 
 #define PAGESIZE 4096
 #define SUPERPAGESIZE (1 << 21) /* 2 MB */
+#define PAGE_SHIFT 12
 
 #define KERNEL_SPACE_BEGIN 0xffff000000000000L
 #define KERNEL_SPACE_END 0xffffffffffffffffL

include/mips/vm_param.h

@@ -12,6 +12,7 @@
 
 #define PAGESIZE 4096
 #define SUPERPAGESIZE (1 << 22) /* 4 MB */
+#define PAGE_SHIFT 12
 
 #define VM_PHYSSEG_NMAX 16
 

include/sys/uvm_amap.h

@@ -0,0 +1,61 @@
+#ifndef _SYS_UVM_AMAP_H_
+#define _SYS_UVM_AMAP_H_
+
+#ifndef _KERNEL
+#error "Do not use this header file outside kernel code!"
+#endif
+
+#include <sys/mutex.h>
+
+typedef struct uvm_anon uvm_anon_t;
+typedef struct uvm_aref uvm_aref_t;
+typedef struct uvm_amap uvm_amap_t;
+
+struct uvm_aref {
+  int ar_pageoff;      /* page offset into amap we start */
+  uvm_amap_t *ar_amap; /* pointer to amap */
+};
+
+/*
+ * The upper layer of UVM’s two-layer mapping scheme.
+ * A uvm_amap describes an area of anonymous memory.
+ *
+ * Field markings and the corresponding locks:
+ *  (@) uvm_amap::am_lock
+ */
+struct uvm_amap {
+  mtx_t am_lock;
+  int am_ref;     /* (@) reference counter */
+  int am_nslot;   /* (@) number of allocated slots */
+  int am_nused;   /* (@) number of used slots */
+  int *am_slot;   /* (@) slots of used anons - refers to am_bckptr */
+  int *am_bckptr; /* (@) stack of used anons - refers to am_anon & am_slots */
+  uvm_anon_t **am_anon; /* (@) anons in that map */
+};
+
+/*
+ * amap interface
+ */
+
+/* Allocate a new amap. */
+uvm_amap_t *uvm_amap_alloc(void);
+/* Acquire amap->am_lock. */
+void uvm_amap_lock(uvm_amap_t *amap);
+/* Release amap->am_lock. */
+void uvm_amap_unlock(uvm_amap_t *amap);
+/* Increase reference counter. */
+void uvm_amap_hold(uvm_amap_t *amap);
+/* Decrement reference counter and destroy amap if it has reached 0. */
+void uvm_amap_drop(uvm_amap_t *amap);
+/* Lookup an anon at offset in amap. */
+uvm_anon_t *uvm_amap_lookup(uvm_aref_t *aref, vaddr_t offset);
+/* Add an annon at offset in amap. */
+void uvm_amap_add(uvm_aref_t *aref, uvm_anon_t *anon, vaddr_t offset);
+/* Remove an annon from an amap. */
+void uvm_amap_remove(uvm_aref_t *aref, vaddr_t offset);
+
+/* TODO: ppref */
+
+/* TODO: aref interface */
+
+#endif /* !_SYS_UVM_AMAP_H_ */

sys/kern/Makefile

@@ -68,6 +68,7 @@ SOURCES = \
 	uart_tty.c \
 	uio.c \
 	ustack.c \
+	uvm_amap.c \
 	vfs.c \
 	vfs_name.c \
 	vfs_readdir.c \

sys/kern/uvm_amap.c

@@ -0,0 +1,158 @@
+#define KL_LOG KL_VM
+#include <sys/klog.h>
+#include <sys/mimiker.h>
+#include <sys/libkern.h>
+#include <sys/pool.h>
+#include <sys/malloc.h>
+#include <sys/uvm_amap.h>
+
+#define AMAP_SLOT_EMPTY (-1)
+
+static POOL_DEFINE(P_AMAP, "uvm_amap", sizeof(uvm_amap_t));
+
+static inline int uvm_amap_slot(uvm_aref_t *aref, vaddr_t offset) {
+  assert(offset % PAGESIZE == 0);
+  return (int)(aref->ar_pageoff + (offset >> PAGE_SHIFT));
+}
+
+uvm_amap_t *uvm_amap_alloc(void) {
+  uvm_amap_t *amap = pool_alloc(P_AMAP, M_ZERO);
+  mtx_init(&amap->am_lock, 0);
+  return amap;
+}
+
+void uvm_amap_lock(uvm_amap_t *amap) {
+  mtx_lock(&amap->am_lock);
+}
+
+void uvm_amap_unlock(uvm_amap_t *amap) {
+  mtx_unlock(&amap->am_lock);
+}
+
+void uvm_amap_hold(uvm_amap_t *amap) {
+  assert(mtx_owned(&amap->am_lock));
+  amap->am_ref++;
+}
+
+/* TODO: revisit after uvm_anon implementation. */
+static void uvm_anon_free(uvm_anon_t *anon) {
+  assert(anon != NULL);
+}
+
+static void uvm_amap_free(uvm_amap_t *amap) {
+  mtx_unlock(&amap->am_lock);
+
+  for (int i = 0; i < amap->am_nused; ++i) {
+    int slot = amap->am_bckptr[i];
+    uvm_anon_free(amap->am_anon[slot]);
+  }
+
+  kfree(M_TEMP, amap->am_slot);
+  kfree(M_TEMP, amap->am_bckptr);
+  kfree(M_TEMP, amap->am_anon);
+
+  pool_free(P_AMAP, amap);
+}
+
+void uvm_amap_drop(uvm_amap_t *amap) {
+  assert(mtx_owned(&amap->am_lock));
+  amap->am_ref--;
+
+  if (amap->am_ref == 0)
+    uvm_amap_free(amap);
+}
+
+static uvm_anon_t *uvm_amap_lookup_nolock(uvm_amap_t *amap, int slot) {
+  if (amap->am_nslot <= slot)
+    return NULL;
+  return amap->am_anon[slot];
+}
+
+uvm_anon_t *uvm_amap_lookup(uvm_aref_t *aref, vaddr_t offset) {
+  uvm_amap_t *amap = aref->ar_amap;
+  int slot = uvm_amap_slot(aref, offset);
+
+  SCOPED_MTX_LOCK(&amap->am_lock);
+  return uvm_amap_lookup_nolock(amap, slot);
+}
+
+static void __uvm_amap_extend(uvm_amap_t *amap, int size) {
+  /* new entries in slot & bckptr will be set to AMAP_SLOT_EMPTY */
+  int *slot = kmalloc(M_TEMP, sizeof(int) * size, 0);
+  int *bckptr = kmalloc(M_TEMP, sizeof(int) * size, 0);
+  /* new entries in anon will be set to NULL */
+  uvm_anon_t **anon = kmalloc(M_TEMP, sizeof(uvm_anon_t *) * size, M_ZERO);
+
+  /* old unused entires will be set to AMAP_SLOT_EMPTY */
+  memcpy(slot, amap->am_slot, sizeof(int) * amap->am_nslot);
+  memcpy(bckptr, amap->am_bckptr, sizeof(int) * amap->am_nslot);
+
+  /* copy pointers only for allocated anons */
+  for (int i = 0; i < amap->am_nused; ++i) {
+    int bslot = amap->am_bckptr[i];
+    anon[bslot] = amap->am_anon[bslot];
+  }
+
+  /* set new entries to AMAP_SLOT_EMPTY */
+  for (int i = amap->am_nslot; i < size; ++i) {
+    slot[i] = AMAP_SLOT_EMPTY;
+    bckptr[i] = AMAP_SLOT_EMPTY;
+  }
+
+  kfree(M_TEMP, amap->am_slot);
+  kfree(M_TEMP, amap->am_bckptr);
+  kfree(M_TEMP, amap->am_anon);
+
+  amap->am_slot = slot;
+  amap->am_bckptr = bckptr;
+  amap->am_anon = anon;
+
+  amap->am_nslot = size;
+}
+
+static void uvm_amap_extend(uvm_amap_t *amap, int size) {
+  int capacity = max(amap->am_nslot, 1);
+  while (capacity < size)
+    capacity *= 2;
+  __uvm_amap_extend(amap, capacity);
+}
+
+static void uvm_amap_add_nolock(uvm_amap_t *amap, uvm_anon_t *anon, int slot) {
+  if (slot >= amap->am_nslot)
+    uvm_amap_extend(amap, slot);
+
+  assert(amap->am_anon[slot] == NULL);
+  amap->am_anon[slot] = anon;
+  amap->am_slot[slot] = amap->am_nused;
+  amap->am_bckptr[amap->am_nused++] = slot;
+}
+
+void uvm_amap_add(uvm_aref_t *aref, uvm_anon_t *anon, vaddr_t offset) {
+  uvm_amap_t *amap = aref->ar_amap;
+  int slot = uvm_amap_slot(aref, offset);
+
+  SCOPED_MTX_LOCK(&amap->am_lock);
+  uvm_amap_add_nolock(amap, anon, slot);
+}
+
+static void uvm_amap_remove_nolock(uvm_amap_t *amap, int slot) {
+  assert(amap->am_nslot > slot);
+  assert(amap->am_anon[slot] != NULL);
+
+  uvm_anon_free(amap->am_anon[slot]);
+  amap->am_anon[slot] = NULL;
+
+  int bslot = amap->am_slot[slot];
+  amap->am_slot[slot] = AMAP_SLOT_EMPTY;
+  amap->am_bckptr[bslot] = AMAP_SLOT_EMPTY;
+  swap(amap->am_bckptr[bslot], amap->am_bckptr[amap->am_nused - 1]);
+  amap->am_nused--;
+}
+
+void uvm_amap_remove(uvm_aref_t *aref, vaddr_t offset) {
+  uvm_amap_t *amap = aref->ar_amap;
+  int slot = uvm_amap_slot(aref, offset);
+
+  SCOPED_MTX_LOCK(&amap->am_lock);
+  uvm_amap_remove_nolock(amap, slot);
+}

sys/tests/Makefile

@@ -30,6 +30,7 @@ SOURCES = \
 	turnstile_propagate_many.c \
 	uiomove.c \
 	utest.c \
+	uvm_amap.c \
 	vm_map.c \
 	devclass.c \
 	vfs.c \

sys/tests/uvm_amap.c

@@ -0,0 +1,56 @@
+#include <sys/klog.h>
+#include <sys/mimiker.h>
+#include <sys/libkern.h>
+#include <sys/uvm_amap.h>
+#include <machine/vm_param.h>
+#include <sys/ktest.h>
+
+/* Test internal state of amap. */
+static int test_amap_simple(void) {
+  uvm_amap_t *amap = uvm_amap_alloc();
+  assert(amap != NULL);
+
+  uvm_aref_t aref = {.ar_pageoff = 0, .ar_amap = amap};
+
+  uvm_amap_add(&aref, (uvm_anon_t *)1, 3 * PAGESIZE);
+  uvm_amap_add(&aref, (uvm_anon_t *)2, 7 * PAGESIZE);
+  uvm_amap_add(&aref, (uvm_anon_t *)3, 2 * PAGESIZE);
+  uvm_amap_add(&aref, (uvm_anon_t *)4, 14 * PAGESIZE);
+
+  uvm_amap_lock(amap);
+  assert(amap->am_nslot == 16);
+  assert(amap->am_nused == 4);
+
+  assert(amap->am_anon[3] == (uvm_anon_t *)1);
+  assert(amap->am_anon[7] == (uvm_anon_t *)2);
+  assert(amap->am_anon[2] == (uvm_anon_t *)3);
+  assert(amap->am_anon[14] == (uvm_anon_t *)4);
+
+  assert(amap->am_bckptr[0] == 3);
+  assert(amap->am_bckptr[1] == 7);
+  assert(amap->am_bckptr[2] == 2);
+  assert(amap->am_bckptr[3] == 14);
+
+  assert(amap->am_slot[3] == 0);
+  assert(amap->am_slot[7] == 1);
+  assert(amap->am_slot[2] == 2);
+  assert(amap->am_slot[14] == 3);
+  uvm_amap_unlock(amap);
+
+  assert(uvm_amap_lookup(&aref, 7 * PAGESIZE) == (uvm_anon_t *)2);
+  uvm_amap_remove(&aref, 7 * PAGESIZE);
+  assert(amap->am_nused == 3);
+  assert(uvm_amap_lookup(&aref, 7 * PAGESIZE) == NULL);
+
+  uvm_amap_lock(amap);
+  assert(amap->am_slot[7] == -1);
+  assert(amap->am_bckptr[1] == 14);
+  assert(amap->am_bckptr[3] == -1);
+  assert(amap->am_anon[7] == NULL);
+
+  uvm_amap_drop(amap);
+
+  return KTEST_SUCCESS;
+}
+
+KTEST_ADD(uvm_amap_simple, test_amap_simple, 0);