Merge remote-tracking branch 'origin/develop'

.github/workflows/release.yml

@@ -86,8 +86,8 @@ jobs:
         include:
           - runner: [self-hosted, linux, normal]
             os: ubuntu-24.04
-          - runner: MacM1
-            os: self-macos-12
+          - runner: [self-hosted, self-macos-latest]
+            os: self-macos-latest
 
     runs-on: ${{ matrix.runner }}
     steps:

.github/workflows/test.yml

@@ -17,8 +17,8 @@ jobs:
         include:
           - runner: [self-hosted, linux, normal]
             os: ubuntu-24.04
-          - runner: MacM1
-            os: self-macos-12
+          - runner: [self-hosted, self-macos-latest]
+            os: self-macos-latest
     runs-on: ${{ matrix.runner }}
     steps:
       - name: 'Check out code'

INSTALL.md

@@ -30,7 +30,7 @@ python3 -m pip install pybind11 lit
 ```
 
 ## macOS / Homebrew
-
+In order to install the dependencies on macOS, you must have Homebrew installed and on your `PATH`.
 ```shell
 brew update
 brew install  \
@@ -40,19 +40,70 @@ brew install  \
   fmt         \
   git         \
   gmp         \
+  grep        \
   jemalloc    \
+  libffi
   libyaml     \
   llvm@15     \
   maven       \
   mpfr        \
   pkg-config  \
   python3     \
   z3
+```
+
+To ensure that the backend can use pybind11 correctly, we must create an virtual
+environment and install the `pybind11` package:
+```shell
+python3 -m venv venv
+source venv/bin/activate
 python3 -m pip install pybind11 lit
 ```
 
+Guarantee that you have the JDK installed and on your `PATH`:
+```shell
+export PATH="/opt/homebrew/opt/openjdk/bin:$PATH"
+```
+
+Some tests rely on GNU Grep options, which are not available on macOS by
+default. To ensure that the tests run correctly, you add the path of
+GNU Grep to your `PATH` in your shell profile:
+```shell
+export PATH=/opt/homebrew/Cellar/grep/3.11/libexec/gnubin/:$PATH
+```
+
 # Building
 
+## Environment Variables
+
+If you're building on macOS, type the following command or epermanently
+add it your `env` (`.zshrc`, `.bashrc`, etc.), so that the Homebrew
+installation of LLVM gets picked up correctly. We recommend adding it to
+your shell profile.
+```shell
+export LLVM_DIR=$($(brew --prefix llvm@15)/bin/llvm-config --cmakedir)
+```
+
+If you don't usually use the `clang` from your Homebrew installation as
+your default compiler, you can set the following CMake flg to use these
+`clang` and `clang++`:
+```shell
+-DCMAKE_C_COMPILER="$(brew --prefix llvm@15)/bin/clang" \
+-DCMAKE_CXX_COMPILER="$(brew --prefix llvm@15)/bin/clang++"
+```
+Once again, we recommend adding them and other llvm binaries to your
+`PATH` in your shell profile:
+```shell
+export PATH="$(brew --prefix llvm@15)/bin:$PATH"
+```
+
+Some tests rely on GNU Grep options, which are not available on macOS by
+default. To ensure that the tests run correctly, you can create an alias
+for GNU Grep:
+```shell
+alias grep=ggrep
+```
+
 Once the system dependencies have been installed, the backend can be built
 locally with:
 ```shell
@@ -66,12 +117,6 @@ cmake .. \
 make -j$(nproc) install
 ```
 
-If you're building on macOS, add the following option to your CMake invocation
-so that the Homebrew installation of LLVM gets picked up correctly.
-```shell
--DLLVM_DIR=$($(brew --prefix llvm@15)/bin/llvm-config --cmakedir)
-```
-
 Additionally, to build the pattern-matching compiler, run:
 ```shell
 cd matching
@@ -91,7 +136,8 @@ To run the integration tests, run:
 ```shell
 lit test
 ```
-from the root source directory.
+from the root source directory. You can use `-v` to see which test is being executed
+and the output of failling tests.
 
 There is also a unit test suite for backend internals;
 Add the following option to your CMake invocation to enable it:
@@ -113,7 +159,7 @@ and conform to best practices.
 
 ```shell
 # Ubuntu
-apt install shellcheck clang-format-12 iwyu
+apt install shellcheck clang-format-15 iwyu
 
 # macOS
 brew install shellcheck clang-format iwyu

cmake/FixHomebrew.cmake

@@ -20,5 +20,15 @@ if(APPLE)
     include_directories(AFTER SYSTEM "${BREW_PREFIX}/include")
     link_directories(AFTER "${BREW_PREFIX}/lib")
     set(ENV{PKG_CONFIG_PATH} "${BREW_PREFIX}/opt/libffi/lib/pkgconfig")
+
+    # Use LLD as the linker
+    # This is necessary as the default linker used by CMake on macOS is
+    # ld64, which currently has some incompatibilities with Homebrew and XCode15.
+    # See: https://github.com/orgs/Homebrew/discussions/4794#discussioncomment-7044468
+    # Adding this flag avoid the following errors:
+    # ld: warning: duplicate -rpath ... ignored
+    # ld: warning: ignoring duplicate libraries ...
+    add_link_options("-fuse-ld=lld")
+
   endif() # USE_NIX
 endif() # APPLE

include/runtime/alloc.h

@@ -15,7 +15,6 @@ extern "C" {
 
 char youngspace_collection_id(void);
 char oldspace_collection_id(void);
-size_t youngspace_size(void);
 
 // allocates exactly requested bytes into the young generation
 void *kore_alloc(size_t requested);

include/runtime/arena.h

@@ -22,37 +22,28 @@ class arena {
     initialize_semispace();
   }
 
+  char *evacuate(char *scan_ptr);
+
   // Allocates the requested number of bytes as a contiguous region and returns a
   // pointer to the first allocated byte.
   void *kore_arena_alloc(size_t requested);
 
   // Returns the address of the first byte that belongs in the given arena.
-  // Returns 0 if nothing has been allocated ever in that arena.
-  char *arena_start_ptr() const {
-    return current_addr_ptr ? current_addr_ptr + sizeof(memory_block_header)
-                            : nullptr;
-  }
+  // Returns nullptr if nothing has been allocated ever in that arena.
+  char *arena_start_ptr() const { return current_addr_ptr; }
 
   // Returns a pointer to a location holding the address of last allocated
   // byte in the given arena plus 1.
-  // This address is 0 if nothing has been allocated ever in that arena.
-  char **arena_end_ptr() { return &allocation_ptr; }
-
-  // return the total number of allocatable bytes currently in the arena in its
-  // active semispace.
-  size_t arena_size() const {
-    update_num_blocks();
-    return BLOCK_SIZE * std::max(num_blocks, num_collection_blocks);
-  }
+  // This address is nullptr if nothing has been allocated ever in that arena.
+  char *arena_end_ptr() { return allocation_ptr; }
 
   // Clears the current allocation space by setting its start back to its first
   // block. It is used during garbage collection to effectively collect all of the
-  // arena.
+  // arena. Resets the tripwire.
   void arena_clear();
 
-  // Resizes the last allocation as long as the resize does not require a new
-  // block allocation.
-  // Returns the address of the byte following the last newlly allocated byte.
+  // Resizes the last allocation.
+  // Returns the address of the byte following the last newly allocated byte.
   void *arena_resize_last_alloc(ssize_t increase) {
     return (allocation_ptr += increase);
   }
@@ -71,10 +62,8 @@ class arena {
   void arena_swap_and_clear();
 
   // Given two pointers to objects allocated in the same arena, return the number
-  // of bytes they are separated by within the virtual block of memory represented
-  // by the blocks of that arena. This difference will include blocks containing
-  // sentinel bytes. Undefined behavior will result if the pointers belong to
-  // different arenas.
+  // of bytes they are apart. Undefined behavior will result if the pointers
+  // don't belong to the same arena
   static ssize_t ptr_diff(char *ptr1, char *ptr2) { return ptr1 - ptr2; }
 
   // Given a starting pointer to an address allocated in an arena and a size in
@@ -84,11 +73,11 @@ class arena {
   // 1st argument: the starting pointer
   // 2nd argument: the size in bytes to add to the starting pointer
   // 3rd argument: the address of last allocated byte in the arena plus 1
-  // Return value: the address allocated in the arena after size bytes from the
-  //               starting pointer, or 0 if this is equal to the 3rd argument.
-  static char *move_ptr(char *ptr, size_t size, char const *arena_end_ptr) {
+  // Return value: starting pointer + size unless this points to unallocated space
+  //               in which case nullptr is returned
+  static char *move_ptr(char *ptr, size_t size, char const *end_ptr) {
     char *next_ptr = ptr + size;
-    return (next_ptr == arena_end_ptr) ? 0 : next_ptr;
+    return (next_ptr == end_ptr) ? nullptr : next_ptr;
   }
 
   // Returns the ID of the semispace where the given address was allocated.
@@ -97,15 +86,6 @@ class arena {
   static char get_arena_semispace_id_of_object(void *ptr);
 
 private:
-  union memory_block_header {
-    //
-    //	Currently the header just holds the semispace id. But we need it to be a
-    //	multiple of sizeof(char*) for alignment purposes so we add a dummy char*.
-    //
-    char semispace;
-    char *alignment_dummy;
-  };
-
   //
   //	We update the number of 1MB blocks actually written to, only when we need this value,
   //	or before a garbage collection rather than trying to determine when we write to a fresh block.
@@ -121,13 +101,6 @@ class arena {
   }
 
   void initialize_semispace();
-
-  static memory_block_header *mem_block_header(void *ptr) {
-    uintptr_t address = reinterpret_cast<uintptr_t>(ptr);
-    return reinterpret_cast<arena::memory_block_header *>(
-        (address - 1) & ~(HYPERBLOCK_SIZE - 1));
-  }
-
   //
   //	Current semispace where allocations are being made.
   //
@@ -146,9 +119,22 @@ class arena {
       = 0; // notional number of BLOCK_SIZE blocks in collection semispace
 };
 
+inline char arena::get_arena_semispace_id_of_object(void *ptr) {
+  //
+  //	We don't have to deal with the "1 past the end of block" case because
+  //	a valid pointer will always point into our hyperblock - we will never return
+  //	an allocation anywhere near the end of our hyperblock.
+  //
+  //	Set the low bits to 1 to get the address of the last byte in the hyperblock.
+  //
+  uintptr_t end_address
+      = reinterpret_cast<uintptr_t>(ptr) | (HYPERBLOCK_SIZE - 1);
+  return *reinterpret_cast<char *>(end_address);
+}
+
 // Macro to define a new arena with the given ID. Supports IDs ranging from 0 to
 // 127.
-#define REGISTER_ARENA(name, id) static thread_local arena name(id)
+#define REGISTER_ARENA(name, id) thread_local arena name(id)
 
 #ifdef __MACH__
 //
@@ -169,8 +155,11 @@ inline void *arena::kore_arena_alloc(size_t requested) {
     //	collect when allowed.
     //
     time_for_collection = true;
-    tripwire = current_addr_ptr
-               + HYPERBLOCK_SIZE; // won't trigger again until arena swap
+    //
+    //	We move the tripwire to 1 past the end of our hyperblock so that we have
+    //	a well defined comparison that will always be false until the next arena swap.
+    //
+    tripwire = current_addr_ptr + HYPERBLOCK_SIZE;
   }
   void *result = allocation_ptr;
   allocation_ptr += requested;

include/runtime/header.h

@@ -91,7 +91,10 @@ struct kore_alloc_heap {
   template <typename... Tags>
   static void *allocate(size_t size, Tags...) {
     if (during_gc()) {
-      return ::operator new(size);
+      auto *result = (string *)::operator new(size + sizeof(blockheader));
+      init_with_len(result, size);
+      result->h.hdr |= NOT_YOUNG_OBJECT_BIT;
+      return result->data;
     }
     bool enabled = gc_enabled;
     gc_enabled = false;
@@ -103,7 +106,7 @@ struct kore_alloc_heap {
 
   static void deallocate(size_t size, void *data) {
     if (during_gc()) {
-      ::operator delete(data);
+      ::operator delete((char *)data - sizeof(blockheader));
     }
   }
 };

runtime/alloc/arena.cpp

@@ -7,15 +7,11 @@
 
 #include "runtime/alloc.h"
 #include "runtime/arena.h"
+#include "runtime/collect.h"
 #include "runtime/header.h"
 
 extern size_t const VAR_BLOCK_SIZE = BLOCK_SIZE;
 
-__attribute__((always_inline)) char
-arena::get_arena_semispace_id_of_object(void *ptr) {
-  return mem_block_header(ptr)->semispace;
-}
-
 #ifdef __MACH__
 //
 //	thread_local disabled for Apple
@@ -46,19 +42,19 @@ void arena::initialize_semispace() {
   }
   //
   //	We allocated 2 * HYPERBLOCK_SIZE worth of address space but we're only going to use 1, aligned on a
-  //	HYPERBLOCK_SIZE boundry. This is so we can get the start of the hyperblock by masking any address within it.
+  //	HYPERBLOCK_SIZE boundry. This is so we can get end of the hyperblock by setting the low bits of any
+  //	address within the space to 1.
   //	We don't worry about unused address space either side of our aligned address space because there will be no
   //	memory mapped to it.
   //
   current_addr_ptr = reinterpret_cast<char *>(
       std::align(HYPERBLOCK_SIZE, HYPERBLOCK_SIZE, addr, request));
   //
-  //	We put a memory_block_header at the beginning so we can identify the semispace a pointer belongs to
-  //	id by masking off the low bits to access this memory_block_header.
+  //	We put a semispace id in the last byte of the hyperblock so we can identify which semispace an address
+  //	belongs to by setting the low bits to 1 to access this id.
   //
-  auto *header = reinterpret_cast<memory_block_header *>(current_addr_ptr);
-  header->semispace = allocation_semispace_id;
-  allocation_ptr = current_addr_ptr + sizeof(arena::memory_block_header);
+  current_addr_ptr[HYPERBLOCK_SIZE - 1] = allocation_semispace_id;
+  allocation_ptr = current_addr_ptr;
   //
   //	We set the tripwire for this space so we get trigger a garbage collection when we pass BLOCK_SIZE of memory
   //	allocated from this space.