Implement ThreadLocals

targettopaz.py

@@ -21,3 +21,4 @@ def handle_config(config, translateconfig):
     elif host_factory.name in ('linux', 'darwin'):
         host_factory.cflags += ('-DMAX_STACK_SIZE=%d' % max_stack_size,)
     config.translation.suggest(check_str_without_nul=True)
+    config.translating = True

topaz/executioncontext.py

@@ -1,23 +1,13 @@
-from rpython.rlib import jit
+import sys
+
+from rpython.rlib import jit, objectmodel
+from rpython.rlib.unroll import unrolling_iterable
 
 from topaz.error import RubyError
 from topaz.frame import Frame
 from topaz.objects.fiberobject import W_FiberObject
 
-
-class ExecutionContextHolder(object):
-    # TODO: convert to be a threadlocal store once we have threads.
-    def __init__(self):
-        self._ec = None
-
-    def get(self):
-        return self._ec
-
-    def set(self, ec):
-        self._ec = ec
-
-    def clear(self):
-        self._ec = None
+TICK_COUNTER_STEP = 100
 
 
 class ExecutionContext(object):
@@ -34,6 +24,16 @@ def __init__(self):
         self.fiber_thread = None
         self.w_main_fiber = None
 
+    @staticmethod
+    def _mark_thread_disappeared(space):
+        # Called in the child process after os.fork() by interp_posix.py.
+        # Marks all ExecutionContexts except the current one
+        # with 'thread_disappeared = True'.
+        me = space.getexecutioncontext()
+        for ec in space.threadlocals.getallvalues().values():
+            if ec is not me:
+                ec.thread_disappeared = True
+
     def getmainfiber(self, space):
         if self.w_main_fiber is None:
             self.w_main_fiber = W_FiberObject.build_main_fiber(space, self)
@@ -48,7 +48,8 @@ def gettraceproc(self):
     def hastraceproc(self):
         return self.w_trace_proc is not None and not self.in_trace_proc
 
-    def invoke_trace_proc(self, space, event, scope_id, classname, frame=None):
+    def invoke_only_trace_proc(self, space, event, scope_id, classname,
+                               frame=None):
         if self.hastraceproc():
             self.in_trace_proc = True
             try:
@@ -68,6 +69,13 @@ def invoke_trace_proc(self, space, event, scope_id, classname, frame=None):
             finally:
                 self.in_trace_proc = False
 
+    def invoke_trace_proc(self, space, event, scope_id, classname, frame=None,
+                          decr_by=TICK_COUNTER_STEP):
+        self.invoke_only_trace_proc(space, event, scope_id, classname, frame)
+        actionflag = space.actionflag
+        if actionflag.decrement_ticker(decr_by) < 0:
+            actionflag.action_dispatcher(self, frame)
+
     def enter(self, frame):
         frame.backref = self.topframeref
         if self.last_instr != -1:
@@ -174,3 +182,142 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_value, tb):
         if self.added:
             del self.ec.catch_names[self.catch_name]
+
+
+class AbstractActionFlag(object):
+    """This holds in an integer the 'ticker'.  If threads are enabled,
+    it is decremented at each bytecode; when it reaches zero, we release
+    the GIL.  And whether we have threads or not, it is forced to zero
+    whenever we fire any of the asynchronous actions.
+    """
+
+    _immutable_fields_ = ["checkinterval_scaled?"]
+
+    def __init__(self):
+        self._periodic_actions = []
+        self._nonperiodic_actions = []
+        self.has_bytecode_counter = False
+        self.fired_actions = None
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
+        self._rebuild_action_dispatcher()
+
+    def fire(self, action):
+        """Request for the action to be run before the next opcode."""
+        if not action._fired:
+            action._fired = True
+            if self.fired_actions is None:
+                self.fired_actions = []
+            self.fired_actions.append(action)
+            # set the ticker to -1 in order to force action_dispatcher()
+            # to run at the next possible bytecode
+            self.reset_ticker(-1)
+
+    def register_periodic_action(self, action, use_bytecode_counter):
+        """NOT_RPYTHON:
+        Register the PeriodicAsyncAction action to be called whenever the
+        tick counter becomes smaller than 0.  If 'use_bytecode_counter' is
+        True, make sure that we decrease the tick counter at every bytecode.
+        This is needed for threads.  Note that 'use_bytecode_counter' can be
+        False for signal handling, because whenever the process receives a
+        signal, the tick counter is set to -1 by C code in signals.h.
+        """
+        assert isinstance(action, PeriodicAsyncAction)
+        # hack to put the release-the-GIL one at the end of the list,
+        # and the report-the-signals one at the start of the list.
+        if use_bytecode_counter:
+            self._periodic_actions.append(action)
+            self.has_bytecode_counter = True
+        else:
+            self._periodic_actions.insert(0, action)
+        self._rebuild_action_dispatcher()
+
+    def getcheckinterval(self):
+        return self.checkinterval_scaled // TICK_COUNTER_STEP
+
+    def setcheckinterval(self, interval):
+        MAX = sys.maxint // TICK_COUNTER_STEP
+        if interval < 1:
+            interval = 1
+        elif interval > MAX:
+            interval = MAX
+        self.checkinterval_scaled = interval * TICK_COUNTER_STEP
+        self.reset_ticker(-1)
+
+    def _rebuild_action_dispatcher(self):
+        periodic_actions = unrolling_iterable(self._periodic_actions)
+
+        @jit.unroll_safe
+        @objectmodel.dont_inline
+        def action_dispatcher(ec, frame):
+            # periodic actions (first reset the bytecode counter)
+            self.reset_ticker(self.checkinterval_scaled)
+            for action in periodic_actions:
+                action.perform(ec, frame)
+
+            # nonperiodic actions
+            actions = self.fired_actions
+            if actions is not None:
+                self.fired_actions = None
+                # NB. in case there are several actions, we reset each
+                # 'action._fired' to false only when we're about to call
+                # 'action.perform()'.  This means that if
+                # 'action.fire()' happens to be called any time before
+                # the corresponding perform(), the fire() has no
+                # effect---which is the effect we want, because
+                # perform() will be called anyway.
+                for action in actions:
+                    action._fired = False
+                    action.perform(ec, frame)
+
+        self.action_dispatcher = action_dispatcher
+
+
+class ActionFlag(AbstractActionFlag):
+    """The normal class for space.actionflag.  The signal module provides
+    a different one."""
+    _ticker = 0
+
+    def get_ticker(self):
+        return self._ticker
+
+    def reset_ticker(self, value):
+        self._ticker = value
+
+    def decrement_ticker(self, by):
+        value = self._ticker
+        if self.has_bytecode_counter:    # this 'if' is constant-folded
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
+        return value
+
+
+class AsyncAction(object):
+    """Abstract base class for actions that must be performed
+    asynchronously with regular bytecode execution, but that still need
+    to occur between two opcodes, not at a completely random time.
+    """
+    _fired = False
+
+    def __init__(self, space):
+        self.space = space
+
+    def fire(self):
+        """Request for the action to be run before the next opcode.
+        The action must have been registered at space initalization time."""
+        self.space.actionflag.fire(self)
+
+    def perform(self, executioncontext, frame):
+        """To be overridden."""
+
+
+class PeriodicAsyncAction(AsyncAction):
+    """Abstract base class for actions that occur automatically
+    every TICK_COUNTER_STEP bytecodes.
+    """

topaz/interpreter.py

@@ -69,8 +69,12 @@ def _interpret(self, space, pc, frame, bytecode):
         frame.last_instr = pc
         if (space.getexecutioncontext().hastraceproc() and
                 bytecode.lineno_table[pc] != bytecode.lineno_table[prev_pc]):
-            space.getexecutioncontext().invoke_trace_proc(
-                space, "line", None, None, frame=frame)
+            if jit.we_are_jitted():
+                space.getexecutioncontext().invoke_only_trace_proc(
+                    space, "line", None, None, frame=frame)
+            else:
+                space.getexecutioncontext().invoke_trace_proc(
+                    space, "line", None, None, frame=frame)
         try:
             pc = self.handle_bytecode(space, pc, frame, bytecode)
         except RubyError as e:

topaz/main.py

@@ -1,7 +1,6 @@
 from __future__ import absolute_import
 
 import os
-import subprocess
 
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
@@ -10,7 +9,7 @@
 from topaz.error import RubyError, print_traceback
 from topaz.objects.exceptionobject import W_SystemExit
 from topaz.objspace import ObjectSpace
-from topaz.system import IS_WINDOWS, IS_64BIT
+from topaz.system import IS_WINDOWS, RUBY_DESCRIPTION
 
 
 USAGE = "\n".join([
@@ -42,11 +41,6 @@
     ""
 ])
 COPYRIGHT = "topaz - Copyright (c) Alex Gaynor and individual contributors\n"
-RUBY_REVISION = subprocess.check_output([
-    "git",
-    "--git-dir", os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, ".git"),
-    "rev-parse", "--short", "HEAD"
-]).rstrip()
 
 if IS_WINDOWS:
     def WinStdinStream():
@@ -72,6 +66,7 @@ def get_topaz_config_options():
     return {
         "translation.continuation": True,
         "translation.jit_opencoder_model": "big",
+        "translation.rweakref": True,
     }
 
 
@@ -128,13 +123,7 @@ def _parse_argv(space, argv):
         elif arg == "--copyright":
             raise ShortCircuitError(COPYRIGHT)
         elif arg == "--version":
-            raise ShortCircuitError("%s\n" % space.str_w(
-                space.send(
-                    space.w_object,
-                    "const_get",
-                    [space.newstr_fromstr("RUBY_DESCRIPTION")]
-                )
-            ))
+            raise ShortCircuitError("%s\n" % RUBY_DESCRIPTION)
         elif arg == "-v":
             flag_globals_w["$-v"] = space.w_true
             flag_globals_w["$VERBOSE"] = space.w_true
@@ -221,23 +210,6 @@ def _parse_argv(space, argv):
 
 
 def _entry_point(space, argv):
-    if IS_WINDOWS:
-        system = "Windows"
-        cpu = "x86_64" if IS_64BIT else "i686"
-    else:
-        system, _, _, _, cpu = os.uname()
-    platform = "%s-%s" % (cpu, system.lower())
-    engine = "topaz"
-    version = "2.4.0"
-    patchlevel = 0
-    description = "%s (ruby-%sp%d) (git rev %s) [%s]" % (engine, version, patchlevel, RUBY_REVISION, platform)
-    space.set_const(space.w_object, "RUBY_ENGINE", space.newstr_fromstr(engine))
-    space.set_const(space.w_object, "RUBY_VERSION", space.newstr_fromstr(version))
-    space.set_const(space.w_object, "RUBY_PATCHLEVEL", space.newint(patchlevel))
-    space.set_const(space.w_object, "RUBY_PLATFORM", space.newstr_fromstr(platform))
-    space.set_const(space.w_object, "RUBY_DESCRIPTION", space.newstr_fromstr(description))
-    space.set_const(space.w_object, "RUBY_REVISION", space.newstr_fromstr(RUBY_REVISION))
-
     try:
         (
             flag_globals_w,
@@ -275,7 +247,7 @@ def _entry_point(space, argv):
     space.set_const(space.w_object, "ARGV", space.newarray(argv_w))
     explicitly_verbose = space.is_true(flag_globals_w["$-v"])
     if explicitly_verbose:
-        os.write(1, "%s\n" % description)
+        os.write(1, "%s\n" % RUBY_DESCRIPTION)
     for varname, w_value in flag_globals_w.iteritems():
         space.globals.set(space, varname, w_value)
 

topaz/modules/ffi/type.py

@@ -1,13 +1,13 @@
-from topaz.objects.objectobject import W_Object
-from topaz.module import ClassDef
-
-from rpython.rlib.jit_libffi import FFI_TYPE_P
 from rpython.rlib import clibffi
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.jit_libffi import FFI_TYPE_P
+from rpython.rlib.objectmodel import not_rpython
 from rpython.rlib.rarithmetic import intmask
-from topaz.coerce import Coerce
+from rpython.rtyper.lltypesystem import rffi, lltype
 
+from topaz.coerce import Coerce
+from topaz.module import ClassDef
 from topaz.modules.ffi import misc
+from topaz.objects.objectobject import W_Object
 
 _native_types = [
     ('VOID', clibffi.ffi_type_void, lltype.Void, []),
@@ -66,14 +66,14 @@
 del _native_types
 
 
+@not_rpython
 def lltype_for_name(name):
-    """NOT_RPYTHON"""
     # XXX maybe use a dictionary
     return lltypes[type_names.index(name)]
 
 
+@not_rpython
 def size_for_name(name):
-    """NOT_RPYTHON"""
     # XXX maybe use a dictionary
     return lltype_sizes[type_names.index(name)]
 

topaz/modules/process.py

@@ -12,6 +12,8 @@
 from topaz.modules.signal import SIGNALS
 from topaz.system import IS_WINDOWS
 from topaz.error import error_for_oserror
+from topaz.executioncontext import ExecutionContext
+from topaz.utils import threadlocals
 
 
 if IS_WINDOWS:
@@ -92,6 +94,9 @@ def method_exit_bang(self, space, status=0):
     def method_fork(self, space, block):
         pid = fork()
         if pid == 0:
+            ExecutionContext._mark_thread_disappeared(space)
+            threadlocals.reinit_threads(space)
+
             if block is not None:
                 space.invoke_block(block, [])
                 space.send(self, "exit")