fix(callable): __call cannot be in a nested metatable (#489)

CHANGELOG.md

@@ -6,6 +6,10 @@ deprecation policy.
 
 see [CONTRIBUTING.md](CONTRIBUTING.md#release-instructions-for-a-new-version) for release instructions
 
+## unreleased
+ - fix(types): callable would return false positive if `__call` was nested
+   [#489](https://github.com/lunarmodules/Penlight/pull/489)
+
 ## 1.14.0 (2024-Apr-15)
  - fix(path): make `path.expanduser` more sturdy
    [#469](https://github.com/lunarmodules/Penlight/pull/469)

lua/pl/types.lua

@@ -8,10 +8,22 @@ local math_ceil = math.ceil
 local assert_arg = utils.assert_arg
 local types = {}
 
---- is the object either a function or a callable object?.
--- @param obj Object to check.
-function types.is_callable (obj)
-    return type(obj) == 'function' or getmetatable(obj) and getmetatable(obj).__call and true
+do
+    -- we prefer debug.getmetatable, but only if available
+    local gmt = (debug or {}).getmetatable or getmetatable
+
+    --- is the object either a function or a callable object?.
+    -- @param obj Object to check.
+    function types.is_callable (obj)
+        if type(obj) == 'function' then
+            return true
+        end
+        local mt = gmt(obj)
+        if not mt then
+            return false
+        end
+        return type(rawget(mt, "__call")) == "function"
+    end
 end
 
 --- is the object of the specified type?.

tests/test-types.lua

@@ -32,6 +32,21 @@ asserteq(types.is_integer(-10.1),false)
 
 asserteq(types.is_callable(asserteq),true)
 asserteq(types.is_callable(List),true)
+do
+    local mt = setmetatable({}, {
+        __index = {
+            __call = function() return "ok" end
+        }
+    })
+    asserteq(type(mt.__call), "function") -- __call is looked-up through another metatable
+    local nc = setmetatable({}, mt)
+    -- proof-of-pudding, let's call it. To verify Lua behaves the same on all engines
+    local success, result = pcall(function() return nc() end)
+    assert(result ~= "ok", "expected result to not be 'ok'")
+    asserteq(success, false)
+    -- real test now
+    asserteq(types.is_callable(nc), false) -- NOT callable, since __call is fetched using RAWget by Lua
+end
 
 asserteq(types.is_indexable(array),true)
 asserteq(types.is_indexable('hello'),nil)