Fix: Stack overflow while attempting to resolve function return value

changelog.md

@@ -7,6 +7,7 @@
 * `FIX` adds the `|lambda|` operator to the `Lua.runtime.nonstandardSymbol` configuration template, which allows the use of that option. Previously, support for it existed in the parser, but we could not actually use the option because it is not recognised in the configuration.
 * `FIX` Typed `@field` (eg `---@field [string] boolean`) should not override other defined field [#2171](https://github.com/LuaLS/lua-language-server/issues/2171), [#2711](https://github.com/LuaLS/lua-language-server/issues/2711)
 * `FIX` don't return empty hover doc when luals failed to find definition
+* `FIX` Prevent stack overflow when attempting to resolve function return values. [#3246](https://github.com/LuaLS/lua-language-server/issues/3246)
 
 ## 3.15.0
 `2025-6-25`

script/vm/sign.lua

@@ -30,10 +30,17 @@ function mt:resolve(uri, args)
 
     ---@type table<string, vm.node>
     local resolved = {}
+    ---@type table<string, boolean>
+    local visited = {}
 
     ---@param object vm.node|vm.node.object
     ---@param node   vm.node
     local function resolve(object, node)
+        local visitedHash = ("%s|%s"):format(object, node)
+        if visited[visitedHash] then
+            return -- prevent circular resolve calls by only visiting each pair once
+        end
+        visited[visitedHash] = true
         if object.type == 'vm.node' then
             for o in object:eachObject() do
                 resolve(o, node)

test/type_inference/common.lua

@@ -4881,3 +4881,25 @@ end
 
 local a, b, <?c?>, d = unpack(t)
 ]]
+
+-- Test for overflow in circular resolve, only pass requirement is no overflow
+TEST 'Callback<<T>>|fun():fun():fun():Success, string' [[
+--- @alias Success fun(): Success
+--- @alias Callback<T> fun(): Success, T
+
+--- @return Success
+local function success()
+    return success
+end
+
+--- @generic T
+--- @param callback Callback<T>
+--- @return Callback<T>
+local function make_callback(callback)
+    return callback
+end
+
+local <?callback?> = make_callback(function()
+    return success, ""
+end)
+]]