From 4fca601e0cf74ce642e4e49ca7d4fe3e59915846 Mon Sep 17 00:00:00 2001 From: Wuzzy Date: Tue, 5 Oct 2021 12:35:55 +0000 Subject: Add get_server_max_lag() (#11671) --- src/script/lua_api/l_server.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'src/script/lua_api/l_server.cpp') diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 473faaa14..6438fa6fd 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -57,6 +57,17 @@ int ModApiServer::l_get_server_uptime(lua_State *L) return 1; } +// get_server_max_lag() +int ModApiServer::l_get_server_max_lag(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + ServerEnvironment *s_env = dynamic_cast(getEnv(L)); + if (!s_env) + lua_pushnil(L); + else + lua_pushnumber(L, s_env->getMaxLagEstimate()); + return 1; +} // print(text) int ModApiServer::l_print(lua_State *L) @@ -512,6 +523,7 @@ void ModApiServer::Initialize(lua_State *L, int top) API_FCT(request_shutdown); API_FCT(get_server_status); API_FCT(get_server_uptime); + API_FCT(get_server_max_lag); API_FCT(get_worldpath); API_FCT(is_singleplayer); -- cgit v1.2.3 From bc71622d2121347ad6dbbe3c174ad485fe1a8f8c Mon Sep 17 00:00:00 2001 From: Wuzzy Date: Tue, 5 Oct 2021 19:53:47 +0000 Subject: Fix crash when calling remove/kick/ban_player on start (#11672) --- src/script/lua_api/l_server.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src/script/lua_api/l_server.cpp') diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 6438fa6fd..476f74c9c 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -293,8 +293,10 @@ int ModApiServer::l_ban_player(lua_State *L) { NO_MAP_LOCK_REQUIRED; - Server *server = getServer(L); + if (!getEnv(L)) + throw LuaError("Can't ban player before server has started up"); + Server *server = getServer(L); const char *name = luaL_checkstring(L, 1); RemotePlayer *player = server->getEnv().getPlayer(name); if (!player) { @@ -312,6 +314,10 @@ int ModApiServer::l_ban_player(lua_State *L) int ModApiServer::l_kick_player(lua_State *L) { NO_MAP_LOCK_REQUIRED; + + if (!getEnv(L)) + throw LuaError("Can't kick player before server has started up"); + const char *name = luaL_checkstring(L, 1); std::string message("Kicked"); if (lua_isstring(L, 2)) @@ -334,7 +340,8 @@ int ModApiServer::l_remove_player(lua_State *L) NO_MAP_LOCK_REQUIRED; std::string name = luaL_checkstring(L, 1); ServerEnvironment *s_env = dynamic_cast(getEnv(L)); - assert(s_env); + if (!s_env) + throw LuaError("Can't remove player before server has started up"); RemotePlayer *player = s_env->getPlayer(name.c_str()); if (!player) -- cgit v1.2.3 From 413be76c63309266d3d271f01cc74385067d7263 Mon Sep 17 00:00:00 2001 From: Corey Powell Date: Fri, 26 Nov 2021 14:19:40 -0500 Subject: Implemented disconnect_player (#10492) Co-authored-by: rubenwardy --- builtin/game/misc.lua | 10 ++++++++++ doc/lua_api.txt | 4 ++++ src/script/lua_api/l_server.cpp | 12 ++++++------ src/script/lua_api/l_server.h | 4 ++-- 4 files changed, 22 insertions(+), 8 deletions(-) (limited to 'src/script/lua_api/l_server.cpp') diff --git a/builtin/game/misc.lua b/builtin/game/misc.lua index 05237662c..ef826eda7 100644 --- a/builtin/game/misc.lua +++ b/builtin/game/misc.lua @@ -6,6 +6,16 @@ local S = core.get_translator("__builtin") -- Misc. API functions -- +-- @spec core.kick_player(String, String) :: Boolean +function core.kick_player(player_name, reason) + if type(reason) == "string" then + reason = "Kicked: " .. reason + else + reason = "Kicked." + end + return core.disconnect_player(player_name, reason) +end + function core.check_player_privs(name, ...) if core.is_player(name) then name = name:get_player_name() diff --git a/doc/lua_api.txt b/doc/lua_api.txt index 0a63642af..aff739cfb 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -5741,6 +5741,10 @@ Bans * `minetest.kick_player(name, [reason])`: disconnect a player with an optional reason. * Returns boolean indicating success (false if player nonexistant) +* `minetest.disconnect_player(name, [reason])`: disconnect a player with an + optional reason, this will not prefix with 'Kicked: ' like kick_player. + If no reason is given, it will default to 'Disconnected.' + * Returns boolean indicating success (false if player nonexistant) Particles --------- diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 476f74c9c..82a692070 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -310,8 +310,8 @@ int ModApiServer::l_ban_player(lua_State *L) return 1; } -// kick_player(name, [reason]) -> success -int ModApiServer::l_kick_player(lua_State *L) +// disconnect_player(name, [reason]) -> success +int ModApiServer::l_disconnect_player(lua_State *L) { NO_MAP_LOCK_REQUIRED; @@ -319,11 +319,11 @@ int ModApiServer::l_kick_player(lua_State *L) throw LuaError("Can't kick player before server has started up"); const char *name = luaL_checkstring(L, 1); - std::string message("Kicked"); + std::string message; if (lua_isstring(L, 2)) - message.append(": ").append(readParam(L, 2)); + message.append(readParam(L, 2)); else - message.append("."); + message.append("Disconnected."); RemotePlayer *player = dynamic_cast(getEnv(L))->getPlayer(name); if (player == NULL) { @@ -554,7 +554,7 @@ void ModApiServer::Initialize(lua_State *L, int top) API_FCT(get_ban_list); API_FCT(get_ban_description); API_FCT(ban_player); - API_FCT(kick_player); + API_FCT(disconnect_player); API_FCT(remove_player); API_FCT(unban_player_or_ip); API_FCT(notify_authentication_modified); diff --git a/src/script/lua_api/l_server.h b/src/script/lua_api/l_server.h index a6f709787..f05c0b7c9 100644 --- a/src/script/lua_api/l_server.h +++ b/src/script/lua_api/l_server.h @@ -97,8 +97,8 @@ private: // unban_player_or_ip() static int l_unban_player_or_ip(lua_State *L); - // kick_player(name, [message]) -> success - static int l_kick_player(lua_State *L); + // disconnect_player(name, [reason]) -> success + static int l_disconnect_player(lua_State *L); // remove_player(name) static int l_remove_player(lua_State *L); -- cgit v1.2.3 From 8472141b79c25092c90dea24aa873bd7ff792142 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Sat, 18 Dec 2021 20:36:43 +0100 Subject: Restructure devtest's unittests and run them in CI (#11859) --- .github/workflows/build.yml | 2 +- games/devtest/mods/unittests/crafting.lua | 18 +- games/devtest/mods/unittests/init.lua | 199 ++++++++++++++++++++++- games/devtest/mods/unittests/itemdescription.lua | 3 +- games/devtest/mods/unittests/misc.lua | 38 +++++ games/devtest/mods/unittests/player.lua | 46 +++--- games/devtest/mods/unittests/random.lua | 10 -- src/script/cpp_api/s_base.h | 9 + src/script/cpp_api/s_server.cpp | 6 +- src/script/cpp_api/s_server.h | 2 +- src/script/lua_api/l_server.cpp | 2 +- util/test_multiplayer.sh | 26 ++- 12 files changed, 289 insertions(+), 72 deletions(-) create mode 100644 games/devtest/mods/unittests/misc.lua delete mode 100644 games/devtest/mods/unittests/random.lua (limited to 'src/script/lua_api/l_server.cpp') diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 417b4f650..af1de15ec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -93,7 +93,7 @@ jobs: run: | ./bin/minetest --run-unittests - - name: Integration test + - name: Integration test + devtest run: | ./util/test_multiplayer.sh diff --git a/games/devtest/mods/unittests/crafting.lua b/games/devtest/mods/unittests/crafting.lua index eff13ce09..8c16d3efb 100644 --- a/games/devtest/mods/unittests/crafting.lua +++ b/games/devtest/mods/unittests/crafting.lua @@ -1,6 +1,7 @@ +dofile(core.get_modpath(core.get_current_modname()) .. "/crafting_prepare.lua") + -- Test minetest.clear_craft function local function test_clear_craft() - minetest.log("info", "[unittests] Testing minetest.clear_craft") -- Clearing by output minetest.register_craft({ output = "foo", @@ -22,11 +23,10 @@ local function test_clear_craft() minetest.clear_craft({recipe={{"foo", "bar"}}}) assert(minetest.get_all_craft_recipes("foo") == nil) end +unittests.register("test_clear_craft", test_clear_craft) -- Test minetest.get_craft_result function local function test_get_craft_result() - minetest.log("info", "[unittests] Testing minetest.get_craft_result") - -- normal local input = { method = "normal", @@ -107,14 +107,6 @@ local function test_get_craft_result() assert(output.item) minetest.log("info", "[unittests] unrepairable tool crafting output.item:to_table(): "..dump(output.item:to_table())) -- unrepairable tool must not yield any output - assert(output.item:get_name() == "") - + assert(output.item:is_empty()) end - -function unittests.test_crafting() - test_clear_craft() - test_get_craft_result() - minetest.log("action", "[unittests] Crafting tests passed!") - return true -end - +unittests.register("test_get_craft_result", test_get_craft_result) diff --git a/games/devtest/mods/unittests/init.lua b/games/devtest/mods/unittests/init.lua index 12c67f78b..0754d507f 100644 --- a/games/devtest/mods/unittests/init.lua +++ b/games/devtest/mods/unittests/init.lua @@ -1,18 +1,199 @@ unittests = {} +unittests.list = {} + +-- name: Name of the test +-- func: +-- for sync: function(player, pos), should error on failure +-- for async: function(callback, player, pos) +-- MUST call callback() or callback("error msg") in case of error once test is finished +-- this means you cannot use assert() in the test implementation +-- opts: { +-- player = false, -- Does test require a player? +-- map = false, -- Does test require map access? +-- async = false, -- Does the test run asynchronously? (read notes above!) +-- } +function unittests.register(name, func, opts) + local def = table.copy(opts or {}) + def.name = name + def.func = func + table.insert(unittests.list, def) +end + +function unittests.on_finished(all_passed) + -- free to override +end + +-- Calls invoke with a callback as argument +-- Suspends coroutine until that callback is called +-- Return values are passed through +local function await(invoke) + local co = coroutine.running() + assert(co) + local called_early = true + invoke(function(...) + if called_early == true then + called_early = {...} + else + coroutine.resume(co, ...) + end + end) + if called_early ~= true then + -- callback was already called before yielding + return unpack(called_early) + end + called_early = nil + return coroutine.yield() +end + +function unittests.run_one(idx, counters, out_callback, player, pos) + local def = unittests.list[idx] + if not def.player then + player = nil + elseif player == nil then + out_callback(false) + return false + end + if not def.map then + pos = nil + elseif pos == nil then + out_callback(false) + return false + end + + local tbegin = core.get_us_time() + local function done(status, err) + local tend = core.get_us_time() + local ms_taken = (tend - tbegin) / 1000 + + if not status then + core.log("error", err) + end + print(string.format("[%s] %s - %dms", + status and "PASS" or "FAIL", def.name, ms_taken)) + counters.time = counters.time + ms_taken + counters.total = counters.total + 1 + if status then + counters.passed = counters.passed + 1 + end + end + + if def.async then + core.log("info", "[unittest] running " .. def.name .. " (async)") + def.func(function(err) + done(err == nil, err) + out_callback(true) + end, player, pos) + else + core.log("info", "[unittest] running " .. def.name) + local status, err = pcall(def.func, player, pos) + done(status, err) + out_callback(true) + end + + return true +end + +local function wait_for_player(callback) + if #core.get_connected_players() > 0 then + return callback(core.get_connected_players()[1]) + end + local first = true + core.register_on_joinplayer(function(player) + if first then + callback(player) + first = false + end + end) +end + +local function wait_for_map(player, callback) + local check = function() + if core.get_node_or_nil(player:get_pos()) ~= nil then + callback() + else + minetest.after(0, check) + end + end + check() +end + +function unittests.run_all() + -- This runs in a coroutine so it uses await(). + local counters = { time = 0, total = 0, passed = 0 } + + -- Run standalone tests first + for idx = 1, #unittests.list do + local def = unittests.list[idx] + def.done = await(function(cb) + unittests.run_one(idx, counters, cb, nil, nil) + end) + end + + -- Wait for a player to join, run tests that require a player + local player = await(wait_for_player) + for idx = 1, #unittests.list do + local def = unittests.list[idx] + if not def.done then + def.done = await(function(cb) + unittests.run_one(idx, counters, cb, player, nil) + end) + end + end + + -- Wait for the world to generate/load, run tests that require map access + await(function(cb) + wait_for_map(player, cb) + end) + local pos = vector.round(player:get_pos()) + for idx = 1, #unittests.list do + local def = unittests.list[idx] + if not def.done then + def.done = await(function(cb) + unittests.run_one(idx, counters, cb, player, pos) + end) + end + end + + -- Print stats + assert(#unittests.list == counters.total) + print(string.rep("+", 80)) + print(string.format("Unit Test Results: %s", + counters.total == counters.passed and "PASSED" or "FAILED")) + print(string.format(" %d / %d failed tests.", + counters.total - counters.passed, counters.total)) + print(string.format(" Testing took %dms total.", counters.time)) + print(string.rep("+", 80)) + unittests.on_finished(counters.total == counters.passed) + return counters.total == counters.passed +end + +-------------- + local modpath = minetest.get_modpath("unittests") -dofile(modpath .. "/random.lua") +dofile(modpath .. "/misc.lua") dofile(modpath .. "/player.lua") -dofile(modpath .. "/crafting_prepare.lua") dofile(modpath .. "/crafting.lua") dofile(modpath .. "/itemdescription.lua") -if minetest.settings:get_bool("devtest_unittests_autostart", false) then - unittests.test_random() - unittests.test_crafting() - unittests.test_short_desc() - minetest.register_on_joinplayer(function(player) - unittests.test_player(player) +-------------- + +if core.settings:get_bool("devtest_unittests_autostart", false) then + core.after(0, function() + coroutine.wrap(unittests.run_all)() end) +else + minetest.register_chatcommand("unittests", { + privs = {basic_privs=true}, + description = "Runs devtest unittests (may modify player or map state)", + func = function(name, param) + unittests.on_finished = function(ok) + core.chat_send_player(name, + (ok and "All tests passed." or "There were test failures.") .. + " Check the console for detailed output.") + end + coroutine.wrap(unittests.run_all)() + return true, "" + end, + }) end - diff --git a/games/devtest/mods/unittests/itemdescription.lua b/games/devtest/mods/unittests/itemdescription.lua index d6ee6551a..dc62de7f0 100644 --- a/games/devtest/mods/unittests/itemdescription.lua +++ b/games/devtest/mods/unittests/itemdescription.lua @@ -25,7 +25,7 @@ minetest.register_chatcommand("item_description", { end }) -function unittests.test_short_desc() +local function test_short_desc() local function get_short_description(item) return ItemStack(item):get_short_description() end @@ -49,3 +49,4 @@ function unittests.test_short_desc() return true end +unittests.register("test_short_desc", test_short_desc) diff --git a/games/devtest/mods/unittests/misc.lua b/games/devtest/mods/unittests/misc.lua new file mode 100644 index 000000000..cf4f92cfa --- /dev/null +++ b/games/devtest/mods/unittests/misc.lua @@ -0,0 +1,38 @@ +local function test_random() + -- Try out PseudoRandom + local pseudo = PseudoRandom(13) + assert(pseudo:next() == 22290) + assert(pseudo:next() == 13854) +end +unittests.register("test_random", test_random) + +local function test_dynamic_media(cb, player) + if core.get_player_information(player:get_player_name()).protocol_version < 40 then + core.log("warning", "test_dynamic_media: Client too old, skipping test.") + return cb() + end + + -- Check that the client acknowledges media transfers + local path = core.get_worldpath() .. "/test_media.obj" + local f = io.open(path, "w") + f:write("# contents don't matter\n") + f:close() + + local call_ok = false + local ok = core.dynamic_add_media({ + filepath = path, + to_player = player:get_player_name(), + }, function(name) + if not call_ok then + cb("impossible condition") + end + cb() + end) + if not ok then + return cb("dynamic_add_media() returned error") + end + call_ok = true + + -- if the callback isn't called this test will just hang :shrug: +end +unittests.register("test_dynamic_media", test_dynamic_media, {async=true, player=true}) diff --git a/games/devtest/mods/unittests/player.lua b/games/devtest/mods/unittests/player.lua index 4a681310d..fa0557960 100644 --- a/games/devtest/mods/unittests/player.lua +++ b/games/devtest/mods/unittests/player.lua @@ -2,6 +2,21 @@ -- HP Change Reasons -- local expect = nil +minetest.register_on_player_hpchange(function(player, hp, reason) + if expect == nil then + return + end + + for key, value in pairs(reason) do + assert(expect[key] == value) + end + for key, value in pairs(expect) do + assert(reason[key] == value) + end + + expect = nil +end) + local function run_hpchangereason_tests(player) local old_hp = player:get_hp() @@ -20,7 +35,11 @@ local function run_hpchangereason_tests(player) player:set_hp(old_hp) end +unittests.register("test_hpchangereason", run_hpchangereason_tests, {player=true}) +-- +-- Player meta +-- local function run_player_meta_tests(player) local meta = player:get_meta() meta:set_string("foo", "bar") @@ -48,29 +67,4 @@ local function run_player_meta_tests(player) assert(meta:get_string("foo") == "") assert(meta:equals(meta2)) end - -function unittests.test_player(player) - minetest.register_on_player_hpchange(function(player, hp, reason) - if not expect then - return - end - - for key, value in pairs(reason) do - assert(expect[key] == value) - end - - for key, value in pairs(expect) do - assert(reason[key] == value) - end - - expect = nil - end) - - run_hpchangereason_tests(player) - run_player_meta_tests(player) - local msg = "Player tests passed for player '"..player:get_player_name().."'!" - minetest.chat_send_all(msg) - minetest.log("action", "[unittests] "..msg) - return true -end - +unittests.register("test_player_meta", run_player_meta_tests, {player=true}) diff --git a/games/devtest/mods/unittests/random.lua b/games/devtest/mods/unittests/random.lua deleted file mode 100644 index f94f0a88e..000000000 --- a/games/devtest/mods/unittests/random.lua +++ /dev/null @@ -1,10 +0,0 @@ -function unittests.test_random() - -- Try out PseudoRandom - minetest.log("action", "[unittests] Testing PseudoRandom ...") - local pseudo = PseudoRandom(13) - assert(pseudo:next() == 22290) - assert(pseudo:next() == 13854) - minetest.log("action", "[unittests] PseudoRandom test passed!") - return true -end - diff --git a/src/script/cpp_api/s_base.h b/src/script/cpp_api/s_base.h index 06df2abe3..244d81605 100644 --- a/src/script/cpp_api/s_base.h +++ b/src/script/cpp_api/s_base.h @@ -125,6 +125,15 @@ protected: friend class ModApiEnvMod; friend class LuaVoxelManip; + /* + Subtle edge case with coroutines: If for whatever reason you have a + method in a subclass that's called from existing lua_CFunction + (any of the l_*.cpp files) then make it static and take the lua_State* + as an argument. This is REQUIRED because getStack() will not return the + correct state if called inside coroutines. + + Also note that src/script/common/ is the better place for such helpers. + */ lua_State* getStack() { return m_luastack; } diff --git a/src/script/cpp_api/s_server.cpp b/src/script/cpp_api/s_server.cpp index 6ddb2630d..c255b0c71 100644 --- a/src/script/cpp_api/s_server.cpp +++ b/src/script/cpp_api/s_server.cpp @@ -198,10 +198,8 @@ std::string ScriptApiServer::formatChatMessage(const std::string &name, return ret; } -u32 ScriptApiServer::allocateDynamicMediaCallback(int f_idx) +u32 ScriptApiServer::allocateDynamicMediaCallback(lua_State *L, int f_idx) { - lua_State *L = getStack(); - if (f_idx < 0) f_idx = lua_gettop(L) + f_idx + 1; @@ -235,7 +233,7 @@ u32 ScriptApiServer::allocateDynamicMediaCallback(int f_idx) void ScriptApiServer::freeDynamicMediaCallback(u32 token) { - lua_State *L = getStack(); + SCRIPTAPI_PRECHECKHEADER verbosestream << "freeDynamicMediaCallback(" << token << ")" << std::endl; diff --git a/src/script/cpp_api/s_server.h b/src/script/cpp_api/s_server.h index c5c3d5596..58c8c0e48 100644 --- a/src/script/cpp_api/s_server.h +++ b/src/script/cpp_api/s_server.h @@ -51,7 +51,7 @@ public: const std::string &password); /* dynamic media handling */ - u32 allocateDynamicMediaCallback(int f_idx); + static u32 allocateDynamicMediaCallback(lua_State *L, int f_idx); void freeDynamicMediaCallback(u32 token); void on_dynamic_media_added(u32 token, const char *playername); diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 82a692070..88ab5e16b 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -496,7 +496,7 @@ int ModApiServer::l_dynamic_add_media(lua_State *L) CHECK_SECURE_PATH(L, filepath.c_str(), false); - u32 token = server->getScriptIface()->allocateDynamicMediaCallback(2); + u32 token = server->getScriptIface()->allocateDynamicMediaCallback(L, 2); bool ok = server->dynamicAddMedia(filepath, token, to_player, ephemeral); if (!ok) diff --git a/util/test_multiplayer.sh b/util/test_multiplayer.sh index 9fb894a30..5ffc044e0 100755 --- a/util/test_multiplayer.sh +++ b/util/test_multiplayer.sh @@ -20,7 +20,7 @@ waitfor () { } gdbrun () { - gdb -q -ex 'set confirm off' -ex 'r' -ex 'bt' -ex 'quit' --args "$@" + gdb -q -batch -ex 'set confirm off' -ex 'r' -ex 'bt' --args "$@" } [ -e $minetest ] || { echo "executable $minetest missing"; exit 1; } @@ -33,17 +33,27 @@ printf '%s\n' >$testspath/client1.conf \ enable_{sound,minimap,shaders}=false printf '%s\n' >$testspath/server.conf \ - max_block_send_distance=1 + max_block_send_distance=1 devtest_unittests_autostart=true cat >$worldpath/worldmods/test/init.lua <<"LUA" core.after(0, function() io.close(io.open(core.get_worldpath() .. "/startup", "w")) end) -core.register_on_joinplayer(function(player) - io.close(io.open(core.get_worldpath() .. "/player_joined", "w")) +local function callback(test_ok) + if not test_ok then + io.close(io.open(core.get_worldpath() .. "/test_failure", "w")) + end + io.close(io.open(core.get_worldpath() .. "/done", "w")) core.request_shutdown("", false, 2) -end) +end +if core.settings:get_bool("devtest_unittests_autostart") then + unittests.on_finished = callback +else + core.register_on_joinplayer(function() callback(true) end) +end LUA +printf '%s\n' >$worldpath/worldmods/test/mod.conf \ + name=test optional_depends=unittests echo "Starting server" gdbrun $minetest --server --config $conf_server --world $worldpath --gameid $gameid 2>&1 | sed -u 's/^/(server) /' & @@ -51,10 +61,14 @@ waitfor $worldpath/startup echo "Starting client" gdbrun $minetest --config $conf_client1 --go --address 127.0.0.1 2>&1 | sed -u 's/^/(client) /' & -waitfor $worldpath/player_joined +waitfor $worldpath/done echo "Waiting for client and server to exit" wait +if [ -f $worldpath/test_failure ]; then + echo "There were test failures." + exit 1 +fi echo "Success" exit 0 -- cgit v1.2.3 From a65f6f07f3a5601207b790edcc8cc945133112f7 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Wed, 27 Apr 2022 19:55:13 +0200 Subject: Clean up some auth packet handling related code --- src/clientiface.cpp | 16 +++------- src/network/clientpackethandler.cpp | 11 ++++--- src/network/networkprotocol.h | 18 +++++------ src/network/serveropcodes.cpp | 2 +- src/network/serverpackethandler.cpp | 24 +++++++-------- src/script/lua_api/l_server.cpp | 9 ++++-- src/server.cpp | 59 +++++++++---------------------------- src/server.h | 5 +--- src/serverenvironment.cpp | 6 ++-- 9 files changed, 52 insertions(+), 98 deletions(-) (limited to 'src/script/lua_api/l_server.cpp') diff --git a/src/clientiface.cpp b/src/clientiface.cpp index a4bfb8242..5733b05de 100644 --- a/src/clientiface.cpp +++ b/src/clientiface.cpp @@ -472,20 +472,14 @@ void RemoteClient::notifyEvent(ClientStateEvent event) { case CSE_AuthAccept: m_state = CS_AwaitingInit2; - if (chosen_mech == AUTH_MECHANISM_SRP || - chosen_mech == AUTH_MECHANISM_LEGACY_PASSWORD) - srp_verifier_delete((SRPVerifier *) auth_data); - chosen_mech = AUTH_MECHANISM_NONE; + resetChosenMech(); break; case CSE_Disconnect: m_state = CS_Disconnecting; break; case CSE_SetDenied: m_state = CS_Denied; - if (chosen_mech == AUTH_MECHANISM_SRP || - chosen_mech == AUTH_MECHANISM_LEGACY_PASSWORD) - srp_verifier_delete((SRPVerifier *) auth_data); - chosen_mech = AUTH_MECHANISM_NONE; + resetChosenMech(); break; default: myerror << "HelloSent: Invalid client state transition! " << event; @@ -561,9 +555,7 @@ void RemoteClient::notifyEvent(ClientStateEvent event) break; case CSE_SudoSuccess: m_state = CS_SudoMode; - if (chosen_mech == AUTH_MECHANISM_SRP) - srp_verifier_delete((SRPVerifier *) auth_data); - chosen_mech = AUTH_MECHANISM_NONE; + resetChosenMech(); break; /* Init GotInit2 SetDefinitionsSent SetMediaSent SetDenied */ default: @@ -598,7 +590,7 @@ void RemoteClient::notifyEvent(ClientStateEvent event) void RemoteClient::resetChosenMech() { - if (chosen_mech == AUTH_MECHANISM_SRP) { + if (auth_data) { srp_verifier_delete((SRPVerifier *) auth_data); auth_data = nullptr; } diff --git a/src/network/clientpackethandler.cpp b/src/network/clientpackethandler.cpp index 15b576640..55d20d673 100644 --- a/src/network/clientpackethandler.cpp +++ b/src/network/clientpackethandler.cpp @@ -183,7 +183,7 @@ void Client::handleCommand_AccessDenied(NetworkPacket* pkt) m_access_denied_reason = "Unknown"; if (pkt->getCommand() != TOCLIENT_ACCESS_DENIED) { - // 13/03/15 Legacy code from 0.4.12 and lesser but is still used + // Legacy code from 0.4.12 and older but is still used // in some places of the server code if (pkt->getSize() >= 2) { std::wstring wide_reason; @@ -196,14 +196,14 @@ void Client::handleCommand_AccessDenied(NetworkPacket* pkt) if (pkt->getSize() < 1) return; - u8 denyCode = SERVER_ACCESSDENIED_UNEXPECTED_DATA; + u8 denyCode; *pkt >> denyCode; + if (denyCode == SERVER_ACCESSDENIED_SHUTDOWN || denyCode == SERVER_ACCESSDENIED_CRASH) { *pkt >> m_access_denied_reason; - if (m_access_denied_reason.empty()) { + if (m_access_denied_reason.empty()) m_access_denied_reason = accessDeniedStrings[denyCode]; - } u8 reconnect; *pkt >> reconnect; m_access_denied_reconnect = reconnect & 1; @@ -220,9 +220,8 @@ void Client::handleCommand_AccessDenied(NetworkPacket* pkt) // Until then (which may be never), this is outside // of the defined protocol. *pkt >> m_access_denied_reason; - if (m_access_denied_reason.empty()) { + if (m_access_denied_reason.empty()) m_access_denied_reason = "Unknown"; - } } } diff --git a/src/network/networkprotocol.h b/src/network/networkprotocol.h index f98a829ba..3923cb858 100644 --- a/src/network/networkprotocol.h +++ b/src/network/networkprotocol.h @@ -1006,7 +1006,7 @@ enum AuthMechanism AUTH_MECHANISM_FIRST_SRP = 1 << 2, }; -enum AccessDeniedCode { +enum AccessDeniedCode : u8 { SERVER_ACCESSDENIED_WRONG_PASSWORD, SERVER_ACCESSDENIED_UNEXPECTED_DATA, SERVER_ACCESSDENIED_SINGLEPLAYER, @@ -1029,18 +1029,18 @@ enum NetProtoCompressionMode { const static std::string accessDeniedStrings[SERVER_ACCESSDENIED_MAX] = { "Invalid password", - "Your client sent something the server didn't expect. Try reconnecting or updating your client", + "Your client sent something the server didn't expect. Try reconnecting or updating your client.", "The server is running in simple singleplayer mode. You cannot connect.", - "Your client's version is not supported.\nPlease contact server administrator.", - "Player name contains disallowed characters.", - "Player name not allowed.", - "Too many users.", + "Your client's version is not supported.\nPlease contact the server administrator.", + "Player name contains disallowed characters", + "Player name not allowed", + "Too many users", "Empty passwords are disallowed. Set a password and try again.", "Another client is connected with this name. If your client closed unexpectedly, try again in a minute.", - "Server authentication failed. This is likely a server error.", + "Internal server error", "", - "Server shutting down.", - "This server has experienced an internal error. You will now be disconnected." + "Server shutting down", + "The server has experienced an internal error. You will now be disconnected." }; enum PlayerListModifer : u8 diff --git a/src/network/serveropcodes.cpp b/src/network/serveropcodes.cpp index 44b65e8da..12665e7f1 100644 --- a/src/network/serveropcodes.cpp +++ b/src/network/serveropcodes.cpp @@ -176,7 +176,7 @@ const ClientCommandFactory clientCommandFactoryTable[TOCLIENT_NUM_MSG_TYPES] = { "TOCLIENT_ACTIVE_OBJECT_MESSAGES", 0, true }, // 0x32 (may be sent as unrel over channel 1 too) { "TOCLIENT_HP", 0, true }, // 0x33 { "TOCLIENT_MOVE_PLAYER", 0, true }, // 0x34 - { "TOCLIENT_ACCESS_DENIED_LEGACY", 0, true }, // 0x35 + null_command_factory, // 0x35 { "TOCLIENT_FOV", 0, true }, // 0x36 { "TOCLIENT_DEATHSCREEN", 0, true }, // 0x37 { "TOCLIENT_MEDIA", 2, true }, // 0x38 diff --git a/src/network/serverpackethandler.cpp b/src/network/serverpackethandler.cpp index 125e85cab..4b9de488c 100644 --- a/src/network/serverpackethandler.cpp +++ b/src/network/serverpackethandler.cpp @@ -227,7 +227,7 @@ void Server::handleCommand_Init(NetworkPacket* pkt) Compose auth methods for answer */ std::string encpwd; // encrypted Password field for the user - bool has_auth = m_script->getAuth(playername, &encpwd, NULL); + bool has_auth = m_script->getAuth(playername, &encpwd, nullptr); u32 auth_mechs = 0; client->chosen_mech = AUTH_MECHANISM_NONE; @@ -1461,11 +1461,9 @@ void Server::handleCommand_FirstSrp(NetworkPacket* pkt) session_t peer_id = pkt->getPeerId(); RemoteClient *client = getClient(peer_id, CS_Invalid); ClientState cstate = client->getState(); + const std::string playername = client->getName(); - std::string playername = client->getName(); - - std::string salt; - std::string verification_key; + std::string salt, verification_key; std::string addr_s = getPeerAddress(peer_id).serializeString(); u8 is_empty; @@ -1551,8 +1549,6 @@ void Server::handleCommand_SrpBytesA(NetworkPacket* pkt) RemoteClient *client = getClient(peer_id, CS_Invalid); ClientState cstate = client->getState(); - bool wantSudo = (cstate == CS_Active); - if (!((cstate == CS_HelloSent) || (cstate == CS_Active))) { actionstream << "Server: got SRP _A packet in wrong state " << cstate << " from " << getPeerAddress(peer_id).serializeString() << @@ -1560,6 +1556,8 @@ void Server::handleCommand_SrpBytesA(NetworkPacket* pkt) return; } + const bool wantSudo = (cstate == CS_Active); + if (client->chosen_mech != AUTH_MECHANISM_NONE) { actionstream << "Server: got SRP _A packet, while auth is already " "going on with mech " << client->chosen_mech << " from " << @@ -1606,8 +1604,7 @@ void Server::handleCommand_SrpBytesA(NetworkPacket* pkt) client->chosen_mech = chosen; - std::string salt; - std::string verifier; + std::string salt, verifier; if (based_on == 0) { @@ -1657,10 +1654,10 @@ void Server::handleCommand_SrpBytesM(NetworkPacket* pkt) session_t peer_id = pkt->getPeerId(); RemoteClient *client = getClient(peer_id, CS_Invalid); ClientState cstate = client->getState(); - std::string addr_s = getPeerAddress(pkt->getPeerId()).serializeString(); - std::string playername = client->getName(); + const std::string addr_s = client->getAddress().serializeString(); + const std::string playername = client->getName(); - bool wantSudo = (cstate == CS_Active); + const bool wantSudo = (cstate == CS_Active); verbosestream << "Server: Received TOSERVER_SRP_BYTES_M." << std::endl; @@ -1720,8 +1717,7 @@ void Server::handleCommand_SrpBytesM(NetworkPacket* pkt) if (client->create_player_on_auth_success) { m_script->createAuth(playername, client->enc_pwd); - std::string checkpwd; // not used, but needed for passing something - if (!m_script->getAuth(playername, &checkpwd, NULL)) { + if (!m_script->getAuth(playername, nullptr, nullptr)) { errorstream << "Server: " << playername << " cannot be authenticated (auth handler does not work?)" << std::endl; diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 88ab5e16b..5b3054d17 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -325,12 +325,15 @@ int ModApiServer::l_disconnect_player(lua_State *L) else message.append("Disconnected."); - RemotePlayer *player = dynamic_cast(getEnv(L))->getPlayer(name); - if (player == NULL) { + Server *server = getServer(L); + + RemotePlayer *player = server->getEnv().getPlayer(name); + if (!player) { lua_pushboolean(L, false); // No such player return 1; } - getServer(L)->DenyAccess_Legacy(player->getPeerId(), utf8_to_wide(message)); + + server->DenyAccess(player->getPeerId(), SERVER_ACCESSDENIED_CUSTOM_STRING, message); lua_pushboolean(L, true); return 1; } diff --git a/src/server.cpp b/src/server.cpp index 8ca8a9bda..9d7e8e563 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -1038,8 +1038,7 @@ void Server::Receive() } catch (const ClientStateError &e) { errorstream << "ProcessData: peer=" << peer_id << " what()=" << e.what() << std::endl; - DenyAccess_Legacy(peer_id, L"Your client sent something server didn't expect." - L"Try reconnecting or updating your client"); + DenyAccess(peer_id, SERVER_ACCESSDENIED_UNEXPECTED_DATA); } catch (const con::PeerNotFoundException &e) { // Do nothing } catch (const con::NoIncomingDataException &e) { @@ -1068,15 +1067,13 @@ PlayerSAO* Server::StageTwoClientInit(session_t peer_id) if (player && player->getPeerId() != PEER_ID_INEXISTENT) { actionstream << "Server: Failed to emerge player \"" << playername << "\" (player allocated to an another client)" << std::endl; - DenyAccess_Legacy(peer_id, L"Another client is connected with this " - L"name. If your client closed unexpectedly, try again in " - L"a minute."); + DenyAccess(peer_id, SERVER_ACCESSDENIED_ALREADY_CONNECTED); } else { errorstream << "Server: " << playername << ": Failed to emerge player" << std::endl; - DenyAccess_Legacy(peer_id, L"Could not allocate player."); + DenyAccess(peer_id, SERVER_ACCESSDENIED_SERVER_FAIL); } - return NULL; + return nullptr; } /* @@ -1141,18 +1138,16 @@ void Server::ProcessData(NetworkPacket *pkt) Address address = getPeerAddress(peer_id); std::string addr_s = address.serializeString(); - if(m_banmanager->isIpBanned(addr_s)) { + // FIXME: Isn't it a bit excessive to check this for every packet? + if (m_banmanager->isIpBanned(addr_s)) { std::string ban_name = m_banmanager->getBanName(addr_s); infostream << "Server: A banned client tried to connect from " - << addr_s << "; banned name was " - << ban_name << std::endl; - // This actually doesn't seem to transfer to the client - DenyAccess_Legacy(peer_id, L"Your ip is banned. Banned name was " - + utf8_to_wide(ban_name)); + << addr_s << "; banned name was " << ban_name << std::endl; + DenyAccess(peer_id, SERVER_ACCESSDENIED_CUSTOM_STRING, + "Your IP is banned. Banned name was " + ban_name); return; } - } - catch(con::PeerNotFoundException &e) { + } catch (con::PeerNotFoundException &e) { /* * no peer for this packet found * most common reason is peer timeout, e.g. peer didn't @@ -1406,13 +1401,6 @@ void Server::SendAccessDenied(session_t peer_id, AccessDeniedCode reason, Send(&pkt); } -void Server::SendAccessDenied_Legacy(session_t peer_id,const std::wstring &reason) -{ - NetworkPacket pkt(TOCLIENT_ACCESS_DENIED_LEGACY, 0, peer_id); - pkt << reason; - Send(&pkt); -} - void Server::SendDeathscreen(session_t peer_id, bool set_camera_point_target, v3f camera_point_target) { @@ -2777,29 +2765,10 @@ void Server::DenySudoAccess(session_t peer_id) } -void Server::DenyAccessVerCompliant(session_t peer_id, u16 proto_ver, AccessDeniedCode reason, - const std::string &str_reason, bool reconnect) -{ - SendAccessDenied(peer_id, reason, str_reason, reconnect); - - m_clients.event(peer_id, CSE_SetDenied); - DisconnectPeer(peer_id); -} - - void Server::DenyAccess(session_t peer_id, AccessDeniedCode reason, - const std::string &custom_reason) -{ - SendAccessDenied(peer_id, reason, custom_reason); - m_clients.event(peer_id, CSE_SetDenied); - DisconnectPeer(peer_id); -} - -// 13/03/15: remove this function when protocol version 25 will become -// the minimum version for MT users, maybe in 1 year -void Server::DenyAccess_Legacy(session_t peer_id, const std::wstring &reason) + const std::string &custom_reason, bool reconnect) { - SendAccessDenied_Legacy(peer_id, reason); + SendAccessDenied(peer_id, reason, custom_reason, reconnect); m_clients.event(peer_id, CSE_SetDenied); DisconnectPeer(peer_id); } @@ -2985,8 +2954,8 @@ std::wstring Server::handleChat(const std::string &name, return ws.str(); } case RPLAYER_CHATRESULT_KICK: - DenyAccess_Legacy(player->getPeerId(), - L"You have been kicked due to message flooding."); + DenyAccess(player->getPeerId(), SERVER_ACCESSDENIED_CUSTOM_STRING, + "You have been kicked due to message flooding."); return L""; case RPLAYER_CHATRESULT_OK: break; diff --git a/src/server.h b/src/server.h index c05393291..008213c5d 100644 --- a/src/server.h +++ b/src/server.h @@ -341,12 +341,9 @@ public: void deletingPeer(con::Peer *peer, bool timeout); void DenySudoAccess(session_t peer_id); - void DenyAccessVerCompliant(session_t peer_id, u16 proto_ver, AccessDeniedCode reason, - const std::string &str_reason = "", bool reconnect = false); void DenyAccess(session_t peer_id, AccessDeniedCode reason, - const std::string &custom_reason = ""); + const std::string &custom_reason = "", bool reconnect = false); void acceptAuth(session_t peer_id, bool forSudoMode); - void DenyAccess_Legacy(session_t peer_id, const std::wstring &reason); void DisconnectPeer(session_t peer_id); bool getClientConInfo(session_t peer_id, con::rtt_stat_type type, float *retval); bool getClientInfo(session_t peer_id, ClientInfo &ret); diff --git a/src/serverenvironment.cpp b/src/serverenvironment.cpp index 34a1e33e5..f8d84604b 100644 --- a/src/serverenvironment.cpp +++ b/src/serverenvironment.cpp @@ -552,10 +552,8 @@ bool ServerEnvironment::removePlayerFromDatabase(const std::string &name) void ServerEnvironment::kickAllPlayers(AccessDeniedCode reason, const std::string &str_reason, bool reconnect) { - for (RemotePlayer *player : m_players) { - m_server->DenyAccessVerCompliant(player->getPeerId(), - player->protocol_version, reason, str_reason, reconnect); - } + for (RemotePlayer *player : m_players) + m_server->DenyAccess(player->getPeerId(), reason, str_reason, reconnect); } void ServerEnvironment::saveLoadedPlayers(bool force) -- cgit v1.2.3 From 56a558baf8ef43810014347ae624cb4ef70b6aa3 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Sat, 9 Apr 2022 14:47:59 +0200 Subject: Refactor some Lua API functions in preparation for async env --- src/gamedef.h | 2 ++ src/script/common/c_content.cpp | 8 ++++---- src/script/common/c_content.h | 6 +++--- src/script/lua_api/l_craft.cpp | 24 +++++++++++------------- src/script/lua_api/l_item.cpp | 6 +++--- src/script/lua_api/l_server.cpp | 28 +++++++++++++--------------- src/server.cpp | 5 ----- src/server.h | 3 +-- 8 files changed, 37 insertions(+), 45 deletions(-) (limited to 'src/script/lua_api/l_server.cpp') diff --git a/src/gamedef.h b/src/gamedef.h index 8a9246da2..45b9c4750 100644 --- a/src/gamedef.h +++ b/src/gamedef.h @@ -63,6 +63,8 @@ public: virtual IRollbackManager* getRollbackManager() { return NULL; } // Shorthands + // TODO: these should be made const-safe so that a const IGameDef* is + // actually usable IItemDefManager *idef() { return getItemDefManager(); } const NodeDefManager *ndef() { return getNodeDefManager(); } ICraftDefManager *cdef() { return getCraftDefManager(); } diff --git a/src/script/common/c_content.cpp b/src/script/common/c_content.cpp index 36f4316ee..a233afb05 100644 --- a/src/script/common/c_content.cpp +++ b/src/script/common/c_content.cpp @@ -1371,7 +1371,7 @@ void push_inventory_lists(lua_State *L, const Inventory &inv) /******************************************************************************/ void read_inventory_list(lua_State *L, int tableindex, - Inventory *inv, const char *name, Server* srv, int forcesize) + Inventory *inv, const char *name, IGameDef *gdef, int forcesize) { if(tableindex < 0) tableindex = lua_gettop(L) + 1 + tableindex; @@ -1383,7 +1383,7 @@ void read_inventory_list(lua_State *L, int tableindex, } // Get Lua-specified items to insert into the list - std::vector items = read_items(L, tableindex,srv); + std::vector items = read_items(L, tableindex, gdef); size_t listsize = (forcesize >= 0) ? forcesize : items.size(); // Create or resize/clear list @@ -1635,7 +1635,7 @@ void push_items(lua_State *L, const std::vector &items) } /******************************************************************************/ -std::vector read_items(lua_State *L, int index, Server *srv) +std::vector read_items(lua_State *L, int index, IGameDef *gdef) { if(index < 0) index = lua_gettop(L) + 1 + index; @@ -1651,7 +1651,7 @@ std::vector read_items(lua_State *L, int index, Server *srv) if (items.size() < (u32) key) { items.resize(key); } - items[key - 1] = read_item(L, -1, srv->idef()); + items[key - 1] = read_item(L, -1, gdef->idef()); lua_pop(L, 1); } return items; diff --git a/src/script/common/c_content.h b/src/script/common/c_content.h index 11b39364f..a7b8709c6 100644 --- a/src/script/common/c_content.h +++ b/src/script/common/c_content.h @@ -59,7 +59,7 @@ class InventoryList; struct NodeBox; struct ContentFeatures; struct TileDef; -class Server; +class IGameDef; struct DigParams; struct HitParams; struct EnumString; @@ -126,7 +126,7 @@ void push_inventory_lists (lua_State *L, const Inventory &inv); void read_inventory_list (lua_State *L, int tableindex, Inventory *inv, const char *name, - Server *srv, int forcesize=-1); + IGameDef *gdef, int forcesize=-1); MapNode readnode (lua_State *L, int index, const NodeDefManager *ndef); @@ -166,7 +166,7 @@ void push_items (lua_State *L, std::vector read_items (lua_State *L, int index, - Server* srv); + IGameDef* gdef); void push_soundspec (lua_State *L, const SimpleSoundSpec &spec); diff --git a/src/script/lua_api/l_craft.cpp b/src/script/lua_api/l_craft.cpp index 18622ee00..c2c5a5551 100644 --- a/src/script/lua_api/l_craft.cpp +++ b/src/script/lua_api/l_craft.cpp @@ -371,8 +371,9 @@ int ModApiCraft::l_clear_craft(lua_State *L) int ModApiCraft::l_get_craft_result(lua_State *L) { NO_MAP_LOCK_REQUIRED; + IGameDef *gdef = getGameDef(L); - int input_i = 1; + const int input_i = 1; std::string method_s = getstringfield_default(L, input_i, "method", "normal"); enum CraftMethod method = (CraftMethod)getenumfield(L, input_i, "method", es_CraftMethod, CRAFT_METHOD_NORMAL); @@ -382,10 +383,9 @@ int ModApiCraft::l_get_craft_result(lua_State *L) width = luaL_checkinteger(L, -1); lua_pop(L, 1); lua_getfield(L, input_i, "items"); - std::vector items = read_items(L, -1,getServer(L)); + std::vector items = read_items(L, -1, gdef); lua_pop(L, 1); // items - IGameDef *gdef = getServer(L); ICraftDefManager *cdef = gdef->cdef(); CraftInput input(method, width, items); CraftOutput output; @@ -465,13 +465,13 @@ static void push_craft_recipes(lua_State *L, IGameDef *gdef, const std::vector &recipes, const CraftOutput &output) { - lua_createtable(L, recipes.size(), 0); - if (recipes.empty()) { lua_pushnil(L); return; } + lua_createtable(L, recipes.size(), 0); + std::vector::const_iterator it = recipes.begin(); for (unsigned i = 0; it != recipes.end(); ++it) { lua_newtable(L); @@ -487,10 +487,9 @@ int ModApiCraft::l_get_craft_recipe(lua_State *L) NO_MAP_LOCK_REQUIRED; std::string item = luaL_checkstring(L, 1); - Server *server = getServer(L); + IGameDef *gdef = getGameDef(L); CraftOutput output(item, 0); - std::vector recipes = server->cdef() - ->getCraftRecipes(output, server, 1); + auto recipes = gdef->cdef()->getCraftRecipes(output, gdef, 1); lua_createtable(L, 1, 0); @@ -500,7 +499,7 @@ int ModApiCraft::l_get_craft_recipe(lua_State *L) setintfield(L, -1, "width", 0); return 1; } - push_craft_recipe(L, server, recipes[0], output); + push_craft_recipe(L, gdef, recipes[0], output); return 1; } @@ -510,12 +509,11 @@ int ModApiCraft::l_get_all_craft_recipes(lua_State *L) NO_MAP_LOCK_REQUIRED; std::string item = luaL_checkstring(L, 1); - Server *server = getServer(L); + IGameDef *gdef = getGameDef(L); CraftOutput output(item, 0); - std::vector recipes = server->cdef() - ->getCraftRecipes(output, server); + auto recipes = gdef->cdef()->getCraftRecipes(output, gdef); - push_craft_recipes(L, server, recipes, output); + push_craft_recipes(L, gdef, recipes, output); return 1; } diff --git a/src/script/lua_api/l_item.cpp b/src/script/lua_api/l_item.cpp index 794d8a6e5..fc97a1736 100644 --- a/src/script/lua_api/l_item.cpp +++ b/src/script/lua_api/l_item.cpp @@ -632,8 +632,8 @@ int ModApiItemMod::l_get_content_id(lua_State *L) NO_MAP_LOCK_REQUIRED; std::string name = luaL_checkstring(L, 1); - const IItemDefManager *idef = getGameDef(L)->getItemDefManager(); - const NodeDefManager *ndef = getGameDef(L)->getNodeDefManager(); + const IItemDefManager *idef = getGameDef(L)->idef(); + const NodeDefManager *ndef = getGameDef(L)->ndef(); // If this is called at mod load time, NodeDefManager isn't aware of // aliases yet, so we need to handle them manually @@ -658,7 +658,7 @@ int ModApiItemMod::l_get_name_from_content_id(lua_State *L) NO_MAP_LOCK_REQUIRED; content_t c = luaL_checkint(L, 1); - const NodeDefManager *ndef = getGameDef(L)->getNodeDefManager(); + const NodeDefManager *ndef = getGameDef(L)->ndef(); const char *name = ndef->get(c).name.c_str(); lua_pushstring(L, name); diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 5b3054d17..42725e5d2 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -61,11 +61,8 @@ int ModApiServer::l_get_server_uptime(lua_State *L) int ModApiServer::l_get_server_max_lag(lua_State *L) { NO_MAP_LOCK_REQUIRED; - ServerEnvironment *s_env = dynamic_cast(getEnv(L)); - if (!s_env) - lua_pushnil(L); - else - lua_pushnumber(L, s_env->getMaxLagEstimate()); + GET_ENV_PTR; + lua_pushnumber(L, env->getMaxLagEstimate()); return 1; } @@ -395,12 +392,11 @@ int ModApiServer::l_get_modpath(lua_State *L) { NO_MAP_LOCK_REQUIRED; std::string modname = luaL_checkstring(L, 1); - const ModSpec *mod = getServer(L)->getModSpec(modname); - if (!mod) { + const ModSpec *mod = getGameDef(L)->getModSpec(modname); + if (!mod) lua_pushnil(L); - return 1; - } - lua_pushstring(L, mod->path.c_str()); + else + lua_pushstring(L, mod->path.c_str()); return 1; } @@ -412,13 +408,14 @@ int ModApiServer::l_get_modnames(lua_State *L) // Get a list of mods std::vector modlist; - getServer(L)->getModNames(modlist); + for (auto &it : getGameDef(L)->getMods()) + modlist.emplace_back(it.name); std::sort(modlist.begin(), modlist.end()); // Package them up for Lua lua_createtable(L, modlist.size(), 0); - std::vector::iterator iter = modlist.begin(); + auto iter = modlist.begin(); for (u16 i = 0; iter != modlist.end(); ++iter) { lua_pushstring(L, iter->c_str()); lua_rawseti(L, -2, ++i); @@ -430,8 +427,8 @@ int ModApiServer::l_get_modnames(lua_State *L) int ModApiServer::l_get_worldpath(lua_State *L) { NO_MAP_LOCK_REQUIRED; - std::string worldpath = getServer(L)->getWorldPath(); - lua_pushstring(L, worldpath.c_str()); + const Server *srv = getServer(L); + lua_pushstring(L, srv->getWorldPath().c_str()); return 1; } @@ -513,7 +510,8 @@ int ModApiServer::l_dynamic_add_media(lua_State *L) int ModApiServer::l_is_singleplayer(lua_State *L) { NO_MAP_LOCK_REQUIRED; - lua_pushboolean(L, getServer(L)->isSingleplayer()); + const Server *srv = getServer(L); + lua_pushboolean(L, srv->isSingleplayer()); return 1; } diff --git a/src/server.cpp b/src/server.cpp index 9d7e8e563..dec6cf44c 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -3658,11 +3658,6 @@ const ModSpec *Server::getModSpec(const std::string &modname) const return m_modmgr->getModSpec(modname); } -void Server::getModNames(std::vector &modlist) -{ - m_modmgr->getModNames(modlist); -} - std::string Server::getBuiltinLuaPath() { return porting::path_share + DIR_DELIM + "builtin"; diff --git a/src/server.h b/src/server.h index 008213c5d..bd799c313 100644 --- a/src/server.h +++ b/src/server.h @@ -292,11 +292,10 @@ public: virtual const std::vector &getMods() const; virtual const ModSpec* getModSpec(const std::string &modname) const; - void getModNames(std::vector &modlist); std::string getBuiltinLuaPath(); virtual std::string getWorldPath() const { return m_path_world; } - inline bool isSingleplayer() + inline bool isSingleplayer() const { return m_simple_singleplayer_mode; } inline void setAsyncFatalError(const std::string &error) -- cgit v1.2.3 From e7659883cc6fca343785da2a1af3890ae273abbf Mon Sep 17 00:00:00 2001 From: sfan5 Date: Mon, 2 May 2022 20:55:04 +0200 Subject: Async environment for mods to do concurrent tasks (#11131) --- builtin/async/game.lua | 46 ++ builtin/async/init.lua | 11 - builtin/async/mainmenu.lua | 9 + builtin/game/async.lua | 22 + builtin/game/init.lua | 1 + builtin/game/misc.lua | 29 ++ builtin/init.lua | 6 +- doc/lua_api.txt | 62 +++ games/devtest/mods/unittests/async_env.lua | 149 ++++++ games/devtest/mods/unittests/init.lua | 1 + games/devtest/mods/unittests/inside_async_env.lua | 15 + src/map.cpp | 33 ++ src/map.h | 17 +- src/script/common/CMakeLists.txt | 1 + src/script/common/c_internal.cpp | 14 + src/script/common/c_internal.h | 5 + src/script/common/c_packer.cpp | 583 ++++++++++++++++++++++ src/script/common/c_packer.h | 123 +++++ src/script/cpp_api/s_async.cpp | 183 +++++-- src/script/cpp_api/s_async.h | 54 +- src/script/lua_api/l_craft.cpp | 8 + src/script/lua_api/l_craft.h | 1 + src/script/lua_api/l_internal.h | 2 +- src/script/lua_api/l_item.cpp | 25 + src/script/lua_api/l_item.h | 7 +- src/script/lua_api/l_noise.cpp | 53 ++ src/script/lua_api/l_noise.h | 6 + src/script/lua_api/l_server.cpp | 85 ++++ src/script/lua_api/l_server.h | 10 + src/script/lua_api/l_util.cpp | 5 + src/script/lua_api/l_util.h | 2 - src/script/lua_api/l_vmanip.cpp | 33 ++ src/script/lua_api/l_vmanip.h | 3 + src/script/scripting_server.cpp | 67 ++- src/script/scripting_server.h | 17 + src/server.cpp | 4 + src/server.h | 8 +- src/serverenvironment.cpp | 2 + src/util/basic_macros.h | 8 +- 39 files changed, 1654 insertions(+), 56 deletions(-) create mode 100644 builtin/async/game.lua delete mode 100644 builtin/async/init.lua create mode 100644 builtin/async/mainmenu.lua create mode 100644 builtin/game/async.lua create mode 100644 games/devtest/mods/unittests/async_env.lua create mode 100644 games/devtest/mods/unittests/inside_async_env.lua create mode 100644 src/script/common/c_packer.cpp create mode 100644 src/script/common/c_packer.h (limited to 'src/script/lua_api/l_server.cpp') diff --git a/builtin/async/game.lua b/builtin/async/game.lua new file mode 100644 index 000000000..212a33e17 --- /dev/null +++ b/builtin/async/game.lua @@ -0,0 +1,46 @@ +core.log("info", "Initializing asynchronous environment (game)") + +local function pack2(...) + return {n=select('#', ...), ...} +end + +-- Entrypoint to run async jobs, called by C++ +function core.job_processor(func, params) + local retval = pack2(func(unpack(params, 1, params.n))) + + return retval +end + +-- Import a bunch of individual files from builtin/game/ +local gamepath = core.get_builtin_path() .. "game" .. DIR_DELIM + +dofile(gamepath .. "constants.lua") +dofile(gamepath .. "item_s.lua") +dofile(gamepath .. "misc_s.lua") +dofile(gamepath .. "features.lua") +dofile(gamepath .. "voxelarea.lua") + +-- Transfer of globals +do + assert(core.transferred_globals) + local all = core.deserialize(core.transferred_globals, true) + core.transferred_globals = nil + + -- reassemble other tables + all.registered_nodes = {} + all.registered_craftitems = {} + all.registered_tools = {} + for k, v in pairs(all.registered_items) do + if v.type == "node" then + all.registered_nodes[k] = v + elseif v.type == "craftitem" then + all.registered_craftitems[k] = v + elseif v.type == "tool" then + all.registered_tools[k] = v + end + end + + for k, v in pairs(all) do + core[k] = v + end +end diff --git a/builtin/async/init.lua b/builtin/async/init.lua deleted file mode 100644 index 3803994d6..000000000 --- a/builtin/async/init.lua +++ /dev/null @@ -1,11 +0,0 @@ - -core.log("info", "Initializing Asynchronous environment") - -function core.job_processor(func, serialized_param) - local param = core.deserialize(serialized_param) - - local retval = core.serialize(func(param)) - - return retval or core.serialize(nil) -end - diff --git a/builtin/async/mainmenu.lua b/builtin/async/mainmenu.lua new file mode 100644 index 000000000..0e9c222d1 --- /dev/null +++ b/builtin/async/mainmenu.lua @@ -0,0 +1,9 @@ +core.log("info", "Initializing asynchronous environment") + +function core.job_processor(func, serialized_param) + local param = core.deserialize(serialized_param) + + local retval = core.serialize(func(param)) + + return retval or core.serialize(nil) +end diff --git a/builtin/game/async.lua b/builtin/game/async.lua new file mode 100644 index 000000000..469f179d7 --- /dev/null +++ b/builtin/game/async.lua @@ -0,0 +1,22 @@ + +core.async_jobs = {} + +function core.async_event_handler(jobid, retval) + local callback = core.async_jobs[jobid] + assert(type(callback) == "function") + callback(unpack(retval, 1, retval.n)) + core.async_jobs[jobid] = nil +end + +function core.handle_async(func, callback, ...) + assert(type(func) == "function" and type(callback) == "function", + "Invalid minetest.handle_async invocation") + local args = {n = select("#", ...), ...} + local mod_origin = core.get_last_run_mod() + + local jobid = core.do_async_callback(func, args, mod_origin) + core.async_jobs[jobid] = callback + + return true +end + diff --git a/builtin/game/init.lua b/builtin/game/init.lua index c5f08113b..68d6a10f8 100644 --- a/builtin/game/init.lua +++ b/builtin/game/init.lua @@ -34,5 +34,6 @@ dofile(gamepath .. "voxelarea.lua") dofile(gamepath .. "forceloading.lua") dofile(gamepath .. "statbars.lua") dofile(gamepath .. "knockback.lua") +dofile(gamepath .. "async.lua") profiler = nil diff --git a/builtin/game/misc.lua b/builtin/game/misc.lua index 18d5a7310..9f5e3312b 100644 --- a/builtin/game/misc.lua +++ b/builtin/game/misc.lua @@ -235,3 +235,32 @@ end -- Used for callback handling with dynamic_add_media core.dynamic_media_callbacks = {} + + +-- Transfer of certain globals into async environment +-- see builtin/async/game.lua for the other side + +local function copy_filtering(t, seen) + if type(t) == "userdata" or type(t) == "function" then + return true -- don't use nil so presence can still be detected + elseif type(t) ~= "table" then + return t + end + local n = {} + seen = seen or {} + seen[t] = n + for k, v in pairs(t) do + local k_ = seen[k] or copy_filtering(k, seen) + local v_ = seen[v] or copy_filtering(v, seen) + n[k_] = v_ + end + return n +end + +function core.get_globals_to_transfer() + local all = { + registered_items = copy_filtering(core.registered_items), + registered_aliases = core.registered_aliases, + } + return core.serialize(all) +end diff --git a/builtin/init.lua b/builtin/init.lua index 7a9b5c427..869136016 100644 --- a/builtin/init.lua +++ b/builtin/init.lua @@ -56,8 +56,10 @@ elseif INIT == "mainmenu" then if not custom_loaded then dofile(core.get_mainmenu_path() .. DIR_DELIM .. "init.lua") end -elseif INIT == "async" then - dofile(asyncpath .. "init.lua") +elseif INIT == "async" then + dofile(asyncpath .. "mainmenu.lua") +elseif INIT == "async_game" then + dofile(asyncpath .. "game.lua") elseif INIT == "client" then dofile(clientpath .. "init.lua") else diff --git a/doc/lua_api.txt b/doc/lua_api.txt index f54672db7..339ce8a27 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -5767,6 +5767,68 @@ Timing * `job:cancel()` * Cancels the job function from being called +Async environment +----------------- + +The engine allows you to submit jobs to be ran in an isolated environment +concurrently with normal server operation. +A job consists of a function to be ran in the async environment, any amount of +arguments (will be serialized) and a callback that will be called with the return +value of the job function once it is finished. + +The async environment does *not* have access to the map, entities, players or any +globals defined in the 'usual' environment. Consequently, functions like +`minetest.get_node()` or `minetest.get_player_by_name()` simply do not exist in it. + +Arguments and return values passed through this can contain certain userdata +objects that will be seamlessly copied (not shared) to the async environment. +This allows you easy interoperability for delegating work to jobs. + +* `minetest.handle_async(func, callback, ...)`: + * Queue the function `func` to be ran in an async environment. + Note that there are multiple persistent workers and any of them may + end up running a given job. The engine will scale the amount of + worker threads automatically. + * When `func` returns the callback is called (in the normal environment) + with all of the return values as arguments. + * Optional: Variable number of arguments that are passed to `func` +* `minetest.register_async_dofile(path)`: + * Register a path to a Lua file to be imported when an async environment + is initialized. You can use this to preload code which you can then call + later using `minetest.handle_async()`. + +### List of APIs available in an async environment + +Classes: +* `ItemStack` +* `PerlinNoise` +* `PerlinNoiseMap` +* `PseudoRandom` +* `PcgRandom` +* `SecureRandom` +* `VoxelArea` +* `VoxelManip` + * only if transferred into environment; can't read/write to map +* `Settings` + +Class instances that can be transferred between environments: +* `ItemStack` +* `PerlinNoise` +* `PerlinNoiseMap` +* `VoxelManip` + +Functions: +* Standalone helpers such as logging, filesystem, encoding, + hashing or compression APIs +* `minetest.request_insecure_environment` (same restrictions apply) + +Variables: +* `minetest.settings` +* `minetest.registered_items`, `registered_nodes`, `registered_tools`, + `registered_craftitems` and `registered_aliases` + * with all functions and userdata values replaced by `true`, calling any + callbacks here is obviously not possible + Server ------ diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua new file mode 100644 index 000000000..aff1fc4d9 --- /dev/null +++ b/games/devtest/mods/unittests/async_env.lua @@ -0,0 +1,149 @@ +-- helper + +core.register_async_dofile(core.get_modpath(core.get_current_modname()) .. + DIR_DELIM .. "inside_async_env.lua") + +local function deepequal(a, b) + if type(a) == "function" then + return type(b) == "function" + elseif type(a) ~= "table" then + return a == b + elseif type(b) ~= "table" then + return false + end + for k, v in pairs(a) do + if not deepequal(v, b[k]) then + return false + end + end + for k, v in pairs(b) do + if not deepequal(a[k], v) then + return false + end + end + return true +end + +-- Object Passing / Serialization + +local test_object = { + name = "stairs:stair_glass", + type = "node", + groups = {oddly_breakable_by_hand = 3, cracky = 3, stair = 1}, + description = "Glass Stair", + sounds = { + dig = {name = "default_glass_footstep", gain = 0.5}, + footstep = {name = "default_glass_footstep", gain = 0.3}, + dug = {name = "default_break_glass", gain = 1} + }, + node_box = { + fixed = { + {-0.5, -0.5, -0.5, 0.5, 0, 0.5}, + {-0.5, 0, 0, 0.5, 0.5, 0.5} + }, + type = "fixed" + }, + tiles = { + {name = "stairs_glass_split.png", backface_culling = true}, + {name = "default_glass.png", backface_culling = true}, + {name = "stairs_glass_stairside.png^[transformFX", backface_culling = true} + }, + on_place = function(itemstack, placer) + return core.is_player(placer) + end, + sunlight_propagates = true, + is_ground_content = false, + light_source = 0, +} + +local function test_object_passing() + local tmp = core.serialize_roundtrip(test_object) + assert(deepequal(test_object, tmp)) + + -- Circular key, should error + tmp = {"foo", "bar"} + tmp[tmp] = true + assert(not pcall(core.serialize_roundtrip, tmp)) + + -- Circular value, should error + tmp = {"foo"} + tmp[2] = tmp + assert(not pcall(core.serialize_roundtrip, tmp)) +end +unittests.register("test_object_passing", test_object_passing) + +local function test_userdata_passing(_, pos) + -- basic userdata passing + local obj = table.copy(test_object.tiles[1]) + obj.test = ItemStack("default:cobble 99") + local tmp = core.serialize_roundtrip(obj) + assert(type(tmp.test) == "userdata") + assert(obj.test:to_string() == tmp.test:to_string()) + + -- object can't be passed, should error + obj = core.raycast(pos, pos) + assert(not pcall(core.serialize_roundtrip, obj)) + + -- VManip + local vm = core.get_voxel_manip(pos, pos) + local expect = vm:get_node_at(pos) + local vm2 = core.serialize_roundtrip(vm) + assert(deepequal(vm2:get_node_at(pos), expect)) +end +unittests.register("test_userdata_passing", test_userdata_passing, {map=true}) + +-- Asynchronous jobs + +local function test_handle_async(cb) + -- Basic test including mod name tracking and unittests.async_test() + -- which is defined inside_async_env.lua + local func = function(x) + return core.get_last_run_mod(), _VERSION, unittests[x]() + end + local expect = {core.get_last_run_mod(), _VERSION, true} + + core.handle_async(func, function(...) + if not deepequal(expect, {...}) then + cb("Values did not equal") + end + if core.get_last_run_mod() ~= expect[1] then + cb("Mod name not tracked correctly") + end + + -- Test passing of nil arguments and return values + core.handle_async(function(a, b) + return a, b + end, function(a, b) + if b ~= 123 then + cb("Argument went missing") + end + cb() + end, nil, 123) + end, "async_test") +end +unittests.register("test_handle_async", test_handle_async, {async=true}) + +local function test_userdata_passing2(cb, _, pos) + -- VManip: check transfer into other env + local vm = core.get_voxel_manip(pos, pos) + local expect = vm:get_node_at(pos) + + core.handle_async(function(vm_, pos_) + return vm_:get_node_at(pos_) + end, function(ret) + if not deepequal(expect, ret) then + cb("Node data mismatch (one-way)") + end + + -- VManip: test a roundtrip + core.handle_async(function(vm_) + return vm_ + end, function(vm2) + if not deepequal(expect, vm2:get_node_at(pos)) then + cb("Node data mismatch (roundtrip)") + end + cb() + end, vm) + end, vm, pos) +end +unittests.register("test_userdata_passing2", test_userdata_passing2, {map=true, async=true}) diff --git a/games/devtest/mods/unittests/init.lua b/games/devtest/mods/unittests/init.lua index 0754d507f..0608f2dd2 100644 --- a/games/devtest/mods/unittests/init.lua +++ b/games/devtest/mods/unittests/init.lua @@ -175,6 +175,7 @@ dofile(modpath .. "/misc.lua") dofile(modpath .. "/player.lua") dofile(modpath .. "/crafting.lua") dofile(modpath .. "/itemdescription.lua") +dofile(modpath .. "/async_env.lua") -------------- diff --git a/games/devtest/mods/unittests/inside_async_env.lua b/games/devtest/mods/unittests/inside_async_env.lua new file mode 100644 index 000000000..9774771f9 --- /dev/null +++ b/games/devtest/mods/unittests/inside_async_env.lua @@ -0,0 +1,15 @@ +unittests = {} + +core.log("info", "Hello World") + +function unittests.async_test() + assert(core == minetest) + -- stuff that should not be here + assert(not core.get_player_by_name) + assert(not core.set_node) + assert(not core.object_refs) + -- stuff that should be here + assert(ItemStack) + assert(core.registered_items[""]) + return true +end diff --git a/src/map.cpp b/src/map.cpp index 9c9324f5f..5153dcaa9 100644 --- a/src/map.cpp +++ b/src/map.cpp @@ -1896,6 +1896,7 @@ MMVManip::MMVManip(Map *map): VoxelManipulator(), m_map(map) { + assert(map); } void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max, @@ -1903,6 +1904,8 @@ void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max, { TimeTaker timer1("initialEmerge", &emerge_time); + assert(m_map); + // Units of these are MapBlocks v3s16 p_min = blockpos_min; v3s16 p_max = blockpos_max; @@ -1986,6 +1989,7 @@ void MMVManip::blitBackAll(std::map *modified_blocks, { if(m_area.getExtent() == v3s16(0,0,0)) return; + assert(m_map); /* Copy data of all blocks @@ -2006,4 +2010,33 @@ void MMVManip::blitBackAll(std::map *modified_blocks, } } +MMVManip *MMVManip::clone() const +{ + MMVManip *ret = new MMVManip(); + + const s32 size = m_area.getVolume(); + ret->m_area = m_area; + if (m_data) { + ret->m_data = new MapNode[size]; + memcpy(ret->m_data, m_data, size * sizeof(MapNode)); + } + if (m_flags) { + ret->m_flags = new u8[size]; + memcpy(ret->m_flags, m_flags, size * sizeof(u8)); + } + + ret->m_is_dirty = m_is_dirty; + // Even if the copy is disconnected from a map object keep the information + // needed to write it back to one + ret->m_loaded_blocks = m_loaded_blocks; + + return ret; +} + +void MMVManip::reparent(Map *map) +{ + assert(map && !m_map); + m_map = map; +} + //END diff --git a/src/map.h b/src/map.h index d8ed29106..21e3dbd6c 100644 --- a/src/map.h +++ b/src/map.h @@ -446,10 +446,25 @@ public: void blitBackAll(std::map * modified_blocks, bool overwrite_generated = true); + /* + Creates a copy of this VManip including contents, the copy will not be + associated with a Map. + */ + MMVManip *clone() const; + + // Reassociates a copied VManip to a map + void reparent(Map *map); + + // Is it impossible to call initialEmerge / blitBackAll? + inline bool isOrphan() const { return !m_map; } + bool m_is_dirty = false; protected: - Map *m_map; + MMVManip() {}; + + // may be null + Map *m_map = nullptr; /* key = blockpos value = flags describing the block diff --git a/src/script/common/CMakeLists.txt b/src/script/common/CMakeLists.txt index d07f6ab1b..3e84b46c7 100644 --- a/src/script/common/CMakeLists.txt +++ b/src/script/common/CMakeLists.txt @@ -3,6 +3,7 @@ set(common_SCRIPT_COMMON_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/c_converter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/c_types.cpp ${CMAKE_CURRENT_SOURCE_DIR}/c_internal.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/c_packer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helper.cpp PARENT_SCOPE) diff --git a/src/script/common/c_internal.cpp b/src/script/common/c_internal.cpp index df82dba14..ddd2d184c 100644 --- a/src/script/common/c_internal.cpp +++ b/src/script/common/c_internal.cpp @@ -166,3 +166,17 @@ void log_deprecated(lua_State *L, std::string message, int stack_depth) infostream << script_get_backtrace(L) << std::endl; } +void call_string_dump(lua_State *L, int idx) +{ + // Retrieve string.dump from insecure env to avoid it being tampered with + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_GLOBALS_BACKUP); + if (!lua_isnil(L, -1)) + lua_getfield(L, -1, "string"); + else + lua_getglobal(L, "string"); + lua_getfield(L, -1, "dump"); + lua_remove(L, -2); // remove _G + lua_remove(L, -2); // remove 'string' table + lua_pushvalue(L, idx); + lua_call(L, 1, 1); +} diff --git a/src/script/common/c_internal.h b/src/script/common/c_internal.h index c43db34aa..272a39941 100644 --- a/src/script/common/c_internal.h +++ b/src/script/common/c_internal.h @@ -56,6 +56,7 @@ extern "C" { #define CUSTOM_RIDX_BACKTRACE (CUSTOM_RIDX_BASE + 3) #define CUSTOM_RIDX_HTTP_API_LUA (CUSTOM_RIDX_BASE + 4) #define CUSTOM_RIDX_VECTOR_METATABLE (CUSTOM_RIDX_BASE + 5) +#define CUSTOM_RIDX_METATABLE_MAP (CUSTOM_RIDX_BASE + 6) // Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata @@ -139,3 +140,7 @@ DeprecatedHandlingMode get_deprecated_handling_mode(); * @param stack_depth How far on the stack to the first user function (ie: not builtin or core) */ void log_deprecated(lua_State *L, std::string message, int stack_depth = 1); + +// Safely call string.dump on a function value +// (does not pop, leaves one value on stack) +void call_string_dump(lua_State *L, int idx); diff --git a/src/script/common/c_packer.cpp b/src/script/common/c_packer.cpp new file mode 100644 index 000000000..fc5277330 --- /dev/null +++ b/src/script/common/c_packer.cpp @@ -0,0 +1,583 @@ +/* +Minetest +Copyright (C) 2022 sfan5 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#include +#include +#include +#include +#include +#include +#include "c_packer.h" +#include "c_internal.h" +#include "log.h" +#include "debug.h" +#include "threading/mutex_auto_lock.h" + +extern "C" { +#include +} + +// +// Helpers +// + +// convert negative index to absolute position on Lua stack +static inline int absidx(lua_State *L, int idx) +{ + assert(idx < 0); + return lua_gettop(L) + idx + 1; +} + +// does the type put anything into PackedInstr::sdata? +static inline bool uses_sdata(int type) +{ + switch (type) { + case LUA_TSTRING: + case LUA_TFUNCTION: + case LUA_TUSERDATA: + return true; + default: + return false; + } +} + +// does the type put anything into PackedInstr::? +static inline bool uses_union(int type) +{ + switch (type) { + case LUA_TNIL: + case LUA_TSTRING: + case LUA_TFUNCTION: + return false; + default: + return true; + } +} + +static inline bool can_set_into(int ktype, int vtype) +{ + switch (ktype) { + case LUA_TNUMBER: + return !uses_union(vtype); + case LUA_TSTRING: + return !uses_sdata(vtype); + default: + return false; + } +} + +// is the key suitable for use with set_into? +static inline bool suitable_key(lua_State *L, int idx) +{ + if (lua_type(L, idx) == LUA_TSTRING) { + // strings may not have a NULL byte (-> lua_setfield) + size_t len; + const char *str = lua_tolstring(L, idx, &len); + return strlen(str) == len; + } else { + assert(lua_type(L, idx) == LUA_TNUMBER); + // numbers must fit into an s32 and be integers (-> lua_rawseti) + lua_Number n = lua_tonumber(L, idx); + return std::floor(n) == n && n >= S32_MIN && n <= S32_MAX; + } +} + +namespace { + // checks if you left any values on the stack, for debugging + class StackChecker { + lua_State *L; + int top; + public: + StackChecker(lua_State *L) : L(L), top(lua_gettop(L)) {} + ~StackChecker() { + assert(lua_gettop(L) >= top); + if (lua_gettop(L) > top) { + rawstream << "Lua stack not cleaned up: " + << lua_gettop(L) << " != " << top + << " (false-positive if exception thrown)" << std::endl; + } + } + }; + + // Since an std::vector may reallocate, this is the only safe way to keep + // a reference to a particular element. + template + class VectorRef { + std::vector *vec; + size_t idx; + VectorRef(std::vector *vec, size_t idx) : vec(vec), idx(idx) {} + public: + static VectorRef front(std::vector &vec) { + return VectorRef(&vec, 0); + } + static VectorRef back(std::vector &vec) { + return VectorRef(&vec, vec.size() - 1); + } + T &operator*() { return (*vec)[idx]; } + T *operator->() { return &(*vec)[idx]; } + }; + + struct Packer { + PackInFunc fin; + PackOutFunc fout; + }; + + typedef std::pair PackerTuple; +} + +static inline auto emplace(PackedValue &pv, s16 type) +{ + pv.i.emplace_back(); + auto ref = VectorRef::back(pv.i); + ref->type = type; + // Initialize fields that may be left untouched + if (type == LUA_TTABLE) { + ref->uidata1 = 0; + ref->uidata2 = 0; + } else if (type == LUA_TUSERDATA) { + ref->ptrdata = nullptr; + } else if (type == INSTR_POP) { + ref->sidata2 = 0; + } + return ref; +} + +// +// Management of registered packers +// + +static std::unordered_map g_packers; +static std::mutex g_packers_lock; + +void script_register_packer(lua_State *L, const char *regname, + PackInFunc fin, PackOutFunc fout) +{ + // Store away callbacks + { + MutexAutoLock autolock(g_packers_lock); + auto it = g_packers.find(regname); + if (it == g_packers.end()) { + auto &ref = g_packers[regname]; + ref.fin = fin; + ref.fout = fout; + } else { + FATAL_ERROR_IF(it->second.fin != fin || it->second.fout != fout, + "Packer registered twice with mismatching callbacks"); + } + } + + // Save metatable so we can identify instances later + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + if (lua_isnil(L, -1)) { + lua_newtable(L); + lua_pushvalue(L, -1); + lua_rawseti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + } + + luaL_getmetatable(L, regname); + FATAL_ERROR_IF(lua_isnil(L, -1), "No metatable registered with that name"); + + // CUSTOM_RIDX_METATABLE_MAP contains { [metatable] = "regname", ... } + // check first + lua_pushstring(L, regname); + lua_rawget(L, -3); + if (!lua_isnil(L, -1)) { + FATAL_ERROR_IF(lua_topointer(L, -1) != lua_topointer(L, -2), + "Packer registered twice with inconsistent metatable"); + } + lua_pop(L, 1); + // then set + lua_pushstring(L, regname); + lua_rawset(L, -3); + + lua_pop(L, 1); +} + +static bool find_packer(const char *regname, PackerTuple &out) +{ + MutexAutoLock autolock(g_packers_lock); + auto it = g_packers.find(regname); + if (it == g_packers.end()) + return false; + // copy data for thread safety + out.first = it->first; + out.second = it->second; + return true; +} + +static bool find_packer(lua_State *L, int idx, PackerTuple &out) +{ +#ifndef NDEBUG + StackChecker checker(L); +#endif + + // retrieve metatable of the object + if (lua_getmetatable(L, idx) != 1) + return false; + + // use our global table to map it to the registry name + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + assert(lua_istable(L, -1)); + lua_pushvalue(L, -2); + lua_rawget(L, -2); + if (lua_isnil(L, -1)) { + lua_pop(L, 3); + return false; + } + + // load the associated data + bool found = find_packer(lua_tostring(L, -1), out); + FATAL_ERROR_IF(!found, "Inconsistent internal state"); + lua_pop(L, 3); + return true; +} + +// +// Packing implementation +// + +// recursively goes through the structure and ensures there are no circular references +static void pack_validate(lua_State *L, int idx, std::unordered_set &seen) +{ +#ifndef NDEBUG + StackChecker checker(L); + assert(idx > 0); +#endif + + if (lua_type(L, idx) != LUA_TTABLE) + return; + + const void *ptr = lua_topointer(L, idx); + assert(ptr); + + if (seen.find(ptr) != seen.end()) + throw LuaError("Circular references cannot be packed (yet)"); + seen.insert(ptr); + + lua_checkstack(L, 5); + lua_pushnil(L); + while (lua_next(L, idx) != 0) { + // key at -2, value at -1 + pack_validate(L, absidx(L, -2), seen); + pack_validate(L, absidx(L, -1), seen); + + lua_pop(L, 1); + } + + seen.erase(ptr); +} + +static VectorRef pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv) +{ +#ifndef NDEBUG + StackChecker checker(L); + assert(idx > 0); + assert(vidx > 0); +#endif + + switch (lua_type(L, idx)) { + case LUA_TNONE: + case LUA_TNIL: + return emplace(pv, LUA_TNIL); + case LUA_TBOOLEAN: { + auto r = emplace(pv, LUA_TBOOLEAN); + r->bdata = lua_toboolean(L, idx); + return r; + } + case LUA_TNUMBER: { + auto r = emplace(pv, LUA_TNUMBER); + r->ndata = lua_tonumber(L, idx); + return r; + } + case LUA_TSTRING: { + auto r = emplace(pv, LUA_TSTRING); + size_t len; + const char *str = lua_tolstring(L, idx, &len); + assert(str); + r->sdata.assign(str, len); + return r; + } + case LUA_TTABLE: + break; // execution continues + case LUA_TFUNCTION: { + auto r = emplace(pv, LUA_TFUNCTION); + call_string_dump(L, idx); + size_t len; + const char *str = lua_tolstring(L, -1, &len); + assert(str); + r->sdata.assign(str, len); + lua_pop(L, 1); + return r; + } + case LUA_TUSERDATA: { + PackerTuple ser; + if (!find_packer(L, idx, ser)) + throw LuaError("Cannot serialize unsupported userdata"); + pv.contains_userdata = true; + auto r = emplace(pv, LUA_TUSERDATA); + r->sdata = ser.first; + r->ptrdata = ser.second.fin(L, idx); + return r; + } + default: { + std::string err = "Cannot serialize type "; + err += lua_typename(L, lua_type(L, idx)); + throw LuaError(err); + } + } + + // LUA_TTABLE + lua_checkstack(L, 5); + + auto rtable = emplace(pv, LUA_TTABLE); + const int vi_table = vidx++; + + lua_pushnil(L); + while (lua_next(L, idx) != 0) { + // key at -2, value at -1 + const int ktype = lua_type(L, -2), vtype = lua_type(L, -1); + if (ktype == LUA_TNUMBER) + rtable->uidata1++; // narr + else + rtable->uidata2++; // nrec + + // check if we can use a shortcut + if (can_set_into(ktype, vtype) && suitable_key(L, -2)) { + // push only the value + auto rval = pack_inner(L, absidx(L, -1), vidx, pv); + rval->pop = vtype != LUA_TTABLE; + // and where to put it: + rval->set_into = vi_table; + if (ktype == LUA_TSTRING) + rval->sdata = lua_tostring(L, -2); + else + rval->sidata1 = lua_tointeger(L, -2); + // pop tables after the fact + if (!rval->pop) { + auto ri1 = emplace(pv, INSTR_POP); + ri1->sidata1 = vidx; + } + } else { + // push the key and value + pack_inner(L, absidx(L, -2), vidx, pv); + vidx++; + pack_inner(L, absidx(L, -1), vidx, pv); + vidx++; + // push an instruction to set them + auto ri1 = emplace(pv, INSTR_SETTABLE); + ri1->set_into = vi_table; + ri1->sidata1 = vidx - 2; + ri1->sidata2 = vidx - 1; + ri1->pop = true; + vidx -= 2; + } + + lua_pop(L, 1); + } + + assert(vidx == vi_table + 1); + return rtable; +} + +PackedValue *script_pack(lua_State *L, int idx) +{ + if (idx < 0) + idx = absidx(L, idx); + + std::unordered_set seen; + pack_validate(L, idx, seen); + assert(seen.size() == 0); + + // Actual serialization + PackedValue pv; + pack_inner(L, idx, 1, pv); + + return new PackedValue(std::move(pv)); +} + +// +// Unpacking implementation +// + +void script_unpack(lua_State *L, PackedValue *pv) +{ + const int top = lua_gettop(L); + int ctr = 0; + + for (auto &i : pv->i) { + // If leaving values on stack make sure there's space (every 5th iteration) + if (!i.pop && (ctr++) >= 5) { + lua_checkstack(L, 5); + ctr = 0; + } + + /* Instructions */ + switch (i.type) { + case INSTR_SETTABLE: + lua_pushvalue(L, top + i.sidata1); // key + lua_pushvalue(L, top + i.sidata2); // value + lua_rawset(L, top + i.set_into); + if (i.pop) { + if (i.sidata1 != i.sidata2) { + // removing moves indices so pop higher index first + lua_remove(L, top + std::max(i.sidata1, i.sidata2)); + lua_remove(L, top + std::min(i.sidata1, i.sidata2)); + } else { + lua_remove(L, top + i.sidata1); + } + } + continue; + case INSTR_POP: + lua_remove(L, top + i.sidata1); + if (i.sidata2 > 0) + lua_remove(L, top + i.sidata2); + continue; + default: + break; + } + + /* Lua types */ + switch (i.type) { + case LUA_TNIL: + lua_pushnil(L); + break; + case LUA_TBOOLEAN: + lua_pushboolean(L, i.bdata); + break; + case LUA_TNUMBER: + lua_pushnumber(L, i.ndata); + break; + case LUA_TSTRING: + lua_pushlstring(L, i.sdata.data(), i.sdata.size()); + break; + case LUA_TTABLE: + lua_createtable(L, i.uidata1, i.uidata2); + break; + case LUA_TFUNCTION: + luaL_loadbuffer(L, i.sdata.data(), i.sdata.size(), nullptr); + break; + case LUA_TUSERDATA: { + PackerTuple ser; + sanity_check(find_packer(i.sdata.c_str(), ser)); + ser.second.fout(L, i.ptrdata); + i.ptrdata = nullptr; // ownership taken by callback + break; + } + default: + assert(0); + break; + } + + if (i.set_into) { + if (!i.pop) + lua_pushvalue(L, -1); + if (uses_sdata(i.type)) + lua_rawseti(L, top + i.set_into, i.sidata1); + else + lua_setfield(L, top + i.set_into, i.sdata.c_str()); + } else { + if (i.pop) + lua_pop(L, 1); + } + } + + // as part of the unpacking process we take ownership of all userdata + pv->contains_userdata = false; + // leave exactly one value on the stack + lua_settop(L, top+1); +} + +// +// PackedValue +// + +PackedValue::~PackedValue() +{ + if (!contains_userdata) + return; + for (auto &i : this->i) { + if (i.type == LUA_TUSERDATA && i.ptrdata) { + PackerTuple ser; + if (find_packer(i.sdata.c_str(), ser)) { + // tell it to deallocate object + ser.second.fout(nullptr, i.ptrdata); + } else { + assert(false); + } + } + } +} + +// +// script_dump_packed +// + +#ifndef NDEBUG +void script_dump_packed(const PackedValue *val) +{ + printf("instruction stream: [\n"); + for (const auto &i : val->i) { + printf("\t("); + switch (i.type) { + case INSTR_SETTABLE: + printf("SETTABLE(%d, %d)", i.sidata1, i.sidata2); + break; + case INSTR_POP: + printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2); + break; + case LUA_TNIL: + printf("nil"); + break; + case LUA_TBOOLEAN: + printf(i.bdata ? "true" : "false"); + break; + case LUA_TNUMBER: + printf("%f", i.ndata); + break; + case LUA_TSTRING: + printf("\"%s\"", i.sdata.c_str()); + break; + case LUA_TTABLE: + printf("table(%d, %d)", i.uidata1, i.uidata2); + break; + case LUA_TFUNCTION: + printf("function(%d byte)", i.sdata.size()); + break; + case LUA_TUSERDATA: + printf("userdata %s %p", i.sdata.c_str(), i.ptrdata); + break; + default: + printf("!!UNKNOWN!!"); + break; + } + if (i.set_into) { + if (i.type >= 0 && uses_sdata(i.type)) + printf(", k=%d, into=%d", i.sidata1, i.set_into); + else if (i.type >= 0) + printf(", k=\"%s\", into=%d", i.sdata.c_str(), i.set_into); + else + printf(", into=%d", i.set_into); + } + if (i.pop) + printf(", pop"); + printf(")\n"); + } + printf("]\n"); +} +#endif diff --git a/src/script/common/c_packer.h b/src/script/common/c_packer.h new file mode 100644 index 000000000..8bccca98d --- /dev/null +++ b/src/script/common/c_packer.h @@ -0,0 +1,123 @@ +/* +Minetest +Copyright (C) 2022 sfan5 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#pragma once + +#include +#include +#include "irrlichttypes.h" +#include "util/basic_macros.h" + +extern "C" { +#include +} + +/* + This file defines an in-memory representation of Lua objects including + support for functions and userdata. It it used to move data between Lua + states and cannot be used for persistence or network transfer. +*/ + +#define INSTR_SETTABLE (-10) +#define INSTR_POP (-11) + +/** + * Represents a single instruction that pushes a new value or works with existing ones. + */ +struct PackedInstr +{ + s16 type; // LUA_T* or INSTR_* + u16 set_into; // set into table on stack + bool pop; // remove from stack? + union { + bool bdata; // boolean: value + lua_Number ndata; // number: value + struct { + u16 uidata1, uidata2; // table: narr, nrec + }; + struct { + /* + SETTABLE: key index, value index + POP: indices to remove + otherwise w/ set_into: numeric key, - + */ + s32 sidata1, sidata2; + }; + void *ptrdata; // userdata: implementation defined + }; + /* + - string: value + - function: buffer + - w/ set_into: string key (no null bytes!) + - userdata: name in registry + */ + std::string sdata; + + PackedInstr() : type(0), set_into(0), pop(false) {} +}; + +/** + * A packed value can be a primitive like a string or number but also a table + * including all of its contents. It is made up of a linear stream of + * 'instructions' that build the final value when executed. + */ +struct PackedValue +{ + std::vector i; + // Indicates whether there are any userdata pointers that need to be deallocated + bool contains_userdata = false; + + PackedValue() = default; + ~PackedValue(); + + DISABLE_CLASS_COPY(PackedValue) + + ALLOW_CLASS_MOVE(PackedValue) +}; + +/* + * Packing callback: Turns a Lua value at given index into a void* + */ +typedef void *(*PackInFunc)(lua_State *L, int idx); +/* + * Unpacking callback: Turns a void* back into the Lua value (left on top of stack) + * + * Note that this function must take ownership of the pointer, so make sure + * to free or keep the memory. + * `L` can be nullptr to indicate that data should just be discarded. + */ +typedef void (*PackOutFunc)(lua_State *L, void *ptr); +/* + * Register a packable type with the name of its metatable. + * + * Even though the callbacks are global this must be called for every Lua state + * that supports objects of this type. + * This function is thread-safe. + */ +void script_register_packer(lua_State *L, const char *regname, + PackInFunc fin, PackOutFunc fout); + +// Pack a Lua value +PackedValue *script_pack(lua_State *L, int idx); +// Unpack a Lua value (left on top of stack) +// Note that this may modify the PackedValue, you can't reuse it! +void script_unpack(lua_State *L, PackedValue *val); + +// Dump contents of PackedValue to stdout for debugging +void script_dump_packed(const PackedValue *val); diff --git a/src/script/cpp_api/s_async.cpp b/src/script/cpp_api/s_async.cpp index dacdcd75a..42a794ceb 100644 --- a/src/script/cpp_api/s_async.cpp +++ b/src/script/cpp_api/s_async.cpp @@ -21,9 +21,9 @@ with this program; if not, write to the Free Software Foundation, Inc., #include extern "C" { -#include "lua.h" -#include "lauxlib.h" -#include "lualib.h" +#include +#include +#include } #include "server.h" @@ -32,6 +32,7 @@ extern "C" { #include "filesys.h" #include "porting.h" #include "common/c_internal.h" +#include "common/c_packer.h" #include "lua_api/l_base.h" /******************************************************************************/ @@ -76,19 +77,34 @@ void AsyncEngine::initialize(unsigned int numEngines) { initDone = true; - for (unsigned int i = 0; i < numEngines; i++) { - AsyncWorkerThread *toAdd = new AsyncWorkerThread(this, - std::string("AsyncWorker-") + itos(i)); - workerThreads.push_back(toAdd); - toAdd->start(); + if (numEngines == 0) { + // Leave one core for the main thread and one for whatever else + autoscaleMaxWorkers = Thread::getNumberOfProcessors(); + if (autoscaleMaxWorkers >= 2) + autoscaleMaxWorkers -= 2; + infostream << "AsyncEngine: using at most " << autoscaleMaxWorkers + << " threads with automatic scaling" << std::endl; + + addWorkerThread(); + } else { + for (unsigned int i = 0; i < numEngines; i++) + addWorkerThread(); } } +void AsyncEngine::addWorkerThread() +{ + AsyncWorkerThread *toAdd = new AsyncWorkerThread(this, + std::string("AsyncWorker-") + itos(workerThreads.size())); + workerThreads.push_back(toAdd); + toAdd->start(); +} + /******************************************************************************/ u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &¶ms, const std::string &mod_origin) { - jobQueueMutex.lock(); + MutexAutoLock autolock(jobQueueMutex); u32 jobId = jobIdCounter++; jobQueue.emplace_back(); @@ -99,7 +115,23 @@ u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &¶ms, to_add.mod_origin = mod_origin; jobQueueCounter.post(); - jobQueueMutex.unlock(); + return jobId; +} + +u32 AsyncEngine::queueAsyncJob(std::string &&func, PackedValue *params, + const std::string &mod_origin) +{ + MutexAutoLock autolock(jobQueueMutex); + u32 jobId = jobIdCounter++; + + jobQueue.emplace_back(); + auto &to_add = jobQueue.back(); + to_add.id = jobId; + to_add.function = std::move(func); + to_add.params_ext.reset(params); + to_add.mod_origin = mod_origin; + + jobQueueCounter.post(); return jobId; } @@ -131,6 +163,12 @@ void AsyncEngine::putJobResult(LuaJobInfo &&result) /******************************************************************************/ void AsyncEngine::step(lua_State *L) +{ + stepJobResults(L); + stepAutoscale(); +} + +void AsyncEngine::stepJobResults(lua_State *L) { int error_handler = PUSH_ERROR_HANDLER(L); lua_getglobal(L, "core"); @@ -148,7 +186,10 @@ void AsyncEngine::step(lua_State *L) luaL_checktype(L, -1, LUA_TFUNCTION); lua_pushinteger(L, j.id); - lua_pushlstring(L, j.result.data(), j.result.size()); + if (j.result_ext) + script_unpack(L, j.result_ext.get()); + else + lua_pushlstring(L, j.result.data(), j.result.size()); // Call handler const char *origin = j.mod_origin.empty() ? nullptr : j.mod_origin.c_str(); @@ -161,12 +202,71 @@ void AsyncEngine::step(lua_State *L) lua_pop(L, 2); // Pop core and error handler } +void AsyncEngine::stepAutoscale() +{ + if (workerThreads.size() >= autoscaleMaxWorkers) + return; + + MutexAutoLock autolock(jobQueueMutex); + + // 2) If the timer elapsed, check again + if (autoscaleTimer && porting::getTimeMs() >= autoscaleTimer) { + autoscaleTimer = 0; + // Determine overlap with previous snapshot + unsigned int n = 0; + for (const auto &it : jobQueue) + n += autoscaleSeenJobs.count(it.id); + autoscaleSeenJobs.clear(); + infostream << "AsyncEngine: " << n << " jobs were still waiting after 1s" << std::endl; + // Start this many new threads + while (workerThreads.size() < autoscaleMaxWorkers && n > 0) { + addWorkerThread(); + n--; + } + return; + } + + // 1) Check if there's anything in the queue + if (!autoscaleTimer && !jobQueue.empty()) { + // Take a snapshot of all jobs we have seen + for (const auto &it : jobQueue) + autoscaleSeenJobs.emplace(it.id); + // and set a timer for 1 second + autoscaleTimer = porting::getTimeMs() + 1000; + } +} + /******************************************************************************/ -void AsyncEngine::prepareEnvironment(lua_State* L, int top) +bool AsyncEngine::prepareEnvironment(lua_State* L, int top) { for (StateInitializer &stateInitializer : stateInitializers) { stateInitializer(L, top); } + + auto *script = ModApiBase::getScriptApiBase(L); + try { + script->loadMod(Server::getBuiltinLuaPath() + DIR_DELIM + "init.lua", + BUILTIN_MOD_NAME); + } catch (const ModError &e) { + errorstream << "Execution of async base environment failed: " + << e.what() << std::endl; + FATAL_ERROR("Execution of async base environment failed"); + } + + // Load per mod stuff + if (server) { + const auto &list = server->m_async_init_files; + try { + for (auto &it : list) + script->loadMod(it.second, it.first); + } catch (const ModError &e) { + errorstream << "Failed to load mod script inside async environment." << std::endl; + server->setAsyncFatalError(e.what()); + return false; + } + } + + return true; } /******************************************************************************/ @@ -178,15 +278,25 @@ AsyncWorkerThread::AsyncWorkerThread(AsyncEngine* jobDispatcher, { lua_State *L = getStack(); + if (jobDispatcher->server) { + setGameDef(jobDispatcher->server); + + if (g_settings->getBool("secure.enable_security")) + initializeSecurity(); + } + // Prepare job lua environment lua_getglobal(L, "core"); int top = lua_gettop(L); // Push builtin initialization type - lua_pushstring(L, "async"); + lua_pushstring(L, jobDispatcher->server ? "async_game" : "async"); lua_setglobal(L, "INIT"); - jobDispatcher->prepareEnvironment(L, top); + if (!jobDispatcher->prepareEnvironment(L, top)) { + // can't throw from here so we're stuck with this + isErrored = true; + } } /******************************************************************************/ @@ -198,19 +308,20 @@ AsyncWorkerThread::~AsyncWorkerThread() /******************************************************************************/ void* AsyncWorkerThread::run() { - lua_State *L = getStack(); + if (isErrored) + return nullptr; - try { - loadMod(getServer()->getBuiltinLuaPath() + DIR_DELIM + "init.lua", - BUILTIN_MOD_NAME); - } catch (const ModError &e) { - errorstream << "Execution of async base environment failed: " - << e.what() << std::endl; - FATAL_ERROR("Execution of async base environment failed"); - } + lua_State *L = getStack(); int error_handler = PUSH_ERROR_HANDLER(L); + auto report_error = [this] (const ModError &e) { + if (jobDispatcher->server) + jobDispatcher->server->setAsyncFatalError(e.what()); + else + errorstream << e.what() << std::endl; + }; + lua_getglobal(L, "core"); if (lua_isnil(L, -1)) { FATAL_ERROR("Unable to find core within async environment!"); @@ -223,6 +334,8 @@ void* AsyncWorkerThread::run() if (!jobDispatcher->getJob(&j) || stopRequested()) continue; + const bool use_ext = !!j.params_ext; + lua_getfield(L, -1, "job_processor"); if (lua_isnil(L, -1)) FATAL_ERROR("Unable to get async job processor!"); @@ -232,7 +345,10 @@ void* AsyncWorkerThread::run() errorstream << "ASYNC WORKER: Unable to deserialize function" << std::endl; lua_pushnil(L); } - lua_pushlstring(L, j.params.data(), j.params.size()); + if (use_ext) + script_unpack(L, j.params_ext.get()); + else + lua_pushlstring(L, j.params.data(), j.params.size()); // Call it setOriginDirect(j.mod_origin.empty() ? nullptr : j.mod_origin.c_str()); @@ -241,19 +357,28 @@ void* AsyncWorkerThread::run() try { scriptError(result, ""); } catch (const ModError &e) { - errorstream << e.what() << std::endl; + report_error(e); } } else { // Fetch result - size_t length; - const char *retval = lua_tolstring(L, -1, &length); - j.result.assign(retval, length); + if (use_ext) { + try { + j.result_ext.reset(script_pack(L, -1)); + } catch (const ModError &e) { + report_error(e); + result = LUA_ERRERR; + } + } else { + size_t length; + const char *retval = lua_tolstring(L, -1, &length); + j.result.assign(retval, length); + } } lua_pop(L, 1); // Pop retval // Put job result - if (!j.result.empty()) + if (result == 0) jobDispatcher->putJobResult(std::move(j)); } diff --git a/src/script/cpp_api/s_async.h b/src/script/cpp_api/s_async.h index 697cb0221..1e34e40ea 100644 --- a/src/script/cpp_api/s_async.h +++ b/src/script/cpp_api/s_async.h @@ -21,11 +21,15 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include +#include +#include +#include #include "threading/semaphore.h" #include "threading/thread.h" -#include "lua.h" +#include "common/c_packer.h" #include "cpp_api/s_base.h" +#include "cpp_api/s_security.h" // Forward declarations class AsyncEngine; @@ -42,8 +46,12 @@ struct LuaJobInfo std::string function; // Parameter to be passed to function (serialized) std::string params; + // Alternative parameters + std::unique_ptr params_ext; // Result of function call (serialized) std::string result; + // Alternative result + std::unique_ptr result_ext; // Name of the mod who invoked this call std::string mod_origin; // JobID used to identify a job and match it to callback @@ -51,7 +59,8 @@ struct LuaJobInfo }; // Asynchronous working environment -class AsyncWorkerThread : public Thread, virtual public ScriptApiBase { +class AsyncWorkerThread : public Thread, + virtual public ScriptApiBase, public ScriptApiSecurity { friend class AsyncEngine; public: virtual ~AsyncWorkerThread(); @@ -63,6 +72,7 @@ protected: private: AsyncEngine *jobDispatcher = nullptr; + bool isErrored = false; }; // Asynchornous thread and job management @@ -71,6 +81,7 @@ class AsyncEngine { typedef void (*StateInitializer)(lua_State *L, int top); public: AsyncEngine() = default; + AsyncEngine(Server *server) : server(server) {}; ~AsyncEngine(); /** @@ -81,7 +92,7 @@ public: /** * Create async engine tasks and lock function registration - * @param numEngines Number of async threads to be started + * @param numEngines Number of worker threads, 0 for automatic scaling */ void initialize(unsigned int numEngines); @@ -94,9 +105,17 @@ public: u32 queueAsyncJob(std::string &&func, std::string &¶ms, const std::string &mod_origin = ""); + /** + * Queue an async job + * @param func Serialized lua function + * @param params Serialized parameters (takes ownership!) + * @return ID of queued job + */ + u32 queueAsyncJob(std::string &&func, PackedValue *params, + const std::string &mod_origin = ""); + /** * Engine step to process finished jobs - * the engine step is one way to pass events back, PushFinishedJobs another * @param L The Lua stack */ void step(lua_State *L); @@ -116,19 +135,44 @@ protected: */ void putJobResult(LuaJobInfo &&result); + /** + * Start an additional worker thread + */ + void addWorkerThread(); + + /** + * Process finished jobs callbacks + */ + void stepJobResults(lua_State *L); + + /** + * Handle automatic scaling of worker threads + */ + void stepAutoscale(); + /** * Initialize environment with current registred functions * this function adds all functions registred by registerFunction to the * passed lua stack * @param L Lua stack to initialize * @param top Stack position + * @return false if a mod error ocurred */ - void prepareEnvironment(lua_State* L, int top); + bool prepareEnvironment(lua_State* L, int top); private: // Variable locking the engine against further modification bool initDone = false; + // Maximum number of worker threads for automatic scaling + // 0 if disabled + unsigned int autoscaleMaxWorkers = 0; + u64 autoscaleTimer = 0; + std::unordered_set autoscaleSeenJobs; + + // Only set for the server async environment (duh) + Server *server = nullptr; + // Internal store for registred state initializers std::vector stateInitializers; diff --git a/src/script/lua_api/l_craft.cpp b/src/script/lua_api/l_craft.cpp index c2c5a5551..137b210be 100644 --- a/src/script/lua_api/l_craft.cpp +++ b/src/script/lua_api/l_craft.cpp @@ -525,3 +525,11 @@ void ModApiCraft::Initialize(lua_State *L, int top) API_FCT(register_craft); API_FCT(clear_craft); } + +void ModApiCraft::InitializeAsync(lua_State *L, int top) +{ + // all read-only functions + API_FCT(get_all_craft_recipes); + API_FCT(get_craft_recipe); + API_FCT(get_craft_result); +} diff --git a/src/script/lua_api/l_craft.h b/src/script/lua_api/l_craft.h index 9002b23ef..5234af56f 100644 --- a/src/script/lua_api/l_craft.h +++ b/src/script/lua_api/l_craft.h @@ -45,4 +45,5 @@ private: public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_internal.h b/src/script/lua_api/l_internal.h index 672e535ca..de73ff42a 100644 --- a/src/script/lua_api/l_internal.h +++ b/src/script/lua_api/l_internal.h @@ -69,7 +69,7 @@ with this program; if not, write to the Free Software Foundation, Inc., // Retrieve Environment pointer as `env` (no map lock) #define GET_PLAIN_ENV_PTR_NO_MAP_LOCK \ - Environment *env = (Environment *)getEnv(L); \ + Environment *env = getEnv(L); \ if (env == NULL) \ return 0 diff --git a/src/script/lua_api/l_item.cpp b/src/script/lua_api/l_item.cpp index fc97a1736..b58b994d9 100644 --- a/src/script/lua_api/l_item.cpp +++ b/src/script/lua_api/l_item.cpp @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "itemdef.h" #include "nodedef.h" #include "server.h" @@ -441,6 +442,7 @@ int LuaItemStack::create_object(lua_State *L) lua_setmetatable(L, -2); return 1; } + // Not callable from Lua int LuaItemStack::create(lua_State *L, const ItemStack &item) { @@ -457,6 +459,20 @@ LuaItemStack *LuaItemStack::checkobject(lua_State *L, int narg) return *(LuaItemStack **)luaL_checkudata(L, narg, className); } +void *LuaItemStack::packIn(lua_State *L, int idx) +{ + LuaItemStack *o = checkobject(L, idx); + return new ItemStack(o->getItem()); +} + +void LuaItemStack::packOut(lua_State *L, void *ptr) +{ + ItemStack *stack = reinterpret_cast(ptr); + if (L) + create(L, *stack); + delete stack; +} + void LuaItemStack::Register(lua_State *L) { lua_newtable(L); @@ -488,6 +504,8 @@ void LuaItemStack::Register(lua_State *L) // Can be created from Lua (ItemStack(itemstack or itemstring or table or nil)) lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } const char LuaItemStack::className[] = "ItemStack"; @@ -673,3 +691,10 @@ void ModApiItemMod::Initialize(lua_State *L, int top) API_FCT(get_content_id); API_FCT(get_name_from_content_id); } + +void ModApiItemMod::InitializeAsync(lua_State *L, int top) +{ + // all read-only functions + API_FCT(get_content_id); + API_FCT(get_name_from_content_id); +} diff --git a/src/script/lua_api/l_item.h b/src/script/lua_api/l_item.h index 16878c101..180975061 100644 --- a/src/script/lua_api/l_item.h +++ b/src/script/lua_api/l_item.h @@ -141,8 +141,11 @@ public: // Not callable from Lua static int create(lua_State *L, const ItemStack &item); static LuaItemStack* checkobject(lua_State *L, int narg); - static void Register(lua_State *L); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + + static void Register(lua_State *L); }; class ModApiItemMod : public ModApiBase { @@ -152,6 +155,8 @@ private: static int l_register_alias_raw(lua_State *L); static int l_get_content_id(lua_State *L); static int l_get_name_from_content_id(lua_State *L); + public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_noise.cpp b/src/script/lua_api/l_noise.cpp index 0eee49b7d..5561eaebf 100644 --- a/src/script/lua_api/l_noise.cpp +++ b/src/script/lua_api/l_noise.cpp @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "log.h" #include "porting.h" #include "util/numeric.h" @@ -101,6 +102,25 @@ LuaPerlinNoise *LuaPerlinNoise::checkobject(lua_State *L, int narg) } +void *LuaPerlinNoise::packIn(lua_State *L, int idx) +{ + LuaPerlinNoise *o = checkobject(L, idx); + return new NoiseParams(o->np); +} + +void LuaPerlinNoise::packOut(lua_State *L, void *ptr) +{ + NoiseParams *np = reinterpret_cast(ptr); + if (L) { + LuaPerlinNoise *o = new LuaPerlinNoise(np); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); + } + delete np; +} + + void LuaPerlinNoise::Register(lua_State *L) { lua_newtable(L); @@ -126,6 +146,8 @@ void LuaPerlinNoise::Register(lua_State *L) lua_pop(L, 1); lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } @@ -357,6 +379,35 @@ LuaPerlinNoiseMap *LuaPerlinNoiseMap::checkobject(lua_State *L, int narg) } +struct NoiseMapParams { + NoiseParams np; + s32 seed; + v3s16 size; +}; + +void *LuaPerlinNoiseMap::packIn(lua_State *L, int idx) +{ + LuaPerlinNoiseMap *o = checkobject(L, idx); + NoiseMapParams *ret = new NoiseMapParams(); + ret->np = o->noise->np; + ret->seed = o->noise->seed; + ret->size = v3s16(o->noise->sx, o->noise->sy, o->noise->sz); + return ret; +} + +void LuaPerlinNoiseMap::packOut(lua_State *L, void *ptr) +{ + NoiseMapParams *p = reinterpret_cast(ptr); + if (L) { + LuaPerlinNoiseMap *o = new LuaPerlinNoiseMap(&p->np, p->seed, p->size); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); + } + delete p; +} + + void LuaPerlinNoiseMap::Register(lua_State *L) { lua_newtable(L); @@ -382,6 +433,8 @@ void LuaPerlinNoiseMap::Register(lua_State *L) lua_pop(L, 1); lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } diff --git a/src/script/lua_api/l_noise.h b/src/script/lua_api/l_noise.h index 29ab41a31..5d34a479b 100644 --- a/src/script/lua_api/l_noise.h +++ b/src/script/lua_api/l_noise.h @@ -52,6 +52,9 @@ public: static LuaPerlinNoise *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; @@ -91,6 +94,9 @@ public: static LuaPerlinNoiseMap *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 42725e5d2..4b0b45887 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "cpp_api/s_base.h" #include "cpp_api/s_security.h" #include "scripting_server.h" @@ -526,6 +527,76 @@ int ModApiServer::l_notify_authentication_modified(lua_State *L) return 0; } +// do_async_callback(func, params, mod_origin) +int ModApiServer::l_do_async_callback(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + ServerScripting *script = getScriptApi(L); + + luaL_checktype(L, 1, LUA_TFUNCTION); + luaL_checktype(L, 2, LUA_TTABLE); + luaL_checktype(L, 3, LUA_TSTRING); + + call_string_dump(L, 1); + size_t func_length; + const char *serialized_func_raw = lua_tolstring(L, -1, &func_length); + + PackedValue *param = script_pack(L, 2); + + std::string mod_origin = readParam(L, 3); + + u32 jobId = script->queueAsync( + std::string(serialized_func_raw, func_length), + param, mod_origin); + + lua_settop(L, 0); + lua_pushinteger(L, jobId); + return 1; +} + +// register_async_dofile(path) +int ModApiServer::l_register_async_dofile(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + + std::string path = readParam(L, 1); + CHECK_SECURE_PATH(L, path.c_str(), false); + + // Find currently running mod name (only at init time) + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME); + if (!lua_isstring(L, -1)) + return 0; + std::string modname = readParam(L, -1); + + getServer(L)->m_async_init_files.emplace_back(modname, path); + lua_pushboolean(L, true); + return 1; +} + +// serialize_roundtrip(value) +// Meant for unit testing the packer from Lua +int ModApiServer::l_serialize_roundtrip(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + + int top = lua_gettop(L); + auto *pv = script_pack(L, 1); + if (top != lua_gettop(L)) + throw LuaError("stack values leaked"); + +#ifndef NDEBUG + script_dump_packed(pv); +#endif + + top = lua_gettop(L); + script_unpack(L, pv); + delete pv; + if (top + 1 != lua_gettop(L)) + throw LuaError("stack values leaked"); + + return 1; +} + void ModApiServer::Initialize(lua_State *L, int top) { API_FCT(request_shutdown); @@ -559,4 +630,18 @@ void ModApiServer::Initialize(lua_State *L, int top) API_FCT(remove_player); API_FCT(unban_player_or_ip); API_FCT(notify_authentication_modified); + + API_FCT(do_async_callback); + API_FCT(register_async_dofile); + API_FCT(serialize_roundtrip); +} + +void ModApiServer::InitializeAsync(lua_State *L, int top) +{ + API_FCT(get_worldpath); + API_FCT(is_singleplayer); + + API_FCT(get_current_modname); + API_FCT(get_modpath); + API_FCT(get_modnames); } diff --git a/src/script/lua_api/l_server.h b/src/script/lua_api/l_server.h index f05c0b7c9..a4f38c34e 100644 --- a/src/script/lua_api/l_server.h +++ b/src/script/lua_api/l_server.h @@ -106,6 +106,16 @@ private: // notify_authentication_modified(name) static int l_notify_authentication_modified(lua_State *L); + // do_async_callback(func, params, mod_origin) + static int l_do_async_callback(lua_State *L); + + // register_async_dofile(path) + static int l_register_async_dofile(lua_State *L); + + // serialize_roundtrip(obj) + static int l_serialize_roundtrip(lua_State *L); + public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_util.cpp b/src/script/lua_api/l_util.cpp index b04f26fda..97068ce4c 100644 --- a/src/script/lua_api/l_util.cpp +++ b/src/script/lua_api/l_util.cpp @@ -671,6 +671,9 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top) API_FCT(cpdir); API_FCT(mvdir); API_FCT(get_dir_list); + API_FCT(safe_file_write); + + API_FCT(request_insecure_environment); API_FCT(encode_base64); API_FCT(decode_base64); @@ -680,6 +683,8 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top) API_FCT(colorspec_to_colorstring); API_FCT(colorspec_to_bytes); + API_FCT(encode_png); + API_FCT(get_last_run_mod); API_FCT(set_last_run_mod); diff --git a/src/script/lua_api/l_util.h b/src/script/lua_api/l_util.h index fcf8a1057..cc5563577 100644 --- a/src/script/lua_api/l_util.h +++ b/src/script/lua_api/l_util.h @@ -129,6 +129,4 @@ public: static void Initialize(lua_State *L, int top); static void InitializeAsync(lua_State *L, int top); static void InitializeClient(lua_State *L, int top); - - static void InitializeAsync(AsyncEngine &engine); }; diff --git a/src/script/lua_api/l_vmanip.cpp b/src/script/lua_api/l_vmanip.cpp index a3ece627c..6187a47db 100644 --- a/src/script/lua_api/l_vmanip.cpp +++ b/src/script/lua_api/l_vmanip.cpp @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_content.h" #include "common/c_converter.h" +#include "common/c_packer.h" #include "emerge.h" #include "environment.h" #include "map.h" @@ -45,6 +46,8 @@ int LuaVoxelManip::l_read_from_map(lua_State *L) LuaVoxelManip *o = checkobject(L, 1); MMVManip *vm = o->vm; + if (vm->isOrphan()) + return 0; v3s16 bp1 = getNodeBlockPos(check_v3s16(L, 2)); v3s16 bp2 = getNodeBlockPos(check_v3s16(L, 3)); @@ -429,6 +432,34 @@ LuaVoxelManip *LuaVoxelManip::checkobject(lua_State *L, int narg) return *(LuaVoxelManip **)ud; // unbox pointer } +void *LuaVoxelManip::packIn(lua_State *L, int idx) +{ + LuaVoxelManip *o = checkobject(L, idx); + + if (o->is_mapgen_vm) + throw LuaError("nope"); + return o->vm->clone(); +} + +void LuaVoxelManip::packOut(lua_State *L, void *ptr) +{ + MMVManip *vm = reinterpret_cast(ptr); + if (!L) { + delete vm; + return; + } + + // Associate vmanip with map if the Lua env has one + Environment *env = getEnv(L); + if (env) + vm->reparent(&(env->getMap())); + + LuaVoxelManip *o = new LuaVoxelManip(vm, false); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); +} + void LuaVoxelManip::Register(lua_State *L) { lua_newtable(L); @@ -455,6 +486,8 @@ void LuaVoxelManip::Register(lua_State *L) // Can be created from Lua (VoxelManip()) lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } const char LuaVoxelManip::className[] = "VoxelManip"; diff --git a/src/script/lua_api/l_vmanip.h b/src/script/lua_api/l_vmanip.h index 5113070dc..005133335 100644 --- a/src/script/lua_api/l_vmanip.h +++ b/src/script/lua_api/l_vmanip.h @@ -75,5 +75,8 @@ public: static LuaVoxelManip *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; diff --git a/src/script/scripting_server.cpp b/src/script/scripting_server.cpp index 85411ded4..5b99468dc 100644 --- a/src/script/scripting_server.cpp +++ b/src/script/scripting_server.cpp @@ -47,11 +47,12 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_storage.h" extern "C" { -#include "lualib.h" +#include } ServerScripting::ServerScripting(Server* server): - ScriptApiBase(ScriptingType::Server) + ScriptApiBase(ScriptingType::Server), + asyncEngine(server) { setGameDef(server); @@ -88,6 +89,47 @@ ServerScripting::ServerScripting(Server* server): infostream << "SCRIPTAPI: Initialized game modules" << std::endl; } +void ServerScripting::initAsync() +{ + // Save globals to transfer + { + lua_State *L = getStack(); + lua_getglobal(L, "core"); + luaL_checktype(L, -1, LUA_TTABLE); + lua_getfield(L, -1, "get_globals_to_transfer"); + lua_call(L, 0, 1); + luaL_checktype(L, -1, LUA_TSTRING); + getServer()->m_async_globals_data.set(readParam(L, -1)); + lua_pushnil(L); + lua_setfield(L, -3, "get_globals_to_transfer"); // unset function too + lua_pop(L, 2); // pop 'core', return value + } + + infostream << "SCRIPTAPI: Initializing async engine" << std::endl; + asyncEngine.registerStateInitializer(InitializeAsync); + asyncEngine.registerStateInitializer(ModApiUtil::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiCraft::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiItemMod::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiServer::InitializeAsync); + // not added: ModApiMapgen is a minefield for thread safety + // not added: ModApiHttp async api can't really work together with our jobs + // not added: ModApiStorage is probably not thread safe(?) + + asyncEngine.initialize(0); +} + +void ServerScripting::stepAsync() +{ + asyncEngine.step(getStack()); +} + +u32 ServerScripting::queueAsync(std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin) +{ + return asyncEngine.queueAsyncJob(std::move(serialized_func), + param, mod_origin); +} + void ServerScripting::InitializeModApi(lua_State *L, int top) { // Register reference classes (userdata) @@ -125,3 +167,24 @@ void ServerScripting::InitializeModApi(lua_State *L, int top) ModApiStorage::Initialize(L, top); ModApiChannels::Initialize(L, top); } + +void ServerScripting::InitializeAsync(lua_State *L, int top) +{ + // classes + LuaItemStack::Register(L); + LuaPerlinNoise::Register(L); + LuaPerlinNoiseMap::Register(L); + LuaPseudoRandom::Register(L); + LuaPcgRandom::Register(L); + LuaSecureRandom::Register(L); + LuaVoxelManip::Register(L); + LuaSettings::Register(L); + + // globals data + lua_getglobal(L, "core"); + luaL_checktype(L, -1, LUA_TTABLE); + std::string s = ModApiBase::getServer(L)->m_async_globals_data.get(); + lua_pushlstring(L, s.c_str(), s.size()); + lua_setfield(L, -2, "transferred_globals"); + lua_pop(L, 1); // pop 'core' +} diff --git a/src/script/scripting_server.h b/src/script/scripting_server.h index bf06ab197..9803397c5 100644 --- a/src/script/scripting_server.h +++ b/src/script/scripting_server.h @@ -27,6 +27,9 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "cpp_api/s_player.h" #include "cpp_api/s_server.h" #include "cpp_api/s_security.h" +#include "cpp_api/s_async.h" + +struct PackedValue; /*****************************************************************************/ /* Scripting <-> Server Game Interface */ @@ -48,6 +51,20 @@ public: // use ScriptApiBase::loadMod() to load mods + // Initialize async engine, call this AFTER loading all mods + void initAsync(); + + // Global step handler to collect async results + void stepAsync(); + + // Pass job to async threads + u32 queueAsync(std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin); + private: void InitializeModApi(lua_State *L, int top); + + static void InitializeAsync(lua_State *L, int top); + + AsyncEngine asyncEngine; }; diff --git a/src/server.cpp b/src/server.cpp index dec6cf44c..c9cd4e398 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -243,6 +243,7 @@ Server::Server( m_clients(m_con), m_admin_chat(iface), m_on_shutdown_errmsg(on_shutdown_errmsg), + m_async_globals_data(""), m_modchannel_mgr(new ModChannelMgr()) { if (m_path_world.empty()) @@ -480,6 +481,9 @@ void Server::init() // Give environment reference to scripting api m_script->initializeEnvironment(m_env); + // Do this after regular script init is done + m_script->initAsync(); + // Register us to receive map edit events servermap->addEventReceiver(this); diff --git a/src/server.h b/src/server.h index bd799c313..5090a3579 100644 --- a/src/server.h +++ b/src/server.h @@ -292,7 +292,7 @@ public: virtual const std::vector &getMods() const; virtual const ModSpec* getModSpec(const std::string &modname) const; - std::string getBuiltinLuaPath(); + static std::string getBuiltinLuaPath(); virtual std::string getWorldPath() const { return m_path_world; } inline bool isSingleplayer() const @@ -385,6 +385,12 @@ public: static bool migrateModStorageDatabase(const GameParams &game_params, const Settings &cmd_args); + // Lua files registered for init of async env, pair of modname + path + std::vector> m_async_init_files; + + // Serialized data transferred into async envs at init time + MutexedVariable m_async_globals_data; + // Bind address Address m_bind_addr; diff --git a/src/serverenvironment.cpp b/src/serverenvironment.cpp index f8d84604b..493210744 100644 --- a/src/serverenvironment.cpp +++ b/src/serverenvironment.cpp @@ -1481,6 +1481,8 @@ void ServerEnvironment::step(float dtime) */ m_script->environment_Step(dtime); + m_script->stepAsync(); + /* Step active objects */ diff --git a/src/util/basic_macros.h b/src/util/basic_macros.h index 334e342e0..3910c6185 100644 --- a/src/util/basic_macros.h +++ b/src/util/basic_macros.h @@ -29,13 +29,19 @@ with this program; if not, write to the Free Software Foundation, Inc., #define CONTAINS(c, v) (std::find((c).begin(), (c).end(), (v)) != (c).end()) // To disable copy constructors and assignment operations for some class -// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) as a private member. +// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) in the class definition. // Note this also disables copying for any classes derived from 'Foobar' as well // as classes having a 'Foobar' member. #define DISABLE_CLASS_COPY(C) \ C(const C &) = delete; \ C &operator=(const C &) = delete; +// If you have used DISABLE_CLASS_COPY with a class but still want to permit moving +// use this macro to add the default move constructors back. +#define ALLOW_CLASS_MOVE(C) \ + C(C &&other) = default; \ + C &operator=(C &&) = default; + #ifndef _MSC_VER #define UNUSED_ATTRIBUTE __attribute__ ((unused)) #else -- cgit v1.2.3