From e7659883cc6fca343785da2a1af3890ae273abbf Mon Sep 17 00:00:00 2001 From: sfan5 Date: Mon, 2 May 2022 20:55:04 +0200 Subject: Async environment for mods to do concurrent tasks (#11131) --- src/script/common/c_packer.h | 123 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 src/script/common/c_packer.h (limited to 'src/script/common/c_packer.h') diff --git a/src/script/common/c_packer.h b/src/script/common/c_packer.h new file mode 100644 index 000000000..8bccca98d --- /dev/null +++ b/src/script/common/c_packer.h @@ -0,0 +1,123 @@ +/* +Minetest +Copyright (C) 2022 sfan5 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#pragma once + +#include +#include +#include "irrlichttypes.h" +#include "util/basic_macros.h" + +extern "C" { +#include +} + +/* + This file defines an in-memory representation of Lua objects including + support for functions and userdata. It it used to move data between Lua + states and cannot be used for persistence or network transfer. +*/ + +#define INSTR_SETTABLE (-10) +#define INSTR_POP (-11) + +/** + * Represents a single instruction that pushes a new value or works with existing ones. + */ +struct PackedInstr +{ + s16 type; // LUA_T* or INSTR_* + u16 set_into; // set into table on stack + bool pop; // remove from stack? + union { + bool bdata; // boolean: value + lua_Number ndata; // number: value + struct { + u16 uidata1, uidata2; // table: narr, nrec + }; + struct { + /* + SETTABLE: key index, value index + POP: indices to remove + otherwise w/ set_into: numeric key, - + */ + s32 sidata1, sidata2; + }; + void *ptrdata; // userdata: implementation defined + }; + /* + - string: value + - function: buffer + - w/ set_into: string key (no null bytes!) + - userdata: name in registry + */ + std::string sdata; + + PackedInstr() : type(0), set_into(0), pop(false) {} +}; + +/** + * A packed value can be a primitive like a string or number but also a table + * including all of its contents. It is made up of a linear stream of + * 'instructions' that build the final value when executed. + */ +struct PackedValue +{ + std::vector i; + // Indicates whether there are any userdata pointers that need to be deallocated + bool contains_userdata = false; + + PackedValue() = default; + ~PackedValue(); + + DISABLE_CLASS_COPY(PackedValue) + + ALLOW_CLASS_MOVE(PackedValue) +}; + +/* + * Packing callback: Turns a Lua value at given index into a void* + */ +typedef void *(*PackInFunc)(lua_State *L, int idx); +/* + * Unpacking callback: Turns a void* back into the Lua value (left on top of stack) + * + * Note that this function must take ownership of the pointer, so make sure + * to free or keep the memory. + * `L` can be nullptr to indicate that data should just be discarded. + */ +typedef void (*PackOutFunc)(lua_State *L, void *ptr); +/* + * Register a packable type with the name of its metatable. + * + * Even though the callbacks are global this must be called for every Lua state + * that supports objects of this type. + * This function is thread-safe. + */ +void script_register_packer(lua_State *L, const char *regname, + PackInFunc fin, PackOutFunc fout); + +// Pack a Lua value +PackedValue *script_pack(lua_State *L, int idx); +// Unpack a Lua value (left on top of stack) +// Note that this may modify the PackedValue, you can't reuse it! +void script_unpack(lua_State *L, PackedValue *val); + +// Dump contents of PackedValue to stdout for debugging +void script_dump_packed(const PackedValue *val); -- cgit v1.2.3 From 7f58887ae33893c981fbdff23d4e1fa4a11c32e4 Mon Sep 17 00:00:00 2001 From: Jude Melton-Houghton Date: Tue, 10 May 2022 16:37:33 -0400 Subject: Support packing arbitrary graphs (#12289) --- games/devtest/mods/unittests/async_env.lua | 37 ++++++++--- src/script/common/c_packer.cpp | 103 ++++++++++++++++------------- src/script/common/c_packer.h | 5 +- 3 files changed, 90 insertions(+), 55 deletions(-) (limited to 'src/script/common/c_packer.h') diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua index aff1fc4d9..3a21bd9e2 100644 --- a/games/devtest/mods/unittests/async_env.lua +++ b/games/devtest/mods/unittests/async_env.lua @@ -60,15 +60,34 @@ local function test_object_passing() local tmp = core.serialize_roundtrip(test_object) assert(deepequal(test_object, tmp)) - -- Circular key, should error - tmp = {"foo", "bar"} - tmp[tmp] = true - assert(not pcall(core.serialize_roundtrip, tmp)) - - -- Circular value, should error - tmp = {"foo"} - tmp[2] = tmp - assert(not pcall(core.serialize_roundtrip, tmp)) + local circular_key = {"foo", "bar"} + circular_key[circular_key] = true + tmp = core.serialize_roundtrip(circular_key) + assert(tmp[1] == "foo") + assert(tmp[2] == "bar") + assert(tmp[tmp] == true) + + local circular_value = {"foo"} + circular_value[2] = circular_value + tmp = core.serialize_roundtrip(circular_value) + assert(tmp[1] == "foo") + assert(tmp[2] == tmp) + + -- Two-segment cycle + local cycle_seg_1, cycle_seg_2 = {}, {} + cycle_seg_1[1] = cycle_seg_2 + cycle_seg_2[1] = cycle_seg_1 + tmp = core.serialize_roundtrip(cycle_seg_1) + assert(tmp[1][1] == tmp) + + -- Duplicated value without a cycle + local acyclic_dup_holder = {} + tmp = ItemStack("") + acyclic_dup_holder[tmp] = tmp + tmp = core.serialize_roundtrip(acyclic_dup_holder) + for k, v in pairs(tmp) do + assert(rawequal(k, v)) + end end unittests.register("test_object_passing", test_object_passing) diff --git a/src/script/common/c_packer.cpp b/src/script/common/c_packer.cpp index fc5277330..ede00c758 100644 --- a/src/script/common/c_packer.cpp +++ b/src/script/common/c_packer.cpp @@ -123,6 +123,7 @@ namespace { size_t idx; VectorRef(std::vector *vec, size_t idx) : vec(vec), idx(idx) {} public: + constexpr VectorRef() : vec(nullptr), idx(0) {} static VectorRef front(std::vector &vec) { return VectorRef(&vec, 0); } @@ -131,6 +132,7 @@ namespace { } T &operator*() { return (*vec)[idx]; } T *operator->() { return &(*vec)[idx]; } + operator bool() const { return vec != nullptr; } }; struct Packer { @@ -252,38 +254,27 @@ static bool find_packer(lua_State *L, int idx, PackerTuple &out) // Packing implementation // -// recursively goes through the structure and ensures there are no circular references -static void pack_validate(lua_State *L, int idx, std::unordered_set &seen) +static VectorRef record_object(lua_State *L, int idx, PackedValue &pv, + std::unordered_map &seen) { -#ifndef NDEBUG - StackChecker checker(L); - assert(idx > 0); -#endif - - if (lua_type(L, idx) != LUA_TTABLE) - return; - const void *ptr = lua_topointer(L, idx); assert(ptr); - - if (seen.find(ptr) != seen.end()) - throw LuaError("Circular references cannot be packed (yet)"); - seen.insert(ptr); - - lua_checkstack(L, 5); - lua_pushnil(L); - while (lua_next(L, idx) != 0) { - // key at -2, value at -1 - pack_validate(L, absidx(L, -2), seen); - pack_validate(L, absidx(L, -1), seen); - - lua_pop(L, 1); + auto found = seen.find(ptr); + if (found == seen.end()) { + seen[ptr] = pv.i.size(); + return VectorRef(); } - - seen.erase(ptr); + s32 ref = found->second; + assert(ref < (s32)pv.i.size()); + // reuse the value from first time + auto r = emplace(pv, INSTR_PUSHREF); + r->ref = ref; + pv.i[ref].keep_ref = true; + return r; } -static VectorRef pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv) +static VectorRef pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv, + std::unordered_map &seen) { #ifndef NDEBUG StackChecker checker(L); @@ -313,10 +304,17 @@ static VectorRef pack_inner(lua_State *L, int idx, int vidx, Packed r->sdata.assign(str, len); return r; } - case LUA_TTABLE: + case LUA_TTABLE: { + auto r = record_object(L, idx, pv, seen); + if (r) + return r; break; // execution continues + } case LUA_TFUNCTION: { - auto r = emplace(pv, LUA_TFUNCTION); + auto r = record_object(L, idx, pv, seen); + if (r) + return r; + r = emplace(pv, LUA_TFUNCTION); call_string_dump(L, idx); size_t len; const char *str = lua_tolstring(L, -1, &len); @@ -326,11 +324,14 @@ static VectorRef pack_inner(lua_State *L, int idx, int vidx, Packed return r; } case LUA_TUSERDATA: { + auto r = record_object(L, idx, pv, seen); + if (r) + return r; PackerTuple ser; if (!find_packer(L, idx, ser)) throw LuaError("Cannot serialize unsupported userdata"); pv.contains_userdata = true; - auto r = emplace(pv, LUA_TUSERDATA); + r = emplace(pv, LUA_TUSERDATA); r->sdata = ser.first; r->ptrdata = ser.second.fin(L, idx); return r; @@ -360,8 +361,8 @@ static VectorRef pack_inner(lua_State *L, int idx, int vidx, Packed // check if we can use a shortcut if (can_set_into(ktype, vtype) && suitable_key(L, -2)) { // push only the value - auto rval = pack_inner(L, absidx(L, -1), vidx, pv); - rval->pop = vtype != LUA_TTABLE; + auto rval = pack_inner(L, absidx(L, -1), vidx, pv, seen); + rval->pop = rval->type != LUA_TTABLE; // and where to put it: rval->set_into = vi_table; if (ktype == LUA_TSTRING) @@ -375,9 +376,9 @@ static VectorRef pack_inner(lua_State *L, int idx, int vidx, Packed } } else { // push the key and value - pack_inner(L, absidx(L, -2), vidx, pv); + pack_inner(L, absidx(L, -2), vidx, pv, seen); vidx++; - pack_inner(L, absidx(L, -1), vidx, pv); + pack_inner(L, absidx(L, -1), vidx, pv, seen); vidx++; // push an instruction to set them auto ri1 = emplace(pv, INSTR_SETTABLE); @@ -400,13 +401,9 @@ PackedValue *script_pack(lua_State *L, int idx) if (idx < 0) idx = absidx(L, idx); - std::unordered_set seen; - pack_validate(L, idx, seen); - assert(seen.size() == 0); - - // Actual serialization PackedValue pv; - pack_inner(L, idx, 1, pv); + std::unordered_map seen; + pack_inner(L, idx, 1, pv, seen); return new PackedValue(std::move(pv)); } @@ -417,18 +414,21 @@ PackedValue *script_pack(lua_State *L, int idx) void script_unpack(lua_State *L, PackedValue *pv) { + lua_newtable(L); // table at index top to track ref indices -> objects const int top = lua_gettop(L); int ctr = 0; - for (auto &i : pv->i) { + for (size_t packed_idx = 0; packed_idx < pv->i.size(); packed_idx++) { + auto &i = pv->i[packed_idx]; + // If leaving values on stack make sure there's space (every 5th iteration) if (!i.pop && (ctr++) >= 5) { lua_checkstack(L, 5); ctr = 0; } - /* Instructions */ switch (i.type) { + /* Instructions */ case INSTR_SETTABLE: lua_pushvalue(L, top + i.sidata1); // key lua_pushvalue(L, top + i.sidata2); // value @@ -448,12 +448,12 @@ void script_unpack(lua_State *L, PackedValue *pv) if (i.sidata2 > 0) lua_remove(L, top + i.sidata2); continue; - default: + case INSTR_PUSHREF: + lua_pushinteger(L, i.ref); + lua_rawget(L, top); break; - } - /* Lua types */ - switch (i.type) { + /* Lua types */ case LUA_TNIL: lua_pushnil(L); break; @@ -479,11 +479,18 @@ void script_unpack(lua_State *L, PackedValue *pv) i.ptrdata = nullptr; // ownership taken by callback break; } + default: assert(0); break; } + if (i.keep_ref) { + lua_pushinteger(L, packed_idx); + lua_pushvalue(L, -2); + lua_rawset(L, top); + } + if (i.set_into) { if (!i.pop) lua_pushvalue(L, -1); @@ -501,6 +508,7 @@ void script_unpack(lua_State *L, PackedValue *pv) pv->contains_userdata = false; // leave exactly one value on the stack lua_settop(L, top+1); + lua_remove(L, top); } // @@ -541,6 +549,9 @@ void script_dump_packed(const PackedValue *val) case INSTR_POP: printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2); break; + case INSTR_PUSHREF: + printf("PUSHREF(%d)", i.ref); + break; case LUA_TNIL: printf("nil"); break; @@ -574,6 +585,8 @@ void script_dump_packed(const PackedValue *val) else printf(", into=%d", i.set_into); } + if (i.keep_ref) + printf(", keep_ref"); if (i.pop) printf(", pop"); printf(")\n"); diff --git a/src/script/common/c_packer.h b/src/script/common/c_packer.h index 8bccca98d..ee732be86 100644 --- a/src/script/common/c_packer.h +++ b/src/script/common/c_packer.h @@ -36,6 +36,7 @@ extern "C" { #define INSTR_SETTABLE (-10) #define INSTR_POP (-11) +#define INSTR_PUSHREF (-12) /** * Represents a single instruction that pushes a new value or works with existing ones. @@ -44,6 +45,7 @@ struct PackedInstr { s16 type; // LUA_T* or INSTR_* u16 set_into; // set into table on stack + bool keep_ref; // is referenced later by INSTR_PUSHREF? bool pop; // remove from stack? union { bool bdata; // boolean: value @@ -60,6 +62,7 @@ struct PackedInstr s32 sidata1, sidata2; }; void *ptrdata; // userdata: implementation defined + s32 ref; // PUSHREF: index of referenced instr }; /* - string: value @@ -69,7 +72,7 @@ struct PackedInstr */ std::string sdata; - PackedInstr() : type(0), set_into(0), pop(false) {} + PackedInstr() : type(0), set_into(0), keep_ref(false), pop(false) {} }; /** -- cgit v1.2.3 From ec9f1575121e3b064b919bca7efddfa8b0fc4e65 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Mon, 9 May 2022 18:20:10 +0200 Subject: Use native packer to transfer globals into async env(s) --- builtin/async/game.lua | 3 +-- builtin/game/misc.lua | 2 +- src/script/common/c_packer.h | 2 +- src/script/scripting_server.cpp | 9 +++++---- src/server.cpp | 1 - src/server.h | 5 +++-- 6 files changed, 11 insertions(+), 11 deletions(-) (limited to 'src/script/common/c_packer.h') diff --git a/builtin/async/game.lua b/builtin/async/game.lua index 212a33e17..8cb9720b6 100644 --- a/builtin/async/game.lua +++ b/builtin/async/game.lua @@ -22,8 +22,7 @@ dofile(gamepath .. "voxelarea.lua") -- Transfer of globals do - assert(core.transferred_globals) - local all = core.deserialize(core.transferred_globals, true) + local all = assert(core.transferred_globals) core.transferred_globals = nil -- reassemble other tables diff --git a/builtin/game/misc.lua b/builtin/game/misc.lua index 9f5e3312b..997b1894a 100644 --- a/builtin/game/misc.lua +++ b/builtin/game/misc.lua @@ -262,5 +262,5 @@ function core.get_globals_to_transfer() registered_items = copy_filtering(core.registered_items), registered_aliases = core.registered_aliases, } - return core.serialize(all) + return all end diff --git a/src/script/common/c_packer.h b/src/script/common/c_packer.h index ee732be86..fe072c10a 100644 --- a/src/script/common/c_packer.h +++ b/src/script/common/c_packer.h @@ -119,7 +119,7 @@ void script_register_packer(lua_State *L, const char *regname, // Pack a Lua value PackedValue *script_pack(lua_State *L, int idx); // Unpack a Lua value (left on top of stack) -// Note that this may modify the PackedValue, you can't reuse it! +// Note that this may modify the PackedValue, reusability is not guaranteed! void script_unpack(lua_State *L, PackedValue *val); // Dump contents of PackedValue to stdout for debugging diff --git a/src/script/scripting_server.cpp b/src/script/scripting_server.cpp index 5b99468dc..b462141b0 100644 --- a/src/script/scripting_server.cpp +++ b/src/script/scripting_server.cpp @@ -98,8 +98,9 @@ void ServerScripting::initAsync() luaL_checktype(L, -1, LUA_TTABLE); lua_getfield(L, -1, "get_globals_to_transfer"); lua_call(L, 0, 1); - luaL_checktype(L, -1, LUA_TSTRING); - getServer()->m_async_globals_data.set(readParam(L, -1)); + auto *data = script_pack(L, -1); + assert(!data->contains_userdata); + getServer()->m_async_globals_data.reset(data); lua_pushnil(L); lua_setfield(L, -3, "get_globals_to_transfer"); // unset function too lua_pop(L, 2); // pop 'core', return value @@ -183,8 +184,8 @@ void ServerScripting::InitializeAsync(lua_State *L, int top) // globals data lua_getglobal(L, "core"); luaL_checktype(L, -1, LUA_TTABLE); - std::string s = ModApiBase::getServer(L)->m_async_globals_data.get(); - lua_pushlstring(L, s.c_str(), s.size()); + auto *data = ModApiBase::getServer(L)->m_async_globals_data.get(); + script_unpack(L, data); lua_setfield(L, -2, "transferred_globals"); lua_pop(L, 1); // pop 'core' } diff --git a/src/server.cpp b/src/server.cpp index ebe1d1f6b..d93f300d2 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -243,7 +243,6 @@ Server::Server( m_clients(m_con), m_admin_chat(iface), m_on_shutdown_errmsg(on_shutdown_errmsg), - m_async_globals_data(""), m_modchannel_mgr(new ModChannelMgr()) { if (m_path_world.empty()) diff --git a/src/server.h b/src/server.h index ecba30b95..2c21f5dfc 100644 --- a/src/server.h +++ b/src/server.h @@ -73,6 +73,7 @@ struct Lighting; class ServerThread; class ServerModManager; class ServerInventoryManager; +struct PackedValue; enum ClientDeletionReason { CDR_LEAVE, @@ -388,8 +389,8 @@ public: // Lua files registered for init of async env, pair of modname + path std::vector> m_async_init_files; - // Serialized data transferred into async envs at init time - MutexedVariable m_async_globals_data; + // Data transferred into async envs at init time + std::unique_ptr m_async_globals_data; // Bind address Address m_bind_addr; -- cgit v1.2.3