From 21113ad4105dd3fb181b3d0638b907af94a352ab Mon Sep 17 00:00:00 2001 From: Wuzzy Date: Fri, 1 Oct 2021 14:21:24 +0000 Subject: Split liquid_viscosity to liquid_viscosity and move_resistance (#10810) --- src/script/common/c_content.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'src/script/common') diff --git a/src/script/common/c_content.cpp b/src/script/common/c_content.cpp index 5a095fd8f..8a5a3fe71 100644 --- a/src/script/common/c_content.cpp +++ b/src/script/common/c_content.cpp @@ -719,6 +719,9 @@ void read_content_features(lua_State *L, ContentFeatures &f, int index) // the slowest possible f.liquid_viscosity = getintfield_default(L, index, "liquid_viscosity", f.liquid_viscosity); + // If move_resistance is not set explicitly, + // move_resistance is equal to liquid_viscosity + f.move_resistance = f.liquid_viscosity; f.liquid_range = getintfield_default(L, index, "liquid_range", f.liquid_range); f.leveled = getintfield_default(L, index, "leveled", f.leveled); @@ -822,6 +825,21 @@ void read_content_features(lua_State *L, ContentFeatures &f, int index) getstringfield(L, index, "node_dig_prediction", f.node_dig_prediction); + // How much the node slows down players, ranging from 1 to 7, + // the higher, the slower. + f.move_resistance = getintfield_default(L, index, + "move_resistance", f.move_resistance); + + // Whether e.g. players in this node will have liquid movement physics + lua_getfield(L, index, "liquid_move_physics"); + if(lua_isboolean(L, -1)) { + f.liquid_move_physics = lua_toboolean(L, -1); + } else if(lua_isnil(L, -1)) { + f.liquid_move_physics = f.liquid_type != LIQUID_NONE; + } else { + errorstream << "Field \"liquid_move_physics\": Invalid type!" << std::endl; + } + lua_pop(L, 1); } void push_content_features(lua_State *L, const ContentFeatures &c) @@ -949,6 +967,10 @@ void push_content_features(lua_State *L, const ContentFeatures &c) lua_setfield(L, -2, "legacy_wallmounted"); lua_pushstring(L, c.node_dig_prediction.c_str()); lua_setfield(L, -2, "node_dig_prediction"); + lua_pushnumber(L, c.move_resistance); + lua_setfield(L, -2, "move_resistance"); + lua_pushboolean(L, c.liquid_move_physics); + lua_setfield(L, -2, "liquid_move_physics"); } /******************************************************************************/ -- cgit v1.2.3 From 8c99f2232bdb52459ccf2a5b751cbe3f7797abc3 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 17 Dec 2021 18:31:29 +0100 Subject: Don't let HTTP API pass through untrusted function This has been a problem since the first day, oops. --- builtin/game/misc.lua | 5 +++-- src/script/common/c_internal.h | 2 ++ src/script/lua_api/l_http.cpp | 23 +++++++++++++++++++---- src/script/lua_api/l_http.h | 3 +++ 4 files changed, 27 insertions(+), 6 deletions(-) (limited to 'src/script/common') diff --git a/builtin/game/misc.lua b/builtin/game/misc.lua index ef826eda7..e86efc50c 100644 --- a/builtin/game/misc.lua +++ b/builtin/game/misc.lua @@ -250,7 +250,7 @@ end -- HTTP callback interface -function core.http_add_fetch(httpenv) +core.set_http_api_lua(function(httpenv) httpenv.fetch = function(req, callback) local handle = httpenv.fetch_async(req) @@ -266,7 +266,8 @@ function core.http_add_fetch(httpenv) end return httpenv -end +end) +core.set_http_api_lua = nil function core.close_formspec(player_name, formname) diff --git a/src/script/common/c_internal.h b/src/script/common/c_internal.h index ab2d7b975..94cfd61fb 100644 --- a/src/script/common/c_internal.h +++ b/src/script/common/c_internal.h @@ -54,6 +54,8 @@ extern "C" { #define CUSTOM_RIDX_GLOBALS_BACKUP (CUSTOM_RIDX_BASE + 1) #define CUSTOM_RIDX_CURRENT_MOD_NAME (CUSTOM_RIDX_BASE + 2) #define CUSTOM_RIDX_BACKTRACE (CUSTOM_RIDX_BASE + 3) +#define CUSTOM_RIDX_HTTP_API_LUA (CUSTOM_RIDX_BASE + 4) + // Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata #if defined(__aarch64__) && USE_LUAJIT diff --git a/src/script/lua_api/l_http.cpp b/src/script/lua_api/l_http.cpp index 751ec9837..b385b698c 100644 --- a/src/script/lua_api/l_http.cpp +++ b/src/script/lua_api/l_http.cpp @@ -163,6 +163,20 @@ int ModApiHttp::l_http_fetch_async_get(lua_State *L) return 1; } +int ModApiHttp::l_set_http_api_lua(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + + // This is called by builtin to give us a function that will later + // populate the http_api table with additional method(s). + // We need this because access to the HTTP api is security-relevant and + // any mod could just mess with a global variable. + luaL_checktype(L, 1, LUA_TFUNCTION); + lua_rawseti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_HTTP_API_LUA); + + return 0; +} + int ModApiHttp::l_request_http_api(lua_State *L) { NO_MAP_LOCK_REQUIRED; @@ -205,16 +219,16 @@ int ModApiHttp::l_request_http_api(lua_State *L) return 1; } - lua_getglobal(L, "core"); - lua_getfield(L, -1, "http_add_fetch"); + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_HTTP_API_LUA); + assert(lua_isfunction(L, -1)); lua_newtable(L); HTTP_API(fetch_async); HTTP_API(fetch_async_get); // Stack now looks like this: - // - // Now call core.http_add_fetch to append .fetch(request, callback) to table + //
+ // Now call it to append .fetch(request, callback) to table lua_call(L, 1, 1); return 1; @@ -247,6 +261,7 @@ void ModApiHttp::Initialize(lua_State *L, int top) API_FCT(get_http_api); } else { API_FCT(request_http_api); + API_FCT(set_http_api_lua); } #endif diff --git a/src/script/lua_api/l_http.h b/src/script/lua_api/l_http.h index c3a2a5276..17fa283ba 100644 --- a/src/script/lua_api/l_http.h +++ b/src/script/lua_api/l_http.h @@ -41,6 +41,9 @@ private: // http_fetch_async_get(handle) static int l_http_fetch_async_get(lua_State *L); + // set_http_api_lua() [internal] + static int l_set_http_api_lua(lua_State *L); + // request_http_api() static int l_request_http_api(lua_State *L); -- cgit v1.2.3 From 1ee37148a8072fe6350124cd51c812c3d3fb069a Mon Sep 17 00:00:00 2001 From: Lars Müller <34514239+appgurueu@users.noreply.github.com> Date: Fri, 4 Feb 2022 20:28:43 +0100 Subject: Fix types of get_mapgen_setting_noiseparams (#12025) --- builtin/mainmenu/dlg_settings_advanced.lua | 30 +++++++++++++----------------- src/script/common/c_content.cpp | 21 ++++++++------------- src/script/common/c_converter.cpp | 27 --------------------------- src/script/common/c_converter.h | 3 --- 4 files changed, 21 insertions(+), 60 deletions(-) (limited to 'src/script/common') diff --git a/builtin/mainmenu/dlg_settings_advanced.lua b/builtin/mainmenu/dlg_settings_advanced.lua index 772509670..83f905446 100644 --- a/builtin/mainmenu/dlg_settings_advanced.lua +++ b/builtin/mainmenu/dlg_settings_advanced.lua @@ -517,24 +517,20 @@ end local function get_current_np_group_as_string(setting) local value = core.settings:get_np_group(setting.name) - local t if value == nil then - t = setting.default - else - t = value.offset .. ", " .. - value.scale .. ", (" .. - value.spread.x .. ", " .. - value.spread.y .. ", " .. - value.spread.z .. "), " .. - value.seed .. ", " .. - value.octaves .. ", " .. - value.persistence .. ", " .. - value.lacunarity - if value.flags ~= "" then - t = t .. ", " .. value.flags - end + return setting.default end - return t + return ("%g, %g, (%g, %g, %g), %g, %g, %g, %g"):format( + value.offset, + value.scale, + value.spread.x, + value.spread.y, + value.spread.z, + value.seed, + value.octaves, + value.persistence, + value.lacunarity + ) .. (value.flags ~= "" and (", " .. value.flags) or "") end local checkboxes = {} -- handle checkboxes events @@ -667,7 +663,7 @@ local function create_change_setting_formspec(dialogdata) elseif setting.type == "v3f" then local val = get_current_value(setting) local v3f = {} - for line in val:gmatch("[+-]?[%d.-e]+") do -- All numeric characters + for line in val:gmatch("[+-]?[%d.+-eE]+") do -- All numeric characters table.insert(v3f, line) end diff --git a/src/script/common/c_content.cpp b/src/script/common/c_content.cpp index 8a5a3fe71..b6eaa6b13 100644 --- a/src/script/common/c_content.cpp +++ b/src/script/common/c_content.cpp @@ -1697,24 +1697,19 @@ bool read_noiseparams(lua_State *L, int index, NoiseParams *np) void push_noiseparams(lua_State *L, NoiseParams *np) { lua_newtable(L); - push_float_string(L, np->offset); - lua_setfield(L, -2, "offset"); - push_float_string(L, np->scale); - lua_setfield(L, -2, "scale"); - push_float_string(L, np->persist); - lua_setfield(L, -2, "persistence"); - push_float_string(L, np->lacunarity); - lua_setfield(L, -2, "lacunarity"); - lua_pushnumber(L, np->seed); - lua_setfield(L, -2, "seed"); - lua_pushnumber(L, np->octaves); - lua_setfield(L, -2, "octaves"); + setfloatfield(L, -1, "offset", np->offset); + setfloatfield(L, -1, "scale", np->scale); + setfloatfield(L, -1, "persist", np->persist); + setfloatfield(L, -1, "persistence", np->persist); + setfloatfield(L, -1, "lacunarity", np->lacunarity); + setintfield( L, -1, "seed", np->seed); + setintfield( L, -1, "octaves", np->octaves); push_flags_string(L, flagdesc_noiseparams, np->flags, np->flags); lua_setfield(L, -2, "flags"); - push_v3_float_string(L, np->spread); + push_v3f(L, np->spread); lua_setfield(L, -2, "spread"); } diff --git a/src/script/common/c_converter.cpp b/src/script/common/c_converter.cpp index 19734b913..716405593 100644 --- a/src/script/common/c_converter.cpp +++ b/src/script/common/c_converter.cpp @@ -73,13 +73,6 @@ static void set_vector_metatable(lua_State *L) lua_pop(L, 1); } - -void push_float_string(lua_State *L, float value) -{ - auto str = ftos(value); - lua_pushstring(L, str.c_str()); -} - void push_v3f(lua_State *L, v3f p) { lua_createtable(L, 0, 3); @@ -101,26 +94,6 @@ void push_v2f(lua_State *L, v2f p) lua_setfield(L, -2, "y"); } -void push_v3_float_string(lua_State *L, v3f p) -{ - lua_createtable(L, 0, 3); - push_float_string(L, p.X); - lua_setfield(L, -2, "x"); - push_float_string(L, p.Y); - lua_setfield(L, -2, "y"); - push_float_string(L, p.Z); - lua_setfield(L, -2, "z"); -} - -void push_v2_float_string(lua_State *L, v2f p) -{ - lua_createtable(L, 0, 2); - push_float_string(L, p.X); - lua_setfield(L, -2, "x"); - push_float_string(L, p.Y); - lua_setfield(L, -2, "y"); -} - v2s16 read_v2s16(lua_State *L, int index) { v2s16 p; diff --git a/src/script/common/c_converter.h b/src/script/common/c_converter.h index 6ad6f3212..a14eb9186 100644 --- a/src/script/common/c_converter.h +++ b/src/script/common/c_converter.h @@ -118,9 +118,6 @@ std::vector read_aabb3f_vector (lua_State *L, int index, f32 scale); size_t read_stringlist (lua_State *L, int index, std::vector *result); -void push_float_string (lua_State *L, float value); -void push_v3_float_string(lua_State *L, v3f p); -void push_v2_float_string(lua_State *L, v2f p); void push_v2s16 (lua_State *L, v2s16 p); void push_v2s32 (lua_State *L, v2s32 p); void push_v3s16 (lua_State *L, v3s16 p); -- cgit v1.2.3 From ba6fbc417ecb812345c1747f42b6606dfc8e1d5b Mon Sep 17 00:00:00 2001 From: sfan5 Date: Thu, 3 Feb 2022 21:35:08 +0100 Subject: Remove awful Mingw32 workarounds Instead a warning is triggered if an affected compiler is detected. closes #12022 --- src/main.cpp | 8 ++++++++ src/script/common/c_converter.cpp | 8 -------- src/serialization.cpp | 28 ++-------------------------- 3 files changed, 10 insertions(+), 34 deletions(-) (limited to 'src/script/common') diff --git a/src/main.cpp b/src/main.cpp index ca95ef874..5ea212d8a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -65,6 +65,14 @@ extern "C" { #error Minetest cannot be built without exceptions or RTTI #endif +#if defined(__MINGW32__) && !defined(__MINGW64__) && !defined(__clang__) && \ + (__GNUC__ < 11 || (__GNUC__ == 11 && __GNUC_MINOR__ < 1)) +// see e.g. https://github.com/minetest/minetest/issues/10137 +#warning ================================== +#warning 32-bit MinGW gcc before 11.1 has known issues with crashes on thread exit, you should upgrade. +#warning ================================== +#endif + #define DEBUGFILE "debug.txt" #define DEFAULT_SERVER_PORT 30000 diff --git a/src/script/common/c_converter.cpp b/src/script/common/c_converter.cpp index 716405593..08fb9ad30 100644 --- a/src/script/common/c_converter.cpp +++ b/src/script/common/c_converter.cpp @@ -452,17 +452,9 @@ size_t read_stringlist(lua_State *L, int index, std::vector *result Table field getters */ -#if defined(__MINGW32__) && !defined(__MINGW64__) -/* MinGW 32-bit somehow crashes in the std::set destructor when this - * variable is thread-local, so just don't do that. */ -static std::set warned_msgs; -#endif - bool check_field_or_nil(lua_State *L, int index, int type, const char *fieldname) { -#if !defined(__MINGW32__) || defined(__MINGW64__) thread_local std::set warned_msgs; -#endif int t = lua_type(L, index); if (t == LUA_TNIL) diff --git a/src/serialization.cpp b/src/serialization.cpp index d4d7b5f6e..d3009bc83 100644 --- a/src/serialization.cpp +++ b/src/serialization.cpp @@ -208,30 +208,11 @@ struct ZSTD_Deleter { } }; -#if defined(__MINGW32__) && !defined(__MINGW64__) -/* - * This is exactly as dumb as it looks. - * Yes, this is a memory leak. No, we don't have better solution right now. - */ -template class leaky_ptr -{ - T *value; -public: - leaky_ptr(T *value) : value(value) {}; - T *get() { return value; } -}; -#endif - void compressZstd(const u8 *data, size_t data_size, std::ostream &os, int level) { -#if defined(__MINGW32__) && !defined(__MINGW64__) - // leaks one context per thread but doesn't crash :shrug: - thread_local leaky_ptr stream(ZSTD_createCStream()); -#else // reusing the context is recommended for performance - // it will destroyed when the thread ends + // it will be destroyed when the thread ends thread_local std::unique_ptr stream(ZSTD_createCStream()); -#endif ZSTD_initCStream(stream.get(), level); @@ -276,14 +257,9 @@ void compressZstd(const std::string &data, std::ostream &os, int level) void decompressZstd(std::istream &is, std::ostream &os) { -#if defined(__MINGW32__) && !defined(__MINGW64__) - // leaks one context per thread but doesn't crash :shrug: - thread_local leaky_ptr stream(ZSTD_createDStream()); -#else // reusing the context is recommended for performance - // it will destroyed when the thread ends + // it will be destroyed when the thread ends thread_local std::unique_ptr stream(ZSTD_createDStream()); -#endif ZSTD_initDStream(stream.get()); -- cgit v1.2.3 From 8d387433b14791db95e59127b5e6e30f58155c1e Mon Sep 17 00:00:00 2001 From: DS Date: Tue, 29 Mar 2022 18:06:16 +0200 Subject: Fix the documentation of InvRef:get_lists() and clean up code (#12150) --- doc/lua_api.txt | 4 ++-- src/database/database-postgresql.cpp | 2 +- src/database/database-sqlite3.cpp | 4 ++-- src/inventory.cpp | 42 ------------------------------------ src/inventory.h | 26 +++++++++++++++------- src/script/common/c_content.cpp | 23 ++++++++++++-------- src/script/common/c_content.h | 6 ++++-- src/script/cpp_api/s_client.cpp | 10 +-------- src/script/lua_api/l_inventory.cpp | 23 +++++++++----------- src/script/lua_api/l_nodemeta.cpp | 12 ++++------- 10 files changed, 56 insertions(+), 96 deletions(-) (limited to 'src/script/common') diff --git a/doc/lua_api.txt b/doc/lua_api.txt index 029c53616..161bdd249 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -6355,9 +6355,9 @@ An `InvRef` is a reference to an inventory. * `set_width(listname, width)`: set width of list; currently used for crafting * `get_stack(listname, i)`: get a copy of stack index `i` in list * `set_stack(listname, i, stack)`: copy `stack` to index `i` in list -* `get_list(listname)`: return full list +* `get_list(listname)`: return full list (list of `ItemStack`s) * `set_list(listname, list)`: set full list (size will not change) -* `get_lists()`: returns list of inventory lists +* `get_lists()`: returns table that maps listnames to inventory lists * `set_lists(lists)`: sets inventory lists (size will not change) * `add_item(listname, stack)`: add item somewhere in list, returns leftover `ItemStack`. diff --git a/src/database/database-postgresql.cpp b/src/database/database-postgresql.cpp index 3469f4242..9d6501e68 100644 --- a/src/database/database-postgresql.cpp +++ b/src/database/database-postgresql.cpp @@ -495,7 +495,7 @@ void PlayerDatabasePostgreSQL::savePlayer(RemotePlayer *player) execPrepared("remove_player_inventories", 1, rmvalues); execPrepared("remove_player_inventory_items", 1, rmvalues); - std::vector inventory_lists = sao->getInventory()->getLists(); + const auto &inventory_lists = sao->getInventory()->getLists(); std::ostringstream oss; for (u16 i = 0; i < inventory_lists.size(); i++) { const InventoryList* list = inventory_lists[i]; diff --git a/src/database/database-sqlite3.cpp b/src/database/database-sqlite3.cpp index 1e63ae9d8..9521085e9 100644 --- a/src/database/database-sqlite3.cpp +++ b/src/database/database-sqlite3.cpp @@ -476,10 +476,10 @@ void PlayerDatabaseSQLite3::savePlayer(RemotePlayer *player) sqlite3_vrfy(sqlite3_step(m_stmt_player_remove_inventory_items), SQLITE_DONE); sqlite3_reset(m_stmt_player_remove_inventory_items); - std::vector inventory_lists = sao->getInventory()->getLists(); + const auto &inventory_lists = sao->getInventory()->getLists(); std::ostringstream oss; for (u16 i = 0; i < inventory_lists.size(); i++) { - const InventoryList* list = inventory_lists[i]; + const InventoryList *list = inventory_lists[i]; str_to_sqlite(m_stmt_player_add_inventory, 1, player->getName()); int_to_sqlite(m_stmt_player_add_inventory, 2, i); diff --git a/src/inventory.cpp b/src/inventory.cpp index d14b12f9d..6d2b7fba3 100644 --- a/src/inventory.cpp +++ b/src/inventory.cpp @@ -503,11 +503,6 @@ void InventoryList::deSerialize(std::istream &is) throw SerializationError(ss.str()); } -InventoryList::InventoryList(const InventoryList &other) -{ - *this = other; -} - InventoryList & InventoryList::operator = (const InventoryList &other) { m_items = other.m_items; @@ -535,21 +530,6 @@ bool InventoryList::operator == (const InventoryList &other) const return true; } -const std::string &InventoryList::getName() const -{ - return m_name; -} - -u32 InventoryList::getSize() const -{ - return m_items.size(); -} - -u32 InventoryList::getWidth() const -{ - return m_width; -} - u32 InventoryList::getUsedSlots() const { u32 num = 0; @@ -560,18 +540,6 @@ u32 InventoryList::getUsedSlots() const return num; } -const ItemStack& InventoryList::getItem(u32 i) const -{ - assert(i < m_size); // Pre-condition - return m_items[i]; -} - -ItemStack& InventoryList::getItem(u32 i) -{ - assert(i < m_size); // Pre-condition - return m_items[i]; -} - ItemStack InventoryList::changeItem(u32 i, const ItemStack &newitem) { if(i >= m_items.size()) @@ -960,16 +928,6 @@ InventoryList * Inventory::getList(const std::string &name) return m_lists[i]; } -std::vector Inventory::getLists() -{ - std::vector lists; - lists.reserve(m_lists.size()); - for (auto list : m_lists) { - lists.push_back(list); - } - return lists; -} - bool Inventory::deleteList(const std::string &name) { s32 i = getListIndex(name); diff --git a/src/inventory.h b/src/inventory.h index eb063d4ad..8b31de3a8 100644 --- a/src/inventory.h +++ b/src/inventory.h @@ -198,7 +198,7 @@ public: void serialize(std::ostream &os, bool incremental) const; void deSerialize(std::istream &is); - InventoryList(const InventoryList &other); + InventoryList(const InventoryList &other) { *this = other; } InventoryList & operator = (const InventoryList &other); bool operator == (const InventoryList &other) const; bool operator != (const InventoryList &other) const @@ -206,15 +206,25 @@ public: return !(*this == other); } - const std::string &getName() const; - u32 getSize() const; - u32 getWidth() const; + const std::string &getName() const { return m_name; } + u32 getSize() const { return static_cast(m_items.size()); } + u32 getWidth() const { return m_width; } // Count used slots u32 getUsedSlots() const; // Get reference to item - const ItemStack& getItem(u32 i) const; - ItemStack& getItem(u32 i); + const ItemStack &getItem(u32 i) const + { + assert(i < m_size); // Pre-condition + return m_items[i]; + } + ItemStack &getItem(u32 i) + { + assert(i < m_size); // Pre-condition + return m_items[i]; + } + // Get reference to all items + const std::vector &getItems() const { return m_items; } // Returns old item. Parameter can be an empty item. ItemStack changeItem(u32 i, const ItemStack &newitem); // Delete item @@ -271,7 +281,7 @@ public: private: std::vector m_items; std::string m_name; - u32 m_size; + u32 m_size; // always the same as m_items.size() u32 m_width = 0; IItemDefManager *m_itemdef; bool m_dirty = true; @@ -301,7 +311,7 @@ public: InventoryList * addList(const std::string &name, u32 size); InventoryList * getList(const std::string &name); const InventoryList * getList(const std::string &name) const; - std::vector getLists(); + const std::vector &getLists() const { return m_lists; } bool deleteList(const std::string &name); // A shorthand for adding items. Returns leftover item (possibly empty). ItemStack addItem(const std::string &listname, const ItemStack &newitem) diff --git a/src/script/common/c_content.cpp b/src/script/common/c_content.cpp index b6eaa6b13..36f4316ee 100644 --- a/src/script/common/c_content.cpp +++ b/src/script/common/c_content.cpp @@ -1351,17 +1351,22 @@ void push_tool_capabilities(lua_State *L, } /******************************************************************************/ -void push_inventory_list(lua_State *L, Inventory *inv, const char *name) +void push_inventory_list(lua_State *L, const InventoryList &invlist) { - InventoryList *invlist = inv->getList(name); - if(invlist == NULL){ - lua_pushnil(L); - return; + push_items(L, invlist.getItems()); +} + +/******************************************************************************/ +void push_inventory_lists(lua_State *L, const Inventory &inv) +{ + const auto &lists = inv.getLists(); + lua_createtable(L, 0, lists.size()); + for(const InventoryList *list : lists) { + const std::string &name = list->getName(); + lua_pushlstring(L, name.c_str(), name.size()); + push_inventory_list(L, *list); + lua_rawset(L, -3); } - std::vector items; - for(u32 i=0; igetSize(); i++) - items.push_back(invlist->getItem(i)); - push_items(L, items); } /******************************************************************************/ diff --git a/src/script/common/c_content.h b/src/script/common/c_content.h index e762604a4..11b39364f 100644 --- a/src/script/common/c_content.h +++ b/src/script/common/c_content.h @@ -55,6 +55,7 @@ struct ObjectProperties; struct SimpleSoundSpec; struct ServerSoundParams; class Inventory; +class InventoryList; struct NodeBox; struct ContentFeatures; struct TileDef; @@ -120,8 +121,9 @@ void push_object_properties (lua_State *L, ObjectProperties *prop); void push_inventory_list (lua_State *L, - Inventory *inv, - const char *name); + const InventoryList &invlist); +void push_inventory_lists (lua_State *L, + const Inventory &inv); void read_inventory_list (lua_State *L, int tableindex, Inventory *inv, const char *name, Server *srv, int forcesize=-1); diff --git a/src/script/cpp_api/s_client.cpp b/src/script/cpp_api/s_client.cpp index c889fffa0..b02a0c7be 100644 --- a/src/script/cpp_api/s_client.cpp +++ b/src/script/cpp_api/s_client.cpp @@ -281,15 +281,7 @@ bool ScriptApiClient::on_inventory_open(Inventory *inventory) lua_getglobal(L, "core"); lua_getfield(L, -1, "registered_on_inventory_open"); - std::vector lists = inventory->getLists(); - std::vector::iterator iter = lists.begin(); - lua_createtable(L, 0, lists.size()); - for (; iter != lists.end(); iter++) { - const char* name = (*iter)->getName().c_str(); - lua_pushstring(L, name); - push_inventory_list(L, inventory, name); - lua_rawset(L, -3); - } + push_inventory_lists(L, *inventory); try { runCallbacks(1, RUN_CALLBACKS_MODE_OR); diff --git a/src/script/lua_api/l_inventory.cpp b/src/script/lua_api/l_inventory.cpp index b0a4ee194..175047e58 100644 --- a/src/script/lua_api/l_inventory.cpp +++ b/src/script/lua_api/l_inventory.cpp @@ -214,11 +214,16 @@ int InvRef::l_get_list(lua_State *L) InvRef *ref = checkobject(L, 1); const char *listname = luaL_checkstring(L, 2); Inventory *inv = getinv(L, ref); - if(inv){ - push_inventory_list(L, inv, listname); - } else { + if (!inv) { lua_pushnil(L); + return 1; } + InventoryList *invlist = inv->getList(listname); + if (!invlist) { + lua_pushnil(L); + return 1; + } + push_inventory_list(L, *invlist); return 1; } @@ -242,7 +247,7 @@ int InvRef::l_set_list(lua_State *L) return 0; } -// get_lists(self) -> list of InventoryLists +// get_lists(self) -> table that maps listnames to InventoryLists int InvRef::l_get_lists(lua_State *L) { NO_MAP_LOCK_REQUIRED; @@ -251,15 +256,7 @@ int InvRef::l_get_lists(lua_State *L) if (!inv) { return 0; } - std::vector lists = inv->getLists(); - std::vector::iterator iter = lists.begin(); - lua_createtable(L, 0, lists.size()); - for (; iter != lists.end(); iter++) { - const char* name = (*iter)->getName().c_str(); - lua_pushstring(L, name); - push_inventory_list(L, inv, name); - lua_rawset(L, -3); - } + push_inventory_lists(L, *inv); return 1; } diff --git a/src/script/lua_api/l_nodemeta.cpp b/src/script/lua_api/l_nodemeta.cpp index 34760157d..1d052685e 100644 --- a/src/script/lua_api/l_nodemeta.cpp +++ b/src/script/lua_api/l_nodemeta.cpp @@ -127,18 +127,14 @@ void NodeMetaRef::handleToTable(lua_State *L, Metadata *_meta) // fields MetaDataRef::handleToTable(L, _meta); - NodeMetadata *meta = (NodeMetadata*) _meta; + NodeMetadata *meta = (NodeMetadata *) _meta; // inventory - lua_newtable(L); Inventory *inv = meta->getInventory(); if (inv) { - std::vector lists = inv->getLists(); - for(std::vector::const_iterator - i = lists.begin(); i != lists.end(); ++i) { - push_inventory_list(L, inv, (*i)->getName().c_str()); - lua_setfield(L, -2, (*i)->getName().c_str()); - } + push_inventory_lists(L, *inv); + } else { + lua_newtable(L); } lua_setfield(L, -2, "inventory"); } -- cgit v1.2.3 From 06d197cdd042392e1551e5e7244c61300a6bb4e3 Mon Sep 17 00:00:00 2001 From: Jude Melton-Houghton Date: Tue, 29 Mar 2022 12:07:00 -0400 Subject: Store vector metatable in registry --- builtin/common/tests/misc_helpers_spec.lua | 1 + builtin/common/tests/serialize_spec.lua | 1 + builtin/common/tests/vector_spec.lua | 2 +- builtin/common/vector.lua | 6 ++---- builtin/mainmenu/tests/serverlistmgr_spec.lua | 1 + games/devtest/mods/unittests/misc.lua | 12 ++++++++++++ src/script/common/c_converter.cpp | 19 +++---------------- src/script/common/c_internal.h | 1 + src/script/cpp_api/s_base.cpp | 8 ++++++++ src/script/cpp_api/s_security.cpp | 6 ++++++ 10 files changed, 36 insertions(+), 21 deletions(-) (limited to 'src/script/common') diff --git a/builtin/common/tests/misc_helpers_spec.lua b/builtin/common/tests/misc_helpers_spec.lua index b16987f0b..b11236860 100644 --- a/builtin/common/tests/misc_helpers_spec.lua +++ b/builtin/common/tests/misc_helpers_spec.lua @@ -1,4 +1,5 @@ _G.core = {} +_G.vector = {metatable = {}} dofile("builtin/common/vector.lua") dofile("builtin/common/misc_helpers.lua") diff --git a/builtin/common/tests/serialize_spec.lua b/builtin/common/tests/serialize_spec.lua index e46b7dcc5..69b2b567c 100644 --- a/builtin/common/tests/serialize_spec.lua +++ b/builtin/common/tests/serialize_spec.lua @@ -1,4 +1,5 @@ _G.core = {} +_G.vector = {metatable = {}} _G.setfenv = require 'busted.compatibility'.setfenv diff --git a/builtin/common/tests/vector_spec.lua b/builtin/common/tests/vector_spec.lua index 2f72f3383..25880236b 100644 --- a/builtin/common/tests/vector_spec.lua +++ b/builtin/common/tests/vector_spec.lua @@ -1,4 +1,4 @@ -_G.vector = {} +_G.vector = {metatable = {}} dofile("builtin/common/vector.lua") describe("vector", function() diff --git a/builtin/common/vector.lua b/builtin/common/vector.lua index 581d014e0..90010f6de 100644 --- a/builtin/common/vector.lua +++ b/builtin/common/vector.lua @@ -6,10 +6,8 @@ Note: The vector.*-functions must be able to accept old vectors that had no meta -- localize functions local setmetatable = setmetatable -vector = {} - -local metatable = {} -vector.metatable = metatable +-- vector.metatable is set by C++. +local metatable = vector.metatable local xyz = {"x", "y", "z"} diff --git a/builtin/mainmenu/tests/serverlistmgr_spec.lua b/builtin/mainmenu/tests/serverlistmgr_spec.lua index a091959fb..ab7a6c60c 100644 --- a/builtin/mainmenu/tests/serverlistmgr_spec.lua +++ b/builtin/mainmenu/tests/serverlistmgr_spec.lua @@ -1,4 +1,5 @@ _G.core = {} +_G.vector = {metatable = {}} _G.unpack = table.unpack _G.serverlistmgr = {} diff --git a/games/devtest/mods/unittests/misc.lua b/games/devtest/mods/unittests/misc.lua index cf4f92cfa..ba980866a 100644 --- a/games/devtest/mods/unittests/misc.lua +++ b/games/devtest/mods/unittests/misc.lua @@ -36,3 +36,15 @@ local function test_dynamic_media(cb, player) -- if the callback isn't called this test will just hang :shrug: end unittests.register("test_dynamic_media", test_dynamic_media, {async=true, player=true}) + +local function test_v3f_metatable(player) + assert(vector.check(player:get_pos())) +end +unittests.register("test_v3f_metatable", test_v3f_metatable, {player=true}) + +local function test_v3s16_metatable(player, pos) + local node = minetest.get_node(pos) + local found_pos = minetest.find_node_near(pos, 0, node.name, true) + assert(vector.check(found_pos)) +end +unittests.register("test_v3s16_metatable", test_v3s16_metatable, {map=true}) diff --git a/src/script/common/c_converter.cpp b/src/script/common/c_converter.cpp index 08fb9ad30..b5ff52f73 100644 --- a/src/script/common/c_converter.cpp +++ b/src/script/common/c_converter.cpp @@ -52,25 +52,12 @@ if (value < F1000_MIN || value > F1000_MAX) { \ /** - * A helper which sets (if available) the vector metatable from builtin as metatable - * for the table on top of the stack + * A helper which sets the vector metatable for the table on top of the stack */ static void set_vector_metatable(lua_State *L) { - // get vector.metatable - lua_getglobal(L, "vector"); - if (!lua_istable(L, -1)) { - // there is no global vector table - lua_pop(L, 1); - errorstream << "set_vector_metatable in c_converter.cpp: " << - "missing global vector table" << std::endl; - return; - } - lua_getfield(L, -1, "metatable"); - // set the metatable - lua_setmetatable(L, -3); - // pop vector global - lua_pop(L, 1); + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_VECTOR_METATABLE); + lua_setmetatable(L, -2); } void push_v3f(lua_State *L, v3f p) diff --git a/src/script/common/c_internal.h b/src/script/common/c_internal.h index 94cfd61fb..c43db34aa 100644 --- a/src/script/common/c_internal.h +++ b/src/script/common/c_internal.h @@ -55,6 +55,7 @@ extern "C" { #define CUSTOM_RIDX_CURRENT_MOD_NAME (CUSTOM_RIDX_BASE + 2) #define CUSTOM_RIDX_BACKTRACE (CUSTOM_RIDX_BASE + 3) #define CUSTOM_RIDX_HTTP_API_LUA (CUSTOM_RIDX_BASE + 4) +#define CUSTOM_RIDX_VECTOR_METATABLE (CUSTOM_RIDX_BASE + 5) // Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata diff --git a/src/script/cpp_api/s_base.cpp b/src/script/cpp_api/s_base.cpp index f7b8a5102..595c9e540 100644 --- a/src/script/cpp_api/s_base.cpp +++ b/src/script/cpp_api/s_base.cpp @@ -121,6 +121,14 @@ ScriptApiBase::ScriptApiBase(ScriptingType type): lua_newtable(m_luastack); lua_setglobal(m_luastack, "core"); + // vector.metatable is stored in the registry for quick access from C++. + lua_newtable(m_luastack); + lua_rawseti(m_luastack, LUA_REGISTRYINDEX, CUSTOM_RIDX_VECTOR_METATABLE); + lua_newtable(m_luastack); + lua_rawgeti(m_luastack, LUA_REGISTRYINDEX, CUSTOM_RIDX_VECTOR_METATABLE); + lua_setfield(m_luastack, -2, "metatable"); + lua_setglobal(m_luastack, "vector"); + if (m_type == ScriptingType::Client) lua_pushstring(m_luastack, "/"); else diff --git a/src/script/cpp_api/s_security.cpp b/src/script/cpp_api/s_security.cpp index a6c5114b2..f68cd1777 100644 --- a/src/script/cpp_api/s_security.cpp +++ b/src/script/cpp_api/s_security.cpp @@ -98,6 +98,7 @@ void ScriptApiSecurity::initializeSecurity() "type", "unpack", "_VERSION", + "vector", "xpcall", }; static const char *whitelist_tables[] = { @@ -254,6 +255,10 @@ void ScriptApiSecurity::initializeSecurity() lua_pushnil(L); lua_setfield(L, old_globals, "core"); + // 'vector' as well. + lua_pushnil(L); + lua_setfield(L, old_globals, "vector"); + lua_pop(L, 1); // Pop globals_backup @@ -296,6 +301,7 @@ void ScriptApiSecurity::initializeSecurityClient() "type", "unpack", "_VERSION", + "vector", "xpcall", // Completely safe libraries "coroutine", -- cgit v1.2.3 From 56a558baf8ef43810014347ae624cb4ef70b6aa3 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Sat, 9 Apr 2022 14:47:59 +0200 Subject: Refactor some Lua API functions in preparation for async env --- src/gamedef.h | 2 ++ src/script/common/c_content.cpp | 8 ++++---- src/script/common/c_content.h | 6 +++--- src/script/lua_api/l_craft.cpp | 24 +++++++++++------------- src/script/lua_api/l_item.cpp | 6 +++--- src/script/lua_api/l_server.cpp | 28 +++++++++++++--------------- src/server.cpp | 5 ----- src/server.h | 3 +-- 8 files changed, 37 insertions(+), 45 deletions(-) (limited to 'src/script/common') diff --git a/src/gamedef.h b/src/gamedef.h index 8a9246da2..45b9c4750 100644 --- a/src/gamedef.h +++ b/src/gamedef.h @@ -63,6 +63,8 @@ public: virtual IRollbackManager* getRollbackManager() { return NULL; } // Shorthands + // TODO: these should be made const-safe so that a const IGameDef* is + // actually usable IItemDefManager *idef() { return getItemDefManager(); } const NodeDefManager *ndef() { return getNodeDefManager(); } ICraftDefManager *cdef() { return getCraftDefManager(); } diff --git a/src/script/common/c_content.cpp b/src/script/common/c_content.cpp index 36f4316ee..a233afb05 100644 --- a/src/script/common/c_content.cpp +++ b/src/script/common/c_content.cpp @@ -1371,7 +1371,7 @@ void push_inventory_lists(lua_State *L, const Inventory &inv) /******************************************************************************/ void read_inventory_list(lua_State *L, int tableindex, - Inventory *inv, const char *name, Server* srv, int forcesize) + Inventory *inv, const char *name, IGameDef *gdef, int forcesize) { if(tableindex < 0) tableindex = lua_gettop(L) + 1 + tableindex; @@ -1383,7 +1383,7 @@ void read_inventory_list(lua_State *L, int tableindex, } // Get Lua-specified items to insert into the list - std::vector items = read_items(L, tableindex,srv); + std::vector items = read_items(L, tableindex, gdef); size_t listsize = (forcesize >= 0) ? forcesize : items.size(); // Create or resize/clear list @@ -1635,7 +1635,7 @@ void push_items(lua_State *L, const std::vector &items) } /******************************************************************************/ -std::vector read_items(lua_State *L, int index, Server *srv) +std::vector read_items(lua_State *L, int index, IGameDef *gdef) { if(index < 0) index = lua_gettop(L) + 1 + index; @@ -1651,7 +1651,7 @@ std::vector read_items(lua_State *L, int index, Server *srv) if (items.size() < (u32) key) { items.resize(key); } - items[key - 1] = read_item(L, -1, srv->idef()); + items[key - 1] = read_item(L, -1, gdef->idef()); lua_pop(L, 1); } return items; diff --git a/src/script/common/c_content.h b/src/script/common/c_content.h index 11b39364f..a7b8709c6 100644 --- a/src/script/common/c_content.h +++ b/src/script/common/c_content.h @@ -59,7 +59,7 @@ class InventoryList; struct NodeBox; struct ContentFeatures; struct TileDef; -class Server; +class IGameDef; struct DigParams; struct HitParams; struct EnumString; @@ -126,7 +126,7 @@ void push_inventory_lists (lua_State *L, const Inventory &inv); void read_inventory_list (lua_State *L, int tableindex, Inventory *inv, const char *name, - Server *srv, int forcesize=-1); + IGameDef *gdef, int forcesize=-1); MapNode readnode (lua_State *L, int index, const NodeDefManager *ndef); @@ -166,7 +166,7 @@ void push_items (lua_State *L, std::vector read_items (lua_State *L, int index, - Server* srv); + IGameDef* gdef); void push_soundspec (lua_State *L, const SimpleSoundSpec &spec); diff --git a/src/script/lua_api/l_craft.cpp b/src/script/lua_api/l_craft.cpp index 18622ee00..c2c5a5551 100644 --- a/src/script/lua_api/l_craft.cpp +++ b/src/script/lua_api/l_craft.cpp @@ -371,8 +371,9 @@ int ModApiCraft::l_clear_craft(lua_State *L) int ModApiCraft::l_get_craft_result(lua_State *L) { NO_MAP_LOCK_REQUIRED; + IGameDef *gdef = getGameDef(L); - int input_i = 1; + const int input_i = 1; std::string method_s = getstringfield_default(L, input_i, "method", "normal"); enum CraftMethod method = (CraftMethod)getenumfield(L, input_i, "method", es_CraftMethod, CRAFT_METHOD_NORMAL); @@ -382,10 +383,9 @@ int ModApiCraft::l_get_craft_result(lua_State *L) width = luaL_checkinteger(L, -1); lua_pop(L, 1); lua_getfield(L, input_i, "items"); - std::vector items = read_items(L, -1,getServer(L)); + std::vector items = read_items(L, -1, gdef); lua_pop(L, 1); // items - IGameDef *gdef = getServer(L); ICraftDefManager *cdef = gdef->cdef(); CraftInput input(method, width, items); CraftOutput output; @@ -465,13 +465,13 @@ static void push_craft_recipes(lua_State *L, IGameDef *gdef, const std::vector &recipes, const CraftOutput &output) { - lua_createtable(L, recipes.size(), 0); - if (recipes.empty()) { lua_pushnil(L); return; } + lua_createtable(L, recipes.size(), 0); + std::vector::const_iterator it = recipes.begin(); for (unsigned i = 0; it != recipes.end(); ++it) { lua_newtable(L); @@ -487,10 +487,9 @@ int ModApiCraft::l_get_craft_recipe(lua_State *L) NO_MAP_LOCK_REQUIRED; std::string item = luaL_checkstring(L, 1); - Server *server = getServer(L); + IGameDef *gdef = getGameDef(L); CraftOutput output(item, 0); - std::vector recipes = server->cdef() - ->getCraftRecipes(output, server, 1); + auto recipes = gdef->cdef()->getCraftRecipes(output, gdef, 1); lua_createtable(L, 1, 0); @@ -500,7 +499,7 @@ int ModApiCraft::l_get_craft_recipe(lua_State *L) setintfield(L, -1, "width", 0); return 1; } - push_craft_recipe(L, server, recipes[0], output); + push_craft_recipe(L, gdef, recipes[0], output); return 1; } @@ -510,12 +509,11 @@ int ModApiCraft::l_get_all_craft_recipes(lua_State *L) NO_MAP_LOCK_REQUIRED; std::string item = luaL_checkstring(L, 1); - Server *server = getServer(L); + IGameDef *gdef = getGameDef(L); CraftOutput output(item, 0); - std::vector recipes = server->cdef() - ->getCraftRecipes(output, server); + auto recipes = gdef->cdef()->getCraftRecipes(output, gdef); - push_craft_recipes(L, server, recipes, output); + push_craft_recipes(L, gdef, recipes, output); return 1; } diff --git a/src/script/lua_api/l_item.cpp b/src/script/lua_api/l_item.cpp index 794d8a6e5..fc97a1736 100644 --- a/src/script/lua_api/l_item.cpp +++ b/src/script/lua_api/l_item.cpp @@ -632,8 +632,8 @@ int ModApiItemMod::l_get_content_id(lua_State *L) NO_MAP_LOCK_REQUIRED; std::string name = luaL_checkstring(L, 1); - const IItemDefManager *idef = getGameDef(L)->getItemDefManager(); - const NodeDefManager *ndef = getGameDef(L)->getNodeDefManager(); + const IItemDefManager *idef = getGameDef(L)->idef(); + const NodeDefManager *ndef = getGameDef(L)->ndef(); // If this is called at mod load time, NodeDefManager isn't aware of // aliases yet, so we need to handle them manually @@ -658,7 +658,7 @@ int ModApiItemMod::l_get_name_from_content_id(lua_State *L) NO_MAP_LOCK_REQUIRED; content_t c = luaL_checkint(L, 1); - const NodeDefManager *ndef = getGameDef(L)->getNodeDefManager(); + const NodeDefManager *ndef = getGameDef(L)->ndef(); const char *name = ndef->get(c).name.c_str(); lua_pushstring(L, name); diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 5b3054d17..42725e5d2 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -61,11 +61,8 @@ int ModApiServer::l_get_server_uptime(lua_State *L) int ModApiServer::l_get_server_max_lag(lua_State *L) { NO_MAP_LOCK_REQUIRED; - ServerEnvironment *s_env = dynamic_cast(getEnv(L)); - if (!s_env) - lua_pushnil(L); - else - lua_pushnumber(L, s_env->getMaxLagEstimate()); + GET_ENV_PTR; + lua_pushnumber(L, env->getMaxLagEstimate()); return 1; } @@ -395,12 +392,11 @@ int ModApiServer::l_get_modpath(lua_State *L) { NO_MAP_LOCK_REQUIRED; std::string modname = luaL_checkstring(L, 1); - const ModSpec *mod = getServer(L)->getModSpec(modname); - if (!mod) { + const ModSpec *mod = getGameDef(L)->getModSpec(modname); + if (!mod) lua_pushnil(L); - return 1; - } - lua_pushstring(L, mod->path.c_str()); + else + lua_pushstring(L, mod->path.c_str()); return 1; } @@ -412,13 +408,14 @@ int ModApiServer::l_get_modnames(lua_State *L) // Get a list of mods std::vector modlist; - getServer(L)->getModNames(modlist); + for (auto &it : getGameDef(L)->getMods()) + modlist.emplace_back(it.name); std::sort(modlist.begin(), modlist.end()); // Package them up for Lua lua_createtable(L, modlist.size(), 0); - std::vector::iterator iter = modlist.begin(); + auto iter = modlist.begin(); for (u16 i = 0; iter != modlist.end(); ++iter) { lua_pushstring(L, iter->c_str()); lua_rawseti(L, -2, ++i); @@ -430,8 +427,8 @@ int ModApiServer::l_get_modnames(lua_State *L) int ModApiServer::l_get_worldpath(lua_State *L) { NO_MAP_LOCK_REQUIRED; - std::string worldpath = getServer(L)->getWorldPath(); - lua_pushstring(L, worldpath.c_str()); + const Server *srv = getServer(L); + lua_pushstring(L, srv->getWorldPath().c_str()); return 1; } @@ -513,7 +510,8 @@ int ModApiServer::l_dynamic_add_media(lua_State *L) int ModApiServer::l_is_singleplayer(lua_State *L) { NO_MAP_LOCK_REQUIRED; - lua_pushboolean(L, getServer(L)->isSingleplayer()); + const Server *srv = getServer(L); + lua_pushboolean(L, srv->isSingleplayer()); return 1; } diff --git a/src/server.cpp b/src/server.cpp index 9d7e8e563..dec6cf44c 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -3658,11 +3658,6 @@ const ModSpec *Server::getModSpec(const std::string &modname) const return m_modmgr->getModSpec(modname); } -void Server::getModNames(std::vector &modlist) -{ - m_modmgr->getModNames(modlist); -} - std::string Server::getBuiltinLuaPath() { return porting::path_share + DIR_DELIM + "builtin"; diff --git a/src/server.h b/src/server.h index 008213c5d..bd799c313 100644 --- a/src/server.h +++ b/src/server.h @@ -292,11 +292,10 @@ public: virtual const std::vector &getMods() const; virtual const ModSpec* getModSpec(const std::string &modname) const; - void getModNames(std::vector &modlist); std::string getBuiltinLuaPath(); virtual std::string getWorldPath() const { return m_path_world; } - inline bool isSingleplayer() + inline bool isSingleplayer() const { return m_simple_singleplayer_mode; } inline void setAsyncFatalError(const std::string &error) -- cgit v1.2.3 From e7659883cc6fca343785da2a1af3890ae273abbf Mon Sep 17 00:00:00 2001 From: sfan5 Date: Mon, 2 May 2022 20:55:04 +0200 Subject: Async environment for mods to do concurrent tasks (#11131) --- builtin/async/game.lua | 46 ++ builtin/async/init.lua | 11 - builtin/async/mainmenu.lua | 9 + builtin/game/async.lua | 22 + builtin/game/init.lua | 1 + builtin/game/misc.lua | 29 ++ builtin/init.lua | 6 +- doc/lua_api.txt | 62 +++ games/devtest/mods/unittests/async_env.lua | 149 ++++++ games/devtest/mods/unittests/init.lua | 1 + games/devtest/mods/unittests/inside_async_env.lua | 15 + src/map.cpp | 33 ++ src/map.h | 17 +- src/script/common/CMakeLists.txt | 1 + src/script/common/c_internal.cpp | 14 + src/script/common/c_internal.h | 5 + src/script/common/c_packer.cpp | 583 ++++++++++++++++++++++ src/script/common/c_packer.h | 123 +++++ src/script/cpp_api/s_async.cpp | 183 +++++-- src/script/cpp_api/s_async.h | 54 +- src/script/lua_api/l_craft.cpp | 8 + src/script/lua_api/l_craft.h | 1 + src/script/lua_api/l_internal.h | 2 +- src/script/lua_api/l_item.cpp | 25 + src/script/lua_api/l_item.h | 7 +- src/script/lua_api/l_noise.cpp | 53 ++ src/script/lua_api/l_noise.h | 6 + src/script/lua_api/l_server.cpp | 85 ++++ src/script/lua_api/l_server.h | 10 + src/script/lua_api/l_util.cpp | 5 + src/script/lua_api/l_util.h | 2 - src/script/lua_api/l_vmanip.cpp | 33 ++ src/script/lua_api/l_vmanip.h | 3 + src/script/scripting_server.cpp | 67 ++- src/script/scripting_server.h | 17 + src/server.cpp | 4 + src/server.h | 8 +- src/serverenvironment.cpp | 2 + src/util/basic_macros.h | 8 +- 39 files changed, 1654 insertions(+), 56 deletions(-) create mode 100644 builtin/async/game.lua delete mode 100644 builtin/async/init.lua create mode 100644 builtin/async/mainmenu.lua create mode 100644 builtin/game/async.lua create mode 100644 games/devtest/mods/unittests/async_env.lua create mode 100644 games/devtest/mods/unittests/inside_async_env.lua create mode 100644 src/script/common/c_packer.cpp create mode 100644 src/script/common/c_packer.h (limited to 'src/script/common') diff --git a/builtin/async/game.lua b/builtin/async/game.lua new file mode 100644 index 000000000..212a33e17 --- /dev/null +++ b/builtin/async/game.lua @@ -0,0 +1,46 @@ +core.log("info", "Initializing asynchronous environment (game)") + +local function pack2(...) + return {n=select('#', ...), ...} +end + +-- Entrypoint to run async jobs, called by C++ +function core.job_processor(func, params) + local retval = pack2(func(unpack(params, 1, params.n))) + + return retval +end + +-- Import a bunch of individual files from builtin/game/ +local gamepath = core.get_builtin_path() .. "game" .. DIR_DELIM + +dofile(gamepath .. "constants.lua") +dofile(gamepath .. "item_s.lua") +dofile(gamepath .. "misc_s.lua") +dofile(gamepath .. "features.lua") +dofile(gamepath .. "voxelarea.lua") + +-- Transfer of globals +do + assert(core.transferred_globals) + local all = core.deserialize(core.transferred_globals, true) + core.transferred_globals = nil + + -- reassemble other tables + all.registered_nodes = {} + all.registered_craftitems = {} + all.registered_tools = {} + for k, v in pairs(all.registered_items) do + if v.type == "node" then + all.registered_nodes[k] = v + elseif v.type == "craftitem" then + all.registered_craftitems[k] = v + elseif v.type == "tool" then + all.registered_tools[k] = v + end + end + + for k, v in pairs(all) do + core[k] = v + end +end diff --git a/builtin/async/init.lua b/builtin/async/init.lua deleted file mode 100644 index 3803994d6..000000000 --- a/builtin/async/init.lua +++ /dev/null @@ -1,11 +0,0 @@ - -core.log("info", "Initializing Asynchronous environment") - -function core.job_processor(func, serialized_param) - local param = core.deserialize(serialized_param) - - local retval = core.serialize(func(param)) - - return retval or core.serialize(nil) -end - diff --git a/builtin/async/mainmenu.lua b/builtin/async/mainmenu.lua new file mode 100644 index 000000000..0e9c222d1 --- /dev/null +++ b/builtin/async/mainmenu.lua @@ -0,0 +1,9 @@ +core.log("info", "Initializing asynchronous environment") + +function core.job_processor(func, serialized_param) + local param = core.deserialize(serialized_param) + + local retval = core.serialize(func(param)) + + return retval or core.serialize(nil) +end diff --git a/builtin/game/async.lua b/builtin/game/async.lua new file mode 100644 index 000000000..469f179d7 --- /dev/null +++ b/builtin/game/async.lua @@ -0,0 +1,22 @@ + +core.async_jobs = {} + +function core.async_event_handler(jobid, retval) + local callback = core.async_jobs[jobid] + assert(type(callback) == "function") + callback(unpack(retval, 1, retval.n)) + core.async_jobs[jobid] = nil +end + +function core.handle_async(func, callback, ...) + assert(type(func) == "function" and type(callback) == "function", + "Invalid minetest.handle_async invocation") + local args = {n = select("#", ...), ...} + local mod_origin = core.get_last_run_mod() + + local jobid = core.do_async_callback(func, args, mod_origin) + core.async_jobs[jobid] = callback + + return true +end + diff --git a/builtin/game/init.lua b/builtin/game/init.lua index c5f08113b..68d6a10f8 100644 --- a/builtin/game/init.lua +++ b/builtin/game/init.lua @@ -34,5 +34,6 @@ dofile(gamepath .. "voxelarea.lua") dofile(gamepath .. "forceloading.lua") dofile(gamepath .. "statbars.lua") dofile(gamepath .. "knockback.lua") +dofile(gamepath .. "async.lua") profiler = nil diff --git a/builtin/game/misc.lua b/builtin/game/misc.lua index 18d5a7310..9f5e3312b 100644 --- a/builtin/game/misc.lua +++ b/builtin/game/misc.lua @@ -235,3 +235,32 @@ end -- Used for callback handling with dynamic_add_media core.dynamic_media_callbacks = {} + + +-- Transfer of certain globals into async environment +-- see builtin/async/game.lua for the other side + +local function copy_filtering(t, seen) + if type(t) == "userdata" or type(t) == "function" then + return true -- don't use nil so presence can still be detected + elseif type(t) ~= "table" then + return t + end + local n = {} + seen = seen or {} + seen[t] = n + for k, v in pairs(t) do + local k_ = seen[k] or copy_filtering(k, seen) + local v_ = seen[v] or copy_filtering(v, seen) + n[k_] = v_ + end + return n +end + +function core.get_globals_to_transfer() + local all = { + registered_items = copy_filtering(core.registered_items), + registered_aliases = core.registered_aliases, + } + return core.serialize(all) +end diff --git a/builtin/init.lua b/builtin/init.lua index 7a9b5c427..869136016 100644 --- a/builtin/init.lua +++ b/builtin/init.lua @@ -56,8 +56,10 @@ elseif INIT == "mainmenu" then if not custom_loaded then dofile(core.get_mainmenu_path() .. DIR_DELIM .. "init.lua") end -elseif INIT == "async" then - dofile(asyncpath .. "init.lua") +elseif INIT == "async" then + dofile(asyncpath .. "mainmenu.lua") +elseif INIT == "async_game" then + dofile(asyncpath .. "game.lua") elseif INIT == "client" then dofile(clientpath .. "init.lua") else diff --git a/doc/lua_api.txt b/doc/lua_api.txt index f54672db7..339ce8a27 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -5767,6 +5767,68 @@ Timing * `job:cancel()` * Cancels the job function from being called +Async environment +----------------- + +The engine allows you to submit jobs to be ran in an isolated environment +concurrently with normal server operation. +A job consists of a function to be ran in the async environment, any amount of +arguments (will be serialized) and a callback that will be called with the return +value of the job function once it is finished. + +The async environment does *not* have access to the map, entities, players or any +globals defined in the 'usual' environment. Consequently, functions like +`minetest.get_node()` or `minetest.get_player_by_name()` simply do not exist in it. + +Arguments and return values passed through this can contain certain userdata +objects that will be seamlessly copied (not shared) to the async environment. +This allows you easy interoperability for delegating work to jobs. + +* `minetest.handle_async(func, callback, ...)`: + * Queue the function `func` to be ran in an async environment. + Note that there are multiple persistent workers and any of them may + end up running a given job. The engine will scale the amount of + worker threads automatically. + * When `func` returns the callback is called (in the normal environment) + with all of the return values as arguments. + * Optional: Variable number of arguments that are passed to `func` +* `minetest.register_async_dofile(path)`: + * Register a path to a Lua file to be imported when an async environment + is initialized. You can use this to preload code which you can then call + later using `minetest.handle_async()`. + +### List of APIs available in an async environment + +Classes: +* `ItemStack` +* `PerlinNoise` +* `PerlinNoiseMap` +* `PseudoRandom` +* `PcgRandom` +* `SecureRandom` +* `VoxelArea` +* `VoxelManip` + * only if transferred into environment; can't read/write to map +* `Settings` + +Class instances that can be transferred between environments: +* `ItemStack` +* `PerlinNoise` +* `PerlinNoiseMap` +* `VoxelManip` + +Functions: +* Standalone helpers such as logging, filesystem, encoding, + hashing or compression APIs +* `minetest.request_insecure_environment` (same restrictions apply) + +Variables: +* `minetest.settings` +* `minetest.registered_items`, `registered_nodes`, `registered_tools`, + `registered_craftitems` and `registered_aliases` + * with all functions and userdata values replaced by `true`, calling any + callbacks here is obviously not possible + Server ------ diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua new file mode 100644 index 000000000..aff1fc4d9 --- /dev/null +++ b/games/devtest/mods/unittests/async_env.lua @@ -0,0 +1,149 @@ +-- helper + +core.register_async_dofile(core.get_modpath(core.get_current_modname()) .. + DIR_DELIM .. "inside_async_env.lua") + +local function deepequal(a, b) + if type(a) == "function" then + return type(b) == "function" + elseif type(a) ~= "table" then + return a == b + elseif type(b) ~= "table" then + return false + end + for k, v in pairs(a) do + if not deepequal(v, b[k]) then + return false + end + end + for k, v in pairs(b) do + if not deepequal(a[k], v) then + return false + end + end + return true +end + +-- Object Passing / Serialization + +local test_object = { + name = "stairs:stair_glass", + type = "node", + groups = {oddly_breakable_by_hand = 3, cracky = 3, stair = 1}, + description = "Glass Stair", + sounds = { + dig = {name = "default_glass_footstep", gain = 0.5}, + footstep = {name = "default_glass_footstep", gain = 0.3}, + dug = {name = "default_break_glass", gain = 1} + }, + node_box = { + fixed = { + {-0.5, -0.5, -0.5, 0.5, 0, 0.5}, + {-0.5, 0, 0, 0.5, 0.5, 0.5} + }, + type = "fixed" + }, + tiles = { + {name = "stairs_glass_split.png", backface_culling = true}, + {name = "default_glass.png", backface_culling = true}, + {name = "stairs_glass_stairside.png^[transformFX", backface_culling = true} + }, + on_place = function(itemstack, placer) + return core.is_player(placer) + end, + sunlight_propagates = true, + is_ground_content = false, + light_source = 0, +} + +local function test_object_passing() + local tmp = core.serialize_roundtrip(test_object) + assert(deepequal(test_object, tmp)) + + -- Circular key, should error + tmp = {"foo", "bar"} + tmp[tmp] = true + assert(not pcall(core.serialize_roundtrip, tmp)) + + -- Circular value, should error + tmp = {"foo"} + tmp[2] = tmp + assert(not pcall(core.serialize_roundtrip, tmp)) +end +unittests.register("test_object_passing", test_object_passing) + +local function test_userdata_passing(_, pos) + -- basic userdata passing + local obj = table.copy(test_object.tiles[1]) + obj.test = ItemStack("default:cobble 99") + local tmp = core.serialize_roundtrip(obj) + assert(type(tmp.test) == "userdata") + assert(obj.test:to_string() == tmp.test:to_string()) + + -- object can't be passed, should error + obj = core.raycast(pos, pos) + assert(not pcall(core.serialize_roundtrip, obj)) + + -- VManip + local vm = core.get_voxel_manip(pos, pos) + local expect = vm:get_node_at(pos) + local vm2 = core.serialize_roundtrip(vm) + assert(deepequal(vm2:get_node_at(pos), expect)) +end +unittests.register("test_userdata_passing", test_userdata_passing, {map=true}) + +-- Asynchronous jobs + +local function test_handle_async(cb) + -- Basic test including mod name tracking and unittests.async_test() + -- which is defined inside_async_env.lua + local func = function(x) + return core.get_last_run_mod(), _VERSION, unittests[x]() + end + local expect = {core.get_last_run_mod(), _VERSION, true} + + core.handle_async(func, function(...) + if not deepequal(expect, {...}) then + cb("Values did not equal") + end + if core.get_last_run_mod() ~= expect[1] then + cb("Mod name not tracked correctly") + end + + -- Test passing of nil arguments and return values + core.handle_async(function(a, b) + return a, b + end, function(a, b) + if b ~= 123 then + cb("Argument went missing") + end + cb() + end, nil, 123) + end, "async_test") +end +unittests.register("test_handle_async", test_handle_async, {async=true}) + +local function test_userdata_passing2(cb, _, pos) + -- VManip: check transfer into other env + local vm = core.get_voxel_manip(pos, pos) + local expect = vm:get_node_at(pos) + + core.handle_async(function(vm_, pos_) + return vm_:get_node_at(pos_) + end, function(ret) + if not deepequal(expect, ret) then + cb("Node data mismatch (one-way)") + end + + -- VManip: test a roundtrip + core.handle_async(function(vm_) + return vm_ + end, function(vm2) + if not deepequal(expect, vm2:get_node_at(pos)) then + cb("Node data mismatch (roundtrip)") + end + cb() + end, vm) + end, vm, pos) +end +unittests.register("test_userdata_passing2", test_userdata_passing2, {map=true, async=true}) diff --git a/games/devtest/mods/unittests/init.lua b/games/devtest/mods/unittests/init.lua index 0754d507f..0608f2dd2 100644 --- a/games/devtest/mods/unittests/init.lua +++ b/games/devtest/mods/unittests/init.lua @@ -175,6 +175,7 @@ dofile(modpath .. "/misc.lua") dofile(modpath .. "/player.lua") dofile(modpath .. "/crafting.lua") dofile(modpath .. "/itemdescription.lua") +dofile(modpath .. "/async_env.lua") -------------- diff --git a/games/devtest/mods/unittests/inside_async_env.lua b/games/devtest/mods/unittests/inside_async_env.lua new file mode 100644 index 000000000..9774771f9 --- /dev/null +++ b/games/devtest/mods/unittests/inside_async_env.lua @@ -0,0 +1,15 @@ +unittests = {} + +core.log("info", "Hello World") + +function unittests.async_test() + assert(core == minetest) + -- stuff that should not be here + assert(not core.get_player_by_name) + assert(not core.set_node) + assert(not core.object_refs) + -- stuff that should be here + assert(ItemStack) + assert(core.registered_items[""]) + return true +end diff --git a/src/map.cpp b/src/map.cpp index 9c9324f5f..5153dcaa9 100644 --- a/src/map.cpp +++ b/src/map.cpp @@ -1896,6 +1896,7 @@ MMVManip::MMVManip(Map *map): VoxelManipulator(), m_map(map) { + assert(map); } void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max, @@ -1903,6 +1904,8 @@ void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max, { TimeTaker timer1("initialEmerge", &emerge_time); + assert(m_map); + // Units of these are MapBlocks v3s16 p_min = blockpos_min; v3s16 p_max = blockpos_max; @@ -1986,6 +1989,7 @@ void MMVManip::blitBackAll(std::map *modified_blocks, { if(m_area.getExtent() == v3s16(0,0,0)) return; + assert(m_map); /* Copy data of all blocks @@ -2006,4 +2010,33 @@ void MMVManip::blitBackAll(std::map *modified_blocks, } } +MMVManip *MMVManip::clone() const +{ + MMVManip *ret = new MMVManip(); + + const s32 size = m_area.getVolume(); + ret->m_area = m_area; + if (m_data) { + ret->m_data = new MapNode[size]; + memcpy(ret->m_data, m_data, size * sizeof(MapNode)); + } + if (m_flags) { + ret->m_flags = new u8[size]; + memcpy(ret->m_flags, m_flags, size * sizeof(u8)); + } + + ret->m_is_dirty = m_is_dirty; + // Even if the copy is disconnected from a map object keep the information + // needed to write it back to one + ret->m_loaded_blocks = m_loaded_blocks; + + return ret; +} + +void MMVManip::reparent(Map *map) +{ + assert(map && !m_map); + m_map = map; +} + //END diff --git a/src/map.h b/src/map.h index d8ed29106..21e3dbd6c 100644 --- a/src/map.h +++ b/src/map.h @@ -446,10 +446,25 @@ public: void blitBackAll(std::map * modified_blocks, bool overwrite_generated = true); + /* + Creates a copy of this VManip including contents, the copy will not be + associated with a Map. + */ + MMVManip *clone() const; + + // Reassociates a copied VManip to a map + void reparent(Map *map); + + // Is it impossible to call initialEmerge / blitBackAll? + inline bool isOrphan() const { return !m_map; } + bool m_is_dirty = false; protected: - Map *m_map; + MMVManip() {}; + + // may be null + Map *m_map = nullptr; /* key = blockpos value = flags describing the block diff --git a/src/script/common/CMakeLists.txt b/src/script/common/CMakeLists.txt index d07f6ab1b..3e84b46c7 100644 --- a/src/script/common/CMakeLists.txt +++ b/src/script/common/CMakeLists.txt @@ -3,6 +3,7 @@ set(common_SCRIPT_COMMON_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/c_converter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/c_types.cpp ${CMAKE_CURRENT_SOURCE_DIR}/c_internal.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/c_packer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helper.cpp PARENT_SCOPE) diff --git a/src/script/common/c_internal.cpp b/src/script/common/c_internal.cpp index df82dba14..ddd2d184c 100644 --- a/src/script/common/c_internal.cpp +++ b/src/script/common/c_internal.cpp @@ -166,3 +166,17 @@ void log_deprecated(lua_State *L, std::string message, int stack_depth) infostream << script_get_backtrace(L) << std::endl; } +void call_string_dump(lua_State *L, int idx) +{ + // Retrieve string.dump from insecure env to avoid it being tampered with + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_GLOBALS_BACKUP); + if (!lua_isnil(L, -1)) + lua_getfield(L, -1, "string"); + else + lua_getglobal(L, "string"); + lua_getfield(L, -1, "dump"); + lua_remove(L, -2); // remove _G + lua_remove(L, -2); // remove 'string' table + lua_pushvalue(L, idx); + lua_call(L, 1, 1); +} diff --git a/src/script/common/c_internal.h b/src/script/common/c_internal.h index c43db34aa..272a39941 100644 --- a/src/script/common/c_internal.h +++ b/src/script/common/c_internal.h @@ -56,6 +56,7 @@ extern "C" { #define CUSTOM_RIDX_BACKTRACE (CUSTOM_RIDX_BASE + 3) #define CUSTOM_RIDX_HTTP_API_LUA (CUSTOM_RIDX_BASE + 4) #define CUSTOM_RIDX_VECTOR_METATABLE (CUSTOM_RIDX_BASE + 5) +#define CUSTOM_RIDX_METATABLE_MAP (CUSTOM_RIDX_BASE + 6) // Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata @@ -139,3 +140,7 @@ DeprecatedHandlingMode get_deprecated_handling_mode(); * @param stack_depth How far on the stack to the first user function (ie: not builtin or core) */ void log_deprecated(lua_State *L, std::string message, int stack_depth = 1); + +// Safely call string.dump on a function value +// (does not pop, leaves one value on stack) +void call_string_dump(lua_State *L, int idx); diff --git a/src/script/common/c_packer.cpp b/src/script/common/c_packer.cpp new file mode 100644 index 000000000..fc5277330 --- /dev/null +++ b/src/script/common/c_packer.cpp @@ -0,0 +1,583 @@ +/* +Minetest +Copyright (C) 2022 sfan5 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#include +#include +#include +#include +#include +#include +#include "c_packer.h" +#include "c_internal.h" +#include "log.h" +#include "debug.h" +#include "threading/mutex_auto_lock.h" + +extern "C" { +#include +} + +// +// Helpers +// + +// convert negative index to absolute position on Lua stack +static inline int absidx(lua_State *L, int idx) +{ + assert(idx < 0); + return lua_gettop(L) + idx + 1; +} + +// does the type put anything into PackedInstr::sdata? +static inline bool uses_sdata(int type) +{ + switch (type) { + case LUA_TSTRING: + case LUA_TFUNCTION: + case LUA_TUSERDATA: + return true; + default: + return false; + } +} + +// does the type put anything into PackedInstr::? +static inline bool uses_union(int type) +{ + switch (type) { + case LUA_TNIL: + case LUA_TSTRING: + case LUA_TFUNCTION: + return false; + default: + return true; + } +} + +static inline bool can_set_into(int ktype, int vtype) +{ + switch (ktype) { + case LUA_TNUMBER: + return !uses_union(vtype); + case LUA_TSTRING: + return !uses_sdata(vtype); + default: + return false; + } +} + +// is the key suitable for use with set_into? +static inline bool suitable_key(lua_State *L, int idx) +{ + if (lua_type(L, idx) == LUA_TSTRING) { + // strings may not have a NULL byte (-> lua_setfield) + size_t len; + const char *str = lua_tolstring(L, idx, &len); + return strlen(str) == len; + } else { + assert(lua_type(L, idx) == LUA_TNUMBER); + // numbers must fit into an s32 and be integers (-> lua_rawseti) + lua_Number n = lua_tonumber(L, idx); + return std::floor(n) == n && n >= S32_MIN && n <= S32_MAX; + } +} + +namespace { + // checks if you left any values on the stack, for debugging + class StackChecker { + lua_State *L; + int top; + public: + StackChecker(lua_State *L) : L(L), top(lua_gettop(L)) {} + ~StackChecker() { + assert(lua_gettop(L) >= top); + if (lua_gettop(L) > top) { + rawstream << "Lua stack not cleaned up: " + << lua_gettop(L) << " != " << top + << " (false-positive if exception thrown)" << std::endl; + } + } + }; + + // Since an std::vector may reallocate, this is the only safe way to keep + // a reference to a particular element. + template + class VectorRef { + std::vector *vec; + size_t idx; + VectorRef(std::vector *vec, size_t idx) : vec(vec), idx(idx) {} + public: + static VectorRef front(std::vector &vec) { + return VectorRef(&vec, 0); + } + static VectorRef back(std::vector &vec) { + return VectorRef(&vec, vec.size() - 1); + } + T &operator*() { return (*vec)[idx]; } + T *operator->() { return &(*vec)[idx]; } + }; + + struct Packer { + PackInFunc fin; + PackOutFunc fout; + }; + + typedef std::pair PackerTuple; +} + +static inline auto emplace(PackedValue &pv, s16 type) +{ + pv.i.emplace_back(); + auto ref = VectorRef::back(pv.i); + ref->type = type; + // Initialize fields that may be left untouched + if (type == LUA_TTABLE) { + ref->uidata1 = 0; + ref->uidata2 = 0; + } else if (type == LUA_TUSERDATA) { + ref->ptrdata = nullptr; + } else if (type == INSTR_POP) { + ref->sidata2 = 0; + } + return ref; +} + +// +// Management of registered packers +// + +static std::unordered_map g_packers; +static std::mutex g_packers_lock; + +void script_register_packer(lua_State *L, const char *regname, + PackInFunc fin, PackOutFunc fout) +{ + // Store away callbacks + { + MutexAutoLock autolock(g_packers_lock); + auto it = g_packers.find(regname); + if (it == g_packers.end()) { + auto &ref = g_packers[regname]; + ref.fin = fin; + ref.fout = fout; + } else { + FATAL_ERROR_IF(it->second.fin != fin || it->second.fout != fout, + "Packer registered twice with mismatching callbacks"); + } + } + + // Save metatable so we can identify instances later + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + if (lua_isnil(L, -1)) { + lua_newtable(L); + lua_pushvalue(L, -1); + lua_rawseti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + } + + luaL_getmetatable(L, regname); + FATAL_ERROR_IF(lua_isnil(L, -1), "No metatable registered with that name"); + + // CUSTOM_RIDX_METATABLE_MAP contains { [metatable] = "regname", ... } + // check first + lua_pushstring(L, regname); + lua_rawget(L, -3); + if (!lua_isnil(L, -1)) { + FATAL_ERROR_IF(lua_topointer(L, -1) != lua_topointer(L, -2), + "Packer registered twice with inconsistent metatable"); + } + lua_pop(L, 1); + // then set + lua_pushstring(L, regname); + lua_rawset(L, -3); + + lua_pop(L, 1); +} + +static bool find_packer(const char *regname, PackerTuple &out) +{ + MutexAutoLock autolock(g_packers_lock); + auto it = g_packers.find(regname); + if (it == g_packers.end()) + return false; + // copy data for thread safety + out.first = it->first; + out.second = it->second; + return true; +} + +static bool find_packer(lua_State *L, int idx, PackerTuple &out) +{ +#ifndef NDEBUG + StackChecker checker(L); +#endif + + // retrieve metatable of the object + if (lua_getmetatable(L, idx) != 1) + return false; + + // use our global table to map it to the registry name + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + assert(lua_istable(L, -1)); + lua_pushvalue(L, -2); + lua_rawget(L, -2); + if (lua_isnil(L, -1)) { + lua_pop(L, 3); + return false; + } + + // load the associated data + bool found = find_packer(lua_tostring(L, -1), out); + FATAL_ERROR_IF(!found, "Inconsistent internal state"); + lua_pop(L, 3); + return true; +} + +// +// Packing implementation +// + +// recursively goes through the structure and ensures there are no circular references +static void pack_validate(lua_State *L, int idx, std::unordered_set &seen) +{ +#ifndef NDEBUG + StackChecker checker(L); + assert(idx > 0); +#endif + + if (lua_type(L, idx) != LUA_TTABLE) + return; + + const void *ptr = lua_topointer(L, idx); + assert(ptr); + + if (seen.find(ptr) != seen.end()) + throw LuaError("Circular references cannot be packed (yet)"); + seen.insert(ptr); + + lua_checkstack(L, 5); + lua_pushnil(L); + while (lua_next(L, idx) != 0) { + // key at -2, value at -1 + pack_validate(L, absidx(L, -2), seen); + pack_validate(L, absidx(L, -1), seen); + + lua_pop(L, 1); + } + + seen.erase(ptr); +} + +static VectorRef pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv) +{ +#ifndef NDEBUG + StackChecker checker(L); + assert(idx > 0); + assert(vidx > 0); +#endif + + switch (lua_type(L, idx)) { + case LUA_TNONE: + case LUA_TNIL: + return emplace(pv, LUA_TNIL); + case LUA_TBOOLEAN: { + auto r = emplace(pv, LUA_TBOOLEAN); + r->bdata = lua_toboolean(L, idx); + return r; + } + case LUA_TNUMBER: { + auto r = emplace(pv, LUA_TNUMBER); + r->ndata = lua_tonumber(L, idx); + return r; + } + case LUA_TSTRING: { + auto r = emplace(pv, LUA_TSTRING); + size_t len; + const char *str = lua_tolstring(L, idx, &len); + assert(str); + r->sdata.assign(str, len); + return r; + } + case LUA_TTABLE: + break; // execution continues + case LUA_TFUNCTION: { + auto r = emplace(pv, LUA_TFUNCTION); + call_string_dump(L, idx); + size_t len; + const char *str = lua_tolstring(L, -1, &len); + assert(str); + r->sdata.assign(str, len); + lua_pop(L, 1); + return r; + } + case LUA_TUSERDATA: { + PackerTuple ser; + if (!find_packer(L, idx, ser)) + throw LuaError("Cannot serialize unsupported userdata"); + pv.contains_userdata = true; + auto r = emplace(pv, LUA_TUSERDATA); + r->sdata = ser.first; + r->ptrdata = ser.second.fin(L, idx); + return r; + } + default: { + std::string err = "Cannot serialize type "; + err += lua_typename(L, lua_type(L, idx)); + throw LuaError(err); + } + } + + // LUA_TTABLE + lua_checkstack(L, 5); + + auto rtable = emplace(pv, LUA_TTABLE); + const int vi_table = vidx++; + + lua_pushnil(L); + while (lua_next(L, idx) != 0) { + // key at -2, value at -1 + const int ktype = lua_type(L, -2), vtype = lua_type(L, -1); + if (ktype == LUA_TNUMBER) + rtable->uidata1++; // narr + else + rtable->uidata2++; // nrec + + // check if we can use a shortcut + if (can_set_into(ktype, vtype) && suitable_key(L, -2)) { + // push only the value + auto rval = pack_inner(L, absidx(L, -1), vidx, pv); + rval->pop = vtype != LUA_TTABLE; + // and where to put it: + rval->set_into = vi_table; + if (ktype == LUA_TSTRING) + rval->sdata = lua_tostring(L, -2); + else + rval->sidata1 = lua_tointeger(L, -2); + // pop tables after the fact + if (!rval->pop) { + auto ri1 = emplace(pv, INSTR_POP); + ri1->sidata1 = vidx; + } + } else { + // push the key and value + pack_inner(L, absidx(L, -2), vidx, pv); + vidx++; + pack_inner(L, absidx(L, -1), vidx, pv); + vidx++; + // push an instruction to set them + auto ri1 = emplace(pv, INSTR_SETTABLE); + ri1->set_into = vi_table; + ri1->sidata1 = vidx - 2; + ri1->sidata2 = vidx - 1; + ri1->pop = true; + vidx -= 2; + } + + lua_pop(L, 1); + } + + assert(vidx == vi_table + 1); + return rtable; +} + +PackedValue *script_pack(lua_State *L, int idx) +{ + if (idx < 0) + idx = absidx(L, idx); + + std::unordered_set seen; + pack_validate(L, idx, seen); + assert(seen.size() == 0); + + // Actual serialization + PackedValue pv; + pack_inner(L, idx, 1, pv); + + return new PackedValue(std::move(pv)); +} + +// +// Unpacking implementation +// + +void script_unpack(lua_State *L, PackedValue *pv) +{ + const int top = lua_gettop(L); + int ctr = 0; + + for (auto &i : pv->i) { + // If leaving values on stack make sure there's space (every 5th iteration) + if (!i.pop && (ctr++) >= 5) { + lua_checkstack(L, 5); + ctr = 0; + } + + /* Instructions */ + switch (i.type) { + case INSTR_SETTABLE: + lua_pushvalue(L, top + i.sidata1); // key + lua_pushvalue(L, top + i.sidata2); // value + lua_rawset(L, top + i.set_into); + if (i.pop) { + if (i.sidata1 != i.sidata2) { + // removing moves indices so pop higher index first + lua_remove(L, top + std::max(i.sidata1, i.sidata2)); + lua_remove(L, top + std::min(i.sidata1, i.sidata2)); + } else { + lua_remove(L, top + i.sidata1); + } + } + continue; + case INSTR_POP: + lua_remove(L, top + i.sidata1); + if (i.sidata2 > 0) + lua_remove(L, top + i.sidata2); + continue; + default: + break; + } + + /* Lua types */ + switch (i.type) { + case LUA_TNIL: + lua_pushnil(L); + break; + case LUA_TBOOLEAN: + lua_pushboolean(L, i.bdata); + break; + case LUA_TNUMBER: + lua_pushnumber(L, i.ndata); + break; + case LUA_TSTRING: + lua_pushlstring(L, i.sdata.data(), i.sdata.size()); + break; + case LUA_TTABLE: + lua_createtable(L, i.uidata1, i.uidata2); + break; + case LUA_TFUNCTION: + luaL_loadbuffer(L, i.sdata.data(), i.sdata.size(), nullptr); + break; + case LUA_TUSERDATA: { + PackerTuple ser; + sanity_check(find_packer(i.sdata.c_str(), ser)); + ser.second.fout(L, i.ptrdata); + i.ptrdata = nullptr; // ownership taken by callback + break; + } + default: + assert(0); + break; + } + + if (i.set_into) { + if (!i.pop) + lua_pushvalue(L, -1); + if (uses_sdata(i.type)) + lua_rawseti(L, top + i.set_into, i.sidata1); + else + lua_setfield(L, top + i.set_into, i.sdata.c_str()); + } else { + if (i.pop) + lua_pop(L, 1); + } + } + + // as part of the unpacking process we take ownership of all userdata + pv->contains_userdata = false; + // leave exactly one value on the stack + lua_settop(L, top+1); +} + +// +// PackedValue +// + +PackedValue::~PackedValue() +{ + if (!contains_userdata) + return; + for (auto &i : this->i) { + if (i.type == LUA_TUSERDATA && i.ptrdata) { + PackerTuple ser; + if (find_packer(i.sdata.c_str(), ser)) { + // tell it to deallocate object + ser.second.fout(nullptr, i.ptrdata); + } else { + assert(false); + } + } + } +} + +// +// script_dump_packed +// + +#ifndef NDEBUG +void script_dump_packed(const PackedValue *val) +{ + printf("instruction stream: [\n"); + for (const auto &i : val->i) { + printf("\t("); + switch (i.type) { + case INSTR_SETTABLE: + printf("SETTABLE(%d, %d)", i.sidata1, i.sidata2); + break; + case INSTR_POP: + printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2); + break; + case LUA_TNIL: + printf("nil"); + break; + case LUA_TBOOLEAN: + printf(i.bdata ? "true" : "false"); + break; + case LUA_TNUMBER: + printf("%f", i.ndata); + break; + case LUA_TSTRING: + printf("\"%s\"", i.sdata.c_str()); + break; + case LUA_TTABLE: + printf("table(%d, %d)", i.uidata1, i.uidata2); + break; + case LUA_TFUNCTION: + printf("function(%d byte)", i.sdata.size()); + break; + case LUA_TUSERDATA: + printf("userdata %s %p", i.sdata.c_str(), i.ptrdata); + break; + default: + printf("!!UNKNOWN!!"); + break; + } + if (i.set_into) { + if (i.type >= 0 && uses_sdata(i.type)) + printf(", k=%d, into=%d", i.sidata1, i.set_into); + else if (i.type >= 0) + printf(", k=\"%s\", into=%d", i.sdata.c_str(), i.set_into); + else + printf(", into=%d", i.set_into); + } + if (i.pop) + printf(", pop"); + printf(")\n"); + } + printf("]\n"); +} +#endif diff --git a/src/script/common/c_packer.h b/src/script/common/c_packer.h new file mode 100644 index 000000000..8bccca98d --- /dev/null +++ b/src/script/common/c_packer.h @@ -0,0 +1,123 @@ +/* +Minetest +Copyright (C) 2022 sfan5 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#pragma once + +#include +#include +#include "irrlichttypes.h" +#include "util/basic_macros.h" + +extern "C" { +#include +} + +/* + This file defines an in-memory representation of Lua objects including + support for functions and userdata. It it used to move data between Lua + states and cannot be used for persistence or network transfer. +*/ + +#define INSTR_SETTABLE (-10) +#define INSTR_POP (-11) + +/** + * Represents a single instruction that pushes a new value or works with existing ones. + */ +struct PackedInstr +{ + s16 type; // LUA_T* or INSTR_* + u16 set_into; // set into table on stack + bool pop; // remove from stack? + union { + bool bdata; // boolean: value + lua_Number ndata; // number: value + struct { + u16 uidata1, uidata2; // table: narr, nrec + }; + struct { + /* + SETTABLE: key index, value index + POP: indices to remove + otherwise w/ set_into: numeric key, - + */ + s32 sidata1, sidata2; + }; + void *ptrdata; // userdata: implementation defined + }; + /* + - string: value + - function: buffer + - w/ set_into: string key (no null bytes!) + - userdata: name in registry + */ + std::string sdata; + + PackedInstr() : type(0), set_into(0), pop(false) {} +}; + +/** + * A packed value can be a primitive like a string or number but also a table + * including all of its contents. It is made up of a linear stream of + * 'instructions' that build the final value when executed. + */ +struct PackedValue +{ + std::vector i; + // Indicates whether there are any userdata pointers that need to be deallocated + bool contains_userdata = false; + + PackedValue() = default; + ~PackedValue(); + + DISABLE_CLASS_COPY(PackedValue) + + ALLOW_CLASS_MOVE(PackedValue) +}; + +/* + * Packing callback: Turns a Lua value at given index into a void* + */ +typedef void *(*PackInFunc)(lua_State *L, int idx); +/* + * Unpacking callback: Turns a void* back into the Lua value (left on top of stack) + * + * Note that this function must take ownership of the pointer, so make sure + * to free or keep the memory. + * `L` can be nullptr to indicate that data should just be discarded. + */ +typedef void (*PackOutFunc)(lua_State *L, void *ptr); +/* + * Register a packable type with the name of its metatable. + * + * Even though the callbacks are global this must be called for every Lua state + * that supports objects of this type. + * This function is thread-safe. + */ +void script_register_packer(lua_State *L, const char *regname, + PackInFunc fin, PackOutFunc fout); + +// Pack a Lua value +PackedValue *script_pack(lua_State *L, int idx); +// Unpack a Lua value (left on top of stack) +// Note that this may modify the PackedValue, you can't reuse it! +void script_unpack(lua_State *L, PackedValue *val); + +// Dump contents of PackedValue to stdout for debugging +void script_dump_packed(const PackedValue *val); diff --git a/src/script/cpp_api/s_async.cpp b/src/script/cpp_api/s_async.cpp index dacdcd75a..42a794ceb 100644 --- a/src/script/cpp_api/s_async.cpp +++ b/src/script/cpp_api/s_async.cpp @@ -21,9 +21,9 @@ with this program; if not, write to the Free Software Foundation, Inc., #include extern "C" { -#include "lua.h" -#include "lauxlib.h" -#include "lualib.h" +#include +#include +#include } #include "server.h" @@ -32,6 +32,7 @@ extern "C" { #include "filesys.h" #include "porting.h" #include "common/c_internal.h" +#include "common/c_packer.h" #include "lua_api/l_base.h" /******************************************************************************/ @@ -76,19 +77,34 @@ void AsyncEngine::initialize(unsigned int numEngines) { initDone = true; - for (unsigned int i = 0; i < numEngines; i++) { - AsyncWorkerThread *toAdd = new AsyncWorkerThread(this, - std::string("AsyncWorker-") + itos(i)); - workerThreads.push_back(toAdd); - toAdd->start(); + if (numEngines == 0) { + // Leave one core for the main thread and one for whatever else + autoscaleMaxWorkers = Thread::getNumberOfProcessors(); + if (autoscaleMaxWorkers >= 2) + autoscaleMaxWorkers -= 2; + infostream << "AsyncEngine: using at most " << autoscaleMaxWorkers + << " threads with automatic scaling" << std::endl; + + addWorkerThread(); + } else { + for (unsigned int i = 0; i < numEngines; i++) + addWorkerThread(); } } +void AsyncEngine::addWorkerThread() +{ + AsyncWorkerThread *toAdd = new AsyncWorkerThread(this, + std::string("AsyncWorker-") + itos(workerThreads.size())); + workerThreads.push_back(toAdd); + toAdd->start(); +} + /******************************************************************************/ u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &¶ms, const std::string &mod_origin) { - jobQueueMutex.lock(); + MutexAutoLock autolock(jobQueueMutex); u32 jobId = jobIdCounter++; jobQueue.emplace_back(); @@ -99,7 +115,23 @@ u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &¶ms, to_add.mod_origin = mod_origin; jobQueueCounter.post(); - jobQueueMutex.unlock(); + return jobId; +} + +u32 AsyncEngine::queueAsyncJob(std::string &&func, PackedValue *params, + const std::string &mod_origin) +{ + MutexAutoLock autolock(jobQueueMutex); + u32 jobId = jobIdCounter++; + + jobQueue.emplace_back(); + auto &to_add = jobQueue.back(); + to_add.id = jobId; + to_add.function = std::move(func); + to_add.params_ext.reset(params); + to_add.mod_origin = mod_origin; + + jobQueueCounter.post(); return jobId; } @@ -131,6 +163,12 @@ void AsyncEngine::putJobResult(LuaJobInfo &&result) /******************************************************************************/ void AsyncEngine::step(lua_State *L) +{ + stepJobResults(L); + stepAutoscale(); +} + +void AsyncEngine::stepJobResults(lua_State *L) { int error_handler = PUSH_ERROR_HANDLER(L); lua_getglobal(L, "core"); @@ -148,7 +186,10 @@ void AsyncEngine::step(lua_State *L) luaL_checktype(L, -1, LUA_TFUNCTION); lua_pushinteger(L, j.id); - lua_pushlstring(L, j.result.data(), j.result.size()); + if (j.result_ext) + script_unpack(L, j.result_ext.get()); + else + lua_pushlstring(L, j.result.data(), j.result.size()); // Call handler const char *origin = j.mod_origin.empty() ? nullptr : j.mod_origin.c_str(); @@ -161,12 +202,71 @@ void AsyncEngine::step(lua_State *L) lua_pop(L, 2); // Pop core and error handler } +void AsyncEngine::stepAutoscale() +{ + if (workerThreads.size() >= autoscaleMaxWorkers) + return; + + MutexAutoLock autolock(jobQueueMutex); + + // 2) If the timer elapsed, check again + if (autoscaleTimer && porting::getTimeMs() >= autoscaleTimer) { + autoscaleTimer = 0; + // Determine overlap with previous snapshot + unsigned int n = 0; + for (const auto &it : jobQueue) + n += autoscaleSeenJobs.count(it.id); + autoscaleSeenJobs.clear(); + infostream << "AsyncEngine: " << n << " jobs were still waiting after 1s" << std::endl; + // Start this many new threads + while (workerThreads.size() < autoscaleMaxWorkers && n > 0) { + addWorkerThread(); + n--; + } + return; + } + + // 1) Check if there's anything in the queue + if (!autoscaleTimer && !jobQueue.empty()) { + // Take a snapshot of all jobs we have seen + for (const auto &it : jobQueue) + autoscaleSeenJobs.emplace(it.id); + // and set a timer for 1 second + autoscaleTimer = porting::getTimeMs() + 1000; + } +} + /******************************************************************************/ -void AsyncEngine::prepareEnvironment(lua_State* L, int top) +bool AsyncEngine::prepareEnvironment(lua_State* L, int top) { for (StateInitializer &stateInitializer : stateInitializers) { stateInitializer(L, top); } + + auto *script = ModApiBase::getScriptApiBase(L); + try { + script->loadMod(Server::getBuiltinLuaPath() + DIR_DELIM + "init.lua", + BUILTIN_MOD_NAME); + } catch (const ModError &e) { + errorstream << "Execution of async base environment failed: " + << e.what() << std::endl; + FATAL_ERROR("Execution of async base environment failed"); + } + + // Load per mod stuff + if (server) { + const auto &list = server->m_async_init_files; + try { + for (auto &it : list) + script->loadMod(it.second, it.first); + } catch (const ModError &e) { + errorstream << "Failed to load mod script inside async environment." << std::endl; + server->setAsyncFatalError(e.what()); + return false; + } + } + + return true; } /******************************************************************************/ @@ -178,15 +278,25 @@ AsyncWorkerThread::AsyncWorkerThread(AsyncEngine* jobDispatcher, { lua_State *L = getStack(); + if (jobDispatcher->server) { + setGameDef(jobDispatcher->server); + + if (g_settings->getBool("secure.enable_security")) + initializeSecurity(); + } + // Prepare job lua environment lua_getglobal(L, "core"); int top = lua_gettop(L); // Push builtin initialization type - lua_pushstring(L, "async"); + lua_pushstring(L, jobDispatcher->server ? "async_game" : "async"); lua_setglobal(L, "INIT"); - jobDispatcher->prepareEnvironment(L, top); + if (!jobDispatcher->prepareEnvironment(L, top)) { + // can't throw from here so we're stuck with this + isErrored = true; + } } /******************************************************************************/ @@ -198,19 +308,20 @@ AsyncWorkerThread::~AsyncWorkerThread() /******************************************************************************/ void* AsyncWorkerThread::run() { - lua_State *L = getStack(); + if (isErrored) + return nullptr; - try { - loadMod(getServer()->getBuiltinLuaPath() + DIR_DELIM + "init.lua", - BUILTIN_MOD_NAME); - } catch (const ModError &e) { - errorstream << "Execution of async base environment failed: " - << e.what() << std::endl; - FATAL_ERROR("Execution of async base environment failed"); - } + lua_State *L = getStack(); int error_handler = PUSH_ERROR_HANDLER(L); + auto report_error = [this] (const ModError &e) { + if (jobDispatcher->server) + jobDispatcher->server->setAsyncFatalError(e.what()); + else + errorstream << e.what() << std::endl; + }; + lua_getglobal(L, "core"); if (lua_isnil(L, -1)) { FATAL_ERROR("Unable to find core within async environment!"); @@ -223,6 +334,8 @@ void* AsyncWorkerThread::run() if (!jobDispatcher->getJob(&j) || stopRequested()) continue; + const bool use_ext = !!j.params_ext; + lua_getfield(L, -1, "job_processor"); if (lua_isnil(L, -1)) FATAL_ERROR("Unable to get async job processor!"); @@ -232,7 +345,10 @@ void* AsyncWorkerThread::run() errorstream << "ASYNC WORKER: Unable to deserialize function" << std::endl; lua_pushnil(L); } - lua_pushlstring(L, j.params.data(), j.params.size()); + if (use_ext) + script_unpack(L, j.params_ext.get()); + else + lua_pushlstring(L, j.params.data(), j.params.size()); // Call it setOriginDirect(j.mod_origin.empty() ? nullptr : j.mod_origin.c_str()); @@ -241,19 +357,28 @@ void* AsyncWorkerThread::run() try { scriptError(result, ""); } catch (const ModError &e) { - errorstream << e.what() << std::endl; + report_error(e); } } else { // Fetch result - size_t length; - const char *retval = lua_tolstring(L, -1, &length); - j.result.assign(retval, length); + if (use_ext) { + try { + j.result_ext.reset(script_pack(L, -1)); + } catch (const ModError &e) { + report_error(e); + result = LUA_ERRERR; + } + } else { + size_t length; + const char *retval = lua_tolstring(L, -1, &length); + j.result.assign(retval, length); + } } lua_pop(L, 1); // Pop retval // Put job result - if (!j.result.empty()) + if (result == 0) jobDispatcher->putJobResult(std::move(j)); } diff --git a/src/script/cpp_api/s_async.h b/src/script/cpp_api/s_async.h index 697cb0221..1e34e40ea 100644 --- a/src/script/cpp_api/s_async.h +++ b/src/script/cpp_api/s_async.h @@ -21,11 +21,15 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include +#include +#include +#include #include "threading/semaphore.h" #include "threading/thread.h" -#include "lua.h" +#include "common/c_packer.h" #include "cpp_api/s_base.h" +#include "cpp_api/s_security.h" // Forward declarations class AsyncEngine; @@ -42,8 +46,12 @@ struct LuaJobInfo std::string function; // Parameter to be passed to function (serialized) std::string params; + // Alternative parameters + std::unique_ptr params_ext; // Result of function call (serialized) std::string result; + // Alternative result + std::unique_ptr result_ext; // Name of the mod who invoked this call std::string mod_origin; // JobID used to identify a job and match it to callback @@ -51,7 +59,8 @@ struct LuaJobInfo }; // Asynchronous working environment -class AsyncWorkerThread : public Thread, virtual public ScriptApiBase { +class AsyncWorkerThread : public Thread, + virtual public ScriptApiBase, public ScriptApiSecurity { friend class AsyncEngine; public: virtual ~AsyncWorkerThread(); @@ -63,6 +72,7 @@ protected: private: AsyncEngine *jobDispatcher = nullptr; + bool isErrored = false; }; // Asynchornous thread and job management @@ -71,6 +81,7 @@ class AsyncEngine { typedef void (*StateInitializer)(lua_State *L, int top); public: AsyncEngine() = default; + AsyncEngine(Server *server) : server(server) {}; ~AsyncEngine(); /** @@ -81,7 +92,7 @@ public: /** * Create async engine tasks and lock function registration - * @param numEngines Number of async threads to be started + * @param numEngines Number of worker threads, 0 for automatic scaling */ void initialize(unsigned int numEngines); @@ -94,9 +105,17 @@ public: u32 queueAsyncJob(std::string &&func, std::string &¶ms, const std::string &mod_origin = ""); + /** + * Queue an async job + * @param func Serialized lua function + * @param params Serialized parameters (takes ownership!) + * @return ID of queued job + */ + u32 queueAsyncJob(std::string &&func, PackedValue *params, + const std::string &mod_origin = ""); + /** * Engine step to process finished jobs - * the engine step is one way to pass events back, PushFinishedJobs another * @param L The Lua stack */ void step(lua_State *L); @@ -116,19 +135,44 @@ protected: */ void putJobResult(LuaJobInfo &&result); + /** + * Start an additional worker thread + */ + void addWorkerThread(); + + /** + * Process finished jobs callbacks + */ + void stepJobResults(lua_State *L); + + /** + * Handle automatic scaling of worker threads + */ + void stepAutoscale(); + /** * Initialize environment with current registred functions * this function adds all functions registred by registerFunction to the * passed lua stack * @param L Lua stack to initialize * @param top Stack position + * @return false if a mod error ocurred */ - void prepareEnvironment(lua_State* L, int top); + bool prepareEnvironment(lua_State* L, int top); private: // Variable locking the engine against further modification bool initDone = false; + // Maximum number of worker threads for automatic scaling + // 0 if disabled + unsigned int autoscaleMaxWorkers = 0; + u64 autoscaleTimer = 0; + std::unordered_set autoscaleSeenJobs; + + // Only set for the server async environment (duh) + Server *server = nullptr; + // Internal store for registred state initializers std::vector stateInitializers; diff --git a/src/script/lua_api/l_craft.cpp b/src/script/lua_api/l_craft.cpp index c2c5a5551..137b210be 100644 --- a/src/script/lua_api/l_craft.cpp +++ b/src/script/lua_api/l_craft.cpp @@ -525,3 +525,11 @@ void ModApiCraft::Initialize(lua_State *L, int top) API_FCT(register_craft); API_FCT(clear_craft); } + +void ModApiCraft::InitializeAsync(lua_State *L, int top) +{ + // all read-only functions + API_FCT(get_all_craft_recipes); + API_FCT(get_craft_recipe); + API_FCT(get_craft_result); +} diff --git a/src/script/lua_api/l_craft.h b/src/script/lua_api/l_craft.h index 9002b23ef..5234af56f 100644 --- a/src/script/lua_api/l_craft.h +++ b/src/script/lua_api/l_craft.h @@ -45,4 +45,5 @@ private: public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_internal.h b/src/script/lua_api/l_internal.h index 672e535ca..de73ff42a 100644 --- a/src/script/lua_api/l_internal.h +++ b/src/script/lua_api/l_internal.h @@ -69,7 +69,7 @@ with this program; if not, write to the Free Software Foundation, Inc., // Retrieve Environment pointer as `env` (no map lock) #define GET_PLAIN_ENV_PTR_NO_MAP_LOCK \ - Environment *env = (Environment *)getEnv(L); \ + Environment *env = getEnv(L); \ if (env == NULL) \ return 0 diff --git a/src/script/lua_api/l_item.cpp b/src/script/lua_api/l_item.cpp index fc97a1736..b58b994d9 100644 --- a/src/script/lua_api/l_item.cpp +++ b/src/script/lua_api/l_item.cpp @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "itemdef.h" #include "nodedef.h" #include "server.h" @@ -441,6 +442,7 @@ int LuaItemStack::create_object(lua_State *L) lua_setmetatable(L, -2); return 1; } + // Not callable from Lua int LuaItemStack::create(lua_State *L, const ItemStack &item) { @@ -457,6 +459,20 @@ LuaItemStack *LuaItemStack::checkobject(lua_State *L, int narg) return *(LuaItemStack **)luaL_checkudata(L, narg, className); } +void *LuaItemStack::packIn(lua_State *L, int idx) +{ + LuaItemStack *o = checkobject(L, idx); + return new ItemStack(o->getItem()); +} + +void LuaItemStack::packOut(lua_State *L, void *ptr) +{ + ItemStack *stack = reinterpret_cast(ptr); + if (L) + create(L, *stack); + delete stack; +} + void LuaItemStack::Register(lua_State *L) { lua_newtable(L); @@ -488,6 +504,8 @@ void LuaItemStack::Register(lua_State *L) // Can be created from Lua (ItemStack(itemstack or itemstring or table or nil)) lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } const char LuaItemStack::className[] = "ItemStack"; @@ -673,3 +691,10 @@ void ModApiItemMod::Initialize(lua_State *L, int top) API_FCT(get_content_id); API_FCT(get_name_from_content_id); } + +void ModApiItemMod::InitializeAsync(lua_State *L, int top) +{ + // all read-only functions + API_FCT(get_content_id); + API_FCT(get_name_from_content_id); +} diff --git a/src/script/lua_api/l_item.h b/src/script/lua_api/l_item.h index 16878c101..180975061 100644 --- a/src/script/lua_api/l_item.h +++ b/src/script/lua_api/l_item.h @@ -141,8 +141,11 @@ public: // Not callable from Lua static int create(lua_State *L, const ItemStack &item); static LuaItemStack* checkobject(lua_State *L, int narg); - static void Register(lua_State *L); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + + static void Register(lua_State *L); }; class ModApiItemMod : public ModApiBase { @@ -152,6 +155,8 @@ private: static int l_register_alias_raw(lua_State *L); static int l_get_content_id(lua_State *L); static int l_get_name_from_content_id(lua_State *L); + public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_noise.cpp b/src/script/lua_api/l_noise.cpp index 0eee49b7d..5561eaebf 100644 --- a/src/script/lua_api/l_noise.cpp +++ b/src/script/lua_api/l_noise.cpp @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "log.h" #include "porting.h" #include "util/numeric.h" @@ -101,6 +102,25 @@ LuaPerlinNoise *LuaPerlinNoise::checkobject(lua_State *L, int narg) } +void *LuaPerlinNoise::packIn(lua_State *L, int idx) +{ + LuaPerlinNoise *o = checkobject(L, idx); + return new NoiseParams(o->np); +} + +void LuaPerlinNoise::packOut(lua_State *L, void *ptr) +{ + NoiseParams *np = reinterpret_cast(ptr); + if (L) { + LuaPerlinNoise *o = new LuaPerlinNoise(np); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); + } + delete np; +} + + void LuaPerlinNoise::Register(lua_State *L) { lua_newtable(L); @@ -126,6 +146,8 @@ void LuaPerlinNoise::Register(lua_State *L) lua_pop(L, 1); lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } @@ -357,6 +379,35 @@ LuaPerlinNoiseMap *LuaPerlinNoiseMap::checkobject(lua_State *L, int narg) } +struct NoiseMapParams { + NoiseParams np; + s32 seed; + v3s16 size; +}; + +void *LuaPerlinNoiseMap::packIn(lua_State *L, int idx) +{ + LuaPerlinNoiseMap *o = checkobject(L, idx); + NoiseMapParams *ret = new NoiseMapParams(); + ret->np = o->noise->np; + ret->seed = o->noise->seed; + ret->size = v3s16(o->noise->sx, o->noise->sy, o->noise->sz); + return ret; +} + +void LuaPerlinNoiseMap::packOut(lua_State *L, void *ptr) +{ + NoiseMapParams *p = reinterpret_cast(ptr); + if (L) { + LuaPerlinNoiseMap *o = new LuaPerlinNoiseMap(&p->np, p->seed, p->size); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); + } + delete p; +} + + void LuaPerlinNoiseMap::Register(lua_State *L) { lua_newtable(L); @@ -382,6 +433,8 @@ void LuaPerlinNoiseMap::Register(lua_State *L) lua_pop(L, 1); lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } diff --git a/src/script/lua_api/l_noise.h b/src/script/lua_api/l_noise.h index 29ab41a31..5d34a479b 100644 --- a/src/script/lua_api/l_noise.h +++ b/src/script/lua_api/l_noise.h @@ -52,6 +52,9 @@ public: static LuaPerlinNoise *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; @@ -91,6 +94,9 @@ public: static LuaPerlinNoiseMap *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 42725e5d2..4b0b45887 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "cpp_api/s_base.h" #include "cpp_api/s_security.h" #include "scripting_server.h" @@ -526,6 +527,76 @@ int ModApiServer::l_notify_authentication_modified(lua_State *L) return 0; } +// do_async_callback(func, params, mod_origin) +int ModApiServer::l_do_async_callback(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + ServerScripting *script = getScriptApi(L); + + luaL_checktype(L, 1, LUA_TFUNCTION); + luaL_checktype(L, 2, LUA_TTABLE); + luaL_checktype(L, 3, LUA_TSTRING); + + call_string_dump(L, 1); + size_t func_length; + const char *serialized_func_raw = lua_tolstring(L, -1, &func_length); + + PackedValue *param = script_pack(L, 2); + + std::string mod_origin = readParam(L, 3); + + u32 jobId = script->queueAsync( + std::string(serialized_func_raw, func_length), + param, mod_origin); + + lua_settop(L, 0); + lua_pushinteger(L, jobId); + return 1; +} + +// register_async_dofile(path) +int ModApiServer::l_register_async_dofile(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + + std::string path = readParam(L, 1); + CHECK_SECURE_PATH(L, path.c_str(), false); + + // Find currently running mod name (only at init time) + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME); + if (!lua_isstring(L, -1)) + return 0; + std::string modname = readParam(L, -1); + + getServer(L)->m_async_init_files.emplace_back(modname, path); + lua_pushboolean(L, true); + return 1; +} + +// serialize_roundtrip(value) +// Meant for unit testing the packer from Lua +int ModApiServer::l_serialize_roundtrip(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + + int top = lua_gettop(L); + auto *pv = script_pack(L, 1); + if (top != lua_gettop(L)) + throw LuaError("stack values leaked"); + +#ifndef NDEBUG + script_dump_packed(pv); +#endif + + top = lua_gettop(L); + script_unpack(L, pv); + delete pv; + if (top + 1 != lua_gettop(L)) + throw LuaError("stack values leaked"); + + return 1; +} + void ModApiServer::Initialize(lua_State *L, int top) { API_FCT(request_shutdown); @@ -559,4 +630,18 @@ void ModApiServer::Initialize(lua_State *L, int top) API_FCT(remove_player); API_FCT(unban_player_or_ip); API_FCT(notify_authentication_modified); + + API_FCT(do_async_callback); + API_FCT(register_async_dofile); + API_FCT(serialize_roundtrip); +} + +void ModApiServer::InitializeAsync(lua_State *L, int top) +{ + API_FCT(get_worldpath); + API_FCT(is_singleplayer); + + API_FCT(get_current_modname); + API_FCT(get_modpath); + API_FCT(get_modnames); } diff --git a/src/script/lua_api/l_server.h b/src/script/lua_api/l_server.h index f05c0b7c9..a4f38c34e 100644 --- a/src/script/lua_api/l_server.h +++ b/src/script/lua_api/l_server.h @@ -106,6 +106,16 @@ private: // notify_authentication_modified(name) static int l_notify_authentication_modified(lua_State *L); + // do_async_callback(func, params, mod_origin) + static int l_do_async_callback(lua_State *L); + + // register_async_dofile(path) + static int l_register_async_dofile(lua_State *L); + + // serialize_roundtrip(obj) + static int l_serialize_roundtrip(lua_State *L); + public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_util.cpp b/src/script/lua_api/l_util.cpp index b04f26fda..97068ce4c 100644 --- a/src/script/lua_api/l_util.cpp +++ b/src/script/lua_api/l_util.cpp @@ -671,6 +671,9 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top) API_FCT(cpdir); API_FCT(mvdir); API_FCT(get_dir_list); + API_FCT(safe_file_write); + + API_FCT(request_insecure_environment); API_FCT(encode_base64); API_FCT(decode_base64); @@ -680,6 +683,8 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top) API_FCT(colorspec_to_colorstring); API_FCT(colorspec_to_bytes); + API_FCT(encode_png); + API_FCT(get_last_run_mod); API_FCT(set_last_run_mod); diff --git a/src/script/lua_api/l_util.h b/src/script/lua_api/l_util.h index fcf8a1057..cc5563577 100644 --- a/src/script/lua_api/l_util.h +++ b/src/script/lua_api/l_util.h @@ -129,6 +129,4 @@ public: static void Initialize(lua_State *L, int top); static void InitializeAsync(lua_State *L, int top); static void InitializeClient(lua_State *L, int top); - - static void InitializeAsync(AsyncEngine &engine); }; diff --git a/src/script/lua_api/l_vmanip.cpp b/src/script/lua_api/l_vmanip.cpp index a3ece627c..6187a47db 100644 --- a/src/script/lua_api/l_vmanip.cpp +++ b/src/script/lua_api/l_vmanip.cpp @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_content.h" #include "common/c_converter.h" +#include "common/c_packer.h" #include "emerge.h" #include "environment.h" #include "map.h" @@ -45,6 +46,8 @@ int LuaVoxelManip::l_read_from_map(lua_State *L) LuaVoxelManip *o = checkobject(L, 1); MMVManip *vm = o->vm; + if (vm->isOrphan()) + return 0; v3s16 bp1 = getNodeBlockPos(check_v3s16(L, 2)); v3s16 bp2 = getNodeBlockPos(check_v3s16(L, 3)); @@ -429,6 +432,34 @@ LuaVoxelManip *LuaVoxelManip::checkobject(lua_State *L, int narg) return *(LuaVoxelManip **)ud; // unbox pointer } +void *LuaVoxelManip::packIn(lua_State *L, int idx) +{ + LuaVoxelManip *o = checkobject(L, idx); + + if (o->is_mapgen_vm) + throw LuaError("nope"); + return o->vm->clone(); +} + +void LuaVoxelManip::packOut(lua_State *L, void *ptr) +{ + MMVManip *vm = reinterpret_cast(ptr); + if (!L) { + delete vm; + return; + } + + // Associate vmanip with map if the Lua env has one + Environment *env = getEnv(L); + if (env) + vm->reparent(&(env->getMap())); + + LuaVoxelManip *o = new LuaVoxelManip(vm, false); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); +} + void LuaVoxelManip::Register(lua_State *L) { lua_newtable(L); @@ -455,6 +486,8 @@ void LuaVoxelManip::Register(lua_State *L) // Can be created from Lua (VoxelManip()) lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } const char LuaVoxelManip::className[] = "VoxelManip"; diff --git a/src/script/lua_api/l_vmanip.h b/src/script/lua_api/l_vmanip.h index 5113070dc..005133335 100644 --- a/src/script/lua_api/l_vmanip.h +++ b/src/script/lua_api/l_vmanip.h @@ -75,5 +75,8 @@ public: static LuaVoxelManip *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; diff --git a/src/script/scripting_server.cpp b/src/script/scripting_server.cpp index 85411ded4..5b99468dc 100644 --- a/src/script/scripting_server.cpp +++ b/src/script/scripting_server.cpp @@ -47,11 +47,12 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_storage.h" extern "C" { -#include "lualib.h" +#include } ServerScripting::ServerScripting(Server* server): - ScriptApiBase(ScriptingType::Server) + ScriptApiBase(ScriptingType::Server), + asyncEngine(server) { setGameDef(server); @@ -88,6 +89,47 @@ ServerScripting::ServerScripting(Server* server): infostream << "SCRIPTAPI: Initialized game modules" << std::endl; } +void ServerScripting::initAsync() +{ + // Save globals to transfer + { + lua_State *L = getStack(); + lua_getglobal(L, "core"); + luaL_checktype(L, -1, LUA_TTABLE); + lua_getfield(L, -1, "get_globals_to_transfer"); + lua_call(L, 0, 1); + luaL_checktype(L, -1, LUA_TSTRING); + getServer()->m_async_globals_data.set(readParam(L, -1)); + lua_pushnil(L); + lua_setfield(L, -3, "get_globals_to_transfer"); // unset function too + lua_pop(L, 2); // pop 'core', return value + } + + infostream << "SCRIPTAPI: Initializing async engine" << std::endl; + asyncEngine.registerStateInitializer(InitializeAsync); + asyncEngine.registerStateInitializer(ModApiUtil::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiCraft::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiItemMod::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiServer::InitializeAsync); + // not added: ModApiMapgen is a minefield for thread safety + // not added: ModApiHttp async api can't really work together with our jobs + // not added: ModApiStorage is probably not thread safe(?) + + asyncEngine.initialize(0); +} + +void ServerScripting::stepAsync() +{ + asyncEngine.step(getStack()); +} + +u32 ServerScripting::queueAsync(std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin) +{ + return asyncEngine.queueAsyncJob(std::move(serialized_func), + param, mod_origin); +} + void ServerScripting::InitializeModApi(lua_State *L, int top) { // Register reference classes (userdata) @@ -125,3 +167,24 @@ void ServerScripting::InitializeModApi(lua_State *L, int top) ModApiStorage::Initialize(L, top); ModApiChannels::Initialize(L, top); } + +void ServerScripting::InitializeAsync(lua_State *L, int top) +{ + // classes + LuaItemStack::Register(L); + LuaPerlinNoise::Register(L); + LuaPerlinNoiseMap::Register(L); + LuaPseudoRandom::Register(L); + LuaPcgRandom::Register(L); + LuaSecureRandom::Register(L); + LuaVoxelManip::Register(L); + LuaSettings::Register(L); + + // globals data + lua_getglobal(L, "core"); + luaL_checktype(L, -1, LUA_TTABLE); + std::string s = ModApiBase::getServer(L)->m_async_globals_data.get(); + lua_pushlstring(L, s.c_str(), s.size()); + lua_setfield(L, -2, "transferred_globals"); + lua_pop(L, 1); // pop 'core' +} diff --git a/src/script/scripting_server.h b/src/script/scripting_server.h index bf06ab197..9803397c5 100644 --- a/src/script/scripting_server.h +++ b/src/script/scripting_server.h @@ -27,6 +27,9 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "cpp_api/s_player.h" #include "cpp_api/s_server.h" #include "cpp_api/s_security.h" +#include "cpp_api/s_async.h" + +struct PackedValue; /*****************************************************************************/ /* Scripting <-> Server Game Interface */ @@ -48,6 +51,20 @@ public: // use ScriptApiBase::loadMod() to load mods + // Initialize async engine, call this AFTER loading all mods + void initAsync(); + + // Global step handler to collect async results + void stepAsync(); + + // Pass job to async threads + u32 queueAsync(std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin); + private: void InitializeModApi(lua_State *L, int top); + + static void InitializeAsync(lua_State *L, int top); + + AsyncEngine asyncEngine; }; diff --git a/src/server.cpp b/src/server.cpp index dec6cf44c..c9cd4e398 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -243,6 +243,7 @@ Server::Server( m_clients(m_con), m_admin_chat(iface), m_on_shutdown_errmsg(on_shutdown_errmsg), + m_async_globals_data(""), m_modchannel_mgr(new ModChannelMgr()) { if (m_path_world.empty()) @@ -480,6 +481,9 @@ void Server::init() // Give environment reference to scripting api m_script->initializeEnvironment(m_env); + // Do this after regular script init is done + m_script->initAsync(); + // Register us to receive map edit events servermap->addEventReceiver(this); diff --git a/src/server.h b/src/server.h index bd799c313..5090a3579 100644 --- a/src/server.h +++ b/src/server.h @@ -292,7 +292,7 @@ public: virtual const std::vector &getMods() const; virtual const ModSpec* getModSpec(const std::string &modname) const; - std::string getBuiltinLuaPath(); + static std::string getBuiltinLuaPath(); virtual std::string getWorldPath() const { return m_path_world; } inline bool isSingleplayer() const @@ -385,6 +385,12 @@ public: static bool migrateModStorageDatabase(const GameParams &game_params, const Settings &cmd_args); + // Lua files registered for init of async env, pair of modname + path + std::vector> m_async_init_files; + + // Serialized data transferred into async envs at init time + MutexedVariable m_async_globals_data; + // Bind address Address m_bind_addr; diff --git a/src/serverenvironment.cpp b/src/serverenvironment.cpp index f8d84604b..493210744 100644 --- a/src/serverenvironment.cpp +++ b/src/serverenvironment.cpp @@ -1481,6 +1481,8 @@ void ServerEnvironment::step(float dtime) */ m_script->environment_Step(dtime); + m_script->stepAsync(); + /* Step active objects */ diff --git a/src/util/basic_macros.h b/src/util/basic_macros.h index 334e342e0..3910c6185 100644 --- a/src/util/basic_macros.h +++ b/src/util/basic_macros.h @@ -29,13 +29,19 @@ with this program; if not, write to the Free Software Foundation, Inc., #define CONTAINS(c, v) (std::find((c).begin(), (c).end(), (v)) != (c).end()) // To disable copy constructors and assignment operations for some class -// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) as a private member. +// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) in the class definition. // Note this also disables copying for any classes derived from 'Foobar' as well // as classes having a 'Foobar' member. #define DISABLE_CLASS_COPY(C) \ C(const C &) = delete; \ C &operator=(const C &) = delete; +// If you have used DISABLE_CLASS_COPY with a class but still want to permit moving +// use this macro to add the default move constructors back. +#define ALLOW_CLASS_MOVE(C) \ + C(C &&other) = default; \ + C &operator=(C &&) = default; + #ifndef _MSC_VER #define UNUSED_ATTRIBUTE __attribute__ ((unused)) #else -- cgit v1.2.3 From 89c82035d88532bb63c9173e0c4edf54b7ea89f8 Mon Sep 17 00:00:00 2001 From: Lars Müller <34514239+appgurueu@users.noreply.github.com> Date: Wed, 4 May 2022 20:55:20 +0200 Subject: hud_get: Return precision field for waypoints (#12215) --- src/script/common/c_content.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/script/common') diff --git a/src/script/common/c_content.cpp b/src/script/common/c_content.cpp index a233afb05..79830ddb4 100644 --- a/src/script/common/c_content.cpp +++ b/src/script/common/c_content.cpp @@ -1977,6 +1977,12 @@ void push_hud_element(lua_State *L, HudElement *elem) lua_pushnumber(L, elem->number); lua_setfield(L, -2, "number"); + if (elem->type == HUD_ELEM_WAYPOINT) { + // waypoints reuse the item field to store precision, precision = item - 1 + lua_pushnumber(L, elem->item - 1); + lua_setfield(L, -2, "precision"); + } + // push the item field for waypoints as well for backwards compatibility lua_pushnumber(L, elem->item); lua_setfield(L, -2, "item"); -- cgit v1.2.3 From 7f58887ae33893c981fbdff23d4e1fa4a11c32e4 Mon Sep 17 00:00:00 2001 From: Jude Melton-Houghton Date: Tue, 10 May 2022 16:37:33 -0400 Subject: Support packing arbitrary graphs (#12289) --- games/devtest/mods/unittests/async_env.lua | 37 ++++++++--- src/script/common/c_packer.cpp | 103 ++++++++++++++++------------- src/script/common/c_packer.h | 5 +- 3 files changed, 90 insertions(+), 55 deletions(-) (limited to 'src/script/common') diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua index aff1fc4d9..3a21bd9e2 100644 --- a/games/devtest/mods/unittests/async_env.lua +++ b/games/devtest/mods/unittests/async_env.lua @@ -60,15 +60,34 @@ local function test_object_passing() local tmp = core.serialize_roundtrip(test_object) assert(deepequal(test_object, tmp)) - -- Circular key, should error - tmp = {"foo", "bar"} - tmp[tmp] = true - assert(not pcall(core.serialize_roundtrip, tmp)) - - -- Circular value, should error - tmp = {"foo"} - tmp[2] = tmp - assert(not pcall(core.serialize_roundtrip, tmp)) + local circular_key = {"foo", "bar"} + circular_key[circular_key] = true + tmp = core.serialize_roundtrip(circular_key) + assert(tmp[1] == "foo") + assert(tmp[2] == "bar") + assert(tmp[tmp] == true) + + local circular_value = {"foo"} + circular_value[2] = circular_value + tmp = core.serialize_roundtrip(circular_value) + assert(tmp[1] == "foo") + assert(tmp[2] == tmp) + + -- Two-segment cycle + local cycle_seg_1, cycle_seg_2 = {}, {} + cycle_seg_1[1] = cycle_seg_2 + cycle_seg_2[1] = cycle_seg_1 + tmp = core.serialize_roundtrip(cycle_seg_1) + assert(tmp[1][1] == tmp) + + -- Duplicated value without a cycle + local acyclic_dup_holder = {} + tmp = ItemStack("") + acyclic_dup_holder[tmp] = tmp + tmp = core.serialize_roundtrip(acyclic_dup_holder) + for k, v in pairs(tmp) do + assert(rawequal(k, v)) + end end unittests.register("test_object_passing", test_object_passing) diff --git a/src/script/common/c_packer.cpp b/src/script/common/c_packer.cpp index fc5277330..ede00c758 100644 --- a/src/script/common/c_packer.cpp +++ b/src/script/common/c_packer.cpp @@ -123,6 +123,7 @@ namespace { size_t idx; VectorRef(std::vector *vec, size_t idx) : vec(vec), idx(idx) {} public: + constexpr VectorRef() : vec(nullptr), idx(0) {} static VectorRef front(std::vector &vec) { return VectorRef(&vec, 0); } @@ -131,6 +132,7 @@ namespace { } T &operator*() { return (*vec)[idx]; } T *operator->() { return &(*vec)[idx]; } + operator bool() const { return vec != nullptr; } }; struct Packer { @@ -252,38 +254,27 @@ static bool find_packer(lua_State *L, int idx, PackerTuple &out) // Packing implementation // -// recursively goes through the structure and ensures there are no circular references -static void pack_validate(lua_State *L, int idx, std::unordered_set &seen) +static VectorRef record_object(lua_State *L, int idx, PackedValue &pv, + std::unordered_map &seen) { -#ifndef NDEBUG - StackChecker checker(L); - assert(idx > 0); -#endif - - if (lua_type(L, idx) != LUA_TTABLE) - return; - const void *ptr = lua_topointer(L, idx); assert(ptr); - - if (seen.find(ptr) != seen.end()) - throw LuaError("Circular references cannot be packed (yet)"); - seen.insert(ptr); - - lua_checkstack(L, 5); - lua_pushnil(L); - while (lua_next(L, idx) != 0) { - // key at -2, value at -1 - pack_validate(L, absidx(L, -2), seen); - pack_validate(L, absidx(L, -1), seen); - - lua_pop(L, 1); + auto found = seen.find(ptr); + if (found == seen.end()) { + seen[ptr] = pv.i.size(); + return VectorRef(); } - - seen.erase(ptr); + s32 ref = found->second; + assert(ref < (s32)pv.i.size()); + // reuse the value from first time + auto r = emplace(pv, INSTR_PUSHREF); + r->ref = ref; + pv.i[ref].keep_ref = true; + return r; } -static VectorRef pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv) +static VectorRef pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv, + std::unordered_map &seen) { #ifndef NDEBUG StackChecker checker(L); @@ -313,10 +304,17 @@ static VectorRef pack_inner(lua_State *L, int idx, int vidx, Packed r->sdata.assign(str, len); return r; } - case LUA_TTABLE: + case LUA_TTABLE: { + auto r = record_object(L, idx, pv, seen); + if (r) + return r; break; // execution continues + } case LUA_TFUNCTION: { - auto r = emplace(pv, LUA_TFUNCTION); + auto r = record_object(L, idx, pv, seen); + if (r) + return r; + r = emplace(pv, LUA_TFUNCTION); call_string_dump(L, idx); size_t len; const char *str = lua_tolstring(L, -1, &len); @@ -326,11 +324,14 @@ static VectorRef pack_inner(lua_State *L, int idx, int vidx, Packed return r; } case LUA_TUSERDATA: { + auto r = record_object(L, idx, pv, seen); + if (r) + return r; PackerTuple ser; if (!find_packer(L, idx, ser)) throw LuaError("Cannot serialize unsupported userdata"); pv.contains_userdata = true; - auto r = emplace(pv, LUA_TUSERDATA); + r = emplace(pv, LUA_TUSERDATA); r->sdata = ser.first; r->ptrdata = ser.second.fin(L, idx); return r; @@ -360,8 +361,8 @@ static VectorRef pack_inner(lua_State *L, int idx, int vidx, Packed // check if we can use a shortcut if (can_set_into(ktype, vtype) && suitable_key(L, -2)) { // push only the value - auto rval = pack_inner(L, absidx(L, -1), vidx, pv); - rval->pop = vtype != LUA_TTABLE; + auto rval = pack_inner(L, absidx(L, -1), vidx, pv, seen); + rval->pop = rval->type != LUA_TTABLE; // and where to put it: rval->set_into = vi_table; if (ktype == LUA_TSTRING) @@ -375,9 +376,9 @@ static VectorRef pack_inner(lua_State *L, int idx, int vidx, Packed } } else { // push the key and value - pack_inner(L, absidx(L, -2), vidx, pv); + pack_inner(L, absidx(L, -2), vidx, pv, seen); vidx++; - pack_inner(L, absidx(L, -1), vidx, pv); + pack_inner(L, absidx(L, -1), vidx, pv, seen); vidx++; // push an instruction to set them auto ri1 = emplace(pv, INSTR_SETTABLE); @@ -400,13 +401,9 @@ PackedValue *script_pack(lua_State *L, int idx) if (idx < 0) idx = absidx(L, idx); - std::unordered_set seen; - pack_validate(L, idx, seen); - assert(seen.size() == 0); - - // Actual serialization PackedValue pv; - pack_inner(L, idx, 1, pv); + std::unordered_map seen; + pack_inner(L, idx, 1, pv, seen); return new PackedValue(std::move(pv)); } @@ -417,18 +414,21 @@ PackedValue *script_pack(lua_State *L, int idx) void script_unpack(lua_State *L, PackedValue *pv) { + lua_newtable(L); // table at index top to track ref indices -> objects const int top = lua_gettop(L); int ctr = 0; - for (auto &i : pv->i) { + for (size_t packed_idx = 0; packed_idx < pv->i.size(); packed_idx++) { + auto &i = pv->i[packed_idx]; + // If leaving values on stack make sure there's space (every 5th iteration) if (!i.pop && (ctr++) >= 5) { lua_checkstack(L, 5); ctr = 0; } - /* Instructions */ switch (i.type) { + /* Instructions */ case INSTR_SETTABLE: lua_pushvalue(L, top + i.sidata1); // key lua_pushvalue(L, top + i.sidata2); // value @@ -448,12 +448,12 @@ void script_unpack(lua_State *L, PackedValue *pv) if (i.sidata2 > 0) lua_remove(L, top + i.sidata2); continue; - default: + case INSTR_PUSHREF: + lua_pushinteger(L, i.ref); + lua_rawget(L, top); break; - } - /* Lua types */ - switch (i.type) { + /* Lua types */ case LUA_TNIL: lua_pushnil(L); break; @@ -479,11 +479,18 @@ void script_unpack(lua_State *L, PackedValue *pv) i.ptrdata = nullptr; // ownership taken by callback break; } + default: assert(0); break; } + if (i.keep_ref) { + lua_pushinteger(L, packed_idx); + lua_pushvalue(L, -2); + lua_rawset(L, top); + } + if (i.set_into) { if (!i.pop) lua_pushvalue(L, -1); @@ -501,6 +508,7 @@ void script_unpack(lua_State *L, PackedValue *pv) pv->contains_userdata = false; // leave exactly one value on the stack lua_settop(L, top+1); + lua_remove(L, top); } // @@ -541,6 +549,9 @@ void script_dump_packed(const PackedValue *val) case INSTR_POP: printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2); break; + case INSTR_PUSHREF: + printf("PUSHREF(%d)", i.ref); + break; case LUA_TNIL: printf("nil"); break; @@ -574,6 +585,8 @@ void script_dump_packed(const PackedValue *val) else printf(", into=%d", i.set_into); } + if (i.keep_ref) + printf(", keep_ref"); if (i.pop) printf(", pop"); printf(")\n"); diff --git a/src/script/common/c_packer.h b/src/script/common/c_packer.h index 8bccca98d..ee732be86 100644 --- a/src/script/common/c_packer.h +++ b/src/script/common/c_packer.h @@ -36,6 +36,7 @@ extern "C" { #define INSTR_SETTABLE (-10) #define INSTR_POP (-11) +#define INSTR_PUSHREF (-12) /** * Represents a single instruction that pushes a new value or works with existing ones. @@ -44,6 +45,7 @@ struct PackedInstr { s16 type; // LUA_T* or INSTR_* u16 set_into; // set into table on stack + bool keep_ref; // is referenced later by INSTR_PUSHREF? bool pop; // remove from stack? union { bool bdata; // boolean: value @@ -60,6 +62,7 @@ struct PackedInstr s32 sidata1, sidata2; }; void *ptrdata; // userdata: implementation defined + s32 ref; // PUSHREF: index of referenced instr }; /* - string: value @@ -69,7 +72,7 @@ struct PackedInstr */ std::string sdata; - PackedInstr() : type(0), set_into(0), pop(false) {} + PackedInstr() : type(0), set_into(0), keep_ref(false), pop(false) {} }; /** -- cgit v1.2.3 From ec9f1575121e3b064b919bca7efddfa8b0fc4e65 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Mon, 9 May 2022 18:20:10 +0200 Subject: Use native packer to transfer globals into async env(s) --- builtin/async/game.lua | 3 +-- builtin/game/misc.lua | 2 +- src/script/common/c_packer.h | 2 +- src/script/scripting_server.cpp | 9 +++++---- src/server.cpp | 1 - src/server.h | 5 +++-- 6 files changed, 11 insertions(+), 11 deletions(-) (limited to 'src/script/common') diff --git a/builtin/async/game.lua b/builtin/async/game.lua index 212a33e17..8cb9720b6 100644 --- a/builtin/async/game.lua +++ b/builtin/async/game.lua @@ -22,8 +22,7 @@ dofile(gamepath .. "voxelarea.lua") -- Transfer of globals do - assert(core.transferred_globals) - local all = core.deserialize(core.transferred_globals, true) + local all = assert(core.transferred_globals) core.transferred_globals = nil -- reassemble other tables diff --git a/builtin/game/misc.lua b/builtin/game/misc.lua index 9f5e3312b..997b1894a 100644 --- a/builtin/game/misc.lua +++ b/builtin/game/misc.lua @@ -262,5 +262,5 @@ function core.get_globals_to_transfer() registered_items = copy_filtering(core.registered_items), registered_aliases = core.registered_aliases, } - return core.serialize(all) + return all end diff --git a/src/script/common/c_packer.h b/src/script/common/c_packer.h index ee732be86..fe072c10a 100644 --- a/src/script/common/c_packer.h +++ b/src/script/common/c_packer.h @@ -119,7 +119,7 @@ void script_register_packer(lua_State *L, const char *regname, // Pack a Lua value PackedValue *script_pack(lua_State *L, int idx); // Unpack a Lua value (left on top of stack) -// Note that this may modify the PackedValue, you can't reuse it! +// Note that this may modify the PackedValue, reusability is not guaranteed! void script_unpack(lua_State *L, PackedValue *val); // Dump contents of PackedValue to stdout for debugging diff --git a/src/script/scripting_server.cpp b/src/script/scripting_server.cpp index 5b99468dc..b462141b0 100644 --- a/src/script/scripting_server.cpp +++ b/src/script/scripting_server.cpp @@ -98,8 +98,9 @@ void ServerScripting::initAsync() luaL_checktype(L, -1, LUA_TTABLE); lua_getfield(L, -1, "get_globals_to_transfer"); lua_call(L, 0, 1); - luaL_checktype(L, -1, LUA_TSTRING); - getServer()->m_async_globals_data.set(readParam(L, -1)); + auto *data = script_pack(L, -1); + assert(!data->contains_userdata); + getServer()->m_async_globals_data.reset(data); lua_pushnil(L); lua_setfield(L, -3, "get_globals_to_transfer"); // unset function too lua_pop(L, 2); // pop 'core', return value @@ -183,8 +184,8 @@ void ServerScripting::InitializeAsync(lua_State *L, int top) // globals data lua_getglobal(L, "core"); luaL_checktype(L, -1, LUA_TTABLE); - std::string s = ModApiBase::getServer(L)->m_async_globals_data.get(); - lua_pushlstring(L, s.c_str(), s.size()); + auto *data = ModApiBase::getServer(L)->m_async_globals_data.get(); + script_unpack(L, data); lua_setfield(L, -2, "transferred_globals"); lua_pop(L, 1); // pop 'core' } diff --git a/src/server.cpp b/src/server.cpp index ebe1d1f6b..d93f300d2 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -243,7 +243,6 @@ Server::Server( m_clients(m_con), m_admin_chat(iface), m_on_shutdown_errmsg(on_shutdown_errmsg), - m_async_globals_data(""), m_modchannel_mgr(new ModChannelMgr()) { if (m_path_world.empty()) diff --git a/src/server.h b/src/server.h index ecba30b95..2c21f5dfc 100644 --- a/src/server.h +++ b/src/server.h @@ -73,6 +73,7 @@ struct Lighting; class ServerThread; class ServerModManager; class ServerInventoryManager; +struct PackedValue; enum ClientDeletionReason { CDR_LEAVE, @@ -388,8 +389,8 @@ public: // Lua files registered for init of async env, pair of modname + path std::vector> m_async_init_files; - // Serialized data transferred into async envs at init time - MutexedVariable m_async_globals_data; + // Data transferred into async envs at init time + std::unique_ptr m_async_globals_data; // Bind address Address m_bind_addr; -- cgit v1.2.3