From 21113ad4105dd3fb181b3d0638b907af94a352ab Mon Sep 17 00:00:00 2001 From: Wuzzy Date: Fri, 1 Oct 2021 14:21:24 +0000 Subject: Split liquid_viscosity to liquid_viscosity and move_resistance (#10810) --- src/script/cpp_api/s_node.h | 1 + 1 file changed, 1 insertion(+) (limited to 'src/script/cpp_api') diff --git a/src/script/cpp_api/s_node.h b/src/script/cpp_api/s_node.h index 3f771c838..3c6a8445b 100644 --- a/src/script/cpp_api/s_node.h +++ b/src/script/cpp_api/s_node.h @@ -53,6 +53,7 @@ public: static struct EnumString es_ContentParamType[]; static struct EnumString es_ContentParamType2[]; static struct EnumString es_LiquidType[]; + static struct EnumString es_LiquidMoveType[]; static struct EnumString es_NodeBoxType[]; static struct EnumString es_TextureAlphaMode[]; }; -- cgit v1.2.3 From 660e63dbae5901f8178b39ab40e6f770b8d7a215 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Mon, 25 Oct 2021 20:30:27 +0200 Subject: Fix item duplication if player dies during interact callback (alternative) (#11662) --- builtin/game/item.lua | 50 +++++++++++++++++++++---------------- doc/lua_api.txt | 11 +++++--- src/network/serverpackethandler.cpp | 44 +++++++++++++++++++------------- src/script/cpp_api/s_item.cpp | 21 +++++++++++----- src/script/cpp_api/s_item.h | 14 ++++++++--- src/script/lua_api/l_env.cpp | 2 +- src/util/Optional.h | 32 ++++++++++++++++++++++-- 7 files changed, 118 insertions(+), 56 deletions(-) (limited to 'src/script/cpp_api') diff --git a/builtin/game/item.lua b/builtin/game/item.lua index c495a67bd..039947584 100644 --- a/builtin/game/item.lua +++ b/builtin/game/item.lua @@ -499,34 +499,40 @@ function core.do_item_eat(hp_change, replace_with_item, itemstack, user, pointed return result end end + if itemstack:take_item():is_empty() then + return itemstack + end + local def = itemstack:get_definition() - if itemstack:take_item() ~= nil then - user:set_hp(user:get_hp() + hp_change) - - if def and def.sound and def.sound.eat then - core.sound_play(def.sound.eat, { - pos = user:get_pos(), - max_hear_distance = 16 - }, true) - end + if def and def.sound and def.sound.eat then + core.sound_play(def.sound.eat, { + pos = user:get_pos(), + max_hear_distance = 16 + }, true) + end - if replace_with_item then - if itemstack:is_empty() then - itemstack:add_item(replace_with_item) + -- Changing hp might kill the player causing mods to do who-knows-what to the + -- inventory, so do this before set_hp(). + if replace_with_item then + if itemstack:is_empty() then + itemstack:add_item(replace_with_item) + else + local inv = user:get_inventory() + -- Check if inv is null, since non-players don't have one + if inv and inv:room_for_item("main", {name=replace_with_item}) then + inv:add_item("main", replace_with_item) else - local inv = user:get_inventory() - -- Check if inv is null, since non-players don't have one - if inv and inv:room_for_item("main", {name=replace_with_item}) then - inv:add_item("main", replace_with_item) - else - local pos = user:get_pos() - pos.y = math.floor(pos.y + 0.5) - core.add_item(pos, replace_with_item) - end + local pos = user:get_pos() + pos.y = math.floor(pos.y + 0.5) + core.add_item(pos, replace_with_item) end end end - return itemstack + user:set_wielded_item(itemstack) + + user:set_hp(user:get_hp() + hp_change) + + return nil -- don't overwrite wield item a second time end function core.item_eat(hp_change, replace_with_item) diff --git a/doc/lua_api.txt b/doc/lua_api.txt index 69ac55493..e47df4686 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -5421,7 +5421,7 @@ Inventory * `minetest.remove_detached_inventory(name)` * Returns a `boolean` indicating whether the removal succeeded. * `minetest.do_item_eat(hp_change, replace_with_item, itemstack, user, pointed_thing)`: - returns left over ItemStack. + returns leftover ItemStack or nil to indicate no inventory change * See `minetest.item_eat` and `minetest.register_on_item_eat` Formspec @@ -7542,12 +7542,15 @@ Used by `minetest.register_node`, `minetest.register_craftitem`, and on_place = function(itemstack, placer, pointed_thing), -- When the 'place' key was pressed with the item in hand -- and a node was pointed at. - -- Shall place item and return the leftover itemstack. + -- Shall place item and return the leftover itemstack + -- or nil to not modify the inventory. -- The placer may be any ObjectRef or nil. -- default: minetest.item_place on_secondary_use = function(itemstack, user, pointed_thing), -- Same as on_place but called when not pointing at a node. + -- Function must return either nil if inventory shall not be modified, + -- or an itemstack to replace the original itemstack. -- The user may be any ObjectRef or nil. -- default: nil @@ -7559,8 +7562,8 @@ Used by `minetest.register_node`, `minetest.register_craftitem`, and on_use = function(itemstack, user, pointed_thing), -- default: nil -- When user pressed the 'punch/mine' key with the item in hand. - -- Function must return either nil if no item shall be removed from - -- inventory, or an itemstack to replace the original itemstack. + -- Function must return either nil if inventory shall not be modified, + -- or an itemstack to replace the original itemstack. -- e.g. itemstack:take_item(); return itemstack -- Otherwise, the function is free to do what it wants. -- The user may be any ObjectRef or nil. diff --git a/src/network/serverpackethandler.cpp b/src/network/serverpackethandler.cpp index dc7be0e23..d4bef3ca2 100644 --- a/src/network/serverpackethandler.cpp +++ b/src/network/serverpackethandler.cpp @@ -921,6 +921,13 @@ bool Server::checkInteractDistance(RemotePlayer *player, const f32 d, const std: return true; } +// Tiny helper to retrieve the selected item into an Optional +static inline void getWieldedItem(const PlayerSAO *playersao, Optional &ret) +{ + ret = ItemStack(); + playersao->getWieldedItem(&(*ret)); +} + void Server::handleCommand_Interact(NetworkPacket *pkt) { /* @@ -1228,14 +1235,17 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) // Place block or right-click object case INTERACT_PLACE: { - ItemStack selected_item; - playersao->getWieldedItem(&selected_item, nullptr); + Optional selected_item; + getWieldedItem(playersao, selected_item); // Reset build time counter if (pointed.type == POINTEDTHING_NODE && - selected_item.getDefinition(m_itemdef).type == ITEM_NODE) + selected_item->getDefinition(m_itemdef).type == ITEM_NODE) getClient(peer_id)->m_time_from_building = 0.0; + const bool had_prediction = !selected_item->getDefinition(m_itemdef). + node_placement_prediction.empty(); + if (pointed.type == POINTEDTHING_OBJECT) { // Right click object @@ -1248,11 +1258,9 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) << pointed_object->getDescription() << std::endl; // Do stuff - if (m_script->item_OnSecondaryUse( - selected_item, playersao, pointed)) { - if (playersao->setWieldedItem(selected_item)) { + if (m_script->item_OnSecondaryUse(selected_item, playersao, pointed)) { + if (selected_item.has_value() && playersao->setWieldedItem(*selected_item)) SendInventory(playersao, true); - } } pointed_object->rightClick(playersao); @@ -1260,7 +1268,7 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) // Placement was handled in lua // Apply returned ItemStack - if (playersao->setWieldedItem(selected_item)) + if (selected_item.has_value() && playersao->setWieldedItem(*selected_item)) SendInventory(playersao, true); } @@ -1272,8 +1280,7 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) RemoteClient *client = getClient(peer_id); v3s16 blockpos = getNodeBlockPos(pointed.node_abovesurface); v3s16 blockpos2 = getNodeBlockPos(pointed.node_undersurface); - if (!selected_item.getDefinition(m_itemdef - ).node_placement_prediction.empty()) { + if (had_prediction) { client->SetBlockNotSent(blockpos); if (blockpos2 != blockpos) client->SetBlockNotSent(blockpos2); @@ -1287,15 +1294,15 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) } // action == INTERACT_PLACE case INTERACT_USE: { - ItemStack selected_item; - playersao->getWieldedItem(&selected_item, nullptr); + Optional selected_item; + getWieldedItem(playersao, selected_item); - actionstream << player->getName() << " uses " << selected_item.name + actionstream << player->getName() << " uses " << selected_item->name << ", pointing at " << pointed.dump() << std::endl; if (m_script->item_OnUse(selected_item, playersao, pointed)) { // Apply returned ItemStack - if (playersao->setWieldedItem(selected_item)) + if (selected_item.has_value() && playersao->setWieldedItem(*selected_item)) SendInventory(playersao, true); } @@ -1304,16 +1311,17 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) // Rightclick air case INTERACT_ACTIVATE: { - ItemStack selected_item; - playersao->getWieldedItem(&selected_item, nullptr); + Optional selected_item; + getWieldedItem(playersao, selected_item); actionstream << player->getName() << " activates " - << selected_item.name << std::endl; + << selected_item->name << std::endl; pointed.type = POINTEDTHING_NOTHING; // can only ever be NOTHING if (m_script->item_OnSecondaryUse(selected_item, playersao, pointed)) { - if (playersao->setWieldedItem(selected_item)) + // Apply returned ItemStack + if (selected_item.has_value() && playersao->setWieldedItem(*selected_item)) SendInventory(playersao, true); } diff --git a/src/script/cpp_api/s_item.cpp b/src/script/cpp_api/s_item.cpp index 48dce14f3..b1916070e 100644 --- a/src/script/cpp_api/s_item.cpp +++ b/src/script/cpp_api/s_item.cpp @@ -59,13 +59,14 @@ bool ScriptApiItem::item_OnDrop(ItemStack &item, return true; } -bool ScriptApiItem::item_OnPlace(ItemStack &item, +bool ScriptApiItem::item_OnPlace(Optional &ret_item, ServerActiveObject *placer, const PointedThing &pointed) { SCRIPTAPI_PRECHECKHEADER int error_handler = PUSH_ERROR_HANDLER(L); + const ItemStack &item = *ret_item; // Push callback function on stack if (!getItemCallback(item.name.c_str(), "on_place")) return false; @@ -82,22 +83,25 @@ bool ScriptApiItem::item_OnPlace(ItemStack &item, PCALL_RES(lua_pcall(L, 3, 1, error_handler)); if (!lua_isnil(L, -1)) { try { - item = read_item(L, -1, getServer()->idef()); + ret_item = read_item(L, -1, getServer()->idef()); } catch (LuaError &e) { throw WRAP_LUAERROR(e, "item=" + item.name); } + } else { + ret_item = nullopt; } lua_pop(L, 2); // Pop item and error handler return true; } -bool ScriptApiItem::item_OnUse(ItemStack &item, +bool ScriptApiItem::item_OnUse(Optional &ret_item, ServerActiveObject *user, const PointedThing &pointed) { SCRIPTAPI_PRECHECKHEADER int error_handler = PUSH_ERROR_HANDLER(L); + const ItemStack &item = *ret_item; // Push callback function on stack if (!getItemCallback(item.name.c_str(), "on_use")) return false; @@ -109,22 +113,25 @@ bool ScriptApiItem::item_OnUse(ItemStack &item, PCALL_RES(lua_pcall(L, 3, 1, error_handler)); if(!lua_isnil(L, -1)) { try { - item = read_item(L, -1, getServer()->idef()); + ret_item = read_item(L, -1, getServer()->idef()); } catch (LuaError &e) { throw WRAP_LUAERROR(e, "item=" + item.name); } + } else { + ret_item = nullopt; } lua_pop(L, 2); // Pop item and error handler return true; } -bool ScriptApiItem::item_OnSecondaryUse(ItemStack &item, +bool ScriptApiItem::item_OnSecondaryUse(Optional &ret_item, ServerActiveObject *user, const PointedThing &pointed) { SCRIPTAPI_PRECHECKHEADER int error_handler = PUSH_ERROR_HANDLER(L); + const ItemStack &item = *ret_item; if (!getItemCallback(item.name.c_str(), "on_secondary_use")) return false; @@ -134,10 +141,12 @@ bool ScriptApiItem::item_OnSecondaryUse(ItemStack &item, PCALL_RES(lua_pcall(L, 3, 1, error_handler)); if (!lua_isnil(L, -1)) { try { - item = read_item(L, -1, getServer()->idef()); + ret_item = read_item(L, -1, getServer()->idef()); } catch (LuaError &e) { throw WRAP_LUAERROR(e, "item=" + item.name); } + } else { + ret_item = nullopt; } lua_pop(L, 2); // Pop item and error handler return true; diff --git a/src/script/cpp_api/s_item.h b/src/script/cpp_api/s_item.h index 25a3501f9..5015d8bd4 100644 --- a/src/script/cpp_api/s_item.h +++ b/src/script/cpp_api/s_item.h @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "cpp_api/s_base.h" #include "irr_v3d.h" +#include "util/Optional.h" struct PointedThing; struct ItemStack; @@ -35,13 +36,20 @@ class ScriptApiItem : virtual public ScriptApiBase { public: + /* + * Functions with Optional are for callbacks where Lua may + * want to prevent the engine from modifying the inventory after it's done. + * This has a longer backstory where on_use may need to empty the player's + * inventory without the engine interfering (see issue #6546). + */ + bool item_OnDrop(ItemStack &item, ServerActiveObject *dropper, v3f pos); - bool item_OnPlace(ItemStack &item, + bool item_OnPlace(Optional &item, ServerActiveObject *placer, const PointedThing &pointed); - bool item_OnUse(ItemStack &item, + bool item_OnUse(Optional &item, ServerActiveObject *user, const PointedThing &pointed); - bool item_OnSecondaryUse(ItemStack &item, + bool item_OnSecondaryUse(Optional &item, ServerActiveObject *user, const PointedThing &pointed); bool item_OnCraft(ItemStack &item, ServerActiveObject *user, const InventoryList *old_craft_grid, const InventoryLocation &craft_inv); diff --git a/src/script/lua_api/l_env.cpp b/src/script/lua_api/l_env.cpp index 98f8861fa..2c709d31b 100644 --- a/src/script/lua_api/l_env.cpp +++ b/src/script/lua_api/l_env.cpp @@ -477,7 +477,7 @@ int ModApiEnvMod::l_place_node(lua_State *L) return 1; } // Create item to place - ItemStack item(ndef->get(n).name, 1, 0, idef); + Optional item = ItemStack(ndef->get(n).name, 1, 0, idef); // Make pointed position PointedThing pointed; pointed.type = POINTEDTHING_NODE; diff --git a/src/util/Optional.h b/src/util/Optional.h index 9c2842b43..eda7fff89 100644 --- a/src/util/Optional.h +++ b/src/util/Optional.h @@ -19,6 +19,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #pragma once +#include #include "debug.h" struct nullopt_t @@ -43,18 +44,38 @@ class Optional public: Optional() noexcept {} Optional(nullopt_t) noexcept {} + Optional(const T &value) noexcept : m_has_value(true), m_value(value) {} + Optional(T &&value) noexcept : m_has_value(true), m_value(std::move(value)) {} + Optional(const Optional &other) noexcept : m_has_value(other.m_has_value), m_value(other.m_value) + {} + Optional(Optional &&other) noexcept : + m_has_value(other.m_has_value), m_value(std::move(other.m_value)) { + other.m_has_value = false; } - void operator=(nullopt_t) noexcept { m_has_value = false; } + Optional &operator=(nullopt_t) noexcept { m_has_value = false; return *this; } - void operator=(const Optional &other) noexcept + Optional &operator=(const Optional &other) noexcept { + if (&other == this) + return *this; m_has_value = other.m_has_value; m_value = other.m_value; + return *this; + } + + Optional &operator=(Optional &&other) noexcept + { + if (&other == this) + return *this; + m_has_value = other.m_has_value; + m_value = std::move(other.m_value); + other.m_has_value = false; + return *this; } T &value() @@ -71,6 +92,13 @@ public: const T &value_or(const T &def) const { return m_has_value ? m_value : def; } + // Unchecked access consistent with std::optional + T* operator->() { return &m_value; } + const T* operator->() const { return &m_value; } + + T& operator*() { return m_value; } + const T& operator*() const { return m_value; } + bool has_value() const noexcept { return m_has_value; } explicit operator bool() const { return m_has_value; } -- cgit v1.2.3 From b9051386ae296a6112383725bc8bfcd96dc9a226 Mon Sep 17 00:00:00 2001 From: Lejo Date: Fri, 26 Nov 2021 19:31:05 +0100 Subject: Add Lua bitop library (#9847) --- CMakeLists.txt | 5 + doc/lua_api.txt | 7 ++ lib/bitop/CMakeLists.txt | 4 + lib/bitop/bit.c | 189 ++++++++++++++++++++++++++++++++++++++ lib/bitop/bit.h | 34 +++++++ src/CMakeLists.txt | 3 + src/script/cpp_api/s_base.cpp | 7 ++ src/script/cpp_api/s_security.cpp | 3 +- 8 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 lib/bitop/CMakeLists.txt create mode 100644 lib/bitop/bit.c create mode 100644 lib/bitop/bit.h (limited to 'src/script/cpp_api') diff --git a/CMakeLists.txt b/CMakeLists.txt index b41738c06..3ba99bc21 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -261,6 +261,11 @@ endif() find_package(GMP REQUIRED) find_package(Json REQUIRED) find_package(Lua REQUIRED) +if(NOT USE_LUAJIT) + set(LUA_BIT_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/bitop) + set(LUA_BIT_LIBRARY bitop) + add_subdirectory(lib/bitop) +endif() # JsonCpp doesn't compile well on GCC 4.8 if(NOT USE_SYSTEM_JSONCPP) diff --git a/doc/lua_api.txt b/doc/lua_api.txt index 36db23b6f..0a63642af 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -8825,3 +8825,10 @@ Used by `minetest.register_authentication_handler`. -- Returns an iterator (use with `for` loops) for all player names -- currently in the auth database } + +Bit Library +----------- + +Functions: bit.tobit, bit.tohex, bit.bnot, bit.band, bit.bor, bit.bxor, bit.lshift, bit.rshift, bit.arshift, bit.rol, bit.ror, bit.bswap + +See http://bitop.luajit.org/ for advanced information. diff --git a/lib/bitop/CMakeLists.txt b/lib/bitop/CMakeLists.txt new file mode 100644 index 000000000..03b4d0b96 --- /dev/null +++ b/lib/bitop/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library(bitop bit.c) +target_link_libraries(bitop) + +include_directories(${LUA_INCLUDE_DIR}) diff --git a/lib/bitop/bit.c b/lib/bitop/bit.c new file mode 100644 index 000000000..f23c31a4c --- /dev/null +++ b/lib/bitop/bit.c @@ -0,0 +1,189 @@ +/* +** Lua BitOp -- a bit operations library for Lua 5.1/5.2. +** http://bitop.luajit.org/ +** +** Copyright (C) 2008-2012 Mike Pall. All rights reserved. +** +** Permission is hereby granted, free of charge, to any person obtaining +** a copy of this software and associated documentation files (the +** "Software"), to deal in the Software without restriction, including +** without limitation the rights to use, copy, modify, merge, publish, +** distribute, sublicense, and/or sell copies of the Software, and to +** permit persons to whom the Software is furnished to do so, subject to +** the following conditions: +** +** The above copyright notice and this permission notice shall be +** included in all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +** SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +** +** [ MIT license: http://www.opensource.org/licenses/mit-license.php ] +*/ + +#include "bit.h" + +#define LUA_BITOP_VERSION "1.0.2" + +#define LUA_LIB +#include "lauxlib.h" + +#ifdef _MSC_VER +/* MSVC is stuck in the last century and doesn't have C99's stdint.h. */ +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int64 uint64_t; +#else +#include +#endif + +typedef int32_t SBits; +typedef uint32_t UBits; + +typedef union { + lua_Number n; +#ifdef LUA_NUMBER_DOUBLE + uint64_t b; +#else + UBits b; +#endif +} BitNum; + +/* Convert argument to bit type. */ +static UBits barg(lua_State *L, int idx) +{ + BitNum bn; + UBits b; +#if LUA_VERSION_NUM < 502 + bn.n = lua_tonumber(L, idx); +#else + bn.n = luaL_checknumber(L, idx); +#endif +#if defined(LUA_NUMBER_DOUBLE) + bn.n += 6755399441055744.0; /* 2^52+2^51 */ +#ifdef SWAPPED_DOUBLE + b = (UBits)(bn.b >> 32); +#else + b = (UBits)bn.b; +#endif +#elif defined(LUA_NUMBER_INT) || defined(LUA_NUMBER_LONG) || \ + defined(LUA_NUMBER_LONGLONG) || defined(LUA_NUMBER_LONG_LONG) || \ + defined(LUA_NUMBER_LLONG) + if (sizeof(UBits) == sizeof(lua_Number)) + b = bn.b; + else + b = (UBits)(SBits)bn.n; +#elif defined(LUA_NUMBER_FLOAT) +#error "A 'float' lua_Number type is incompatible with this library" +#else +#error "Unknown number type, check LUA_NUMBER_* in luaconf.h" +#endif +#if LUA_VERSION_NUM < 502 + if (b == 0 && !lua_isnumber(L, idx)) { + luaL_typerror(L, idx, "number"); + } +#endif + return b; +} + +/* Return bit type. */ +#define BRET(b) lua_pushnumber(L, (lua_Number)(SBits)(b)); return 1; + +static int bit_tobit(lua_State *L) { BRET(barg(L, 1)) } +static int bit_bnot(lua_State *L) { BRET(~barg(L, 1)) } + +#define BIT_OP(func, opr) \ + static int func(lua_State *L) { int i; UBits b = barg(L, 1); \ + for (i = lua_gettop(L); i > 1; i--) b opr barg(L, i); BRET(b) } +BIT_OP(bit_band, &=) +BIT_OP(bit_bor, |=) +BIT_OP(bit_bxor, ^=) + +#define bshl(b, n) (b << n) +#define bshr(b, n) (b >> n) +#define bsar(b, n) ((SBits)b >> n) +#define brol(b, n) ((b << n) | (b >> (32-n))) +#define bror(b, n) ((b << (32-n)) | (b >> n)) +#define BIT_SH(func, fn) \ + static int func(lua_State *L) { \ + UBits b = barg(L, 1); UBits n = barg(L, 2) & 31; BRET(fn(b, n)) } +BIT_SH(bit_lshift, bshl) +BIT_SH(bit_rshift, bshr) +BIT_SH(bit_arshift, bsar) +BIT_SH(bit_rol, brol) +BIT_SH(bit_ror, bror) + +static int bit_bswap(lua_State *L) +{ + UBits b = barg(L, 1); + b = (b >> 24) | ((b >> 8) & 0xff00) | ((b & 0xff00) << 8) | (b << 24); + BRET(b) +} + +static int bit_tohex(lua_State *L) +{ + UBits b = barg(L, 1); + SBits n = lua_isnone(L, 2) ? 8 : (SBits)barg(L, 2); + const char *hexdigits = "0123456789abcdef"; + char buf[8]; + int i; + if (n < 0) { n = -n; hexdigits = "0123456789ABCDEF"; } + if (n > 8) n = 8; + for (i = (int)n; --i >= 0; ) { buf[i] = hexdigits[b & 15]; b >>= 4; } + lua_pushlstring(L, buf, (size_t)n); + return 1; +} + +static const struct luaL_Reg bit_funcs[] = { + { "tobit", bit_tobit }, + { "bnot", bit_bnot }, + { "band", bit_band }, + { "bor", bit_bor }, + { "bxor", bit_bxor }, + { "lshift", bit_lshift }, + { "rshift", bit_rshift }, + { "arshift", bit_arshift }, + { "rol", bit_rol }, + { "ror", bit_ror }, + { "bswap", bit_bswap }, + { "tohex", bit_tohex }, + { NULL, NULL } +}; + +/* Signed right-shifts are implementation-defined per C89/C99. +** But the de facto standard are arithmetic right-shifts on two's +** complement CPUs. This behaviour is required here, so test for it. +*/ +#define BAD_SAR (bsar(-8, 2) != (SBits)-2) + +LUALIB_API int luaopen_bit(lua_State *L) +{ + UBits b; + lua_pushnumber(L, (lua_Number)1437217655L); + b = barg(L, -1); + if (b != (UBits)1437217655L || BAD_SAR) { /* Perform a simple self-test. */ + const char *msg = "compiled with incompatible luaconf.h"; +#ifdef LUA_NUMBER_DOUBLE +#ifdef _WIN32 + if (b == (UBits)1610612736L) + msg = "use D3DCREATE_FPU_PRESERVE with DirectX"; +#endif + if (b == (UBits)1127743488L) + msg = "not compiled with SWAPPED_DOUBLE"; +#endif + if (BAD_SAR) + msg = "arithmetic right-shift broken"; + luaL_error(L, "bit library self-test failed (%s)", msg); + } +#if LUA_VERSION_NUM < 502 + luaL_register(L, "bit", bit_funcs); +#else + luaL_newlib(L, bit_funcs); +#endif + return 1; +} diff --git a/lib/bitop/bit.h b/lib/bitop/bit.h new file mode 100644 index 000000000..3e5845ee5 --- /dev/null +++ b/lib/bitop/bit.h @@ -0,0 +1,34 @@ +/* +** Lua BitOp -- a bit operations library for Lua 5.1/5.2. +** http://bitop.luajit.org/ +** +** Copyright (C) 2008-2012 Mike Pall. All rights reserved. +** +** Permission is hereby granted, free of charge, to any person obtaining +** a copy of this software and associated documentation files (the +** "Software"), to deal in the Software without restriction, including +** without limitation the rights to use, copy, modify, merge, publish, +** distribute, sublicense, and/or sell copies of the Software, and to +** permit persons to whom the Software is furnished to do so, subject to +** the following conditions: +** +** The above copyright notice and this permission notice shall be +** included in all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +** SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +** +** [ MIT license: http://www.opensource.org/licenses/mit-license.php ] +*/ + +#pragma once + +#include "lua.h" + +#define LUA_BITLIBNAME "bit" +LUALIB_API int luaopen_bit(lua_State *L); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1549587b7..4803b475b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -500,6 +500,7 @@ include_directories( ${LUA_INCLUDE_DIR} ${GMP_INCLUDE_DIR} ${JSON_INCLUDE_DIR} + ${LUA_BIT_INCLUDE_DIR} ${X11_INCLUDE_DIR} ${PROJECT_SOURCE_DIR}/script ) @@ -537,6 +538,7 @@ if(BUILD_CLIENT) ${LUA_LIBRARY} ${GMP_LIBRARY} ${JSON_LIBRARY} + ${LUA_BIT_LIBRARY} ${PLATFORM_LIBS} ) if(NOT USE_LUAJIT) @@ -619,6 +621,7 @@ if(BUILD_SERVER) ${SQLITE3_LIBRARY} ${JSON_LIBRARY} ${LUA_LIBRARY} + ${LUA_BIT_LIBRARY} ${GMP_LIBRARY} ${PLATFORM_LIBS} ) diff --git a/src/script/cpp_api/s_base.cpp b/src/script/cpp_api/s_base.cpp index 921f713c0..f7b8a5102 100644 --- a/src/script/cpp_api/s_base.cpp +++ b/src/script/cpp_api/s_base.cpp @@ -37,6 +37,8 @@ extern "C" { #include "lualib.h" #if USE_LUAJIT #include "luajit.h" +#else + #include "bit.h" #endif } @@ -88,6 +90,11 @@ ScriptApiBase::ScriptApiBase(ScriptingType type): else luaL_openlibs(m_luastack); + // Load bit library + lua_pushcfunction(m_luastack, luaopen_bit); + lua_pushstring(m_luastack, LUA_BITLIBNAME); + lua_call(m_luastack, 1, 0); + // Make the ScriptApiBase* accessible to ModApiBase #if INDIRECT_SCRIPTAPI_RIDX *(void **)(lua_newuserdata(m_luastack, sizeof(void *))) = this; diff --git a/src/script/cpp_api/s_security.cpp b/src/script/cpp_api/s_security.cpp index 580042ec2..5faf8cc80 100644 --- a/src/script/cpp_api/s_security.cpp +++ b/src/script/cpp_api/s_security.cpp @@ -106,6 +106,7 @@ void ScriptApiSecurity::initializeSecurity() "string", "table", "math", + "bit" }; static const char *io_whitelist[] = { "close", @@ -298,6 +299,7 @@ void ScriptApiSecurity::initializeSecurityClient() "string", "table", "math", + "bit", }; static const char *os_whitelist[] = { "clock", @@ -834,4 +836,3 @@ int ScriptApiSecurity::sl_os_remove(lua_State *L) lua_call(L, 1, 2); return 2; } - -- cgit v1.2.3 From 8472141b79c25092c90dea24aa873bd7ff792142 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Sat, 18 Dec 2021 20:36:43 +0100 Subject: Restructure devtest's unittests and run them in CI (#11859) --- .github/workflows/build.yml | 2 +- games/devtest/mods/unittests/crafting.lua | 18 +- games/devtest/mods/unittests/init.lua | 199 ++++++++++++++++++++++- games/devtest/mods/unittests/itemdescription.lua | 3 +- games/devtest/mods/unittests/misc.lua | 38 +++++ games/devtest/mods/unittests/player.lua | 46 +++--- games/devtest/mods/unittests/random.lua | 10 -- src/script/cpp_api/s_base.h | 9 + src/script/cpp_api/s_server.cpp | 6 +- src/script/cpp_api/s_server.h | 2 +- src/script/lua_api/l_server.cpp | 2 +- util/test_multiplayer.sh | 26 ++- 12 files changed, 289 insertions(+), 72 deletions(-) create mode 100644 games/devtest/mods/unittests/misc.lua delete mode 100644 games/devtest/mods/unittests/random.lua (limited to 'src/script/cpp_api') diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 417b4f650..af1de15ec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -93,7 +93,7 @@ jobs: run: | ./bin/minetest --run-unittests - - name: Integration test + - name: Integration test + devtest run: | ./util/test_multiplayer.sh diff --git a/games/devtest/mods/unittests/crafting.lua b/games/devtest/mods/unittests/crafting.lua index eff13ce09..8c16d3efb 100644 --- a/games/devtest/mods/unittests/crafting.lua +++ b/games/devtest/mods/unittests/crafting.lua @@ -1,6 +1,7 @@ +dofile(core.get_modpath(core.get_current_modname()) .. "/crafting_prepare.lua") + -- Test minetest.clear_craft function local function test_clear_craft() - minetest.log("info", "[unittests] Testing minetest.clear_craft") -- Clearing by output minetest.register_craft({ output = "foo", @@ -22,11 +23,10 @@ local function test_clear_craft() minetest.clear_craft({recipe={{"foo", "bar"}}}) assert(minetest.get_all_craft_recipes("foo") == nil) end +unittests.register("test_clear_craft", test_clear_craft) -- Test minetest.get_craft_result function local function test_get_craft_result() - minetest.log("info", "[unittests] Testing minetest.get_craft_result") - -- normal local input = { method = "normal", @@ -107,14 +107,6 @@ local function test_get_craft_result() assert(output.item) minetest.log("info", "[unittests] unrepairable tool crafting output.item:to_table(): "..dump(output.item:to_table())) -- unrepairable tool must not yield any output - assert(output.item:get_name() == "") - + assert(output.item:is_empty()) end - -function unittests.test_crafting() - test_clear_craft() - test_get_craft_result() - minetest.log("action", "[unittests] Crafting tests passed!") - return true -end - +unittests.register("test_get_craft_result", test_get_craft_result) diff --git a/games/devtest/mods/unittests/init.lua b/games/devtest/mods/unittests/init.lua index 12c67f78b..0754d507f 100644 --- a/games/devtest/mods/unittests/init.lua +++ b/games/devtest/mods/unittests/init.lua @@ -1,18 +1,199 @@ unittests = {} +unittests.list = {} + +-- name: Name of the test +-- func: +-- for sync: function(player, pos), should error on failure +-- for async: function(callback, player, pos) +-- MUST call callback() or callback("error msg") in case of error once test is finished +-- this means you cannot use assert() in the test implementation +-- opts: { +-- player = false, -- Does test require a player? +-- map = false, -- Does test require map access? +-- async = false, -- Does the test run asynchronously? (read notes above!) +-- } +function unittests.register(name, func, opts) + local def = table.copy(opts or {}) + def.name = name + def.func = func + table.insert(unittests.list, def) +end + +function unittests.on_finished(all_passed) + -- free to override +end + +-- Calls invoke with a callback as argument +-- Suspends coroutine until that callback is called +-- Return values are passed through +local function await(invoke) + local co = coroutine.running() + assert(co) + local called_early = true + invoke(function(...) + if called_early == true then + called_early = {...} + else + coroutine.resume(co, ...) + end + end) + if called_early ~= true then + -- callback was already called before yielding + return unpack(called_early) + end + called_early = nil + return coroutine.yield() +end + +function unittests.run_one(idx, counters, out_callback, player, pos) + local def = unittests.list[idx] + if not def.player then + player = nil + elseif player == nil then + out_callback(false) + return false + end + if not def.map then + pos = nil + elseif pos == nil then + out_callback(false) + return false + end + + local tbegin = core.get_us_time() + local function done(status, err) + local tend = core.get_us_time() + local ms_taken = (tend - tbegin) / 1000 + + if not status then + core.log("error", err) + end + print(string.format("[%s] %s - %dms", + status and "PASS" or "FAIL", def.name, ms_taken)) + counters.time = counters.time + ms_taken + counters.total = counters.total + 1 + if status then + counters.passed = counters.passed + 1 + end + end + + if def.async then + core.log("info", "[unittest] running " .. def.name .. " (async)") + def.func(function(err) + done(err == nil, err) + out_callback(true) + end, player, pos) + else + core.log("info", "[unittest] running " .. def.name) + local status, err = pcall(def.func, player, pos) + done(status, err) + out_callback(true) + end + + return true +end + +local function wait_for_player(callback) + if #core.get_connected_players() > 0 then + return callback(core.get_connected_players()[1]) + end + local first = true + core.register_on_joinplayer(function(player) + if first then + callback(player) + first = false + end + end) +end + +local function wait_for_map(player, callback) + local check = function() + if core.get_node_or_nil(player:get_pos()) ~= nil then + callback() + else + minetest.after(0, check) + end + end + check() +end + +function unittests.run_all() + -- This runs in a coroutine so it uses await(). + local counters = { time = 0, total = 0, passed = 0 } + + -- Run standalone tests first + for idx = 1, #unittests.list do + local def = unittests.list[idx] + def.done = await(function(cb) + unittests.run_one(idx, counters, cb, nil, nil) + end) + end + + -- Wait for a player to join, run tests that require a player + local player = await(wait_for_player) + for idx = 1, #unittests.list do + local def = unittests.list[idx] + if not def.done then + def.done = await(function(cb) + unittests.run_one(idx, counters, cb, player, nil) + end) + end + end + + -- Wait for the world to generate/load, run tests that require map access + await(function(cb) + wait_for_map(player, cb) + end) + local pos = vector.round(player:get_pos()) + for idx = 1, #unittests.list do + local def = unittests.list[idx] + if not def.done then + def.done = await(function(cb) + unittests.run_one(idx, counters, cb, player, pos) + end) + end + end + + -- Print stats + assert(#unittests.list == counters.total) + print(string.rep("+", 80)) + print(string.format("Unit Test Results: %s", + counters.total == counters.passed and "PASSED" or "FAILED")) + print(string.format(" %d / %d failed tests.", + counters.total - counters.passed, counters.total)) + print(string.format(" Testing took %dms total.", counters.time)) + print(string.rep("+", 80)) + unittests.on_finished(counters.total == counters.passed) + return counters.total == counters.passed +end + +-------------- + local modpath = minetest.get_modpath("unittests") -dofile(modpath .. "/random.lua") +dofile(modpath .. "/misc.lua") dofile(modpath .. "/player.lua") -dofile(modpath .. "/crafting_prepare.lua") dofile(modpath .. "/crafting.lua") dofile(modpath .. "/itemdescription.lua") -if minetest.settings:get_bool("devtest_unittests_autostart", false) then - unittests.test_random() - unittests.test_crafting() - unittests.test_short_desc() - minetest.register_on_joinplayer(function(player) - unittests.test_player(player) +-------------- + +if core.settings:get_bool("devtest_unittests_autostart", false) then + core.after(0, function() + coroutine.wrap(unittests.run_all)() end) +else + minetest.register_chatcommand("unittests", { + privs = {basic_privs=true}, + description = "Runs devtest unittests (may modify player or map state)", + func = function(name, param) + unittests.on_finished = function(ok) + core.chat_send_player(name, + (ok and "All tests passed." or "There were test failures.") .. + " Check the console for detailed output.") + end + coroutine.wrap(unittests.run_all)() + return true, "" + end, + }) end - diff --git a/games/devtest/mods/unittests/itemdescription.lua b/games/devtest/mods/unittests/itemdescription.lua index d6ee6551a..dc62de7f0 100644 --- a/games/devtest/mods/unittests/itemdescription.lua +++ b/games/devtest/mods/unittests/itemdescription.lua @@ -25,7 +25,7 @@ minetest.register_chatcommand("item_description", { end }) -function unittests.test_short_desc() +local function test_short_desc() local function get_short_description(item) return ItemStack(item):get_short_description() end @@ -49,3 +49,4 @@ function unittests.test_short_desc() return true end +unittests.register("test_short_desc", test_short_desc) diff --git a/games/devtest/mods/unittests/misc.lua b/games/devtest/mods/unittests/misc.lua new file mode 100644 index 000000000..cf4f92cfa --- /dev/null +++ b/games/devtest/mods/unittests/misc.lua @@ -0,0 +1,38 @@ +local function test_random() + -- Try out PseudoRandom + local pseudo = PseudoRandom(13) + assert(pseudo:next() == 22290) + assert(pseudo:next() == 13854) +end +unittests.register("test_random", test_random) + +local function test_dynamic_media(cb, player) + if core.get_player_information(player:get_player_name()).protocol_version < 40 then + core.log("warning", "test_dynamic_media: Client too old, skipping test.") + return cb() + end + + -- Check that the client acknowledges media transfers + local path = core.get_worldpath() .. "/test_media.obj" + local f = io.open(path, "w") + f:write("# contents don't matter\n") + f:close() + + local call_ok = false + local ok = core.dynamic_add_media({ + filepath = path, + to_player = player:get_player_name(), + }, function(name) + if not call_ok then + cb("impossible condition") + end + cb() + end) + if not ok then + return cb("dynamic_add_media() returned error") + end + call_ok = true + + -- if the callback isn't called this test will just hang :shrug: +end +unittests.register("test_dynamic_media", test_dynamic_media, {async=true, player=true}) diff --git a/games/devtest/mods/unittests/player.lua b/games/devtest/mods/unittests/player.lua index 4a681310d..fa0557960 100644 --- a/games/devtest/mods/unittests/player.lua +++ b/games/devtest/mods/unittests/player.lua @@ -2,6 +2,21 @@ -- HP Change Reasons -- local expect = nil +minetest.register_on_player_hpchange(function(player, hp, reason) + if expect == nil then + return + end + + for key, value in pairs(reason) do + assert(expect[key] == value) + end + for key, value in pairs(expect) do + assert(reason[key] == value) + end + + expect = nil +end) + local function run_hpchangereason_tests(player) local old_hp = player:get_hp() @@ -20,7 +35,11 @@ local function run_hpchangereason_tests(player) player:set_hp(old_hp) end +unittests.register("test_hpchangereason", run_hpchangereason_tests, {player=true}) +-- +-- Player meta +-- local function run_player_meta_tests(player) local meta = player:get_meta() meta:set_string("foo", "bar") @@ -48,29 +67,4 @@ local function run_player_meta_tests(player) assert(meta:get_string("foo") == "") assert(meta:equals(meta2)) end - -function unittests.test_player(player) - minetest.register_on_player_hpchange(function(player, hp, reason) - if not expect then - return - end - - for key, value in pairs(reason) do - assert(expect[key] == value) - end - - for key, value in pairs(expect) do - assert(reason[key] == value) - end - - expect = nil - end) - - run_hpchangereason_tests(player) - run_player_meta_tests(player) - local msg = "Player tests passed for player '"..player:get_player_name().."'!" - minetest.chat_send_all(msg) - minetest.log("action", "[unittests] "..msg) - return true -end - +unittests.register("test_player_meta", run_player_meta_tests, {player=true}) diff --git a/games/devtest/mods/unittests/random.lua b/games/devtest/mods/unittests/random.lua deleted file mode 100644 index f94f0a88e..000000000 --- a/games/devtest/mods/unittests/random.lua +++ /dev/null @@ -1,10 +0,0 @@ -function unittests.test_random() - -- Try out PseudoRandom - minetest.log("action", "[unittests] Testing PseudoRandom ...") - local pseudo = PseudoRandom(13) - assert(pseudo:next() == 22290) - assert(pseudo:next() == 13854) - minetest.log("action", "[unittests] PseudoRandom test passed!") - return true -end - diff --git a/src/script/cpp_api/s_base.h b/src/script/cpp_api/s_base.h index 06df2abe3..244d81605 100644 --- a/src/script/cpp_api/s_base.h +++ b/src/script/cpp_api/s_base.h @@ -125,6 +125,15 @@ protected: friend class ModApiEnvMod; friend class LuaVoxelManip; + /* + Subtle edge case with coroutines: If for whatever reason you have a + method in a subclass that's called from existing lua_CFunction + (any of the l_*.cpp files) then make it static and take the lua_State* + as an argument. This is REQUIRED because getStack() will not return the + correct state if called inside coroutines. + + Also note that src/script/common/ is the better place for such helpers. + */ lua_State* getStack() { return m_luastack; } diff --git a/src/script/cpp_api/s_server.cpp b/src/script/cpp_api/s_server.cpp index 6ddb2630d..c255b0c71 100644 --- a/src/script/cpp_api/s_server.cpp +++ b/src/script/cpp_api/s_server.cpp @@ -198,10 +198,8 @@ std::string ScriptApiServer::formatChatMessage(const std::string &name, return ret; } -u32 ScriptApiServer::allocateDynamicMediaCallback(int f_idx) +u32 ScriptApiServer::allocateDynamicMediaCallback(lua_State *L, int f_idx) { - lua_State *L = getStack(); - if (f_idx < 0) f_idx = lua_gettop(L) + f_idx + 1; @@ -235,7 +233,7 @@ u32 ScriptApiServer::allocateDynamicMediaCallback(int f_idx) void ScriptApiServer::freeDynamicMediaCallback(u32 token) { - lua_State *L = getStack(); + SCRIPTAPI_PRECHECKHEADER verbosestream << "freeDynamicMediaCallback(" << token << ")" << std::endl; diff --git a/src/script/cpp_api/s_server.h b/src/script/cpp_api/s_server.h index c5c3d5596..58c8c0e48 100644 --- a/src/script/cpp_api/s_server.h +++ b/src/script/cpp_api/s_server.h @@ -51,7 +51,7 @@ public: const std::string &password); /* dynamic media handling */ - u32 allocateDynamicMediaCallback(int f_idx); + static u32 allocateDynamicMediaCallback(lua_State *L, int f_idx); void freeDynamicMediaCallback(u32 token); void on_dynamic_media_added(u32 token, const char *playername); diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 82a692070..88ab5e16b 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -496,7 +496,7 @@ int ModApiServer::l_dynamic_add_media(lua_State *L) CHECK_SECURE_PATH(L, filepath.c_str(), false); - u32 token = server->getScriptIface()->allocateDynamicMediaCallback(2); + u32 token = server->getScriptIface()->allocateDynamicMediaCallback(L, 2); bool ok = server->dynamicAddMedia(filepath, token, to_player, ephemeral); if (!ok) diff --git a/util/test_multiplayer.sh b/util/test_multiplayer.sh index 9fb894a30..5ffc044e0 100755 --- a/util/test_multiplayer.sh +++ b/util/test_multiplayer.sh @@ -20,7 +20,7 @@ waitfor () { } gdbrun () { - gdb -q -ex 'set confirm off' -ex 'r' -ex 'bt' -ex 'quit' --args "$@" + gdb -q -batch -ex 'set confirm off' -ex 'r' -ex 'bt' --args "$@" } [ -e $minetest ] || { echo "executable $minetest missing"; exit 1; } @@ -33,17 +33,27 @@ printf '%s\n' >$testspath/client1.conf \ enable_{sound,minimap,shaders}=false printf '%s\n' >$testspath/server.conf \ - max_block_send_distance=1 + max_block_send_distance=1 devtest_unittests_autostart=true cat >$worldpath/worldmods/test/init.lua <<"LUA" core.after(0, function() io.close(io.open(core.get_worldpath() .. "/startup", "w")) end) -core.register_on_joinplayer(function(player) - io.close(io.open(core.get_worldpath() .. "/player_joined", "w")) +local function callback(test_ok) + if not test_ok then + io.close(io.open(core.get_worldpath() .. "/test_failure", "w")) + end + io.close(io.open(core.get_worldpath() .. "/done", "w")) core.request_shutdown("", false, 2) -end) +end +if core.settings:get_bool("devtest_unittests_autostart") then + unittests.on_finished = callback +else + core.register_on_joinplayer(function() callback(true) end) +end LUA +printf '%s\n' >$worldpath/worldmods/test/mod.conf \ + name=test optional_depends=unittests echo "Starting server" gdbrun $minetest --server --config $conf_server --world $worldpath --gameid $gameid 2>&1 | sed -u 's/^/(server) /' & @@ -51,10 +61,14 @@ waitfor $worldpath/startup echo "Starting client" gdbrun $minetest --config $conf_client1 --go --address 127.0.0.1 2>&1 | sed -u 's/^/(client) /' & -waitfor $worldpath/player_joined +waitfor $worldpath/done echo "Waiting for client and server to exit" wait +if [ -f $worldpath/test_failure ]; then + echo "There were test failures." + exit 1 +fi echo "Success" exit 0 -- cgit v1.2.3 From f4054595482bf4573075f45d3ca56076a0d6113e Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 17 Dec 2021 18:35:30 +0100 Subject: Remove setlocal and setupvalue from `debug` table whitelist It's likely that these could be used trick mods into revealing the insecure environment even if they do everything right (which is already hard enough). --- src/script/cpp_api/s_security.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/script/cpp_api') diff --git a/src/script/cpp_api/s_security.cpp b/src/script/cpp_api/s_security.cpp index 5faf8cc80..11c277839 100644 --- a/src/script/cpp_api/s_security.cpp +++ b/src/script/cpp_api/s_security.cpp @@ -129,12 +129,10 @@ void ScriptApiSecurity::initializeSecurity() "traceback", "getinfo", "getmetatable", - "setupvalue", "setmetatable", "upvalueid", "sethook", "debug", - "setlocal", }; static const char *package_whitelist[] = { "config", -- cgit v1.2.3 From b2409b14d0682655363c1b3b3b6bafbaa7e7c1bf Mon Sep 17 00:00:00 2001 From: sfan5 Date: Fri, 17 Dec 2021 19:04:46 +0100 Subject: Refactor trusted mod checking code --- src/script/cpp_api/s_security.cpp | 33 +++++++++++++++++++++++++++++++++ src/script/cpp_api/s_security.h | 14 +++++++++----- src/script/lua_api/l_http.cpp | 39 +++------------------------------------ src/script/lua_api/l_util.cpp | 32 +------------------------------- 4 files changed, 46 insertions(+), 72 deletions(-) (limited to 'src/script/cpp_api') diff --git a/src/script/cpp_api/s_security.cpp b/src/script/cpp_api/s_security.cpp index 11c277839..ccd1214e3 100644 --- a/src/script/cpp_api/s_security.cpp +++ b/src/script/cpp_api/s_security.cpp @@ -27,6 +27,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include +#include #include @@ -604,6 +605,38 @@ bool ScriptApiSecurity::checkPath(lua_State *L, const char *path, return false; } +bool ScriptApiSecurity::checkWhitelisted(lua_State *L, const std::string &setting) +{ + assert(str_starts_with(setting, "secure.")); + + // We have to make sure that this function is being called directly by + // a mod, otherwise a malicious mod could override this function and + // steal its return value. + lua_Debug info; + + // Make sure there's only one item below this function on the stack... + if (lua_getstack(L, 2, &info)) + return false; + FATAL_ERROR_IF(!lua_getstack(L, 1, &info), "lua_getstack() failed"); + FATAL_ERROR_IF(!lua_getinfo(L, "S", &info), "lua_getinfo() failed"); + + // ...and that that item is the main file scope. + if (strcmp(info.what, "main") != 0) + return false; + + // Mod must be listed in secure.http_mods or secure.trusted_mods + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME); + if (!lua_isstring(L, -1)) + return false; + std::string mod_name = readParam(L, -1); + + std::string value = g_settings->get(setting); + value.erase(std::remove(value.begin(), value.end(), ' '), value.end()); + auto mod_list = str_split(value, ','); + + return CONTAINS(mod_list, mod_name); +} + int ScriptApiSecurity::sl_g_dofile(lua_State *L) { diff --git a/src/script/cpp_api/s_security.h b/src/script/cpp_api/s_security.h index 73e763548..619bf824f 100644 --- a/src/script/cpp_api/s_security.h +++ b/src/script/cpp_api/s_security.h @@ -40,11 +40,6 @@ with this program; if not, write to the Free Software Foundation, Inc., class ScriptApiSecurity : virtual public ScriptApiBase { public: - int getThread(lua_State *L); - // creates an empty Lua environment - void createEmptyEnv(lua_State *L); - // sets the enviroment to the table thats on top of the stack - void setLuaEnv(lua_State *L, int thread); // Sets up security on the ScriptApi's Lua state void initializeSecurity(); void initializeSecurityClient(); @@ -57,8 +52,17 @@ public: // Checks if mods are allowed to read (and optionally write) to the path static bool checkPath(lua_State *L, const char *path, bool write_required, bool *write_allowed=NULL); + // Check if mod is whitelisted in the given setting + // This additionally checks that the mod's main file scope is executing. + static bool checkWhitelisted(lua_State *L, const std::string &setting); private: + int getThread(lua_State *L); + // sets the enviroment to the table thats on top of the stack + void setLuaEnv(lua_State *L, int thread); + // creates an empty Lua environment + void createEmptyEnv(lua_State *L); + // Syntax: "sl_" '_' // (sl stands for Secure Lua) diff --git a/src/script/lua_api/l_http.cpp b/src/script/lua_api/l_http.cpp index b385b698c..bd359b3cc 100644 --- a/src/script/lua_api/l_http.cpp +++ b/src/script/lua_api/l_http.cpp @@ -21,14 +21,13 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "common/c_converter.h" #include "common/c_content.h" #include "lua_api/l_http.h" +#include "cpp_api/s_security.h" #include "httpfetch.h" #include "settings.h" #include "debug.h" #include "log.h" -#include #include -#include #define HTTP_API(name) \ lua_pushstring(L, #name); \ @@ -181,40 +180,8 @@ int ModApiHttp::l_request_http_api(lua_State *L) { NO_MAP_LOCK_REQUIRED; - // We have to make sure that this function is being called directly by - // a mod, otherwise a malicious mod could override this function and - // steal its return value. - lua_Debug info; - - // Make sure there's only one item below this function on the stack... - if (lua_getstack(L, 2, &info)) { - return 0; - } - FATAL_ERROR_IF(!lua_getstack(L, 1, &info), "lua_getstack() failed"); - FATAL_ERROR_IF(!lua_getinfo(L, "S", &info), "lua_getinfo() failed"); - - // ...and that that item is the main file scope. - if (strcmp(info.what, "main") != 0) { - return 0; - } - - // Mod must be listed in secure.http_mods or secure.trusted_mods - lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME); - if (!lua_isstring(L, -1)) { - return 0; - } - - std::string mod_name = readParam(L, -1); - std::string http_mods = g_settings->get("secure.http_mods"); - http_mods.erase(std::remove(http_mods.begin(), http_mods.end(), ' '), http_mods.end()); - std::vector mod_list_http = str_split(http_mods, ','); - - std::string trusted_mods = g_settings->get("secure.trusted_mods"); - trusted_mods.erase(std::remove(trusted_mods.begin(), trusted_mods.end(), ' '), trusted_mods.end()); - std::vector mod_list_trusted = str_split(trusted_mods, ','); - - mod_list_http.insert(mod_list_http.end(), mod_list_trusted.begin(), mod_list_trusted.end()); - if (std::find(mod_list_http.begin(), mod_list_http.end(), mod_name) == mod_list_http.end()) { + if (!ScriptApiSecurity::checkWhitelisted(L, "secure.http_mods") && + !ScriptApiSecurity::checkWhitelisted(L, "secure.trusted_mods")) { lua_pushnil(L); return 1; } diff --git a/src/script/lua_api/l_util.cpp b/src/script/lua_api/l_util.cpp index 528d9c6dd..b04f26fda 100644 --- a/src/script/lua_api/l_util.cpp +++ b/src/script/lua_api/l_util.cpp @@ -41,7 +41,6 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "util/hex.h" #include "util/sha1.h" #include "util/png.h" -#include #include // log([level,] text) @@ -444,36 +443,7 @@ int ModApiUtil::l_request_insecure_environment(lua_State *L) return 1; } - // We have to make sure that this function is being called directly by - // a mod, otherwise a malicious mod could override this function and - // steal its return value. - lua_Debug info; - // Make sure there's only one item below this function on the stack... - if (lua_getstack(L, 2, &info)) { - return 0; - } - FATAL_ERROR_IF(!lua_getstack(L, 1, &info), "lua_getstack() failed"); - FATAL_ERROR_IF(!lua_getinfo(L, "S", &info), "lua_getinfo() failed"); - // ...and that that item is the main file scope. - if (strcmp(info.what, "main") != 0) { - return 0; - } - - // Get mod name - lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME); - if (!lua_isstring(L, -1)) { - return 0; - } - - // Check secure.trusted_mods - std::string mod_name = readParam(L, -1); - std::string trusted_mods = g_settings->get("secure.trusted_mods"); - trusted_mods.erase(std::remove_if(trusted_mods.begin(), - trusted_mods.end(), static_cast(&std::isspace)), - trusted_mods.end()); - std::vector mod_list = str_split(trusted_mods, ','); - if (std::find(mod_list.begin(), mod_list.end(), mod_name) == - mod_list.end()) { + if (!ScriptApiSecurity::checkWhitelisted(L, "secure.trusted_mods")) { return 0; } -- cgit v1.2.3 From b81948a14c138517f6a227dac5b71f0b2facb33c Mon Sep 17 00:00:00 2001 From: Wuzzy Date: Thu, 6 Jan 2022 20:16:35 +0000 Subject: Fix damage wraparound if very high damage (#11872) --- doc/lua_api.txt | 2 +- src/script/cpp_api/s_entity.cpp | 2 +- src/script/cpp_api/s_entity.h | 2 +- src/script/cpp_api/s_player.cpp | 2 +- src/script/cpp_api/s_player.h | 2 +- src/tool.cpp | 4 +++- src/tool.h | 4 ++-- 7 files changed, 10 insertions(+), 8 deletions(-) (limited to 'src/script/cpp_api') diff --git a/doc/lua_api.txt b/doc/lua_api.txt index d4dc19fdd..3edfd5bb1 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -3524,7 +3524,7 @@ Helper functions * `minetest.get_hit_params(groups, tool_capabilities [, time_from_last_punch [, wear]])`: Simulates an item that punches an object. Returns a table with the following fields: - * `hp`: How much damage the punch would cause. + * `hp`: How much damage the punch would cause (between -65535 and 65535). * `wear`: How much wear would be added to the tool (ignored for non-tools). Parameters: * `groups`: Damage groups of the object diff --git a/src/script/cpp_api/s_entity.cpp b/src/script/cpp_api/s_entity.cpp index 746f7013e..06337b9e8 100644 --- a/src/script/cpp_api/s_entity.cpp +++ b/src/script/cpp_api/s_entity.cpp @@ -240,7 +240,7 @@ void ScriptApiEntity::luaentity_Step(u16 id, float dtime, // tool_capabilities, direction, damage) bool ScriptApiEntity::luaentity_Punch(u16 id, ServerActiveObject *puncher, float time_from_last_punch, - const ToolCapabilities *toolcap, v3f dir, s16 damage) + const ToolCapabilities *toolcap, v3f dir, s32 damage) { SCRIPTAPI_PRECHECKHEADER diff --git a/src/script/cpp_api/s_entity.h b/src/script/cpp_api/s_entity.h index b52f6e447..7658ae922 100644 --- a/src/script/cpp_api/s_entity.h +++ b/src/script/cpp_api/s_entity.h @@ -42,7 +42,7 @@ public: const collisionMoveResult *moveresult); bool luaentity_Punch(u16 id, ServerActiveObject *puncher, float time_from_last_punch, - const ToolCapabilities *toolcap, v3f dir, s16 damage); + const ToolCapabilities *toolcap, v3f dir, s32 damage); bool luaentity_on_death(u16 id, ServerActiveObject *killer); void luaentity_Rightclick(u16 id, ServerActiveObject *clicker); void luaentity_on_attach_child(u16 id, ServerActiveObject *child); diff --git a/src/script/cpp_api/s_player.cpp b/src/script/cpp_api/s_player.cpp index d3e6138dc..22b24f363 100644 --- a/src/script/cpp_api/s_player.cpp +++ b/src/script/cpp_api/s_player.cpp @@ -60,7 +60,7 @@ bool ScriptApiPlayer::on_punchplayer(ServerActiveObject *player, float time_from_last_punch, const ToolCapabilities *toolcap, v3f dir, - s16 damage) + s32 damage) { SCRIPTAPI_PRECHECKHEADER // Get core.registered_on_punchplayers diff --git a/src/script/cpp_api/s_player.h b/src/script/cpp_api/s_player.h index c0f141862..e866aee46 100644 --- a/src/script/cpp_api/s_player.h +++ b/src/script/cpp_api/s_player.h @@ -46,7 +46,7 @@ public: void on_cheat(ServerActiveObject *player, const std::string &cheat_type); bool on_punchplayer(ServerActiveObject *player, ServerActiveObject *hitter, float time_from_last_punch, const ToolCapabilities *toolcap, - v3f dir, s16 damage); + v3f dir, s32 damage); void on_rightclickplayer(ServerActiveObject *player, ServerActiveObject *clicker); s32 on_player_hpchange(ServerActiveObject *player, s32 hp_change, const PlayerHPChangeReason &reason); diff --git a/src/tool.cpp b/src/tool.cpp index b0749286d..075c6b3c5 100644 --- a/src/tool.cpp +++ b/src/tool.cpp @@ -306,7 +306,7 @@ HitParams getHitParams(const ItemGroupList &armor_groups, const ToolCapabilities *tp, float time_from_last_punch, u16 initial_wear) { - s16 damage = 0; + s32 damage = 0; float result_wear = 0.0f; float punch_interval_multiplier = rangelim(time_from_last_punch / tp->full_punch_interval, 0.0f, 1.0f); @@ -320,6 +320,8 @@ HitParams getHitParams(const ItemGroupList &armor_groups, result_wear = calculateResultWear(tp->punch_attack_uses, initial_wear); result_wear *= punch_interval_multiplier; } + // Keep damage in sane bounds for simplicity + damage = rangelim(damage, -U16_MAX, U16_MAX); u32 wear_i = (u32) result_wear; return {damage, wear_i}; diff --git a/src/tool.h b/src/tool.h index 0e3388485..8409f59af 100644 --- a/src/tool.h +++ b/src/tool.h @@ -106,11 +106,11 @@ DigParams getDigParams(const ItemGroupList &groups, struct HitParams { - s16 hp; + s32 hp; // Caused wear u32 wear; // u32 because wear could be 65536 (single-use weapon) - HitParams(s16 hp_ = 0, u32 wear_ = 0): + HitParams(s32 hp_ = 0, u32 wear_ = 0): hp(hp_), wear(wear_) {} -- cgit v1.2.3 From 9a12e4499ecf5c1a3467af9c831d0d350a21923d Mon Sep 17 00:00:00 2001 From: sfan5 Date: Thu, 13 Jan 2022 22:12:44 +0100 Subject: Minor improvements to Lua sandbox --- src/script/cpp_api/s_security.cpp | 30 ++++++++++++++++++++++++++---- src/script/cpp_api/s_security.h | 1 + 2 files changed, 27 insertions(+), 4 deletions(-) (limited to 'src/script/cpp_api') diff --git a/src/script/cpp_api/s_security.cpp b/src/script/cpp_api/s_security.cpp index ccd1214e3..a6c5114b2 100644 --- a/src/script/cpp_api/s_security.cpp +++ b/src/script/cpp_api/s_security.cpp @@ -121,9 +121,7 @@ void ScriptApiSecurity::initializeSecurity() "date", "difftime", "getenv", - "setlocale", "time", - "tmpname", }; static const char *debug_whitelist[] = { "gethook", @@ -219,6 +217,7 @@ void ScriptApiSecurity::initializeSecurity() // And replace unsafe ones SECURE_API(os, remove); SECURE_API(os, rename); + SECURE_API(os, setlocale); lua_setglobal(L, "os"); lua_pop(L, 1); // Pop old OS @@ -250,6 +249,11 @@ void ScriptApiSecurity::initializeSecurity() lua_pop(L, 1); // Pop old jit #endif + // Get rid of 'core' in the old globals, we don't want anyone thinking it's + // safe or even usable. + lua_pushnil(L); + lua_setfield(L, old_globals, "core"); + lua_pop(L, 1); // Pop globals_backup @@ -285,7 +289,7 @@ void ScriptApiSecurity::initializeSecurityClient() "rawset", "select", "setfenv", - // getmetatable can be used to escape the sandbox + // getmetatable can be used to escape the sandbox <- ??? "setmetatable", "tonumber", "tostring", @@ -307,7 +311,7 @@ void ScriptApiSecurity::initializeSecurityClient() "time" }; static const char *debug_whitelist[] = { - "getinfo", + "getinfo", // used by builtin and unset before mods load "traceback" }; @@ -867,3 +871,21 @@ int ScriptApiSecurity::sl_os_remove(lua_State *L) lua_call(L, 1, 2); return 2; } + + +int ScriptApiSecurity::sl_os_setlocale(lua_State *L) +{ + const bool cat = lua_gettop(L) > 1; + // Don't allow changes + if (!lua_isnoneornil(L, 1)) { + lua_pushnil(L); + return 1; + } + + push_original(L, "os", "setlocale"); + lua_pushnil(L); + if (cat) + lua_pushvalue(L, 2); + lua_call(L, cat ? 2 : 1, 1); + return 1; +} diff --git a/src/script/cpp_api/s_security.h b/src/script/cpp_api/s_security.h index 619bf824f..880ce1638 100644 --- a/src/script/cpp_api/s_security.h +++ b/src/script/cpp_api/s_security.h @@ -79,4 +79,5 @@ private: static int sl_os_rename(lua_State *L); static int sl_os_remove(lua_State *L); + static int sl_os_setlocale(lua_State *L); }; -- cgit v1.2.3 From 8d387433b14791db95e59127b5e6e30f58155c1e Mon Sep 17 00:00:00 2001 From: DS Date: Tue, 29 Mar 2022 18:06:16 +0200 Subject: Fix the documentation of InvRef:get_lists() and clean up code (#12150) --- doc/lua_api.txt | 4 ++-- src/database/database-postgresql.cpp | 2 +- src/database/database-sqlite3.cpp | 4 ++-- src/inventory.cpp | 42 ------------------------------------ src/inventory.h | 26 +++++++++++++++------- src/script/common/c_content.cpp | 23 ++++++++++++-------- src/script/common/c_content.h | 6 ++++-- src/script/cpp_api/s_client.cpp | 10 +-------- src/script/lua_api/l_inventory.cpp | 23 +++++++++----------- src/script/lua_api/l_nodemeta.cpp | 12 ++++------- 10 files changed, 56 insertions(+), 96 deletions(-) (limited to 'src/script/cpp_api') diff --git a/doc/lua_api.txt b/doc/lua_api.txt index 029c53616..161bdd249 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -6355,9 +6355,9 @@ An `InvRef` is a reference to an inventory. * `set_width(listname, width)`: set width of list; currently used for crafting * `get_stack(listname, i)`: get a copy of stack index `i` in list * `set_stack(listname, i, stack)`: copy `stack` to index `i` in list -* `get_list(listname)`: return full list +* `get_list(listname)`: return full list (list of `ItemStack`s) * `set_list(listname, list)`: set full list (size will not change) -* `get_lists()`: returns list of inventory lists +* `get_lists()`: returns table that maps listnames to inventory lists * `set_lists(lists)`: sets inventory lists (size will not change) * `add_item(listname, stack)`: add item somewhere in list, returns leftover `ItemStack`. diff --git a/src/database/database-postgresql.cpp b/src/database/database-postgresql.cpp index 3469f4242..9d6501e68 100644 --- a/src/database/database-postgresql.cpp +++ b/src/database/database-postgresql.cpp @@ -495,7 +495,7 @@ void PlayerDatabasePostgreSQL::savePlayer(RemotePlayer *player) execPrepared("remove_player_inventories", 1, rmvalues); execPrepared("remove_player_inventory_items", 1, rmvalues); - std::vector inventory_lists = sao->getInventory()->getLists(); + const auto &inventory_lists = sao->getInventory()->getLists(); std::ostringstream oss; for (u16 i = 0; i < inventory_lists.size(); i++) { const InventoryList* list = inventory_lists[i]; diff --git a/src/database/database-sqlite3.cpp b/src/database/database-sqlite3.cpp index 1e63ae9d8..9521085e9 100644 --- a/src/database/database-sqlite3.cpp +++ b/src/database/database-sqlite3.cpp @@ -476,10 +476,10 @@ void PlayerDatabaseSQLite3::savePlayer(RemotePlayer *player) sqlite3_vrfy(sqlite3_step(m_stmt_player_remove_inventory_items), SQLITE_DONE); sqlite3_reset(m_stmt_player_remove_inventory_items); - std::vector inventory_lists = sao->getInventory()->getLists(); + const auto &inventory_lists = sao->getInventory()->getLists(); std::ostringstream oss; for (u16 i = 0; i < inventory_lists.size(); i++) { - const InventoryList* list = inventory_lists[i]; + const InventoryList *list = inventory_lists[i]; str_to_sqlite(m_stmt_player_add_inventory, 1, player->getName()); int_to_sqlite(m_stmt_player_add_inventory, 2, i); diff --git a/src/inventory.cpp b/src/inventory.cpp index d14b12f9d..6d2b7fba3 100644 --- a/src/inventory.cpp +++ b/src/inventory.cpp @@ -503,11 +503,6 @@ void InventoryList::deSerialize(std::istream &is) throw SerializationError(ss.str()); } -InventoryList::InventoryList(const InventoryList &other) -{ - *this = other; -} - InventoryList & InventoryList::operator = (const InventoryList &other) { m_items = other.m_items; @@ -535,21 +530,6 @@ bool InventoryList::operator == (const InventoryList &other) const return true; } -const std::string &InventoryList::getName() const -{ - return m_name; -} - -u32 InventoryList::getSize() const -{ - return m_items.size(); -} - -u32 InventoryList::getWidth() const -{ - return m_width; -} - u32 InventoryList::getUsedSlots() const { u32 num = 0; @@ -560,18 +540,6 @@ u32 InventoryList::getUsedSlots() const return num; } -const ItemStack& InventoryList::getItem(u32 i) const -{ - assert(i < m_size); // Pre-condition - return m_items[i]; -} - -ItemStack& InventoryList::getItem(u32 i) -{ - assert(i < m_size); // Pre-condition - return m_items[i]; -} - ItemStack InventoryList::changeItem(u32 i, const ItemStack &newitem) { if(i >= m_items.size()) @@ -960,16 +928,6 @@ InventoryList * Inventory::getList(const std::string &name) return m_lists[i]; } -std::vector Inventory::getLists() -{ - std::vector lists; - lists.reserve(m_lists.size()); - for (auto list : m_lists) { - lists.push_back(list); - } - return lists; -} - bool Inventory::deleteList(const std::string &name) { s32 i = getListIndex(name); diff --git a/src/inventory.h b/src/inventory.h index eb063d4ad..8b31de3a8 100644 --- a/src/inventory.h +++ b/src/inventory.h @@ -198,7 +198,7 @@ public: void serialize(std::ostream &os, bool incremental) const; void deSerialize(std::istream &is); - InventoryList(const InventoryList &other); + InventoryList(const InventoryList &other) { *this = other; } InventoryList & operator = (const InventoryList &other); bool operator == (const InventoryList &other) const; bool operator != (const InventoryList &other) const @@ -206,15 +206,25 @@ public: return !(*this == other); } - const std::string &getName() const; - u32 getSize() const; - u32 getWidth() const; + const std::string &getName() const { return m_name; } + u32 getSize() const { return static_cast(m_items.size()); } + u32 getWidth() const { return m_width; } // Count used slots u32 getUsedSlots() const; // Get reference to item - const ItemStack& getItem(u32 i) const; - ItemStack& getItem(u32 i); + const ItemStack &getItem(u32 i) const + { + assert(i < m_size); // Pre-condition + return m_items[i]; + } + ItemStack &getItem(u32 i) + { + assert(i < m_size); // Pre-condition + return m_items[i]; + } + // Get reference to all items + const std::vector &getItems() const { return m_items; } // Returns old item. Parameter can be an empty item. ItemStack changeItem(u32 i, const ItemStack &newitem); // Delete item @@ -271,7 +281,7 @@ public: private: std::vector m_items; std::string m_name; - u32 m_size; + u32 m_size; // always the same as m_items.size() u32 m_width = 0; IItemDefManager *m_itemdef; bool m_dirty = true; @@ -301,7 +311,7 @@ public: InventoryList * addList(const std::string &name, u32 size); InventoryList * getList(const std::string &name); const InventoryList * getList(const std::string &name) const; - std::vector getLists(); + const std::vector &getLists() const { return m_lists; } bool deleteList(const std::string &name); // A shorthand for adding items. Returns leftover item (possibly empty). ItemStack addItem(const std::string &listname, const ItemStack &newitem) diff --git a/src/script/common/c_content.cpp b/src/script/common/c_content.cpp index b6eaa6b13..36f4316ee 100644 --- a/src/script/common/c_content.cpp +++ b/src/script/common/c_content.cpp @@ -1351,17 +1351,22 @@ void push_tool_capabilities(lua_State *L, } /******************************************************************************/ -void push_inventory_list(lua_State *L, Inventory *inv, const char *name) +void push_inventory_list(lua_State *L, const InventoryList &invlist) { - InventoryList *invlist = inv->getList(name); - if(invlist == NULL){ - lua_pushnil(L); - return; + push_items(L, invlist.getItems()); +} + +/******************************************************************************/ +void push_inventory_lists(lua_State *L, const Inventory &inv) +{ + const auto &lists = inv.getLists(); + lua_createtable(L, 0, lists.size()); + for(const InventoryList *list : lists) { + const std::string &name = list->getName(); + lua_pushlstring(L, name.c_str(), name.size()); + push_inventory_list(L, *list); + lua_rawset(L, -3); } - std::vector items; - for(u32 i=0; igetSize(); i++) - items.push_back(invlist->getItem(i)); - push_items(L, items); } /******************************************************************************/ diff --git a/src/script/common/c_content.h b/src/script/common/c_content.h index e762604a4..11b39364f 100644 --- a/src/script/common/c_content.h +++ b/src/script/common/c_content.h @@ -55,6 +55,7 @@ struct ObjectProperties; struct SimpleSoundSpec; struct ServerSoundParams; class Inventory; +class InventoryList; struct NodeBox; struct ContentFeatures; struct TileDef; @@ -120,8 +121,9 @@ void push_object_properties (lua_State *L, ObjectProperties *prop); void push_inventory_list (lua_State *L, - Inventory *inv, - const char *name); + const InventoryList &invlist); +void push_inventory_lists (lua_State *L, + const Inventory &inv); void read_inventory_list (lua_State *L, int tableindex, Inventory *inv, const char *name, Server *srv, int forcesize=-1); diff --git a/src/script/cpp_api/s_client.cpp b/src/script/cpp_api/s_client.cpp index c889fffa0..b02a0c7be 100644 --- a/src/script/cpp_api/s_client.cpp +++ b/src/script/cpp_api/s_client.cpp @@ -281,15 +281,7 @@ bool ScriptApiClient::on_inventory_open(Inventory *inventory) lua_getglobal(L, "core"); lua_getfield(L, -1, "registered_on_inventory_open"); - std::vector lists = inventory->getLists(); - std::vector::iterator iter = lists.begin(); - lua_createtable(L, 0, lists.size()); - for (; iter != lists.end(); iter++) { - const char* name = (*iter)->getName().c_str(); - lua_pushstring(L, name); - push_inventory_list(L, inventory, name); - lua_rawset(L, -3); - } + push_inventory_lists(L, *inventory); try { runCallbacks(1, RUN_CALLBACKS_MODE_OR); diff --git a/src/script/lua_api/l_inventory.cpp b/src/script/lua_api/l_inventory.cpp index b0a4ee194..175047e58 100644 --- a/src/script/lua_api/l_inventory.cpp +++ b/src/script/lua_api/l_inventory.cpp @@ -214,11 +214,16 @@ int InvRef::l_get_list(lua_State *L) InvRef *ref = checkobject(L, 1); const char *listname = luaL_checkstring(L, 2); Inventory *inv = getinv(L, ref); - if(inv){ - push_inventory_list(L, inv, listname); - } else { + if (!inv) { lua_pushnil(L); + return 1; } + InventoryList *invlist = inv->getList(listname); + if (!invlist) { + lua_pushnil(L); + return 1; + } + push_inventory_list(L, *invlist); return 1; } @@ -242,7 +247,7 @@ int InvRef::l_set_list(lua_State *L) return 0; } -// get_lists(self) -> list of InventoryLists +// get_lists(self) -> table that maps listnames to InventoryLists int InvRef::l_get_lists(lua_State *L) { NO_MAP_LOCK_REQUIRED; @@ -251,15 +256,7 @@ int InvRef::l_get_lists(lua_State *L) if (!inv) { return 0; } - std::vector lists = inv->getLists(); - std::vector::iterator iter = lists.begin(); - lua_createtable(L, 0, lists.size()); - for (; iter != lists.end(); iter++) { - const char* name = (*iter)->getName().c_str(); - lua_pushstring(L, name); - push_inventory_list(L, inv, name); - lua_rawset(L, -3); - } + push_inventory_lists(L, *inv); return 1; } diff --git a/src/script/lua_api/l_nodemeta.cpp b/src/script/lua_api/l_nodemeta.cpp index 34760157d..1d052685e 100644 --- a/src/script/lua_api/l_nodemeta.cpp +++ b/src/script/lua_api/l_nodemeta.cpp @@ -127,18 +127,14 @@ void NodeMetaRef::handleToTable(lua_State *L, Metadata *_meta) // fields MetaDataRef::handleToTable(L, _meta); - NodeMetadata *meta = (NodeMetadata*) _meta; + NodeMetadata *meta = (NodeMetadata *) _meta; // inventory - lua_newtable(L); Inventory *inv = meta->getInventory(); if (inv) { - std::vector lists = inv->getLists(); - for(std::vector::const_iterator - i = lists.begin(); i != lists.end(); ++i) { - push_inventory_list(L, inv, (*i)->getName().c_str()); - lua_setfield(L, -2, (*i)->getName().c_str()); - } + push_inventory_lists(L, *inv); + } else { + lua_newtable(L); } lua_setfield(L, -2, "inventory"); } -- cgit v1.2.3 From 06d197cdd042392e1551e5e7244c61300a6bb4e3 Mon Sep 17 00:00:00 2001 From: Jude Melton-Houghton Date: Tue, 29 Mar 2022 12:07:00 -0400 Subject: Store vector metatable in registry --- builtin/common/tests/misc_helpers_spec.lua | 1 + builtin/common/tests/serialize_spec.lua | 1 + builtin/common/tests/vector_spec.lua | 2 +- builtin/common/vector.lua | 6 ++---- builtin/mainmenu/tests/serverlistmgr_spec.lua | 1 + games/devtest/mods/unittests/misc.lua | 12 ++++++++++++ src/script/common/c_converter.cpp | 19 +++---------------- src/script/common/c_internal.h | 1 + src/script/cpp_api/s_base.cpp | 8 ++++++++ src/script/cpp_api/s_security.cpp | 6 ++++++ 10 files changed, 36 insertions(+), 21 deletions(-) (limited to 'src/script/cpp_api') diff --git a/builtin/common/tests/misc_helpers_spec.lua b/builtin/common/tests/misc_helpers_spec.lua index b16987f0b..b11236860 100644 --- a/builtin/common/tests/misc_helpers_spec.lua +++ b/builtin/common/tests/misc_helpers_spec.lua @@ -1,4 +1,5 @@ _G.core = {} +_G.vector = {metatable = {}} dofile("builtin/common/vector.lua") dofile("builtin/common/misc_helpers.lua") diff --git a/builtin/common/tests/serialize_spec.lua b/builtin/common/tests/serialize_spec.lua index e46b7dcc5..69b2b567c 100644 --- a/builtin/common/tests/serialize_spec.lua +++ b/builtin/common/tests/serialize_spec.lua @@ -1,4 +1,5 @@ _G.core = {} +_G.vector = {metatable = {}} _G.setfenv = require 'busted.compatibility'.setfenv diff --git a/builtin/common/tests/vector_spec.lua b/builtin/common/tests/vector_spec.lua index 2f72f3383..25880236b 100644 --- a/builtin/common/tests/vector_spec.lua +++ b/builtin/common/tests/vector_spec.lua @@ -1,4 +1,4 @@ -_G.vector = {} +_G.vector = {metatable = {}} dofile("builtin/common/vector.lua") describe("vector", function() diff --git a/builtin/common/vector.lua b/builtin/common/vector.lua index 581d014e0..90010f6de 100644 --- a/builtin/common/vector.lua +++ b/builtin/common/vector.lua @@ -6,10 +6,8 @@ Note: The vector.*-functions must be able to accept old vectors that had no meta -- localize functions local setmetatable = setmetatable -vector = {} - -local metatable = {} -vector.metatable = metatable +-- vector.metatable is set by C++. +local metatable = vector.metatable local xyz = {"x", "y", "z"} diff --git a/builtin/mainmenu/tests/serverlistmgr_spec.lua b/builtin/mainmenu/tests/serverlistmgr_spec.lua index a091959fb..ab7a6c60c 100644 --- a/builtin/mainmenu/tests/serverlistmgr_spec.lua +++ b/builtin/mainmenu/tests/serverlistmgr_spec.lua @@ -1,4 +1,5 @@ _G.core = {} +_G.vector = {metatable = {}} _G.unpack = table.unpack _G.serverlistmgr = {} diff --git a/games/devtest/mods/unittests/misc.lua b/games/devtest/mods/unittests/misc.lua index cf4f92cfa..ba980866a 100644 --- a/games/devtest/mods/unittests/misc.lua +++ b/games/devtest/mods/unittests/misc.lua @@ -36,3 +36,15 @@ local function test_dynamic_media(cb, player) -- if the callback isn't called this test will just hang :shrug: end unittests.register("test_dynamic_media", test_dynamic_media, {async=true, player=true}) + +local function test_v3f_metatable(player) + assert(vector.check(player:get_pos())) +end +unittests.register("test_v3f_metatable", test_v3f_metatable, {player=true}) + +local function test_v3s16_metatable(player, pos) + local node = minetest.get_node(pos) + local found_pos = minetest.find_node_near(pos, 0, node.name, true) + assert(vector.check(found_pos)) +end +unittests.register("test_v3s16_metatable", test_v3s16_metatable, {map=true}) diff --git a/src/script/common/c_converter.cpp b/src/script/common/c_converter.cpp index 08fb9ad30..b5ff52f73 100644 --- a/src/script/common/c_converter.cpp +++ b/src/script/common/c_converter.cpp @@ -52,25 +52,12 @@ if (value < F1000_MIN || value > F1000_MAX) { \ /** - * A helper which sets (if available) the vector metatable from builtin as metatable - * for the table on top of the stack + * A helper which sets the vector metatable for the table on top of the stack */ static void set_vector_metatable(lua_State *L) { - // get vector.metatable - lua_getglobal(L, "vector"); - if (!lua_istable(L, -1)) { - // there is no global vector table - lua_pop(L, 1); - errorstream << "set_vector_metatable in c_converter.cpp: " << - "missing global vector table" << std::endl; - return; - } - lua_getfield(L, -1, "metatable"); - // set the metatable - lua_setmetatable(L, -3); - // pop vector global - lua_pop(L, 1); + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_VECTOR_METATABLE); + lua_setmetatable(L, -2); } void push_v3f(lua_State *L, v3f p) diff --git a/src/script/common/c_internal.h b/src/script/common/c_internal.h index 94cfd61fb..c43db34aa 100644 --- a/src/script/common/c_internal.h +++ b/src/script/common/c_internal.h @@ -55,6 +55,7 @@ extern "C" { #define CUSTOM_RIDX_CURRENT_MOD_NAME (CUSTOM_RIDX_BASE + 2) #define CUSTOM_RIDX_BACKTRACE (CUSTOM_RIDX_BASE + 3) #define CUSTOM_RIDX_HTTP_API_LUA (CUSTOM_RIDX_BASE + 4) +#define CUSTOM_RIDX_VECTOR_METATABLE (CUSTOM_RIDX_BASE + 5) // Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata diff --git a/src/script/cpp_api/s_base.cpp b/src/script/cpp_api/s_base.cpp index f7b8a5102..595c9e540 100644 --- a/src/script/cpp_api/s_base.cpp +++ b/src/script/cpp_api/s_base.cpp @@ -121,6 +121,14 @@ ScriptApiBase::ScriptApiBase(ScriptingType type): lua_newtable(m_luastack); lua_setglobal(m_luastack, "core"); + // vector.metatable is stored in the registry for quick access from C++. + lua_newtable(m_luastack); + lua_rawseti(m_luastack, LUA_REGISTRYINDEX, CUSTOM_RIDX_VECTOR_METATABLE); + lua_newtable(m_luastack); + lua_rawgeti(m_luastack, LUA_REGISTRYINDEX, CUSTOM_RIDX_VECTOR_METATABLE); + lua_setfield(m_luastack, -2, "metatable"); + lua_setglobal(m_luastack, "vector"); + if (m_type == ScriptingType::Client) lua_pushstring(m_luastack, "/"); else diff --git a/src/script/cpp_api/s_security.cpp b/src/script/cpp_api/s_security.cpp index a6c5114b2..f68cd1777 100644 --- a/src/script/cpp_api/s_security.cpp +++ b/src/script/cpp_api/s_security.cpp @@ -98,6 +98,7 @@ void ScriptApiSecurity::initializeSecurity() "type", "unpack", "_VERSION", + "vector", "xpcall", }; static const char *whitelist_tables[] = { @@ -254,6 +255,10 @@ void ScriptApiSecurity::initializeSecurity() lua_pushnil(L); lua_setfield(L, old_globals, "core"); + // 'vector' as well. + lua_pushnil(L); + lua_setfield(L, old_globals, "vector"); + lua_pop(L, 1); // Pop globals_backup @@ -296,6 +301,7 @@ void ScriptApiSecurity::initializeSecurityClient() "type", "unpack", "_VERSION", + "vector", "xpcall", // Completely safe libraries "coroutine", -- cgit v1.2.3 From 7993909fabce4f796ca67b5a8139936667de25df Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Wed, 1 Dec 2021 18:54:12 -0500 Subject: Spacing fixes --- CMakeLists.txt | 2 +- client/shaders/nodes_shader/opengl_fragment.glsl | 2 +- client/shaders/nodes_shader/opengl_vertex.glsl | 4 +- clientmods/preview/mod.conf | 2 +- cmake/Modules/FindSQLite3.cmake | 2 +- cmake/Modules/FindVorbis.cmake | 17 ++++--- doc/lgpl-2.1.txt | 18 ++++---- minetest.conf.example | 58 ++++++++++++------------ src/CMakeLists.txt | 2 +- src/client/CMakeLists.txt | 4 +- src/client/clientmap.cpp | 4 +- src/client/imagefilters.cpp | 2 +- src/client/render/core.cpp | 2 +- src/clientiface.h | 2 +- src/collision.cpp | 4 +- src/config.h | 2 +- src/database/database-leveldb.cpp | 2 +- src/inventorymanager.cpp | 2 +- src/irrlicht_changes/CGUITTFont.cpp | 2 +- src/mapgen/dungeongen.cpp | 2 +- src/mapgen/mapgen_flat.cpp | 2 +- src/mapgen/mg_biome.cpp | 2 +- src/mapgen/mg_ore.cpp | 4 +- src/network/connection.h | 4 +- src/script/cpp_api/s_env.cpp | 4 +- src/script/lua_api/l_env.cpp | 4 +- src/script/lua_api/l_env.h | 2 +- src/serverenvironment.cpp | 2 +- src/util/ieee_float.cpp | 2 +- src/util/string.h | 2 +- util/generate-texture-normals.sh | 2 +- 31 files changed, 82 insertions(+), 83 deletions(-) (limited to 'src/script/cpp_api') diff --git a/CMakeLists.txt b/CMakeLists.txt index 827191835..5dfc857d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,7 +69,7 @@ if(NOT "${IRRLICHTMT_BUILD_DIR}" STREQUAL "") find_package(IrrlichtMt QUIET PATHS "${IRRLICHTMT_BUILD_DIR}" NO_DEFAULT_PATH -) + ) if(NOT TARGET IrrlichtMt::IrrlichtMt) # find_package() searches certain subdirectories. ${PATH}/cmake is not diff --git a/client/shaders/nodes_shader/opengl_fragment.glsl b/client/shaders/nodes_shader/opengl_fragment.glsl index 4d0d107d1..fea350788 100644 --- a/client/shaders/nodes_shader/opengl_fragment.glsl +++ b/client/shaders/nodes_shader/opengl_fragment.glsl @@ -557,6 +557,6 @@ void main(void) - fogShadingParameter * length(eyeVec) / fogDistance, 0.0, 1.0); col = mix(skyBgColor, col, clarity); col = vec4(col.rgb, base.a); - + gl_FragColor = col; } diff --git a/client/shaders/nodes_shader/opengl_vertex.glsl b/client/shaders/nodes_shader/opengl_vertex.glsl index 935fbf043..8c7e27459 100644 --- a/client/shaders/nodes_shader/opengl_vertex.glsl +++ b/client/shaders/nodes_shader/opengl_vertex.glsl @@ -199,13 +199,13 @@ void main(void) vec3 nNormal = normalize(vNormal); cosLight = dot(nNormal, -v_LightDirection); - // Calculate normal offset scale based on the texel size adjusted for + // Calculate normal offset scale based on the texel size adjusted for // curvature of the SM texture. This code must be change together with // getPerspectiveFactor or any light-space transformation. vec3 eyeToVertex = worldPosition - eyePosition + cameraOffset; // Distance from the vertex to the player float distanceToPlayer = length(eyeToVertex - v_LightDirection * dot(eyeToVertex, v_LightDirection)) / f_shadowfar; - // perspective factor estimation according to the + // perspective factor estimation according to the float perspectiveFactor = distanceToPlayer * xyPerspectiveBias0 + xyPerspectiveBias1; float texelSize = f_shadowfar * perspectiveFactor * perspectiveFactor / (f_textureresolution * xyPerspectiveBias1 - perspectiveFactor * xyPerspectiveBias0); diff --git a/clientmods/preview/mod.conf b/clientmods/preview/mod.conf index 4e56ec293..23a5c3e90 100644 --- a/clientmods/preview/mod.conf +++ b/clientmods/preview/mod.conf @@ -1 +1 @@ -name = preview +name = preview diff --git a/cmake/Modules/FindSQLite3.cmake b/cmake/Modules/FindSQLite3.cmake index b23553a80..8a66cb241 100644 --- a/cmake/Modules/FindSQLite3.cmake +++ b/cmake/Modules/FindSQLite3.cmake @@ -1,4 +1,4 @@ -mark_as_advanced(SQLITE3_LIBRARY SQLITE3_INCLUDE_DIR) +mark_as_advanced(SQLITE3_LIBRARY SQLITE3_INCLUDE_DIR) find_path(SQLITE3_INCLUDE_DIR sqlite3.h) diff --git a/cmake/Modules/FindVorbis.cmake b/cmake/Modules/FindVorbis.cmake index e5fe7f25e..222ddd9d5 100644 --- a/cmake/Modules/FindVorbis.cmake +++ b/cmake/Modules/FindVorbis.cmake @@ -29,18 +29,17 @@ else(NOT GP2XWIZ) find_package_handle_standard_args(Vorbis DEFAULT_MSG VORBIS_INCLUDE_DIR VORBIS_LIBRARY) endif(NOT GP2XWIZ) - + if(VORBIS_FOUND) - if(NOT GP2XWIZ) - set(VORBIS_LIBRARIES ${VORBISFILE_LIBRARY} ${VORBIS_LIBRARY} - ${OGG_LIBRARY}) - else(NOT GP2XWIZ) - set(VORBIS_LIBRARIES ${VORBIS_LIBRARY}) - endif(NOT GP2XWIZ) + if(NOT GP2XWIZ) + set(VORBIS_LIBRARIES ${VORBISFILE_LIBRARY} ${VORBIS_LIBRARY} + ${OGG_LIBRARY}) + else(NOT GP2XWIZ) + set(VORBIS_LIBRARIES ${VORBIS_LIBRARY}) + endif(NOT GP2XWIZ) else(VORBIS_FOUND) - set(VORBIS_LIBRARIES) + set(VORBIS_LIBRARIES) endif(VORBIS_FOUND) mark_as_advanced(OGG_INCLUDE_DIR VORBIS_INCLUDE_DIR) mark_as_advanced(OGG_LIBRARY VORBIS_LIBRARY VORBISFILE_LIBRARY) - diff --git a/doc/lgpl-2.1.txt b/doc/lgpl-2.1.txt index 4362b4915..e5ab03e12 100644 --- a/doc/lgpl-2.1.txt +++ b/doc/lgpl-2.1.txt @@ -55,7 +55,7 @@ modified by someone else and passed on, the recipients should know that what they have is not the original version, so that the original author's reputation will not be affected by problems that might be introduced by others. - + Finally, software patents pose a constant threat to the existence of any free program. We wish to make sure that a company cannot effectively restrict the users of a free program by obtaining a @@ -111,7 +111,7 @@ modification follow. Pay close attention to the difference between a "work based on the library" and a "work that uses the library". The former contains code derived from the library, whereas the latter must be combined with the library in order to run. - + GNU LESSER GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION @@ -158,7 +158,7 @@ Library. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. - + 2. You may modify your copy or copies of the Library or any portion of it, thus forming a work based on the Library, and copy and distribute such modifications or work under the terms of Section 1 @@ -216,7 +216,7 @@ instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify that version instead if you wish.) Do not make any other change in these notices. - + Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all subsequent copies and derivative works made from that copy. @@ -267,7 +267,7 @@ Library will still fall under Section 6.) distribute the object code for the work under the terms of Section 6. Any executables containing that work also fall under Section 6, whether or not they are linked directly with the Library itself. - + 6. As an exception to the Sections above, you may also combine or link a "work that uses the Library" with the Library to produce a work containing portions of the Library, and distribute that work @@ -329,7 +329,7 @@ restrictions of other proprietary libraries that do not normally accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you distribute. - + 7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library facilities not covered by this License, and distribute such a combined @@ -370,7 +370,7 @@ subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties with this License. - + 11. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or @@ -422,7 +422,7 @@ conditions either of that version or of any later version published by the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by the Free Software Foundation. - + 14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, write to the author to ask for permission. For software which is @@ -456,7 +456,7 @@ SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS - + How to Apply These Terms to Your New Libraries If you develop a new library, and you want it to be of the greatest diff --git a/minetest.conf.example b/minetest.conf.example index ed2ebc969..21aeb3546 100644 --- a/minetest.conf.example +++ b/minetest.conf.example @@ -1385,7 +1385,7 @@ # ask_reconnect_on_crash = false # From how far clients know about objects, stated in mapblocks (16 nodes). -# +# # Setting this larger than active_block_range will also cause the server # to maintain active objects up to this distance in the direction the # player is looking. (This can avoid mobs suddenly disappearing from view) @@ -1938,7 +1938,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -1951,7 +1951,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining giant caverns. @@ -1964,7 +1964,7 @@ # octaves = 5, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining terrain. @@ -1990,7 +1990,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Mapgen V6 @@ -2377,7 +2377,7 @@ # octaves = 5, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining structure of river canyon walls. @@ -2390,7 +2390,7 @@ # octaves = 4, # persistence = 0.75, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining structure of floatlands. @@ -2406,7 +2406,7 @@ # octaves = 4, # persistence = 0.75, # lacunarity = 1.618, -# flags = +# flags = # } # 3D noise defining giant caverns. @@ -2419,7 +2419,7 @@ # octaves = 5, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # First of two 3D noises that together define tunnels. @@ -2432,7 +2432,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -2445,7 +2445,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise that determines number of dungeons per mapchunk. @@ -2458,7 +2458,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Mapgen Carpathian @@ -2701,7 +2701,7 @@ # octaves = 5, # persistence = 0.55, # lacunarity = 2.0, -# flags = +# flags = # } # First of two 3D noises that together define tunnels. @@ -2714,7 +2714,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -2727,7 +2727,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining giant caverns. @@ -2740,7 +2740,7 @@ # octaves = 5, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise that determines number of dungeons per mapchunk. @@ -2753,7 +2753,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Mapgen Flat @@ -2875,7 +2875,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -2888,7 +2888,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining giant caverns. @@ -2901,7 +2901,7 @@ # octaves = 5, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise that determines number of dungeons per mapchunk. @@ -2914,7 +2914,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Mapgen Fractal @@ -3088,7 +3088,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -3101,7 +3101,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise that determines number of dungeons per mapchunk. @@ -3114,7 +3114,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Mapgen Valleys @@ -3204,7 +3204,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -3217,7 +3217,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # The depth of dirt or other biome filler node. @@ -3243,7 +3243,7 @@ # octaves = 6, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # Defines large-scale river channel structure. @@ -3295,7 +3295,7 @@ # octaves = 6, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } # Amplifies the valleys. @@ -3334,7 +3334,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Advanced diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2de68a8f0..0323603fc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -711,7 +711,7 @@ else() # Move text segment below LuaJIT's 47-bit limit (see issue #9367) if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") # FreeBSD uses lld, and lld does not support -Ttext-segment, suggesting - # --image-base instead. Not sure if it's equivalent change for the purpose + # --image-base instead. Not sure if it's equivalent change for the purpose # but at least if fixes build on FreeBSD/aarch64 # XXX: the condition should also be changed to check for lld regardless of # os, bit CMake doesn't have anything like CMAKE_LINKER_IS_LLD yet diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 8d058852a..656ad45ce 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -60,7 +60,7 @@ set(client_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/wieldmesh.cpp ${CMAKE_CURRENT_SOURCE_DIR}/shadows/dynamicshadows.cpp ${CMAKE_CURRENT_SOURCE_DIR}/shadows/dynamicshadowsrender.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/shadows/shadowsshadercallbacks.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/shadows/shadowsScreenQuad.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/shadows/shadowsshadercallbacks.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/shadows/shadowsScreenQuad.cpp PARENT_SCOPE ) diff --git a/src/client/clientmap.cpp b/src/client/clientmap.cpp index 99ff0aefb..10967c0cb 100644 --- a/src/client/clientmap.cpp +++ b/src/client/clientmap.cpp @@ -847,12 +847,12 @@ void ClientMap::renderMapShadows(video::IVideoDriver *driver, vertex_count += buf->getIndexCount(); } - // restore the driver material state + // restore the driver material state video::SMaterial clean; clean.BlendOperation = video::EBO_ADD; driver->setMaterial(clean); // reset material to defaults driver->draw3DLine(v3f(), v3f(), video::SColor(0)); - + g_profiler->avg(prefix + "draw meshes [ms]", draw.stop(true)); g_profiler->avg(prefix + "vertices drawn [#]", vertex_count); g_profiler->avg(prefix + "drawcalls [#]", drawcall_count); diff --git a/src/client/imagefilters.cpp b/src/client/imagefilters.cpp index b62e336f7..c9d1504ad 100644 --- a/src/client/imagefilters.cpp +++ b/src/client/imagefilters.cpp @@ -124,7 +124,7 @@ void imageCleanTransparent(video::IImage *src, u32 threshold) // Ignore pixels we haven't processed if (!bitmap.get(sx, sy)) continue; - + // Add RGB values weighted by alpha IF the pixel is opaque, otherwise // use full weight since we want to propagate colors. video::SColor d = src->getPixel(sx, sy); diff --git a/src/client/render/core.cpp b/src/client/render/core.cpp index f151832f3..c67f297c4 100644 --- a/src/client/render/core.cpp +++ b/src/client/render/core.cpp @@ -103,7 +103,7 @@ void RenderingCore::drawHUD() if (show_hud) { if (draw_crosshair) hud->drawCrosshair(); - + hud->drawHotbar(client->getEnv().getLocalPlayer()->getWieldIndex()); hud->drawLuaElements(camera->getOffset()); camera->drawNametags(); diff --git a/src/clientiface.h b/src/clientiface.h index 1be9c972a..947952e82 100644 --- a/src/clientiface.h +++ b/src/clientiface.h @@ -341,7 +341,7 @@ public: u8 getMinor() const { return m_version_minor; } u8 getPatch() const { return m_version_patch; } const std::string &getFullVer() const { return m_full_version; } - + void setLangCode(const std::string &code) { m_lang_code = code; } const std::string &getLangCode() const { return m_lang_code; } diff --git a/src/collision.cpp b/src/collision.cpp index ccc3a058d..be135a225 100644 --- a/src/collision.cpp +++ b/src/collision.cpp @@ -153,7 +153,7 @@ CollisionAxis axisAlignedCollision( (std::max(movingbox.MaxEdge.Z + speed.Z * time, staticbox.MaxEdge.Z) - std::min(movingbox.MinEdge.Z + speed.Z * time, staticbox.MinEdge.Z) - relbox.MinEdge.Z < 0) - ) + ) return COLLISION_AXIS_X; } } else { @@ -180,7 +180,7 @@ CollisionAxis axisAlignedCollision( (std::max(movingbox.MaxEdge.Y + speed.Y * time, staticbox.MaxEdge.Y) - std::min(movingbox.MinEdge.Y + speed.Y * time, staticbox.MinEdge.Y) - relbox.MinEdge.Y < 0) - ) + ) return COLLISION_AXIS_Z; } } diff --git a/src/config.h b/src/config.h index 5e1164642..8d920b150 100644 --- a/src/config.h +++ b/src/config.h @@ -16,7 +16,7 @@ #define PROJECT_NAME_C "Minetest" #define STATIC_SHAREDIR "" #define VERSION_STRING STR(VERSION_MAJOR) "." STR(VERSION_MINOR) "." STR(VERSION_PATCH) STR(VERSION_EXTRA) -#ifdef NDEBUG + #ifdef NDEBUG #define BUILD_TYPE "Release" #else #define BUILD_TYPE "Debug" diff --git a/src/database/database-leveldb.cpp b/src/database/database-leveldb.cpp index 39f4c8442..6e59daab3 100644 --- a/src/database/database-leveldb.cpp +++ b/src/database/database-leveldb.cpp @@ -74,7 +74,7 @@ void Database_LevelDB::loadBlock(const v3s16 &pos, std::string *block) i64tos(getBlockAsInteger(pos)), block); if (!status.ok()) - block->clear(); + block->clear(); } bool Database_LevelDB::deleteBlock(const v3s16 &pos) diff --git a/src/inventorymanager.cpp b/src/inventorymanager.cpp index a159bf786..ecdb56a97 100644 --- a/src/inventorymanager.cpp +++ b/src/inventorymanager.cpp @@ -172,7 +172,7 @@ void IMoveAction::onPutAndOnTake(const ItemStack &src_item, ServerActiveObject * sa->player_inventory_OnPut(*this, src_item, player); else assert(false); - + if (from_inv.type == InventoryLocation::DETACHED) sa->detached_inventory_OnTake(*this, src_item, player); else if (from_inv.type == InventoryLocation::NODEMETA) diff --git a/src/irrlicht_changes/CGUITTFont.cpp b/src/irrlicht_changes/CGUITTFont.cpp index e785ea837..787f4cd5a 100644 --- a/src/irrlicht_changes/CGUITTFont.cpp +++ b/src/irrlicht_changes/CGUITTFont.cpp @@ -478,7 +478,7 @@ CGUITTGlyphPage* CGUITTFont::getLastGlyphPage() const CGUITTGlyphPage* CGUITTFont::createGlyphPage(const u8& pixel_mode) { CGUITTGlyphPage* page = 0; - + // Name of our page. io::path name("TTFontGlyphPage_"); name += tt_face->family_name; diff --git a/src/mapgen/dungeongen.cpp b/src/mapgen/dungeongen.cpp index acdb1a0f0..1d439abeb 100644 --- a/src/mapgen/dungeongen.cpp +++ b/src/mapgen/dungeongen.cpp @@ -71,7 +71,7 @@ DungeonGen::DungeonGen(const NodeDefManager *ndef, dp.num_dungeons = 1; dp.notifytype = GENNOTIFY_DUNGEON; - dp.np_alt_wall = + dp.np_alt_wall = NoiseParams(-0.4, 1.0, v3f(40.0, 40.0, 40.0), 32474, 6, 1.1, 2.0); } } diff --git a/src/mapgen/mapgen_flat.cpp b/src/mapgen/mapgen_flat.cpp index 342455029..6b249ea1f 100644 --- a/src/mapgen/mapgen_flat.cpp +++ b/src/mapgen/mapgen_flat.cpp @@ -177,7 +177,7 @@ void MapgenFlatParams::setDefaultSettings(Settings *settings) int MapgenFlat::getSpawnLevelAtPoint(v2s16 p) { s16 stone_level = ground_level; - float n_terrain = + float n_terrain = ((spflags & MGFLAT_LAKES) || (spflags & MGFLAT_HILLS)) ? NoisePerlin2D(&noise_terrain->np, p.X, p.Y, seed) : 0.0f; diff --git a/src/mapgen/mg_biome.cpp b/src/mapgen/mg_biome.cpp index f08cc190f..8b4c96cd5 100644 --- a/src/mapgen/mg_biome.cpp +++ b/src/mapgen/mg_biome.cpp @@ -273,7 +273,7 @@ Biome *BiomeGenOriginal::calcBiomeFromNoise(float heat, float humidity, v3s16 po pos.Y - biome_closest_blend->max_pos.Y) return biome_closest_blend; - return (biome_closest) ? biome_closest : (Biome *)m_bmgr->getRaw(BIOME_NONE); + return (biome_closest) ? biome_closest : (Biome *)m_bmgr->getRaw(BIOME_NONE); } diff --git a/src/mapgen/mg_ore.cpp b/src/mapgen/mg_ore.cpp index 5814f433a..4f0c35548 100644 --- a/src/mapgen/mg_ore.cpp +++ b/src/mapgen/mg_ore.cpp @@ -498,8 +498,8 @@ void OreVein::generate(MMVManip *vm, int mapseed, u32 blockseed, } // randval ranges from -1..1 - /* - Note: can generate values slightly larger than 1 + /* + Note: can generate values slightly larger than 1 but this can't be changed as mapgen must be deterministic accross versions. */ float randval = (float)pr.next() / float(pr.RANDOM_RANGE / 2) - 1.f; diff --git a/src/network/connection.h b/src/network/connection.h index 1afb4ae84..88e323bb1 100644 --- a/src/network/connection.h +++ b/src/network/connection.h @@ -752,8 +752,8 @@ protected: void putEvent(ConnectionEventPtr e); void TriggerSend(); - - bool ConnectedToServer() + + bool ConnectedToServer() { return getPeerNoEx(PEER_ID_SERVER) != nullptr; } diff --git a/src/script/cpp_api/s_env.cpp b/src/script/cpp_api/s_env.cpp index 874c37b6e..af68f689f 100644 --- a/src/script/cpp_api/s_env.cpp +++ b/src/script/cpp_api/s_env.cpp @@ -140,10 +140,10 @@ void ScriptApiEnv::initializeEnvironment(ServerEnvironment *env) bool simple_catch_up = true; getboolfield(L, current_abm, "catch_up", simple_catch_up); - + s16 min_y = INT16_MIN; getintfield(L, current_abm, "min_y", min_y); - + s16 max_y = INT16_MAX; getintfield(L, current_abm, "max_y", max_y); diff --git a/src/script/lua_api/l_env.cpp b/src/script/lua_api/l_env.cpp index 54821df24..7640f2782 100644 --- a/src/script/lua_api/l_env.cpp +++ b/src/script/lua_api/l_env.cpp @@ -757,7 +757,7 @@ int ModApiEnvMod::l_get_objects_in_area(lua_State *L) { GET_ENV_PTR; ScriptApiBase *script = getScriptApiBase(L); - + v3f minp = read_v3f(L, 1) * BS; v3f maxp = read_v3f(L, 2) * BS; aabb3f box(minp, maxp); @@ -1409,7 +1409,7 @@ int ModApiEnvMod::l_compare_block_status(lua_State *L) v3s16 nodepos = check_v3s16(L, 1); std::string condition_s = luaL_checkstring(L, 2); auto status = env->getBlockStatus(getNodeBlockPos(nodepos)); - + int condition_i = -1; if (!string_to_enum(es_BlockStatusType, condition_i, condition_s)) return 0; // Unsupported diff --git a/src/script/lua_api/l_env.h b/src/script/lua_api/l_env.h index 67c76faae..a7d406d2a 100644 --- a/src/script/lua_api/l_env.h +++ b/src/script/lua_api/l_env.h @@ -114,7 +114,7 @@ private: // get_objects_inside_radius(pos, radius) static int l_get_objects_inside_radius(lua_State *L); - + // get_objects_in_area(pos, minp, maxp) static int l_get_objects_in_area(lua_State *L); diff --git a/src/serverenvironment.cpp b/src/serverenvironment.cpp index f3711652c..34a1e33e5 100644 --- a/src/serverenvironment.cpp +++ b/src/serverenvironment.cpp @@ -892,7 +892,7 @@ public: for (ActiveABM &aabm : *m_aabms[c]) { if ((p.Y < aabm.min_y) || (p.Y > aabm.max_y)) continue; - + if (myrand() % aabm.chance != 0) continue; diff --git a/src/util/ieee_float.cpp b/src/util/ieee_float.cpp index 887258921..b73763c55 100644 --- a/src/util/ieee_float.cpp +++ b/src/util/ieee_float.cpp @@ -39,7 +39,7 @@ f32 u32Tof32Slow(u32 i) if (exp == 0xFF) { // Inf/NaN if (imant == 0) { - if (std::numeric_limits::has_infinity) + if (std::numeric_limits::has_infinity) return sign ? -std::numeric_limits::infinity() : std::numeric_limits::infinity(); return sign ? std::numeric_limits::max() : diff --git a/src/util/string.h b/src/util/string.h index 8a9e83f22..f4ca1a7de 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -410,7 +410,7 @@ DEFINE_STD_TOSTRING_FLOATINGPOINT(long double) template inline wstring to_wstring(T val) { - return utf8_to_wide(to_string(val)); + return utf8_to_wide(to_string(val)); } } #endif diff --git a/util/generate-texture-normals.sh b/util/generate-texture-normals.sh index 6279dfa69..b2fcbf77b 100755 --- a/util/generate-texture-normals.sh +++ b/util/generate-texture-normals.sh @@ -197,7 +197,7 @@ normalMap() (gimp-convert-rgb image) () ) - (plug-in-normalmap + (plug-in-normalmap RUN-NONINTERACTIVE image drawable -- cgit v1.2.3 From e7659883cc6fca343785da2a1af3890ae273abbf Mon Sep 17 00:00:00 2001 From: sfan5 Date: Mon, 2 May 2022 20:55:04 +0200 Subject: Async environment for mods to do concurrent tasks (#11131) --- builtin/async/game.lua | 46 ++ builtin/async/init.lua | 11 - builtin/async/mainmenu.lua | 9 + builtin/game/async.lua | 22 + builtin/game/init.lua | 1 + builtin/game/misc.lua | 29 ++ builtin/init.lua | 6 +- doc/lua_api.txt | 62 +++ games/devtest/mods/unittests/async_env.lua | 149 ++++++ games/devtest/mods/unittests/init.lua | 1 + games/devtest/mods/unittests/inside_async_env.lua | 15 + src/map.cpp | 33 ++ src/map.h | 17 +- src/script/common/CMakeLists.txt | 1 + src/script/common/c_internal.cpp | 14 + src/script/common/c_internal.h | 5 + src/script/common/c_packer.cpp | 583 ++++++++++++++++++++++ src/script/common/c_packer.h | 123 +++++ src/script/cpp_api/s_async.cpp | 183 +++++-- src/script/cpp_api/s_async.h | 54 +- src/script/lua_api/l_craft.cpp | 8 + src/script/lua_api/l_craft.h | 1 + src/script/lua_api/l_internal.h | 2 +- src/script/lua_api/l_item.cpp | 25 + src/script/lua_api/l_item.h | 7 +- src/script/lua_api/l_noise.cpp | 53 ++ src/script/lua_api/l_noise.h | 6 + src/script/lua_api/l_server.cpp | 85 ++++ src/script/lua_api/l_server.h | 10 + src/script/lua_api/l_util.cpp | 5 + src/script/lua_api/l_util.h | 2 - src/script/lua_api/l_vmanip.cpp | 33 ++ src/script/lua_api/l_vmanip.h | 3 + src/script/scripting_server.cpp | 67 ++- src/script/scripting_server.h | 17 + src/server.cpp | 4 + src/server.h | 8 +- src/serverenvironment.cpp | 2 + src/util/basic_macros.h | 8 +- 39 files changed, 1654 insertions(+), 56 deletions(-) create mode 100644 builtin/async/game.lua delete mode 100644 builtin/async/init.lua create mode 100644 builtin/async/mainmenu.lua create mode 100644 builtin/game/async.lua create mode 100644 games/devtest/mods/unittests/async_env.lua create mode 100644 games/devtest/mods/unittests/inside_async_env.lua create mode 100644 src/script/common/c_packer.cpp create mode 100644 src/script/common/c_packer.h (limited to 'src/script/cpp_api') diff --git a/builtin/async/game.lua b/builtin/async/game.lua new file mode 100644 index 000000000..212a33e17 --- /dev/null +++ b/builtin/async/game.lua @@ -0,0 +1,46 @@ +core.log("info", "Initializing asynchronous environment (game)") + +local function pack2(...) + return {n=select('#', ...), ...} +end + +-- Entrypoint to run async jobs, called by C++ +function core.job_processor(func, params) + local retval = pack2(func(unpack(params, 1, params.n))) + + return retval +end + +-- Import a bunch of individual files from builtin/game/ +local gamepath = core.get_builtin_path() .. "game" .. DIR_DELIM + +dofile(gamepath .. "constants.lua") +dofile(gamepath .. "item_s.lua") +dofile(gamepath .. "misc_s.lua") +dofile(gamepath .. "features.lua") +dofile(gamepath .. "voxelarea.lua") + +-- Transfer of globals +do + assert(core.transferred_globals) + local all = core.deserialize(core.transferred_globals, true) + core.transferred_globals = nil + + -- reassemble other tables + all.registered_nodes = {} + all.registered_craftitems = {} + all.registered_tools = {} + for k, v in pairs(all.registered_items) do + if v.type == "node" then + all.registered_nodes[k] = v + elseif v.type == "craftitem" then + all.registered_craftitems[k] = v + elseif v.type == "tool" then + all.registered_tools[k] = v + end + end + + for k, v in pairs(all) do + core[k] = v + end +end diff --git a/builtin/async/init.lua b/builtin/async/init.lua deleted file mode 100644 index 3803994d6..000000000 --- a/builtin/async/init.lua +++ /dev/null @@ -1,11 +0,0 @@ - -core.log("info", "Initializing Asynchronous environment") - -function core.job_processor(func, serialized_param) - local param = core.deserialize(serialized_param) - - local retval = core.serialize(func(param)) - - return retval or core.serialize(nil) -end - diff --git a/builtin/async/mainmenu.lua b/builtin/async/mainmenu.lua new file mode 100644 index 000000000..0e9c222d1 --- /dev/null +++ b/builtin/async/mainmenu.lua @@ -0,0 +1,9 @@ +core.log("info", "Initializing asynchronous environment") + +function core.job_processor(func, serialized_param) + local param = core.deserialize(serialized_param) + + local retval = core.serialize(func(param)) + + return retval or core.serialize(nil) +end diff --git a/builtin/game/async.lua b/builtin/game/async.lua new file mode 100644 index 000000000..469f179d7 --- /dev/null +++ b/builtin/game/async.lua @@ -0,0 +1,22 @@ + +core.async_jobs = {} + +function core.async_event_handler(jobid, retval) + local callback = core.async_jobs[jobid] + assert(type(callback) == "function") + callback(unpack(retval, 1, retval.n)) + core.async_jobs[jobid] = nil +end + +function core.handle_async(func, callback, ...) + assert(type(func) == "function" and type(callback) == "function", + "Invalid minetest.handle_async invocation") + local args = {n = select("#", ...), ...} + local mod_origin = core.get_last_run_mod() + + local jobid = core.do_async_callback(func, args, mod_origin) + core.async_jobs[jobid] = callback + + return true +end + diff --git a/builtin/game/init.lua b/builtin/game/init.lua index c5f08113b..68d6a10f8 100644 --- a/builtin/game/init.lua +++ b/builtin/game/init.lua @@ -34,5 +34,6 @@ dofile(gamepath .. "voxelarea.lua") dofile(gamepath .. "forceloading.lua") dofile(gamepath .. "statbars.lua") dofile(gamepath .. "knockback.lua") +dofile(gamepath .. "async.lua") profiler = nil diff --git a/builtin/game/misc.lua b/builtin/game/misc.lua index 18d5a7310..9f5e3312b 100644 --- a/builtin/game/misc.lua +++ b/builtin/game/misc.lua @@ -235,3 +235,32 @@ end -- Used for callback handling with dynamic_add_media core.dynamic_media_callbacks = {} + + +-- Transfer of certain globals into async environment +-- see builtin/async/game.lua for the other side + +local function copy_filtering(t, seen) + if type(t) == "userdata" or type(t) == "function" then + return true -- don't use nil so presence can still be detected + elseif type(t) ~= "table" then + return t + end + local n = {} + seen = seen or {} + seen[t] = n + for k, v in pairs(t) do + local k_ = seen[k] or copy_filtering(k, seen) + local v_ = seen[v] or copy_filtering(v, seen) + n[k_] = v_ + end + return n +end + +function core.get_globals_to_transfer() + local all = { + registered_items = copy_filtering(core.registered_items), + registered_aliases = core.registered_aliases, + } + return core.serialize(all) +end diff --git a/builtin/init.lua b/builtin/init.lua index 7a9b5c427..869136016 100644 --- a/builtin/init.lua +++ b/builtin/init.lua @@ -56,8 +56,10 @@ elseif INIT == "mainmenu" then if not custom_loaded then dofile(core.get_mainmenu_path() .. DIR_DELIM .. "init.lua") end -elseif INIT == "async" then - dofile(asyncpath .. "init.lua") +elseif INIT == "async" then + dofile(asyncpath .. "mainmenu.lua") +elseif INIT == "async_game" then + dofile(asyncpath .. "game.lua") elseif INIT == "client" then dofile(clientpath .. "init.lua") else diff --git a/doc/lua_api.txt b/doc/lua_api.txt index f54672db7..339ce8a27 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -5767,6 +5767,68 @@ Timing * `job:cancel()` * Cancels the job function from being called +Async environment +----------------- + +The engine allows you to submit jobs to be ran in an isolated environment +concurrently with normal server operation. +A job consists of a function to be ran in the async environment, any amount of +arguments (will be serialized) and a callback that will be called with the return +value of the job function once it is finished. + +The async environment does *not* have access to the map, entities, players or any +globals defined in the 'usual' environment. Consequently, functions like +`minetest.get_node()` or `minetest.get_player_by_name()` simply do not exist in it. + +Arguments and return values passed through this can contain certain userdata +objects that will be seamlessly copied (not shared) to the async environment. +This allows you easy interoperability for delegating work to jobs. + +* `minetest.handle_async(func, callback, ...)`: + * Queue the function `func` to be ran in an async environment. + Note that there are multiple persistent workers and any of them may + end up running a given job. The engine will scale the amount of + worker threads automatically. + * When `func` returns the callback is called (in the normal environment) + with all of the return values as arguments. + * Optional: Variable number of arguments that are passed to `func` +* `minetest.register_async_dofile(path)`: + * Register a path to a Lua file to be imported when an async environment + is initialized. You can use this to preload code which you can then call + later using `minetest.handle_async()`. + +### List of APIs available in an async environment + +Classes: +* `ItemStack` +* `PerlinNoise` +* `PerlinNoiseMap` +* `PseudoRandom` +* `PcgRandom` +* `SecureRandom` +* `VoxelArea` +* `VoxelManip` + * only if transferred into environment; can't read/write to map +* `Settings` + +Class instances that can be transferred between environments: +* `ItemStack` +* `PerlinNoise` +* `PerlinNoiseMap` +* `VoxelManip` + +Functions: +* Standalone helpers such as logging, filesystem, encoding, + hashing or compression APIs +* `minetest.request_insecure_environment` (same restrictions apply) + +Variables: +* `minetest.settings` +* `minetest.registered_items`, `registered_nodes`, `registered_tools`, + `registered_craftitems` and `registered_aliases` + * with all functions and userdata values replaced by `true`, calling any + callbacks here is obviously not possible + Server ------ diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua new file mode 100644 index 000000000..aff1fc4d9 --- /dev/null +++ b/games/devtest/mods/unittests/async_env.lua @@ -0,0 +1,149 @@ +-- helper + +core.register_async_dofile(core.get_modpath(core.get_current_modname()) .. + DIR_DELIM .. "inside_async_env.lua") + +local function deepequal(a, b) + if type(a) == "function" then + return type(b) == "function" + elseif type(a) ~= "table" then + return a == b + elseif type(b) ~= "table" then + return false + end + for k, v in pairs(a) do + if not deepequal(v, b[k]) then + return false + end + end + for k, v in pairs(b) do + if not deepequal(a[k], v) then + return false + end + end + return true +end + +-- Object Passing / Serialization + +local test_object = { + name = "stairs:stair_glass", + type = "node", + groups = {oddly_breakable_by_hand = 3, cracky = 3, stair = 1}, + description = "Glass Stair", + sounds = { + dig = {name = "default_glass_footstep", gain = 0.5}, + footstep = {name = "default_glass_footstep", gain = 0.3}, + dug = {name = "default_break_glass", gain = 1} + }, + node_box = { + fixed = { + {-0.5, -0.5, -0.5, 0.5, 0, 0.5}, + {-0.5, 0, 0, 0.5, 0.5, 0.5} + }, + type = "fixed" + }, + tiles = { + {name = "stairs_glass_split.png", backface_culling = true}, + {name = "default_glass.png", backface_culling = true}, + {name = "stairs_glass_stairside.png^[transformFX", backface_culling = true} + }, + on_place = function(itemstack, placer) + return core.is_player(placer) + end, + sunlight_propagates = true, + is_ground_content = false, + light_source = 0, +} + +local function test_object_passing() + local tmp = core.serialize_roundtrip(test_object) + assert(deepequal(test_object, tmp)) + + -- Circular key, should error + tmp = {"foo", "bar"} + tmp[tmp] = true + assert(not pcall(core.serialize_roundtrip, tmp)) + + -- Circular value, should error + tmp = {"foo"} + tmp[2] = tmp + assert(not pcall(core.serialize_roundtrip, tmp)) +end +unittests.register("test_object_passing", test_object_passing) + +local function test_userdata_passing(_, pos) + -- basic userdata passing + local obj = table.copy(test_object.tiles[1]) + obj.test = ItemStack("default:cobble 99") + local tmp = core.serialize_roundtrip(obj) + assert(type(tmp.test) == "userdata") + assert(obj.test:to_string() == tmp.test:to_string()) + + -- object can't be passed, should error + obj = core.raycast(pos, pos) + assert(not pcall(core.serialize_roundtrip, obj)) + + -- VManip + local vm = core.get_voxel_manip(pos, pos) + local expect = vm:get_node_at(pos) + local vm2 = core.serialize_roundtrip(vm) + assert(deepequal(vm2:get_node_at(pos), expect)) +end +unittests.register("test_userdata_passing", test_userdata_passing, {map=true}) + +-- Asynchronous jobs + +local function test_handle_async(cb) + -- Basic test including mod name tracking and unittests.async_test() + -- which is defined inside_async_env.lua + local func = function(x) + return core.get_last_run_mod(), _VERSION, unittests[x]() + end + local expect = {core.get_last_run_mod(), _VERSION, true} + + core.handle_async(func, function(...) + if not deepequal(expect, {...}) then + cb("Values did not equal") + end + if core.get_last_run_mod() ~= expect[1] then + cb("Mod name not tracked correctly") + end + + -- Test passing of nil arguments and return values + core.handle_async(function(a, b) + return a, b + end, function(a, b) + if b ~= 123 then + cb("Argument went missing") + end + cb() + end, nil, 123) + end, "async_test") +end +unittests.register("test_handle_async", test_handle_async, {async=true}) + +local function test_userdata_passing2(cb, _, pos) + -- VManip: check transfer into other env + local vm = core.get_voxel_manip(pos, pos) + local expect = vm:get_node_at(pos) + + core.handle_async(function(vm_, pos_) + return vm_:get_node_at(pos_) + end, function(ret) + if not deepequal(expect, ret) then + cb("Node data mismatch (one-way)") + end + + -- VManip: test a roundtrip + core.handle_async(function(vm_) + return vm_ + end, function(vm2) + if not deepequal(expect, vm2:get_node_at(pos)) then + cb("Node data mismatch (roundtrip)") + end + cb() + end, vm) + end, vm, pos) +end +unittests.register("test_userdata_passing2", test_userdata_passing2, {map=true, async=true}) diff --git a/games/devtest/mods/unittests/init.lua b/games/devtest/mods/unittests/init.lua index 0754d507f..0608f2dd2 100644 --- a/games/devtest/mods/unittests/init.lua +++ b/games/devtest/mods/unittests/init.lua @@ -175,6 +175,7 @@ dofile(modpath .. "/misc.lua") dofile(modpath .. "/player.lua") dofile(modpath .. "/crafting.lua") dofile(modpath .. "/itemdescription.lua") +dofile(modpath .. "/async_env.lua") -------------- diff --git a/games/devtest/mods/unittests/inside_async_env.lua b/games/devtest/mods/unittests/inside_async_env.lua new file mode 100644 index 000000000..9774771f9 --- /dev/null +++ b/games/devtest/mods/unittests/inside_async_env.lua @@ -0,0 +1,15 @@ +unittests = {} + +core.log("info", "Hello World") + +function unittests.async_test() + assert(core == minetest) + -- stuff that should not be here + assert(not core.get_player_by_name) + assert(not core.set_node) + assert(not core.object_refs) + -- stuff that should be here + assert(ItemStack) + assert(core.registered_items[""]) + return true +end diff --git a/src/map.cpp b/src/map.cpp index 9c9324f5f..5153dcaa9 100644 --- a/src/map.cpp +++ b/src/map.cpp @@ -1896,6 +1896,7 @@ MMVManip::MMVManip(Map *map): VoxelManipulator(), m_map(map) { + assert(map); } void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max, @@ -1903,6 +1904,8 @@ void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max, { TimeTaker timer1("initialEmerge", &emerge_time); + assert(m_map); + // Units of these are MapBlocks v3s16 p_min = blockpos_min; v3s16 p_max = blockpos_max; @@ -1986,6 +1989,7 @@ void MMVManip::blitBackAll(std::map *modified_blocks, { if(m_area.getExtent() == v3s16(0,0,0)) return; + assert(m_map); /* Copy data of all blocks @@ -2006,4 +2010,33 @@ void MMVManip::blitBackAll(std::map *modified_blocks, } } +MMVManip *MMVManip::clone() const +{ + MMVManip *ret = new MMVManip(); + + const s32 size = m_area.getVolume(); + ret->m_area = m_area; + if (m_data) { + ret->m_data = new MapNode[size]; + memcpy(ret->m_data, m_data, size * sizeof(MapNode)); + } + if (m_flags) { + ret->m_flags = new u8[size]; + memcpy(ret->m_flags, m_flags, size * sizeof(u8)); + } + + ret->m_is_dirty = m_is_dirty; + // Even if the copy is disconnected from a map object keep the information + // needed to write it back to one + ret->m_loaded_blocks = m_loaded_blocks; + + return ret; +} + +void MMVManip::reparent(Map *map) +{ + assert(map && !m_map); + m_map = map; +} + //END diff --git a/src/map.h b/src/map.h index d8ed29106..21e3dbd6c 100644 --- a/src/map.h +++ b/src/map.h @@ -446,10 +446,25 @@ public: void blitBackAll(std::map * modified_blocks, bool overwrite_generated = true); + /* + Creates a copy of this VManip including contents, the copy will not be + associated with a Map. + */ + MMVManip *clone() const; + + // Reassociates a copied VManip to a map + void reparent(Map *map); + + // Is it impossible to call initialEmerge / blitBackAll? + inline bool isOrphan() const { return !m_map; } + bool m_is_dirty = false; protected: - Map *m_map; + MMVManip() {}; + + // may be null + Map *m_map = nullptr; /* key = blockpos value = flags describing the block diff --git a/src/script/common/CMakeLists.txt b/src/script/common/CMakeLists.txt index d07f6ab1b..3e84b46c7 100644 --- a/src/script/common/CMakeLists.txt +++ b/src/script/common/CMakeLists.txt @@ -3,6 +3,7 @@ set(common_SCRIPT_COMMON_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/c_converter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/c_types.cpp ${CMAKE_CURRENT_SOURCE_DIR}/c_internal.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/c_packer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helper.cpp PARENT_SCOPE) diff --git a/src/script/common/c_internal.cpp b/src/script/common/c_internal.cpp index df82dba14..ddd2d184c 100644 --- a/src/script/common/c_internal.cpp +++ b/src/script/common/c_internal.cpp @@ -166,3 +166,17 @@ void log_deprecated(lua_State *L, std::string message, int stack_depth) infostream << script_get_backtrace(L) << std::endl; } +void call_string_dump(lua_State *L, int idx) +{ + // Retrieve string.dump from insecure env to avoid it being tampered with + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_GLOBALS_BACKUP); + if (!lua_isnil(L, -1)) + lua_getfield(L, -1, "string"); + else + lua_getglobal(L, "string"); + lua_getfield(L, -1, "dump"); + lua_remove(L, -2); // remove _G + lua_remove(L, -2); // remove 'string' table + lua_pushvalue(L, idx); + lua_call(L, 1, 1); +} diff --git a/src/script/common/c_internal.h b/src/script/common/c_internal.h index c43db34aa..272a39941 100644 --- a/src/script/common/c_internal.h +++ b/src/script/common/c_internal.h @@ -56,6 +56,7 @@ extern "C" { #define CUSTOM_RIDX_BACKTRACE (CUSTOM_RIDX_BASE + 3) #define CUSTOM_RIDX_HTTP_API_LUA (CUSTOM_RIDX_BASE + 4) #define CUSTOM_RIDX_VECTOR_METATABLE (CUSTOM_RIDX_BASE + 5) +#define CUSTOM_RIDX_METATABLE_MAP (CUSTOM_RIDX_BASE + 6) // Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata @@ -139,3 +140,7 @@ DeprecatedHandlingMode get_deprecated_handling_mode(); * @param stack_depth How far on the stack to the first user function (ie: not builtin or core) */ void log_deprecated(lua_State *L, std::string message, int stack_depth = 1); + +// Safely call string.dump on a function value +// (does not pop, leaves one value on stack) +void call_string_dump(lua_State *L, int idx); diff --git a/src/script/common/c_packer.cpp b/src/script/common/c_packer.cpp new file mode 100644 index 000000000..fc5277330 --- /dev/null +++ b/src/script/common/c_packer.cpp @@ -0,0 +1,583 @@ +/* +Minetest +Copyright (C) 2022 sfan5 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#include +#include +#include +#include +#include +#include +#include "c_packer.h" +#include "c_internal.h" +#include "log.h" +#include "debug.h" +#include "threading/mutex_auto_lock.h" + +extern "C" { +#include +} + +// +// Helpers +// + +// convert negative index to absolute position on Lua stack +static inline int absidx(lua_State *L, int idx) +{ + assert(idx < 0); + return lua_gettop(L) + idx + 1; +} + +// does the type put anything into PackedInstr::sdata? +static inline bool uses_sdata(int type) +{ + switch (type) { + case LUA_TSTRING: + case LUA_TFUNCTION: + case LUA_TUSERDATA: + return true; + default: + return false; + } +} + +// does the type put anything into PackedInstr::? +static inline bool uses_union(int type) +{ + switch (type) { + case LUA_TNIL: + case LUA_TSTRING: + case LUA_TFUNCTION: + return false; + default: + return true; + } +} + +static inline bool can_set_into(int ktype, int vtype) +{ + switch (ktype) { + case LUA_TNUMBER: + return !uses_union(vtype); + case LUA_TSTRING: + return !uses_sdata(vtype); + default: + return false; + } +} + +// is the key suitable for use with set_into? +static inline bool suitable_key(lua_State *L, int idx) +{ + if (lua_type(L, idx) == LUA_TSTRING) { + // strings may not have a NULL byte (-> lua_setfield) + size_t len; + const char *str = lua_tolstring(L, idx, &len); + return strlen(str) == len; + } else { + assert(lua_type(L, idx) == LUA_TNUMBER); + // numbers must fit into an s32 and be integers (-> lua_rawseti) + lua_Number n = lua_tonumber(L, idx); + return std::floor(n) == n && n >= S32_MIN && n <= S32_MAX; + } +} + +namespace { + // checks if you left any values on the stack, for debugging + class StackChecker { + lua_State *L; + int top; + public: + StackChecker(lua_State *L) : L(L), top(lua_gettop(L)) {} + ~StackChecker() { + assert(lua_gettop(L) >= top); + if (lua_gettop(L) > top) { + rawstream << "Lua stack not cleaned up: " + << lua_gettop(L) << " != " << top + << " (false-positive if exception thrown)" << std::endl; + } + } + }; + + // Since an std::vector may reallocate, this is the only safe way to keep + // a reference to a particular element. + template + class VectorRef { + std::vector *vec; + size_t idx; + VectorRef(std::vector *vec, size_t idx) : vec(vec), idx(idx) {} + public: + static VectorRef front(std::vector &vec) { + return VectorRef(&vec, 0); + } + static VectorRef back(std::vector &vec) { + return VectorRef(&vec, vec.size() - 1); + } + T &operator*() { return (*vec)[idx]; } + T *operator->() { return &(*vec)[idx]; } + }; + + struct Packer { + PackInFunc fin; + PackOutFunc fout; + }; + + typedef std::pair PackerTuple; +} + +static inline auto emplace(PackedValue &pv, s16 type) +{ + pv.i.emplace_back(); + auto ref = VectorRef::back(pv.i); + ref->type = type; + // Initialize fields that may be left untouched + if (type == LUA_TTABLE) { + ref->uidata1 = 0; + ref->uidata2 = 0; + } else if (type == LUA_TUSERDATA) { + ref->ptrdata = nullptr; + } else if (type == INSTR_POP) { + ref->sidata2 = 0; + } + return ref; +} + +// +// Management of registered packers +// + +static std::unordered_map g_packers; +static std::mutex g_packers_lock; + +void script_register_packer(lua_State *L, const char *regname, + PackInFunc fin, PackOutFunc fout) +{ + // Store away callbacks + { + MutexAutoLock autolock(g_packers_lock); + auto it = g_packers.find(regname); + if (it == g_packers.end()) { + auto &ref = g_packers[regname]; + ref.fin = fin; + ref.fout = fout; + } else { + FATAL_ERROR_IF(it->second.fin != fin || it->second.fout != fout, + "Packer registered twice with mismatching callbacks"); + } + } + + // Save metatable so we can identify instances later + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + if (lua_isnil(L, -1)) { + lua_newtable(L); + lua_pushvalue(L, -1); + lua_rawseti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + } + + luaL_getmetatable(L, regname); + FATAL_ERROR_IF(lua_isnil(L, -1), "No metatable registered with that name"); + + // CUSTOM_RIDX_METATABLE_MAP contains { [metatable] = "regname", ... } + // check first + lua_pushstring(L, regname); + lua_rawget(L, -3); + if (!lua_isnil(L, -1)) { + FATAL_ERROR_IF(lua_topointer(L, -1) != lua_topointer(L, -2), + "Packer registered twice with inconsistent metatable"); + } + lua_pop(L, 1); + // then set + lua_pushstring(L, regname); + lua_rawset(L, -3); + + lua_pop(L, 1); +} + +static bool find_packer(const char *regname, PackerTuple &out) +{ + MutexAutoLock autolock(g_packers_lock); + auto it = g_packers.find(regname); + if (it == g_packers.end()) + return false; + // copy data for thread safety + out.first = it->first; + out.second = it->second; + return true; +} + +static bool find_packer(lua_State *L, int idx, PackerTuple &out) +{ +#ifndef NDEBUG + StackChecker checker(L); +#endif + + // retrieve metatable of the object + if (lua_getmetatable(L, idx) != 1) + return false; + + // use our global table to map it to the registry name + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + assert(lua_istable(L, -1)); + lua_pushvalue(L, -2); + lua_rawget(L, -2); + if (lua_isnil(L, -1)) { + lua_pop(L, 3); + return false; + } + + // load the associated data + bool found = find_packer(lua_tostring(L, -1), out); + FATAL_ERROR_IF(!found, "Inconsistent internal state"); + lua_pop(L, 3); + return true; +} + +// +// Packing implementation +// + +// recursively goes through the structure and ensures there are no circular references +static void pack_validate(lua_State *L, int idx, std::unordered_set &seen) +{ +#ifndef NDEBUG + StackChecker checker(L); + assert(idx > 0); +#endif + + if (lua_type(L, idx) != LUA_TTABLE) + return; + + const void *ptr = lua_topointer(L, idx); + assert(ptr); + + if (seen.find(ptr) != seen.end()) + throw LuaError("Circular references cannot be packed (yet)"); + seen.insert(ptr); + + lua_checkstack(L, 5); + lua_pushnil(L); + while (lua_next(L, idx) != 0) { + // key at -2, value at -1 + pack_validate(L, absidx(L, -2), seen); + pack_validate(L, absidx(L, -1), seen); + + lua_pop(L, 1); + } + + seen.erase(ptr); +} + +static VectorRef pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv) +{ +#ifndef NDEBUG + StackChecker checker(L); + assert(idx > 0); + assert(vidx > 0); +#endif + + switch (lua_type(L, idx)) { + case LUA_TNONE: + case LUA_TNIL: + return emplace(pv, LUA_TNIL); + case LUA_TBOOLEAN: { + auto r = emplace(pv, LUA_TBOOLEAN); + r->bdata = lua_toboolean(L, idx); + return r; + } + case LUA_TNUMBER: { + auto r = emplace(pv, LUA_TNUMBER); + r->ndata = lua_tonumber(L, idx); + return r; + } + case LUA_TSTRING: { + auto r = emplace(pv, LUA_TSTRING); + size_t len; + const char *str = lua_tolstring(L, idx, &len); + assert(str); + r->sdata.assign(str, len); + return r; + } + case LUA_TTABLE: + break; // execution continues + case LUA_TFUNCTION: { + auto r = emplace(pv, LUA_TFUNCTION); + call_string_dump(L, idx); + size_t len; + const char *str = lua_tolstring(L, -1, &len); + assert(str); + r->sdata.assign(str, len); + lua_pop(L, 1); + return r; + } + case LUA_TUSERDATA: { + PackerTuple ser; + if (!find_packer(L, idx, ser)) + throw LuaError("Cannot serialize unsupported userdata"); + pv.contains_userdata = true; + auto r = emplace(pv, LUA_TUSERDATA); + r->sdata = ser.first; + r->ptrdata = ser.second.fin(L, idx); + return r; + } + default: { + std::string err = "Cannot serialize type "; + err += lua_typename(L, lua_type(L, idx)); + throw LuaError(err); + } + } + + // LUA_TTABLE + lua_checkstack(L, 5); + + auto rtable = emplace(pv, LUA_TTABLE); + const int vi_table = vidx++; + + lua_pushnil(L); + while (lua_next(L, idx) != 0) { + // key at -2, value at -1 + const int ktype = lua_type(L, -2), vtype = lua_type(L, -1); + if (ktype == LUA_TNUMBER) + rtable->uidata1++; // narr + else + rtable->uidata2++; // nrec + + // check if we can use a shortcut + if (can_set_into(ktype, vtype) && suitable_key(L, -2)) { + // push only the value + auto rval = pack_inner(L, absidx(L, -1), vidx, pv); + rval->pop = vtype != LUA_TTABLE; + // and where to put it: + rval->set_into = vi_table; + if (ktype == LUA_TSTRING) + rval->sdata = lua_tostring(L, -2); + else + rval->sidata1 = lua_tointeger(L, -2); + // pop tables after the fact + if (!rval->pop) { + auto ri1 = emplace(pv, INSTR_POP); + ri1->sidata1 = vidx; + } + } else { + // push the key and value + pack_inner(L, absidx(L, -2), vidx, pv); + vidx++; + pack_inner(L, absidx(L, -1), vidx, pv); + vidx++; + // push an instruction to set them + auto ri1 = emplace(pv, INSTR_SETTABLE); + ri1->set_into = vi_table; + ri1->sidata1 = vidx - 2; + ri1->sidata2 = vidx - 1; + ri1->pop = true; + vidx -= 2; + } + + lua_pop(L, 1); + } + + assert(vidx == vi_table + 1); + return rtable; +} + +PackedValue *script_pack(lua_State *L, int idx) +{ + if (idx < 0) + idx = absidx(L, idx); + + std::unordered_set seen; + pack_validate(L, idx, seen); + assert(seen.size() == 0); + + // Actual serialization + PackedValue pv; + pack_inner(L, idx, 1, pv); + + return new PackedValue(std::move(pv)); +} + +// +// Unpacking implementation +// + +void script_unpack(lua_State *L, PackedValue *pv) +{ + const int top = lua_gettop(L); + int ctr = 0; + + for (auto &i : pv->i) { + // If leaving values on stack make sure there's space (every 5th iteration) + if (!i.pop && (ctr++) >= 5) { + lua_checkstack(L, 5); + ctr = 0; + } + + /* Instructions */ + switch (i.type) { + case INSTR_SETTABLE: + lua_pushvalue(L, top + i.sidata1); // key + lua_pushvalue(L, top + i.sidata2); // value + lua_rawset(L, top + i.set_into); + if (i.pop) { + if (i.sidata1 != i.sidata2) { + // removing moves indices so pop higher index first + lua_remove(L, top + std::max(i.sidata1, i.sidata2)); + lua_remove(L, top + std::min(i.sidata1, i.sidata2)); + } else { + lua_remove(L, top + i.sidata1); + } + } + continue; + case INSTR_POP: + lua_remove(L, top + i.sidata1); + if (i.sidata2 > 0) + lua_remove(L, top + i.sidata2); + continue; + default: + break; + } + + /* Lua types */ + switch (i.type) { + case LUA_TNIL: + lua_pushnil(L); + break; + case LUA_TBOOLEAN: + lua_pushboolean(L, i.bdata); + break; + case LUA_TNUMBER: + lua_pushnumber(L, i.ndata); + break; + case LUA_TSTRING: + lua_pushlstring(L, i.sdata.data(), i.sdata.size()); + break; + case LUA_TTABLE: + lua_createtable(L, i.uidata1, i.uidata2); + break; + case LUA_TFUNCTION: + luaL_loadbuffer(L, i.sdata.data(), i.sdata.size(), nullptr); + break; + case LUA_TUSERDATA: { + PackerTuple ser; + sanity_check(find_packer(i.sdata.c_str(), ser)); + ser.second.fout(L, i.ptrdata); + i.ptrdata = nullptr; // ownership taken by callback + break; + } + default: + assert(0); + break; + } + + if (i.set_into) { + if (!i.pop) + lua_pushvalue(L, -1); + if (uses_sdata(i.type)) + lua_rawseti(L, top + i.set_into, i.sidata1); + else + lua_setfield(L, top + i.set_into, i.sdata.c_str()); + } else { + if (i.pop) + lua_pop(L, 1); + } + } + + // as part of the unpacking process we take ownership of all userdata + pv->contains_userdata = false; + // leave exactly one value on the stack + lua_settop(L, top+1); +} + +// +// PackedValue +// + +PackedValue::~PackedValue() +{ + if (!contains_userdata) + return; + for (auto &i : this->i) { + if (i.type == LUA_TUSERDATA && i.ptrdata) { + PackerTuple ser; + if (find_packer(i.sdata.c_str(), ser)) { + // tell it to deallocate object + ser.second.fout(nullptr, i.ptrdata); + } else { + assert(false); + } + } + } +} + +// +// script_dump_packed +// + +#ifndef NDEBUG +void script_dump_packed(const PackedValue *val) +{ + printf("instruction stream: [\n"); + for (const auto &i : val->i) { + printf("\t("); + switch (i.type) { + case INSTR_SETTABLE: + printf("SETTABLE(%d, %d)", i.sidata1, i.sidata2); + break; + case INSTR_POP: + printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2); + break; + case LUA_TNIL: + printf("nil"); + break; + case LUA_TBOOLEAN: + printf(i.bdata ? "true" : "false"); + break; + case LUA_TNUMBER: + printf("%f", i.ndata); + break; + case LUA_TSTRING: + printf("\"%s\"", i.sdata.c_str()); + break; + case LUA_TTABLE: + printf("table(%d, %d)", i.uidata1, i.uidata2); + break; + case LUA_TFUNCTION: + printf("function(%d byte)", i.sdata.size()); + break; + case LUA_TUSERDATA: + printf("userdata %s %p", i.sdata.c_str(), i.ptrdata); + break; + default: + printf("!!UNKNOWN!!"); + break; + } + if (i.set_into) { + if (i.type >= 0 && uses_sdata(i.type)) + printf(", k=%d, into=%d", i.sidata1, i.set_into); + else if (i.type >= 0) + printf(", k=\"%s\", into=%d", i.sdata.c_str(), i.set_into); + else + printf(", into=%d", i.set_into); + } + if (i.pop) + printf(", pop"); + printf(")\n"); + } + printf("]\n"); +} +#endif diff --git a/src/script/common/c_packer.h b/src/script/common/c_packer.h new file mode 100644 index 000000000..8bccca98d --- /dev/null +++ b/src/script/common/c_packer.h @@ -0,0 +1,123 @@ +/* +Minetest +Copyright (C) 2022 sfan5 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#pragma once + +#include +#include +#include "irrlichttypes.h" +#include "util/basic_macros.h" + +extern "C" { +#include +} + +/* + This file defines an in-memory representation of Lua objects including + support for functions and userdata. It it used to move data between Lua + states and cannot be used for persistence or network transfer. +*/ + +#define INSTR_SETTABLE (-10) +#define INSTR_POP (-11) + +/** + * Represents a single instruction that pushes a new value or works with existing ones. + */ +struct PackedInstr +{ + s16 type; // LUA_T* or INSTR_* + u16 set_into; // set into table on stack + bool pop; // remove from stack? + union { + bool bdata; // boolean: value + lua_Number ndata; // number: value + struct { + u16 uidata1, uidata2; // table: narr, nrec + }; + struct { + /* + SETTABLE: key index, value index + POP: indices to remove + otherwise w/ set_into: numeric key, - + */ + s32 sidata1, sidata2; + }; + void *ptrdata; // userdata: implementation defined + }; + /* + - string: value + - function: buffer + - w/ set_into: string key (no null bytes!) + - userdata: name in registry + */ + std::string sdata; + + PackedInstr() : type(0), set_into(0), pop(false) {} +}; + +/** + * A packed value can be a primitive like a string or number but also a table + * including all of its contents. It is made up of a linear stream of + * 'instructions' that build the final value when executed. + */ +struct PackedValue +{ + std::vector i; + // Indicates whether there are any userdata pointers that need to be deallocated + bool contains_userdata = false; + + PackedValue() = default; + ~PackedValue(); + + DISABLE_CLASS_COPY(PackedValue) + + ALLOW_CLASS_MOVE(PackedValue) +}; + +/* + * Packing callback: Turns a Lua value at given index into a void* + */ +typedef void *(*PackInFunc)(lua_State *L, int idx); +/* + * Unpacking callback: Turns a void* back into the Lua value (left on top of stack) + * + * Note that this function must take ownership of the pointer, so make sure + * to free or keep the memory. + * `L` can be nullptr to indicate that data should just be discarded. + */ +typedef void (*PackOutFunc)(lua_State *L, void *ptr); +/* + * Register a packable type with the name of its metatable. + * + * Even though the callbacks are global this must be called for every Lua state + * that supports objects of this type. + * This function is thread-safe. + */ +void script_register_packer(lua_State *L, const char *regname, + PackInFunc fin, PackOutFunc fout); + +// Pack a Lua value +PackedValue *script_pack(lua_State *L, int idx); +// Unpack a Lua value (left on top of stack) +// Note that this may modify the PackedValue, you can't reuse it! +void script_unpack(lua_State *L, PackedValue *val); + +// Dump contents of PackedValue to stdout for debugging +void script_dump_packed(const PackedValue *val); diff --git a/src/script/cpp_api/s_async.cpp b/src/script/cpp_api/s_async.cpp index dacdcd75a..42a794ceb 100644 --- a/src/script/cpp_api/s_async.cpp +++ b/src/script/cpp_api/s_async.cpp @@ -21,9 +21,9 @@ with this program; if not, write to the Free Software Foundation, Inc., #include extern "C" { -#include "lua.h" -#include "lauxlib.h" -#include "lualib.h" +#include +#include +#include } #include "server.h" @@ -32,6 +32,7 @@ extern "C" { #include "filesys.h" #include "porting.h" #include "common/c_internal.h" +#include "common/c_packer.h" #include "lua_api/l_base.h" /******************************************************************************/ @@ -76,19 +77,34 @@ void AsyncEngine::initialize(unsigned int numEngines) { initDone = true; - for (unsigned int i = 0; i < numEngines; i++) { - AsyncWorkerThread *toAdd = new AsyncWorkerThread(this, - std::string("AsyncWorker-") + itos(i)); - workerThreads.push_back(toAdd); - toAdd->start(); + if (numEngines == 0) { + // Leave one core for the main thread and one for whatever else + autoscaleMaxWorkers = Thread::getNumberOfProcessors(); + if (autoscaleMaxWorkers >= 2) + autoscaleMaxWorkers -= 2; + infostream << "AsyncEngine: using at most " << autoscaleMaxWorkers + << " threads with automatic scaling" << std::endl; + + addWorkerThread(); + } else { + for (unsigned int i = 0; i < numEngines; i++) + addWorkerThread(); } } +void AsyncEngine::addWorkerThread() +{ + AsyncWorkerThread *toAdd = new AsyncWorkerThread(this, + std::string("AsyncWorker-") + itos(workerThreads.size())); + workerThreads.push_back(toAdd); + toAdd->start(); +} + /******************************************************************************/ u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &¶ms, const std::string &mod_origin) { - jobQueueMutex.lock(); + MutexAutoLock autolock(jobQueueMutex); u32 jobId = jobIdCounter++; jobQueue.emplace_back(); @@ -99,7 +115,23 @@ u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &¶ms, to_add.mod_origin = mod_origin; jobQueueCounter.post(); - jobQueueMutex.unlock(); + return jobId; +} + +u32 AsyncEngine::queueAsyncJob(std::string &&func, PackedValue *params, + const std::string &mod_origin) +{ + MutexAutoLock autolock(jobQueueMutex); + u32 jobId = jobIdCounter++; + + jobQueue.emplace_back(); + auto &to_add = jobQueue.back(); + to_add.id = jobId; + to_add.function = std::move(func); + to_add.params_ext.reset(params); + to_add.mod_origin = mod_origin; + + jobQueueCounter.post(); return jobId; } @@ -131,6 +163,12 @@ void AsyncEngine::putJobResult(LuaJobInfo &&result) /******************************************************************************/ void AsyncEngine::step(lua_State *L) +{ + stepJobResults(L); + stepAutoscale(); +} + +void AsyncEngine::stepJobResults(lua_State *L) { int error_handler = PUSH_ERROR_HANDLER(L); lua_getglobal(L, "core"); @@ -148,7 +186,10 @@ void AsyncEngine::step(lua_State *L) luaL_checktype(L, -1, LUA_TFUNCTION); lua_pushinteger(L, j.id); - lua_pushlstring(L, j.result.data(), j.result.size()); + if (j.result_ext) + script_unpack(L, j.result_ext.get()); + else + lua_pushlstring(L, j.result.data(), j.result.size()); // Call handler const char *origin = j.mod_origin.empty() ? nullptr : j.mod_origin.c_str(); @@ -161,12 +202,71 @@ void AsyncEngine::step(lua_State *L) lua_pop(L, 2); // Pop core and error handler } +void AsyncEngine::stepAutoscale() +{ + if (workerThreads.size() >= autoscaleMaxWorkers) + return; + + MutexAutoLock autolock(jobQueueMutex); + + // 2) If the timer elapsed, check again + if (autoscaleTimer && porting::getTimeMs() >= autoscaleTimer) { + autoscaleTimer = 0; + // Determine overlap with previous snapshot + unsigned int n = 0; + for (const auto &it : jobQueue) + n += autoscaleSeenJobs.count(it.id); + autoscaleSeenJobs.clear(); + infostream << "AsyncEngine: " << n << " jobs were still waiting after 1s" << std::endl; + // Start this many new threads + while (workerThreads.size() < autoscaleMaxWorkers && n > 0) { + addWorkerThread(); + n--; + } + return; + } + + // 1) Check if there's anything in the queue + if (!autoscaleTimer && !jobQueue.empty()) { + // Take a snapshot of all jobs we have seen + for (const auto &it : jobQueue) + autoscaleSeenJobs.emplace(it.id); + // and set a timer for 1 second + autoscaleTimer = porting::getTimeMs() + 1000; + } +} + /******************************************************************************/ -void AsyncEngine::prepareEnvironment(lua_State* L, int top) +bool AsyncEngine::prepareEnvironment(lua_State* L, int top) { for (StateInitializer &stateInitializer : stateInitializers) { stateInitializer(L, top); } + + auto *script = ModApiBase::getScriptApiBase(L); + try { + script->loadMod(Server::getBuiltinLuaPath() + DIR_DELIM + "init.lua", + BUILTIN_MOD_NAME); + } catch (const ModError &e) { + errorstream << "Execution of async base environment failed: " + << e.what() << std::endl; + FATAL_ERROR("Execution of async base environment failed"); + } + + // Load per mod stuff + if (server) { + const auto &list = server->m_async_init_files; + try { + for (auto &it : list) + script->loadMod(it.second, it.first); + } catch (const ModError &e) { + errorstream << "Failed to load mod script inside async environment." << std::endl; + server->setAsyncFatalError(e.what()); + return false; + } + } + + return true; } /******************************************************************************/ @@ -178,15 +278,25 @@ AsyncWorkerThread::AsyncWorkerThread(AsyncEngine* jobDispatcher, { lua_State *L = getStack(); + if (jobDispatcher->server) { + setGameDef(jobDispatcher->server); + + if (g_settings->getBool("secure.enable_security")) + initializeSecurity(); + } + // Prepare job lua environment lua_getglobal(L, "core"); int top = lua_gettop(L); // Push builtin initialization type - lua_pushstring(L, "async"); + lua_pushstring(L, jobDispatcher->server ? "async_game" : "async"); lua_setglobal(L, "INIT"); - jobDispatcher->prepareEnvironment(L, top); + if (!jobDispatcher->prepareEnvironment(L, top)) { + // can't throw from here so we're stuck with this + isErrored = true; + } } /******************************************************************************/ @@ -198,19 +308,20 @@ AsyncWorkerThread::~AsyncWorkerThread() /******************************************************************************/ void* AsyncWorkerThread::run() { - lua_State *L = getStack(); + if (isErrored) + return nullptr; - try { - loadMod(getServer()->getBuiltinLuaPath() + DIR_DELIM + "init.lua", - BUILTIN_MOD_NAME); - } catch (const ModError &e) { - errorstream << "Execution of async base environment failed: " - << e.what() << std::endl; - FATAL_ERROR("Execution of async base environment failed"); - } + lua_State *L = getStack(); int error_handler = PUSH_ERROR_HANDLER(L); + auto report_error = [this] (const ModError &e) { + if (jobDispatcher->server) + jobDispatcher->server->setAsyncFatalError(e.what()); + else + errorstream << e.what() << std::endl; + }; + lua_getglobal(L, "core"); if (lua_isnil(L, -1)) { FATAL_ERROR("Unable to find core within async environment!"); @@ -223,6 +334,8 @@ void* AsyncWorkerThread::run() if (!jobDispatcher->getJob(&j) || stopRequested()) continue; + const bool use_ext = !!j.params_ext; + lua_getfield(L, -1, "job_processor"); if (lua_isnil(L, -1)) FATAL_ERROR("Unable to get async job processor!"); @@ -232,7 +345,10 @@ void* AsyncWorkerThread::run() errorstream << "ASYNC WORKER: Unable to deserialize function" << std::endl; lua_pushnil(L); } - lua_pushlstring(L, j.params.data(), j.params.size()); + if (use_ext) + script_unpack(L, j.params_ext.get()); + else + lua_pushlstring(L, j.params.data(), j.params.size()); // Call it setOriginDirect(j.mod_origin.empty() ? nullptr : j.mod_origin.c_str()); @@ -241,19 +357,28 @@ void* AsyncWorkerThread::run() try { scriptError(result, ""); } catch (const ModError &e) { - errorstream << e.what() << std::endl; + report_error(e); } } else { // Fetch result - size_t length; - const char *retval = lua_tolstring(L, -1, &length); - j.result.assign(retval, length); + if (use_ext) { + try { + j.result_ext.reset(script_pack(L, -1)); + } catch (const ModError &e) { + report_error(e); + result = LUA_ERRERR; + } + } else { + size_t length; + const char *retval = lua_tolstring(L, -1, &length); + j.result.assign(retval, length); + } } lua_pop(L, 1); // Pop retval // Put job result - if (!j.result.empty()) + if (result == 0) jobDispatcher->putJobResult(std::move(j)); } diff --git a/src/script/cpp_api/s_async.h b/src/script/cpp_api/s_async.h index 697cb0221..1e34e40ea 100644 --- a/src/script/cpp_api/s_async.h +++ b/src/script/cpp_api/s_async.h @@ -21,11 +21,15 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include +#include +#include +#include #include "threading/semaphore.h" #include "threading/thread.h" -#include "lua.h" +#include "common/c_packer.h" #include "cpp_api/s_base.h" +#include "cpp_api/s_security.h" // Forward declarations class AsyncEngine; @@ -42,8 +46,12 @@ struct LuaJobInfo std::string function; // Parameter to be passed to function (serialized) std::string params; + // Alternative parameters + std::unique_ptr params_ext; // Result of function call (serialized) std::string result; + // Alternative result + std::unique_ptr result_ext; // Name of the mod who invoked this call std::string mod_origin; // JobID used to identify a job and match it to callback @@ -51,7 +59,8 @@ struct LuaJobInfo }; // Asynchronous working environment -class AsyncWorkerThread : public Thread, virtual public ScriptApiBase { +class AsyncWorkerThread : public Thread, + virtual public ScriptApiBase, public ScriptApiSecurity { friend class AsyncEngine; public: virtual ~AsyncWorkerThread(); @@ -63,6 +72,7 @@ protected: private: AsyncEngine *jobDispatcher = nullptr; + bool isErrored = false; }; // Asynchornous thread and job management @@ -71,6 +81,7 @@ class AsyncEngine { typedef void (*StateInitializer)(lua_State *L, int top); public: AsyncEngine() = default; + AsyncEngine(Server *server) : server(server) {}; ~AsyncEngine(); /** @@ -81,7 +92,7 @@ public: /** * Create async engine tasks and lock function registration - * @param numEngines Number of async threads to be started + * @param numEngines Number of worker threads, 0 for automatic scaling */ void initialize(unsigned int numEngines); @@ -94,9 +105,17 @@ public: u32 queueAsyncJob(std::string &&func, std::string &¶ms, const std::string &mod_origin = ""); + /** + * Queue an async job + * @param func Serialized lua function + * @param params Serialized parameters (takes ownership!) + * @return ID of queued job + */ + u32 queueAsyncJob(std::string &&func, PackedValue *params, + const std::string &mod_origin = ""); + /** * Engine step to process finished jobs - * the engine step is one way to pass events back, PushFinishedJobs another * @param L The Lua stack */ void step(lua_State *L); @@ -116,19 +135,44 @@ protected: */ void putJobResult(LuaJobInfo &&result); + /** + * Start an additional worker thread + */ + void addWorkerThread(); + + /** + * Process finished jobs callbacks + */ + void stepJobResults(lua_State *L); + + /** + * Handle automatic scaling of worker threads + */ + void stepAutoscale(); + /** * Initialize environment with current registred functions * this function adds all functions registred by registerFunction to the * passed lua stack * @param L Lua stack to initialize * @param top Stack position + * @return false if a mod error ocurred */ - void prepareEnvironment(lua_State* L, int top); + bool prepareEnvironment(lua_State* L, int top); private: // Variable locking the engine against further modification bool initDone = false; + // Maximum number of worker threads for automatic scaling + // 0 if disabled + unsigned int autoscaleMaxWorkers = 0; + u64 autoscaleTimer = 0; + std::unordered_set autoscaleSeenJobs; + + // Only set for the server async environment (duh) + Server *server = nullptr; + // Internal store for registred state initializers std::vector stateInitializers; diff --git a/src/script/lua_api/l_craft.cpp b/src/script/lua_api/l_craft.cpp index c2c5a5551..137b210be 100644 --- a/src/script/lua_api/l_craft.cpp +++ b/src/script/lua_api/l_craft.cpp @@ -525,3 +525,11 @@ void ModApiCraft::Initialize(lua_State *L, int top) API_FCT(register_craft); API_FCT(clear_craft); } + +void ModApiCraft::InitializeAsync(lua_State *L, int top) +{ + // all read-only functions + API_FCT(get_all_craft_recipes); + API_FCT(get_craft_recipe); + API_FCT(get_craft_result); +} diff --git a/src/script/lua_api/l_craft.h b/src/script/lua_api/l_craft.h index 9002b23ef..5234af56f 100644 --- a/src/script/lua_api/l_craft.h +++ b/src/script/lua_api/l_craft.h @@ -45,4 +45,5 @@ private: public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_internal.h b/src/script/lua_api/l_internal.h index 672e535ca..de73ff42a 100644 --- a/src/script/lua_api/l_internal.h +++ b/src/script/lua_api/l_internal.h @@ -69,7 +69,7 @@ with this program; if not, write to the Free Software Foundation, Inc., // Retrieve Environment pointer as `env` (no map lock) #define GET_PLAIN_ENV_PTR_NO_MAP_LOCK \ - Environment *env = (Environment *)getEnv(L); \ + Environment *env = getEnv(L); \ if (env == NULL) \ return 0 diff --git a/src/script/lua_api/l_item.cpp b/src/script/lua_api/l_item.cpp index fc97a1736..b58b994d9 100644 --- a/src/script/lua_api/l_item.cpp +++ b/src/script/lua_api/l_item.cpp @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "itemdef.h" #include "nodedef.h" #include "server.h" @@ -441,6 +442,7 @@ int LuaItemStack::create_object(lua_State *L) lua_setmetatable(L, -2); return 1; } + // Not callable from Lua int LuaItemStack::create(lua_State *L, const ItemStack &item) { @@ -457,6 +459,20 @@ LuaItemStack *LuaItemStack::checkobject(lua_State *L, int narg) return *(LuaItemStack **)luaL_checkudata(L, narg, className); } +void *LuaItemStack::packIn(lua_State *L, int idx) +{ + LuaItemStack *o = checkobject(L, idx); + return new ItemStack(o->getItem()); +} + +void LuaItemStack::packOut(lua_State *L, void *ptr) +{ + ItemStack *stack = reinterpret_cast(ptr); + if (L) + create(L, *stack); + delete stack; +} + void LuaItemStack::Register(lua_State *L) { lua_newtable(L); @@ -488,6 +504,8 @@ void LuaItemStack::Register(lua_State *L) // Can be created from Lua (ItemStack(itemstack or itemstring or table or nil)) lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } const char LuaItemStack::className[] = "ItemStack"; @@ -673,3 +691,10 @@ void ModApiItemMod::Initialize(lua_State *L, int top) API_FCT(get_content_id); API_FCT(get_name_from_content_id); } + +void ModApiItemMod::InitializeAsync(lua_State *L, int top) +{ + // all read-only functions + API_FCT(get_content_id); + API_FCT(get_name_from_content_id); +} diff --git a/src/script/lua_api/l_item.h b/src/script/lua_api/l_item.h index 16878c101..180975061 100644 --- a/src/script/lua_api/l_item.h +++ b/src/script/lua_api/l_item.h @@ -141,8 +141,11 @@ public: // Not callable from Lua static int create(lua_State *L, const ItemStack &item); static LuaItemStack* checkobject(lua_State *L, int narg); - static void Register(lua_State *L); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + + static void Register(lua_State *L); }; class ModApiItemMod : public ModApiBase { @@ -152,6 +155,8 @@ private: static int l_register_alias_raw(lua_State *L); static int l_get_content_id(lua_State *L); static int l_get_name_from_content_id(lua_State *L); + public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_noise.cpp b/src/script/lua_api/l_noise.cpp index 0eee49b7d..5561eaebf 100644 --- a/src/script/lua_api/l_noise.cpp +++ b/src/script/lua_api/l_noise.cpp @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "log.h" #include "porting.h" #include "util/numeric.h" @@ -101,6 +102,25 @@ LuaPerlinNoise *LuaPerlinNoise::checkobject(lua_State *L, int narg) } +void *LuaPerlinNoise::packIn(lua_State *L, int idx) +{ + LuaPerlinNoise *o = checkobject(L, idx); + return new NoiseParams(o->np); +} + +void LuaPerlinNoise::packOut(lua_State *L, void *ptr) +{ + NoiseParams *np = reinterpret_cast(ptr); + if (L) { + LuaPerlinNoise *o = new LuaPerlinNoise(np); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); + } + delete np; +} + + void LuaPerlinNoise::Register(lua_State *L) { lua_newtable(L); @@ -126,6 +146,8 @@ void LuaPerlinNoise::Register(lua_State *L) lua_pop(L, 1); lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } @@ -357,6 +379,35 @@ LuaPerlinNoiseMap *LuaPerlinNoiseMap::checkobject(lua_State *L, int narg) } +struct NoiseMapParams { + NoiseParams np; + s32 seed; + v3s16 size; +}; + +void *LuaPerlinNoiseMap::packIn(lua_State *L, int idx) +{ + LuaPerlinNoiseMap *o = checkobject(L, idx); + NoiseMapParams *ret = new NoiseMapParams(); + ret->np = o->noise->np; + ret->seed = o->noise->seed; + ret->size = v3s16(o->noise->sx, o->noise->sy, o->noise->sz); + return ret; +} + +void LuaPerlinNoiseMap::packOut(lua_State *L, void *ptr) +{ + NoiseMapParams *p = reinterpret_cast(ptr); + if (L) { + LuaPerlinNoiseMap *o = new LuaPerlinNoiseMap(&p->np, p->seed, p->size); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); + } + delete p; +} + + void LuaPerlinNoiseMap::Register(lua_State *L) { lua_newtable(L); @@ -382,6 +433,8 @@ void LuaPerlinNoiseMap::Register(lua_State *L) lua_pop(L, 1); lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } diff --git a/src/script/lua_api/l_noise.h b/src/script/lua_api/l_noise.h index 29ab41a31..5d34a479b 100644 --- a/src/script/lua_api/l_noise.h +++ b/src/script/lua_api/l_noise.h @@ -52,6 +52,9 @@ public: static LuaPerlinNoise *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; @@ -91,6 +94,9 @@ public: static LuaPerlinNoiseMap *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 42725e5d2..4b0b45887 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "cpp_api/s_base.h" #include "cpp_api/s_security.h" #include "scripting_server.h" @@ -526,6 +527,76 @@ int ModApiServer::l_notify_authentication_modified(lua_State *L) return 0; } +// do_async_callback(func, params, mod_origin) +int ModApiServer::l_do_async_callback(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + ServerScripting *script = getScriptApi(L); + + luaL_checktype(L, 1, LUA_TFUNCTION); + luaL_checktype(L, 2, LUA_TTABLE); + luaL_checktype(L, 3, LUA_TSTRING); + + call_string_dump(L, 1); + size_t func_length; + const char *serialized_func_raw = lua_tolstring(L, -1, &func_length); + + PackedValue *param = script_pack(L, 2); + + std::string mod_origin = readParam(L, 3); + + u32 jobId = script->queueAsync( + std::string(serialized_func_raw, func_length), + param, mod_origin); + + lua_settop(L, 0); + lua_pushinteger(L, jobId); + return 1; +} + +// register_async_dofile(path) +int ModApiServer::l_register_async_dofile(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + + std::string path = readParam(L, 1); + CHECK_SECURE_PATH(L, path.c_str(), false); + + // Find currently running mod name (only at init time) + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME); + if (!lua_isstring(L, -1)) + return 0; + std::string modname = readParam(L, -1); + + getServer(L)->m_async_init_files.emplace_back(modname, path); + lua_pushboolean(L, true); + return 1; +} + +// serialize_roundtrip(value) +// Meant for unit testing the packer from Lua +int ModApiServer::l_serialize_roundtrip(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + + int top = lua_gettop(L); + auto *pv = script_pack(L, 1); + if (top != lua_gettop(L)) + throw LuaError("stack values leaked"); + +#ifndef NDEBUG + script_dump_packed(pv); +#endif + + top = lua_gettop(L); + script_unpack(L, pv); + delete pv; + if (top + 1 != lua_gettop(L)) + throw LuaError("stack values leaked"); + + return 1; +} + void ModApiServer::Initialize(lua_State *L, int top) { API_FCT(request_shutdown); @@ -559,4 +630,18 @@ void ModApiServer::Initialize(lua_State *L, int top) API_FCT(remove_player); API_FCT(unban_player_or_ip); API_FCT(notify_authentication_modified); + + API_FCT(do_async_callback); + API_FCT(register_async_dofile); + API_FCT(serialize_roundtrip); +} + +void ModApiServer::InitializeAsync(lua_State *L, int top) +{ + API_FCT(get_worldpath); + API_FCT(is_singleplayer); + + API_FCT(get_current_modname); + API_FCT(get_modpath); + API_FCT(get_modnames); } diff --git a/src/script/lua_api/l_server.h b/src/script/lua_api/l_server.h index f05c0b7c9..a4f38c34e 100644 --- a/src/script/lua_api/l_server.h +++ b/src/script/lua_api/l_server.h @@ -106,6 +106,16 @@ private: // notify_authentication_modified(name) static int l_notify_authentication_modified(lua_State *L); + // do_async_callback(func, params, mod_origin) + static int l_do_async_callback(lua_State *L); + + // register_async_dofile(path) + static int l_register_async_dofile(lua_State *L); + + // serialize_roundtrip(obj) + static int l_serialize_roundtrip(lua_State *L); + public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_util.cpp b/src/script/lua_api/l_util.cpp index b04f26fda..97068ce4c 100644 --- a/src/script/lua_api/l_util.cpp +++ b/src/script/lua_api/l_util.cpp @@ -671,6 +671,9 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top) API_FCT(cpdir); API_FCT(mvdir); API_FCT(get_dir_list); + API_FCT(safe_file_write); + + API_FCT(request_insecure_environment); API_FCT(encode_base64); API_FCT(decode_base64); @@ -680,6 +683,8 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top) API_FCT(colorspec_to_colorstring); API_FCT(colorspec_to_bytes); + API_FCT(encode_png); + API_FCT(get_last_run_mod); API_FCT(set_last_run_mod); diff --git a/src/script/lua_api/l_util.h b/src/script/lua_api/l_util.h index fcf8a1057..cc5563577 100644 --- a/src/script/lua_api/l_util.h +++ b/src/script/lua_api/l_util.h @@ -129,6 +129,4 @@ public: static void Initialize(lua_State *L, int top); static void InitializeAsync(lua_State *L, int top); static void InitializeClient(lua_State *L, int top); - - static void InitializeAsync(AsyncEngine &engine); }; diff --git a/src/script/lua_api/l_vmanip.cpp b/src/script/lua_api/l_vmanip.cpp index a3ece627c..6187a47db 100644 --- a/src/script/lua_api/l_vmanip.cpp +++ b/src/script/lua_api/l_vmanip.cpp @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_content.h" #include "common/c_converter.h" +#include "common/c_packer.h" #include "emerge.h" #include "environment.h" #include "map.h" @@ -45,6 +46,8 @@ int LuaVoxelManip::l_read_from_map(lua_State *L) LuaVoxelManip *o = checkobject(L, 1); MMVManip *vm = o->vm; + if (vm->isOrphan()) + return 0; v3s16 bp1 = getNodeBlockPos(check_v3s16(L, 2)); v3s16 bp2 = getNodeBlockPos(check_v3s16(L, 3)); @@ -429,6 +432,34 @@ LuaVoxelManip *LuaVoxelManip::checkobject(lua_State *L, int narg) return *(LuaVoxelManip **)ud; // unbox pointer } +void *LuaVoxelManip::packIn(lua_State *L, int idx) +{ + LuaVoxelManip *o = checkobject(L, idx); + + if (o->is_mapgen_vm) + throw LuaError("nope"); + return o->vm->clone(); +} + +void LuaVoxelManip::packOut(lua_State *L, void *ptr) +{ + MMVManip *vm = reinterpret_cast(ptr); + if (!L) { + delete vm; + return; + } + + // Associate vmanip with map if the Lua env has one + Environment *env = getEnv(L); + if (env) + vm->reparent(&(env->getMap())); + + LuaVoxelManip *o = new LuaVoxelManip(vm, false); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); +} + void LuaVoxelManip::Register(lua_State *L) { lua_newtable(L); @@ -455,6 +486,8 @@ void LuaVoxelManip::Register(lua_State *L) // Can be created from Lua (VoxelManip()) lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } const char LuaVoxelManip::className[] = "VoxelManip"; diff --git a/src/script/lua_api/l_vmanip.h b/src/script/lua_api/l_vmanip.h index 5113070dc..005133335 100644 --- a/src/script/lua_api/l_vmanip.h +++ b/src/script/lua_api/l_vmanip.h @@ -75,5 +75,8 @@ public: static LuaVoxelManip *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; diff --git a/src/script/scripting_server.cpp b/src/script/scripting_server.cpp index 85411ded4..5b99468dc 100644 --- a/src/script/scripting_server.cpp +++ b/src/script/scripting_server.cpp @@ -47,11 +47,12 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_storage.h" extern "C" { -#include "lualib.h" +#include } ServerScripting::ServerScripting(Server* server): - ScriptApiBase(ScriptingType::Server) + ScriptApiBase(ScriptingType::Server), + asyncEngine(server) { setGameDef(server); @@ -88,6 +89,47 @@ ServerScripting::ServerScripting(Server* server): infostream << "SCRIPTAPI: Initialized game modules" << std::endl; } +void ServerScripting::initAsync() +{ + // Save globals to transfer + { + lua_State *L = getStack(); + lua_getglobal(L, "core"); + luaL_checktype(L, -1, LUA_TTABLE); + lua_getfield(L, -1, "get_globals_to_transfer"); + lua_call(L, 0, 1); + luaL_checktype(L, -1, LUA_TSTRING); + getServer()->m_async_globals_data.set(readParam(L, -1)); + lua_pushnil(L); + lua_setfield(L, -3, "get_globals_to_transfer"); // unset function too + lua_pop(L, 2); // pop 'core', return value + } + + infostream << "SCRIPTAPI: Initializing async engine" << std::endl; + asyncEngine.registerStateInitializer(InitializeAsync); + asyncEngine.registerStateInitializer(ModApiUtil::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiCraft::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiItemMod::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiServer::InitializeAsync); + // not added: ModApiMapgen is a minefield for thread safety + // not added: ModApiHttp async api can't really work together with our jobs + // not added: ModApiStorage is probably not thread safe(?) + + asyncEngine.initialize(0); +} + +void ServerScripting::stepAsync() +{ + asyncEngine.step(getStack()); +} + +u32 ServerScripting::queueAsync(std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin) +{ + return asyncEngine.queueAsyncJob(std::move(serialized_func), + param, mod_origin); +} + void ServerScripting::InitializeModApi(lua_State *L, int top) { // Register reference classes (userdata) @@ -125,3 +167,24 @@ void ServerScripting::InitializeModApi(lua_State *L, int top) ModApiStorage::Initialize(L, top); ModApiChannels::Initialize(L, top); } + +void ServerScripting::InitializeAsync(lua_State *L, int top) +{ + // classes + LuaItemStack::Register(L); + LuaPerlinNoise::Register(L); + LuaPerlinNoiseMap::Register(L); + LuaPseudoRandom::Register(L); + LuaPcgRandom::Register(L); + LuaSecureRandom::Register(L); + LuaVoxelManip::Register(L); + LuaSettings::Register(L); + + // globals data + lua_getglobal(L, "core"); + luaL_checktype(L, -1, LUA_TTABLE); + std::string s = ModApiBase::getServer(L)->m_async_globals_data.get(); + lua_pushlstring(L, s.c_str(), s.size()); + lua_setfield(L, -2, "transferred_globals"); + lua_pop(L, 1); // pop 'core' +} diff --git a/src/script/scripting_server.h b/src/script/scripting_server.h index bf06ab197..9803397c5 100644 --- a/src/script/scripting_server.h +++ b/src/script/scripting_server.h @@ -27,6 +27,9 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "cpp_api/s_player.h" #include "cpp_api/s_server.h" #include "cpp_api/s_security.h" +#include "cpp_api/s_async.h" + +struct PackedValue; /*****************************************************************************/ /* Scripting <-> Server Game Interface */ @@ -48,6 +51,20 @@ public: // use ScriptApiBase::loadMod() to load mods + // Initialize async engine, call this AFTER loading all mods + void initAsync(); + + // Global step handler to collect async results + void stepAsync(); + + // Pass job to async threads + u32 queueAsync(std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin); + private: void InitializeModApi(lua_State *L, int top); + + static void InitializeAsync(lua_State *L, int top); + + AsyncEngine asyncEngine; }; diff --git a/src/server.cpp b/src/server.cpp index dec6cf44c..c9cd4e398 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -243,6 +243,7 @@ Server::Server( m_clients(m_con), m_admin_chat(iface), m_on_shutdown_errmsg(on_shutdown_errmsg), + m_async_globals_data(""), m_modchannel_mgr(new ModChannelMgr()) { if (m_path_world.empty()) @@ -480,6 +481,9 @@ void Server::init() // Give environment reference to scripting api m_script->initializeEnvironment(m_env); + // Do this after regular script init is done + m_script->initAsync(); + // Register us to receive map edit events servermap->addEventReceiver(this); diff --git a/src/server.h b/src/server.h index bd799c313..5090a3579 100644 --- a/src/server.h +++ b/src/server.h @@ -292,7 +292,7 @@ public: virtual const std::vector &getMods() const; virtual const ModSpec* getModSpec(const std::string &modname) const; - std::string getBuiltinLuaPath(); + static std::string getBuiltinLuaPath(); virtual std::string getWorldPath() const { return m_path_world; } inline bool isSingleplayer() const @@ -385,6 +385,12 @@ public: static bool migrateModStorageDatabase(const GameParams &game_params, const Settings &cmd_args); + // Lua files registered for init of async env, pair of modname + path + std::vector> m_async_init_files; + + // Serialized data transferred into async envs at init time + MutexedVariable m_async_globals_data; + // Bind address Address m_bind_addr; diff --git a/src/serverenvironment.cpp b/src/serverenvironment.cpp index f8d84604b..493210744 100644 --- a/src/serverenvironment.cpp +++ b/src/serverenvironment.cpp @@ -1481,6 +1481,8 @@ void ServerEnvironment::step(float dtime) */ m_script->environment_Step(dtime); + m_script->stepAsync(); + /* Step active objects */ diff --git a/src/util/basic_macros.h b/src/util/basic_macros.h index 334e342e0..3910c6185 100644 --- a/src/util/basic_macros.h +++ b/src/util/basic_macros.h @@ -29,13 +29,19 @@ with this program; if not, write to the Free Software Foundation, Inc., #define CONTAINS(c, v) (std::find((c).begin(), (c).end(), (v)) != (c).end()) // To disable copy constructors and assignment operations for some class -// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) as a private member. +// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) in the class definition. // Note this also disables copying for any classes derived from 'Foobar' as well // as classes having a 'Foobar' member. #define DISABLE_CLASS_COPY(C) \ C(const C &) = delete; \ C &operator=(const C &) = delete; +// If you have used DISABLE_CLASS_COPY with a class but still want to permit moving +// use this macro to add the default move constructors back. +#define ALLOW_CLASS_MOVE(C) \ + C(C &&other) = default; \ + C &operator=(C &&) = default; + #ifndef _MSC_VER #define UNUSED_ATTRIBUTE __attribute__ ((unused)) #else -- cgit v1.2.3