From 2628316842e9c3fc1c54c6d57505c7851fa83490 Mon Sep 17 00:00:00 2001 From: nia <29542929+alarixnia@users.noreply.github.com> Date: Sun, 19 Sep 2021 18:23:52 +0000 Subject: Fix src/util/string.cpp on NetBSD - iconv() prototype changed from traditional Unix defintion to POSIX definition in 9.99.x. - wchar_t is not a valid character set for iconv. Share code with Android for using UTF-32. --- src/util/string.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'src/util') diff --git a/src/util/string.cpp b/src/util/string.cpp index eec5ab4cd..8be5e320a 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -39,8 +39,13 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #endif -#if defined(_ICONV_H_) && (defined(__FreeBSD__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) || defined(__DragonFly__)) +#ifdef __NetBSD__ + #include + #if __NetBSD_Version__ <= 999001500 + #define BSD_ICONV_USED + #endif +#elif defined(_ICONV_H_) && (defined(__FreeBSD__) || defined(__OpenBSD__) || \ + defined(__DragonFly__)) #define BSD_ICONV_USED #endif @@ -79,6 +84,14 @@ static bool convert(const char *to, const char *from, char *outbuf, #ifdef __ANDROID__ // On Android iconv disagrees how big a wchar_t is for whatever reason const char *DEFAULT_ENCODING = "UTF-32LE"; +#elif defined(__NetBSD__) + // NetBSD does not allow "WCHAR_T" as a charset input to iconv. + #include + #if BYTE_ORDER == BIG_ENDIAN + const char *DEFAULT_ENCODING = "UTF-32BE"; + #else + const char *DEFAULT_ENCODING = "UTF-32LE"; + #endif #else const char *DEFAULT_ENCODING = "WCHAR_T"; #endif @@ -94,7 +107,7 @@ std::wstring utf8_to_wide(const std::string &input) std::wstring out; out.resize(outbuf_size / sizeof(wchar_t)); -#ifdef __ANDROID__ +#if defined(__ANDROID__) || defined(__NetBSD__) SANITY_CHECK(sizeof(wchar_t) == 4); #endif -- cgit v1.2.3 From fe7195badb2801f4957d6dea2c961a3ffcf7debf Mon Sep 17 00:00:00 2001 From: Wuzzy Date: Wed, 17 Mar 2021 19:03:00 +0100 Subject: Make /status message easier to read --- src/server.cpp | 10 +++++----- src/util/string.h | 31 ++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 10 deletions(-) (limited to 'src/util') diff --git a/src/server.cpp b/src/server.cpp index 7fb9a78e9..5022221ee 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -3119,15 +3119,16 @@ std::string Server::getStatusString() std::ostringstream os(std::ios_base::binary); os << "# Server: "; // Version - os << "version=" << g_version_string; + os << "version: " << g_version_string; // Uptime - os << ", uptime=" << m_uptime_counter->get(); + os << " | uptime: " << duration_to_string((int) m_uptime_counter->get()); // Max lag estimate - os << ", max_lag=" << (m_env ? m_env->getMaxLagEstimate() : 0); + os << " | max lag: " << std::setprecision(3); + os << (m_env ? m_env->getMaxLagEstimate() : 0) << "s"; // Information about clients bool first = true; - os << ", clients={"; + os << " | clients: "; if (m_env) { std::vector clients = m_clients.getClientIDs(); for (session_t client_id : clients) { @@ -3144,7 +3145,6 @@ std::string Server::getStatusString() os << name; } } - os << "}"; if (m_env && !((ServerMap*)(&m_env->getMap()))->isSavingEnabled()) os << std::endl << "# Server: " << " WARNING: Map saving is disabled."; diff --git a/src/util/string.h b/src/util/string.h index 21f1d6877..bca998f56 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -661,28 +661,49 @@ inline const char *bool_to_cstr(bool val) return val ? "true" : "false"; } +/** + * Converts a duration in seconds to a pretty-printed duration in + * days, hours, minutes and seconds. + * + * @param sec duration in seconds + * @return pretty-printed duration + */ inline const std::string duration_to_string(int sec) { + std::ostringstream ss; + const char *neg = ""; + if (sec < 0) { + sec = -sec; + neg = "-"; + } + int total_sec = sec; int min = sec / 60; sec %= 60; int hour = min / 60; min %= 60; + int day = hour / 24; + hour %= 24; + + if (day > 0) { + ss << neg << day << "d"; + if (hour > 0 || min > 0 || sec > 0) + ss << " "; + } - std::stringstream ss; if (hour > 0) { - ss << hour << "h"; + ss << neg << hour << "h"; if (min > 0 || sec > 0) ss << " "; } if (min > 0) { - ss << min << "min"; + ss << neg << min << "min"; if (sec > 0) ss << " "; } - if (sec > 0) { - ss << sec << "s"; + if (sec > 0 || total_sec == 0) { + ss << neg << sec << "s"; } return ss.str(); -- cgit v1.2.3 From 660e63dbae5901f8178b39ab40e6f770b8d7a215 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Mon, 25 Oct 2021 20:30:27 +0200 Subject: Fix item duplication if player dies during interact callback (alternative) (#11662) --- builtin/game/item.lua | 50 +++++++++++++++++++++---------------- doc/lua_api.txt | 11 +++++--- src/network/serverpackethandler.cpp | 44 +++++++++++++++++++------------- src/script/cpp_api/s_item.cpp | 21 +++++++++++----- src/script/cpp_api/s_item.h | 14 ++++++++--- src/script/lua_api/l_env.cpp | 2 +- src/util/Optional.h | 32 ++++++++++++++++++++++-- 7 files changed, 118 insertions(+), 56 deletions(-) (limited to 'src/util') diff --git a/builtin/game/item.lua b/builtin/game/item.lua index c495a67bd..039947584 100644 --- a/builtin/game/item.lua +++ b/builtin/game/item.lua @@ -499,34 +499,40 @@ function core.do_item_eat(hp_change, replace_with_item, itemstack, user, pointed return result end end + if itemstack:take_item():is_empty() then + return itemstack + end + local def = itemstack:get_definition() - if itemstack:take_item() ~= nil then - user:set_hp(user:get_hp() + hp_change) - - if def and def.sound and def.sound.eat then - core.sound_play(def.sound.eat, { - pos = user:get_pos(), - max_hear_distance = 16 - }, true) - end + if def and def.sound and def.sound.eat then + core.sound_play(def.sound.eat, { + pos = user:get_pos(), + max_hear_distance = 16 + }, true) + end - if replace_with_item then - if itemstack:is_empty() then - itemstack:add_item(replace_with_item) + -- Changing hp might kill the player causing mods to do who-knows-what to the + -- inventory, so do this before set_hp(). + if replace_with_item then + if itemstack:is_empty() then + itemstack:add_item(replace_with_item) + else + local inv = user:get_inventory() + -- Check if inv is null, since non-players don't have one + if inv and inv:room_for_item("main", {name=replace_with_item}) then + inv:add_item("main", replace_with_item) else - local inv = user:get_inventory() - -- Check if inv is null, since non-players don't have one - if inv and inv:room_for_item("main", {name=replace_with_item}) then - inv:add_item("main", replace_with_item) - else - local pos = user:get_pos() - pos.y = math.floor(pos.y + 0.5) - core.add_item(pos, replace_with_item) - end + local pos = user:get_pos() + pos.y = math.floor(pos.y + 0.5) + core.add_item(pos, replace_with_item) end end end - return itemstack + user:set_wielded_item(itemstack) + + user:set_hp(user:get_hp() + hp_change) + + return nil -- don't overwrite wield item a second time end function core.item_eat(hp_change, replace_with_item) diff --git a/doc/lua_api.txt b/doc/lua_api.txt index 69ac55493..e47df4686 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -5421,7 +5421,7 @@ Inventory * `minetest.remove_detached_inventory(name)` * Returns a `boolean` indicating whether the removal succeeded. * `minetest.do_item_eat(hp_change, replace_with_item, itemstack, user, pointed_thing)`: - returns left over ItemStack. + returns leftover ItemStack or nil to indicate no inventory change * See `minetest.item_eat` and `minetest.register_on_item_eat` Formspec @@ -7542,12 +7542,15 @@ Used by `minetest.register_node`, `minetest.register_craftitem`, and on_place = function(itemstack, placer, pointed_thing), -- When the 'place' key was pressed with the item in hand -- and a node was pointed at. - -- Shall place item and return the leftover itemstack. + -- Shall place item and return the leftover itemstack + -- or nil to not modify the inventory. -- The placer may be any ObjectRef or nil. -- default: minetest.item_place on_secondary_use = function(itemstack, user, pointed_thing), -- Same as on_place but called when not pointing at a node. + -- Function must return either nil if inventory shall not be modified, + -- or an itemstack to replace the original itemstack. -- The user may be any ObjectRef or nil. -- default: nil @@ -7559,8 +7562,8 @@ Used by `minetest.register_node`, `minetest.register_craftitem`, and on_use = function(itemstack, user, pointed_thing), -- default: nil -- When user pressed the 'punch/mine' key with the item in hand. - -- Function must return either nil if no item shall be removed from - -- inventory, or an itemstack to replace the original itemstack. + -- Function must return either nil if inventory shall not be modified, + -- or an itemstack to replace the original itemstack. -- e.g. itemstack:take_item(); return itemstack -- Otherwise, the function is free to do what it wants. -- The user may be any ObjectRef or nil. diff --git a/src/network/serverpackethandler.cpp b/src/network/serverpackethandler.cpp index dc7be0e23..d4bef3ca2 100644 --- a/src/network/serverpackethandler.cpp +++ b/src/network/serverpackethandler.cpp @@ -921,6 +921,13 @@ bool Server::checkInteractDistance(RemotePlayer *player, const f32 d, const std: return true; } +// Tiny helper to retrieve the selected item into an Optional +static inline void getWieldedItem(const PlayerSAO *playersao, Optional &ret) +{ + ret = ItemStack(); + playersao->getWieldedItem(&(*ret)); +} + void Server::handleCommand_Interact(NetworkPacket *pkt) { /* @@ -1228,14 +1235,17 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) // Place block or right-click object case INTERACT_PLACE: { - ItemStack selected_item; - playersao->getWieldedItem(&selected_item, nullptr); + Optional selected_item; + getWieldedItem(playersao, selected_item); // Reset build time counter if (pointed.type == POINTEDTHING_NODE && - selected_item.getDefinition(m_itemdef).type == ITEM_NODE) + selected_item->getDefinition(m_itemdef).type == ITEM_NODE) getClient(peer_id)->m_time_from_building = 0.0; + const bool had_prediction = !selected_item->getDefinition(m_itemdef). + node_placement_prediction.empty(); + if (pointed.type == POINTEDTHING_OBJECT) { // Right click object @@ -1248,11 +1258,9 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) << pointed_object->getDescription() << std::endl; // Do stuff - if (m_script->item_OnSecondaryUse( - selected_item, playersao, pointed)) { - if (playersao->setWieldedItem(selected_item)) { + if (m_script->item_OnSecondaryUse(selected_item, playersao, pointed)) { + if (selected_item.has_value() && playersao->setWieldedItem(*selected_item)) SendInventory(playersao, true); - } } pointed_object->rightClick(playersao); @@ -1260,7 +1268,7 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) // Placement was handled in lua // Apply returned ItemStack - if (playersao->setWieldedItem(selected_item)) + if (selected_item.has_value() && playersao->setWieldedItem(*selected_item)) SendInventory(playersao, true); } @@ -1272,8 +1280,7 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) RemoteClient *client = getClient(peer_id); v3s16 blockpos = getNodeBlockPos(pointed.node_abovesurface); v3s16 blockpos2 = getNodeBlockPos(pointed.node_undersurface); - if (!selected_item.getDefinition(m_itemdef - ).node_placement_prediction.empty()) { + if (had_prediction) { client->SetBlockNotSent(blockpos); if (blockpos2 != blockpos) client->SetBlockNotSent(blockpos2); @@ -1287,15 +1294,15 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) } // action == INTERACT_PLACE case INTERACT_USE: { - ItemStack selected_item; - playersao->getWieldedItem(&selected_item, nullptr); + Optional selected_item; + getWieldedItem(playersao, selected_item); - actionstream << player->getName() << " uses " << selected_item.name + actionstream << player->getName() << " uses " << selected_item->name << ", pointing at " << pointed.dump() << std::endl; if (m_script->item_OnUse(selected_item, playersao, pointed)) { // Apply returned ItemStack - if (playersao->setWieldedItem(selected_item)) + if (selected_item.has_value() && playersao->setWieldedItem(*selected_item)) SendInventory(playersao, true); } @@ -1304,16 +1311,17 @@ void Server::handleCommand_Interact(NetworkPacket *pkt) // Rightclick air case INTERACT_ACTIVATE: { - ItemStack selected_item; - playersao->getWieldedItem(&selected_item, nullptr); + Optional selected_item; + getWieldedItem(playersao, selected_item); actionstream << player->getName() << " activates " - << selected_item.name << std::endl; + << selected_item->name << std::endl; pointed.type = POINTEDTHING_NOTHING; // can only ever be NOTHING if (m_script->item_OnSecondaryUse(selected_item, playersao, pointed)) { - if (playersao->setWieldedItem(selected_item)) + // Apply returned ItemStack + if (selected_item.has_value() && playersao->setWieldedItem(*selected_item)) SendInventory(playersao, true); } diff --git a/src/script/cpp_api/s_item.cpp b/src/script/cpp_api/s_item.cpp index 48dce14f3..b1916070e 100644 --- a/src/script/cpp_api/s_item.cpp +++ b/src/script/cpp_api/s_item.cpp @@ -59,13 +59,14 @@ bool ScriptApiItem::item_OnDrop(ItemStack &item, return true; } -bool ScriptApiItem::item_OnPlace(ItemStack &item, +bool ScriptApiItem::item_OnPlace(Optional &ret_item, ServerActiveObject *placer, const PointedThing &pointed) { SCRIPTAPI_PRECHECKHEADER int error_handler = PUSH_ERROR_HANDLER(L); + const ItemStack &item = *ret_item; // Push callback function on stack if (!getItemCallback(item.name.c_str(), "on_place")) return false; @@ -82,22 +83,25 @@ bool ScriptApiItem::item_OnPlace(ItemStack &item, PCALL_RES(lua_pcall(L, 3, 1, error_handler)); if (!lua_isnil(L, -1)) { try { - item = read_item(L, -1, getServer()->idef()); + ret_item = read_item(L, -1, getServer()->idef()); } catch (LuaError &e) { throw WRAP_LUAERROR(e, "item=" + item.name); } + } else { + ret_item = nullopt; } lua_pop(L, 2); // Pop item and error handler return true; } -bool ScriptApiItem::item_OnUse(ItemStack &item, +bool ScriptApiItem::item_OnUse(Optional &ret_item, ServerActiveObject *user, const PointedThing &pointed) { SCRIPTAPI_PRECHECKHEADER int error_handler = PUSH_ERROR_HANDLER(L); + const ItemStack &item = *ret_item; // Push callback function on stack if (!getItemCallback(item.name.c_str(), "on_use")) return false; @@ -109,22 +113,25 @@ bool ScriptApiItem::item_OnUse(ItemStack &item, PCALL_RES(lua_pcall(L, 3, 1, error_handler)); if(!lua_isnil(L, -1)) { try { - item = read_item(L, -1, getServer()->idef()); + ret_item = read_item(L, -1, getServer()->idef()); } catch (LuaError &e) { throw WRAP_LUAERROR(e, "item=" + item.name); } + } else { + ret_item = nullopt; } lua_pop(L, 2); // Pop item and error handler return true; } -bool ScriptApiItem::item_OnSecondaryUse(ItemStack &item, +bool ScriptApiItem::item_OnSecondaryUse(Optional &ret_item, ServerActiveObject *user, const PointedThing &pointed) { SCRIPTAPI_PRECHECKHEADER int error_handler = PUSH_ERROR_HANDLER(L); + const ItemStack &item = *ret_item; if (!getItemCallback(item.name.c_str(), "on_secondary_use")) return false; @@ -134,10 +141,12 @@ bool ScriptApiItem::item_OnSecondaryUse(ItemStack &item, PCALL_RES(lua_pcall(L, 3, 1, error_handler)); if (!lua_isnil(L, -1)) { try { - item = read_item(L, -1, getServer()->idef()); + ret_item = read_item(L, -1, getServer()->idef()); } catch (LuaError &e) { throw WRAP_LUAERROR(e, "item=" + item.name); } + } else { + ret_item = nullopt; } lua_pop(L, 2); // Pop item and error handler return true; diff --git a/src/script/cpp_api/s_item.h b/src/script/cpp_api/s_item.h index 25a3501f9..5015d8bd4 100644 --- a/src/script/cpp_api/s_item.h +++ b/src/script/cpp_api/s_item.h @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "cpp_api/s_base.h" #include "irr_v3d.h" +#include "util/Optional.h" struct PointedThing; struct ItemStack; @@ -35,13 +36,20 @@ class ScriptApiItem : virtual public ScriptApiBase { public: + /* + * Functions with Optional are for callbacks where Lua may + * want to prevent the engine from modifying the inventory after it's done. + * This has a longer backstory where on_use may need to empty the player's + * inventory without the engine interfering (see issue #6546). + */ + bool item_OnDrop(ItemStack &item, ServerActiveObject *dropper, v3f pos); - bool item_OnPlace(ItemStack &item, + bool item_OnPlace(Optional &item, ServerActiveObject *placer, const PointedThing &pointed); - bool item_OnUse(ItemStack &item, + bool item_OnUse(Optional &item, ServerActiveObject *user, const PointedThing &pointed); - bool item_OnSecondaryUse(ItemStack &item, + bool item_OnSecondaryUse(Optional &item, ServerActiveObject *user, const PointedThing &pointed); bool item_OnCraft(ItemStack &item, ServerActiveObject *user, const InventoryList *old_craft_grid, const InventoryLocation &craft_inv); diff --git a/src/script/lua_api/l_env.cpp b/src/script/lua_api/l_env.cpp index 98f8861fa..2c709d31b 100644 --- a/src/script/lua_api/l_env.cpp +++ b/src/script/lua_api/l_env.cpp @@ -477,7 +477,7 @@ int ModApiEnvMod::l_place_node(lua_State *L) return 1; } // Create item to place - ItemStack item(ndef->get(n).name, 1, 0, idef); + Optional item = ItemStack(ndef->get(n).name, 1, 0, idef); // Make pointed position PointedThing pointed; pointed.type = POINTEDTHING_NODE; diff --git a/src/util/Optional.h b/src/util/Optional.h index 9c2842b43..eda7fff89 100644 --- a/src/util/Optional.h +++ b/src/util/Optional.h @@ -19,6 +19,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #pragma once +#include #include "debug.h" struct nullopt_t @@ -43,18 +44,38 @@ class Optional public: Optional() noexcept {} Optional(nullopt_t) noexcept {} + Optional(const T &value) noexcept : m_has_value(true), m_value(value) {} + Optional(T &&value) noexcept : m_has_value(true), m_value(std::move(value)) {} + Optional(const Optional &other) noexcept : m_has_value(other.m_has_value), m_value(other.m_value) + {} + Optional(Optional &&other) noexcept : + m_has_value(other.m_has_value), m_value(std::move(other.m_value)) { + other.m_has_value = false; } - void operator=(nullopt_t) noexcept { m_has_value = false; } + Optional &operator=(nullopt_t) noexcept { m_has_value = false; return *this; } - void operator=(const Optional &other) noexcept + Optional &operator=(const Optional &other) noexcept { + if (&other == this) + return *this; m_has_value = other.m_has_value; m_value = other.m_value; + return *this; + } + + Optional &operator=(Optional &&other) noexcept + { + if (&other == this) + return *this; + m_has_value = other.m_has_value; + m_value = std::move(other.m_value); + other.m_has_value = false; + return *this; } T &value() @@ -71,6 +92,13 @@ public: const T &value_or(const T &def) const { return m_has_value ? m_value : def; } + // Unchecked access consistent with std::optional + T* operator->() { return &m_value; } + const T* operator->() const { return &m_value; } + + T& operator*() { return m_value; } + const T& operator*() const { return m_value; } + bool has_value() const noexcept { return m_has_value; } explicit operator bool() const { return m_has_value; } -- cgit v1.2.3 From 57a59ae92d4bbfa4fdd60d7acd72c6440f63a49c Mon Sep 17 00:00:00 2001 From: SmallJoker Date: Wed, 1 Dec 2021 20:22:33 +0100 Subject: Network: Delete copy constructor and use std::move instead (#11642) This is a follow-up change which disables class copies where possible to avoid unnecessary memory movements. --- src/client/client.cpp | 2 +- src/network/connection.cpp | 445 ++++++++++++++++++++++---------------- src/network/connection.h | 416 ++++++++++++++++------------------- src/network/connectionthreads.cpp | 190 ++++++++-------- src/network/connectionthreads.h | 31 ++- src/unittest/test_connection.cpp | 10 +- src/util/pointer.h | 58 ++--- 7 files changed, 600 insertions(+), 552 deletions(-) (limited to 'src/util') diff --git a/src/client/client.cpp b/src/client/client.cpp index 45cc62a33..3ee1298ff 100644 --- a/src/client/client.cpp +++ b/src/client/client.cpp @@ -877,7 +877,7 @@ void Client::ProcessData(NetworkPacket *pkt) */ if(sender_peer_id != PEER_ID_SERVER) { infostream << "Client::ProcessData(): Discarding data not " - "coming from server: peer_id=" << sender_peer_id + "coming from server: peer_id=" << sender_peer_id << " command=" << pkt->getCommand() << std::endl; return; } diff --git a/src/network/connection.cpp b/src/network/connection.cpp index 548b2e3a0..2d3cf6e88 100644 --- a/src/network/connection.cpp +++ b/src/network/connection.cpp @@ -62,18 +62,27 @@ namespace con #define PING_TIMEOUT 5.0 -BufferedPacket makePacket(Address &address, const SharedBuffer &data, +u16 BufferedPacket::getSeqnum() const +{ + if (size() < BASE_HEADER_SIZE + 3) + return 0; // should never happen + + return readU16(&data[BASE_HEADER_SIZE + 1]); +} + +BufferedPacketPtr makePacket(Address &address, const SharedBuffer &data, u32 protocol_id, session_t sender_peer_id, u8 channel) { u32 packet_size = data.getSize() + BASE_HEADER_SIZE; - BufferedPacket p(packet_size); - p.address = address; - writeU32(&p.data[0], protocol_id); - writeU16(&p.data[4], sender_peer_id); - writeU8(&p.data[6], channel); + BufferedPacketPtr p(new BufferedPacket(packet_size)); + p->address = address; + + writeU32(&p->data[0], protocol_id); + writeU16(&p->data[4], sender_peer_id); + writeU8(&p->data[6], channel); - memcpy(&p.data[BASE_HEADER_SIZE], *data, data.getSize()); + memcpy(&p->data[BASE_HEADER_SIZE], *data, data.getSize()); return p; } @@ -169,9 +178,8 @@ void ReliablePacketBuffer::print() MutexAutoLock listlock(m_list_mutex); LOG(dout_con<<"Dump of ReliablePacketBuffer:" << std::endl); unsigned int index = 0; - for (BufferedPacket &bufferedPacket : m_list) { - u16 s = readU16(&(bufferedPacket.data[BASE_HEADER_SIZE+1])); - LOG(dout_con<getSeqnum() << std::endl); index++; } } @@ -188,16 +196,13 @@ u32 ReliablePacketBuffer::size() return m_list.size(); } -RPBSearchResult ReliablePacketBuffer::findPacket(u16 seqnum) +RPBSearchResult ReliablePacketBuffer::findPacketNoLock(u16 seqnum) { - std::list::iterator i = m_list.begin(); - for(; i != m_list.end(); ++i) - { - u16 s = readU16(&(i->data[BASE_HEADER_SIZE+1])); - if (s == seqnum) - break; + for (auto it = m_list.begin(); it != m_list.end(); ++it) { + if ((*it)->getSeqnum() == seqnum) + return it; } - return i; + return m_list.end(); } bool ReliablePacketBuffer::getFirstSeqnum(u16& result) @@ -205,54 +210,54 @@ bool ReliablePacketBuffer::getFirstSeqnum(u16& result) MutexAutoLock listlock(m_list_mutex); if (m_list.empty()) return false; - const BufferedPacket &p = m_list.front(); - result = readU16(&p.data[BASE_HEADER_SIZE + 1]); + result = m_list.front()->getSeqnum(); return true; } -BufferedPacket ReliablePacketBuffer::popFirst() +BufferedPacketPtr ReliablePacketBuffer::popFirst() { MutexAutoLock listlock(m_list_mutex); if (m_list.empty()) throw NotFoundException("Buffer is empty"); - BufferedPacket p = std::move(m_list.front()); + + BufferedPacketPtr p(m_list.front()); m_list.pop_front(); if (m_list.empty()) { m_oldest_non_answered_ack = 0; } else { - m_oldest_non_answered_ack = - readU16(&m_list.front().data[BASE_HEADER_SIZE + 1]); + m_oldest_non_answered_ack = m_list.front()->getSeqnum(); } return p; } -BufferedPacket ReliablePacketBuffer::popSeqnum(u16 seqnum) +BufferedPacketPtr ReliablePacketBuffer::popSeqnum(u16 seqnum) { MutexAutoLock listlock(m_list_mutex); - RPBSearchResult r = findPacket(seqnum); - if (r == notFound()) { + RPBSearchResult r = findPacketNoLock(seqnum); + if (r == m_list.end()) { LOG(dout_con<<"Sequence number: " << seqnum << " not found in reliable buffer"<getSeqnum(); } return p; } -void ReliablePacketBuffer::insert(const BufferedPacket &p, u16 next_expected) +void ReliablePacketBuffer::insert(BufferedPacketPtr &p_ptr, u16 next_expected) { MutexAutoLock listlock(m_list_mutex); - if (p.data.getSize() < BASE_HEADER_SIZE + 3) { + const BufferedPacket &p = *p_ptr; + + if (p.size() < BASE_HEADER_SIZE + 3) { errorstream << "ReliablePacketBuffer::insert(): Invalid data size for " "reliable packet" << std::endl; return; @@ -263,7 +268,7 @@ void ReliablePacketBuffer::insert(const BufferedPacket &p, u16 next_expected) << std::endl; return; } - u16 seqnum = readU16(&p.data[BASE_HEADER_SIZE + 1]); + const u16 seqnum = p.getSeqnum(); if (!seqnum_in_window(seqnum, next_expected, MAX_RELIABLE_WINDOW_SIZE)) { errorstream << "ReliablePacketBuffer::insert(): seqnum is outside of " @@ -280,44 +285,44 @@ void ReliablePacketBuffer::insert(const BufferedPacket &p, u16 next_expected) // Find the right place for the packet and insert it there // If list is empty, just add it - if (m_list.empty()) - { - m_list.push_back(p); + if (m_list.empty()) { + m_list.push_back(p_ptr); m_oldest_non_answered_ack = seqnum; // Done. return; } // Otherwise find the right place - std::list::iterator i = m_list.begin(); + auto it = m_list.begin(); // Find the first packet in the list which has a higher seqnum - u16 s = readU16(&(i->data[BASE_HEADER_SIZE+1])); + u16 s = (*it)->getSeqnum(); /* case seqnum is smaller then next_expected seqnum */ /* this is true e.g. on wrap around */ if (seqnum < next_expected) { - while(((s < seqnum) || (s >= next_expected)) && (i != m_list.end())) { - ++i; - if (i != m_list.end()) - s = readU16(&(i->data[BASE_HEADER_SIZE+1])); + while(((s < seqnum) || (s >= next_expected)) && (it != m_list.end())) { + ++it; + if (it != m_list.end()) + s = (*it)->getSeqnum(); } } /* non wrap around case (at least for incoming and next_expected */ else { - while(((s < seqnum) && (s >= next_expected)) && (i != m_list.end())) { - ++i; - if (i != m_list.end()) - s = readU16(&(i->data[BASE_HEADER_SIZE+1])); + while(((s < seqnum) && (s >= next_expected)) && (it != m_list.end())) { + ++it; + if (it != m_list.end()) + s = (*it)->getSeqnum(); } } if (s == seqnum) { /* nothing to do this seems to be a resent packet */ /* for paranoia reason data should be compared */ + auto &i = *it; if ( - (readU16(&(i->data[BASE_HEADER_SIZE+1])) != seqnum) || - (i->data.getSize() != p.data.getSize()) || + (i->getSeqnum() != seqnum) || + (i->size() != p.size()) || (i->address != p.address) ) { @@ -325,51 +330,52 @@ void ReliablePacketBuffer::insert(const BufferedPacket &p, u16 next_expected) fprintf(stderr, "Duplicated seqnum %d non matching packet detected:\n", seqnum); - fprintf(stderr, "Old: seqnum: %05d size: %04d, address: %s\n", - readU16(&(i->data[BASE_HEADER_SIZE+1])),i->data.getSize(), + fprintf(stderr, "Old: seqnum: %05d size: %04zu, address: %s\n", + i->getSeqnum(), i->size(), i->address.serializeString().c_str()); - fprintf(stderr, "New: seqnum: %05d size: %04u, address: %s\n", - readU16(&(p.data[BASE_HEADER_SIZE+1])),p.data.getSize(), + fprintf(stderr, "New: seqnum: %05d size: %04zu, address: %s\n", + p.getSeqnum(), p.size(), p.address.serializeString().c_str()); throw IncomingDataCorruption("duplicated packet isn't same as original one"); } } /* insert or push back */ - else if (i != m_list.end()) { - m_list.insert(i, p); + else if (it != m_list.end()) { + m_list.insert(it, p_ptr); } else { - m_list.push_back(p); + m_list.push_back(p_ptr); } /* update last packet number */ - m_oldest_non_answered_ack = readU16(&m_list.front().data[BASE_HEADER_SIZE+1]); + m_oldest_non_answered_ack = m_list.front()->getSeqnum(); } void ReliablePacketBuffer::incrementTimeouts(float dtime) { MutexAutoLock listlock(m_list_mutex); - for (BufferedPacket &bufferedPacket : m_list) { - bufferedPacket.time += dtime; - bufferedPacket.totaltime += dtime; + for (auto &packet : m_list) { + packet->time += dtime; + packet->totaltime += dtime; } } -std::list +std::list> ReliablePacketBuffer::getTimedOuts(float timeout, u32 max_packets) { MutexAutoLock listlock(m_list_mutex); - std::list timed_outs; - for (BufferedPacket &bufferedPacket : m_list) { - if (bufferedPacket.time >= timeout) { - // caller will resend packet so reset time and increase counter - bufferedPacket.time = 0.0f; - bufferedPacket.resend_count++; + std::list> timed_outs; + for (auto &packet : m_list) { + if (packet->time < timeout) + continue; - timed_outs.push_back(bufferedPacket); + // caller will resend packet so reset time and increase counter + packet->time = 0.0f; + packet->resend_count++; - if (timed_outs.size() >= max_packets) - break; - } + timed_outs.emplace_back(packet); + + if (timed_outs.size() >= max_packets) + break; } return timed_outs; } @@ -428,11 +434,13 @@ IncomingSplitBuffer::~IncomingSplitBuffer() } } -SharedBuffer IncomingSplitBuffer::insert(const BufferedPacket &p, bool reliable) +SharedBuffer IncomingSplitBuffer::insert(BufferedPacketPtr &p_ptr, bool reliable) { MutexAutoLock listlock(m_map_mutex); + const BufferedPacket &p = *p_ptr; + u32 headersize = BASE_HEADER_SIZE + 7; - if (p.data.getSize() < headersize) { + if (p.size() < headersize) { errorstream << "Invalid data size for split packet" << std::endl; return SharedBuffer(); } @@ -473,7 +481,7 @@ SharedBuffer IncomingSplitBuffer::insert(const BufferedPacket &p, bool relia < chunkdata(chunkdatasize); memcpy(*chunkdata, &(p.data[headersize]), chunkdatasize); @@ -520,14 +528,67 @@ void IncomingSplitBuffer::removeUnreliableTimedOuts(float dtime, float timeout) ConnectionCommand */ -void ConnectionCommand::send(session_t peer_id_, u8 channelnum_, NetworkPacket *pkt, - bool reliable_) +ConnectionCommandPtr ConnectionCommand::create(ConnectionCommandType type) +{ + return ConnectionCommandPtr(new ConnectionCommand(type)); +} + +ConnectionCommandPtr ConnectionCommand::serve(Address address) +{ + auto c = create(CONNCMD_SERVE); + c->address = address; + return c; +} + +ConnectionCommandPtr ConnectionCommand::connect(Address address) { - type = CONNCMD_SEND; - peer_id = peer_id_; - channelnum = channelnum_; - data = pkt->oldForgePacket(); - reliable = reliable_; + auto c = create(CONNCMD_CONNECT); + c->address = address; + return c; +} + +ConnectionCommandPtr ConnectionCommand::disconnect() +{ + return create(CONNCMD_DISCONNECT); +} + +ConnectionCommandPtr ConnectionCommand::disconnect_peer(session_t peer_id) +{ + auto c = create(CONNCMD_DISCONNECT_PEER); + c->peer_id = peer_id; + return c; +} + +ConnectionCommandPtr ConnectionCommand::send(session_t peer_id, u8 channelnum, + NetworkPacket *pkt, bool reliable) +{ + auto c = create(CONNCMD_SEND); + c->peer_id = peer_id; + c->channelnum = channelnum; + c->reliable = reliable; + c->data = pkt->oldForgePacket(); + return c; +} + +ConnectionCommandPtr ConnectionCommand::ack(session_t peer_id, u8 channelnum, const Buffer &data) +{ + auto c = create(CONCMD_ACK); + c->peer_id = peer_id; + c->channelnum = channelnum; + c->reliable = false; + data.copyTo(c->data); + return c; +} + +ConnectionCommandPtr ConnectionCommand::createPeer(session_t peer_id, const Buffer &data) +{ + auto c = create(CONCMD_CREATE_PEER); + c->peer_id = peer_id; + c->channelnum = 0; + c->reliable = true; + c->raw = true; + data.copyTo(c->data); + return c; } /* @@ -562,39 +623,38 @@ void Channel::setNextSplitSeqNum(u16 seqnum) u16 Channel::getOutgoingSequenceNumber(bool& successful) { MutexAutoLock internal(m_internal_mutex); + u16 retval = next_outgoing_seqnum; - u16 lowest_unacked_seqnumber; + successful = false; /* shortcut if there ain't any packet in outgoing list */ - if (outgoing_reliables_sent.empty()) - { + if (outgoing_reliables_sent.empty()) { + successful = true; next_outgoing_seqnum++; return retval; } - if (outgoing_reliables_sent.getFirstSeqnum(lowest_unacked_seqnumber)) - { + u16 lowest_unacked_seqnumber; + if (outgoing_reliables_sent.getFirstSeqnum(lowest_unacked_seqnumber)) { if (lowest_unacked_seqnumber < next_outgoing_seqnum) { // ugly cast but this one is required in order to tell compiler we // know about difference of two unsigned may be negative in general // but we already made sure it won't happen in this case if (((u16)(next_outgoing_seqnum - lowest_unacked_seqnumber)) > m_window_size) { - successful = false; return 0; } - } - else { + } else { // ugly cast but this one is required in order to tell compiler we // know about difference of two unsigned may be negative in general // but we already made sure it won't happen in this case if ((next_outgoing_seqnum + (u16)(SEQNUM_MAX - lowest_unacked_seqnumber)) > m_window_size) { - successful = false; return 0; } } } + successful = true; next_outgoing_seqnum++; return retval; } @@ -946,45 +1006,45 @@ bool UDPPeer::Ping(float dtime,SharedBuffer& data) return false; } -void UDPPeer::PutReliableSendCommand(ConnectionCommand &c, +void UDPPeer::PutReliableSendCommand(ConnectionCommandPtr &c, unsigned int max_packet_size) { if (m_pending_disconnect) return; - Channel &chan = channels[c.channelnum]; + Channel &chan = channels[c->channelnum]; if (chan.queued_commands.empty() && /* don't queue more packets then window size */ - (chan.queued_reliables.size() < chan.getWindowSize() / 2)) { + (chan.queued_reliables.size() + 1 < chan.getWindowSize() / 2)) { LOG(dout_con<getDesc() - <<" processing reliable command for peer id: " << c.peer_id - <<" data size: " << c.data.getSize() << std::endl); - if (!processReliableSendCommand(c,max_packet_size)) { - chan.queued_commands.push_back(c); - } - } - else { + <<" processing reliable command for peer id: " << c->peer_id + <<" data size: " << c->data.getSize() << std::endl); + if (processReliableSendCommand(c, max_packet_size)) + return; + } else { LOG(dout_con<getDesc() - <<" Queueing reliable command for peer id: " << c.peer_id - <<" data size: " << c.data.getSize() <= chan.getWindowSize() / 2) { + <<" Queueing reliable command for peer id: " << c->peer_id + <<" data size: " << c->data.getSize() <= chan.getWindowSize() / 2) { LOG(derr_con << m_connection->getDesc() - << "Possible packet stall to peer id: " << c.peer_id + << "Possible packet stall to peer id: " << c->peer_id << " queued_commands=" << chan.queued_commands.size() << std::endl); } } + chan.queued_commands.push_back(c); } bool UDPPeer::processReliableSendCommand( - ConnectionCommand &c, + ConnectionCommandPtr &c_ptr, unsigned int max_packet_size) { if (m_pending_disconnect) return true; + const auto &c = *c_ptr; Channel &chan = channels[c.channelnum]; u32 chunksize_max = max_packet_size @@ -1003,9 +1063,9 @@ bool UDPPeer::processReliableSendCommand( chan.setNextSplitSeqNum(split_sequence_number); } - bool have_sequence_number = true; + bool have_sequence_number = false; bool have_initial_sequence_number = false; - std::queue toadd; + std::queue toadd; volatile u16 initial_sequence_number = 0; for (SharedBuffer &original : originals) { @@ -1024,25 +1084,23 @@ bool UDPPeer::processReliableSendCommand( SharedBuffer reliable = makeReliablePacket(original, seqnum); // Add base headers and make a packet - BufferedPacket p = con::makePacket(address, reliable, + BufferedPacketPtr p = con::makePacket(address, reliable, m_connection->GetProtocolID(), m_connection->GetPeerID(), c.channelnum); - toadd.push(std::move(p)); + toadd.push(p); } if (have_sequence_number) { - volatile u16 pcount = 0; while (!toadd.empty()) { - BufferedPacket p = std::move(toadd.front()); + BufferedPacketPtr p = toadd.front(); toadd.pop(); // LOG(dout_con<getDesc() // << " queuing reliable packet for peer_id: " << c.peer_id // << " channel: " << (c.channelnum&0xFF) // << " seqnum: " << readU16(&p.data[BASE_HEADER_SIZE+1]) // << std::endl) - chan.queued_reliables.push(std::move(p)); - pcount++; + chan.queued_reliables.push(p); } sanity_check(chan.queued_reliables.size() < 0xFFFF); return true; @@ -1051,6 +1109,7 @@ bool UDPPeer::processReliableSendCommand( volatile u16 packets_available = toadd.size(); /* we didn't get a single sequence number no need to fill queue */ if (!have_initial_sequence_number) { + LOG(derr_con << m_connection->getDesc() << "Ran out of sequence numbers!" << std::endl); return false; } @@ -1096,18 +1155,18 @@ void UDPPeer::RunCommandQueues( (channel.queued_reliables.size() < maxtransfer) && (commands_processed < maxcommands)) { try { - ConnectionCommand c = channel.queued_commands.front(); + ConnectionCommandPtr c = channel.queued_commands.front(); LOG(dout_con << m_connection->getDesc() << " processing queued reliable command " << std::endl); // Packet is processed, remove it from queue - if (processReliableSendCommand(c,max_packet_size)) { + if (processReliableSendCommand(c, max_packet_size)) { channel.queued_commands.pop_front(); } else { LOG(dout_con << m_connection->getDesc() - << " Failed to queue packets for peer_id: " << c.peer_id - << ", delaying sending of " << c.data.getSize() + << " Failed to queue packets for peer_id: " << c->peer_id + << ", delaying sending of " << c->data.getSize() << " bytes" << std::endl); } } @@ -1130,13 +1189,70 @@ void UDPPeer::setNextSplitSequenceNumber(u8 channel, u16 seqnum) channels[channel].setNextSplitSeqNum(seqnum); } -SharedBuffer UDPPeer::addSplitPacket(u8 channel, const BufferedPacket &toadd, +SharedBuffer UDPPeer::addSplitPacket(u8 channel, BufferedPacketPtr &toadd, bool reliable) { assert(channel < CHANNEL_COUNT); // Pre-condition return channels[channel].incoming_splits.insert(toadd, reliable); } +/* + ConnectionEvent +*/ + +const char *ConnectionEvent::describe() const +{ + switch(type) { + case CONNEVENT_NONE: + return "CONNEVENT_NONE"; + case CONNEVENT_DATA_RECEIVED: + return "CONNEVENT_DATA_RECEIVED"; + case CONNEVENT_PEER_ADDED: + return "CONNEVENT_PEER_ADDED"; + case CONNEVENT_PEER_REMOVED: + return "CONNEVENT_PEER_REMOVED"; + case CONNEVENT_BIND_FAILED: + return "CONNEVENT_BIND_FAILED"; + } + return "Invalid ConnectionEvent"; +} + + +ConnectionEventPtr ConnectionEvent::create(ConnectionEventType type) +{ + return std::shared_ptr(new ConnectionEvent(type)); +} + +ConnectionEventPtr ConnectionEvent::dataReceived(session_t peer_id, const Buffer &data) +{ + auto e = create(CONNEVENT_DATA_RECEIVED); + e->peer_id = peer_id; + data.copyTo(e->data); + return e; +} + +ConnectionEventPtr ConnectionEvent::peerAdded(session_t peer_id, Address address) +{ + auto e = create(CONNEVENT_PEER_ADDED); + e->peer_id = peer_id; + e->address = address; + return e; +} + +ConnectionEventPtr ConnectionEvent::peerRemoved(session_t peer_id, bool is_timeout, Address address) +{ + auto e = create(CONNEVENT_PEER_REMOVED); + e->peer_id = peer_id; + e->timeout = is_timeout; + e->address = address; + return e; +} + +ConnectionEventPtr ConnectionEvent::bindFailed() +{ + return create(CONNEVENT_BIND_FAILED); +} + /* Connection */ @@ -1186,18 +1302,12 @@ Connection::~Connection() /* Internal stuff */ -void Connection::putEvent(const ConnectionEvent &e) +void Connection::putEvent(ConnectionEventPtr e) { - assert(e.type != CONNEVENT_NONE); // Pre-condition + assert(e->type != CONNEVENT_NONE); // Pre-condition m_event_queue.push_back(e); } -void Connection::putEvent(ConnectionEvent &&e) -{ - assert(e.type != CONNEVENT_NONE); // Pre-condition - m_event_queue.push_back(std::move(e)); -} - void Connection::TriggerSend() { m_sendThread->Trigger(); @@ -1260,11 +1370,9 @@ bool Connection::deletePeer(session_t peer_id, bool timeout) Address peer_address; //any peer has a primary address this never fails! peer->getAddress(MTP_PRIMARY, peer_address); - // Create event - ConnectionEvent e; - e.peerRemoved(peer_id, timeout, peer_address); - putEvent(e); + // Create event + putEvent(ConnectionEvent::peerRemoved(peer_id, timeout, peer_address)); peer->Drop(); return true; @@ -1272,18 +1380,16 @@ bool Connection::deletePeer(session_t peer_id, bool timeout) /* Interface */ -ConnectionEvent Connection::waitEvent(u32 timeout_ms) +ConnectionEventPtr Connection::waitEvent(u32 timeout_ms) { try { return m_event_queue.pop_front(timeout_ms); } catch(ItemNotFoundException &ex) { - ConnectionEvent e; - e.type = CONNEVENT_NONE; - return e; + return ConnectionEvent::create(CONNEVENT_NONE); } } -void Connection::putCommand(const ConnectionCommand &c) +void Connection::putCommand(ConnectionCommandPtr c) { if (!m_shutting_down) { m_command_queue.push_back(c); @@ -1291,26 +1397,14 @@ void Connection::putCommand(const ConnectionCommand &c) } } -void Connection::putCommand(ConnectionCommand &&c) -{ - if (!m_shutting_down) { - m_command_queue.push_back(std::move(c)); - m_sendThread->Trigger(); - } -} - void Connection::Serve(Address bind_addr) { - ConnectionCommand c; - c.serve(bind_addr); - putCommand(c); + putCommand(ConnectionCommand::serve(bind_addr)); } void Connection::Connect(Address address) { - ConnectionCommand c; - c.connect(address); - putCommand(c); + putCommand(ConnectionCommand::connect(address)); } bool Connection::Connected() @@ -1332,9 +1426,7 @@ bool Connection::Connected() void Connection::Disconnect() { - ConnectionCommand c; - c.disconnect(); - putCommand(c); + putCommand(ConnectionCommand::disconnect()); } bool Connection::Receive(NetworkPacket *pkt, u32 timeout) @@ -1345,11 +1437,15 @@ bool Connection::Receive(NetworkPacket *pkt, u32 timeout) This is not considered to be a problem (is it?) */ for(;;) { - ConnectionEvent e = waitEvent(timeout); - if (e.type != CONNEVENT_NONE) + ConnectionEventPtr e_ptr = waitEvent(timeout); + const ConnectionEvent &e = *e_ptr; + + if (e.type != CONNEVENT_NONE) { LOG(dout_con << getDesc() << ": Receive: got event: " << e.describe() << std::endl); - switch(e.type) { + } + + switch (e.type) { case CONNEVENT_NONE: return false; case CONNEVENT_DATA_RECEIVED: @@ -1397,10 +1493,7 @@ void Connection::Send(session_t peer_id, u8 channelnum, { assert(channelnum < CHANNEL_COUNT); // Pre-condition - ConnectionCommand c; - - c.send(peer_id, channelnum, pkt, reliable); - putCommand(std::move(c)); + putCommand(ConnectionCommand::send(peer_id, channelnum, pkt, reliable)); } Address Connection::GetPeerAddress(session_t peer_id) @@ -1499,41 +1592,31 @@ u16 Connection::createPeer(Address& sender, MTProtocols protocol, int fd) LOG(dout_con << getDesc() << "createPeer(): giving peer_id=" << peer_id_new << std::endl); - ConnectionCommand cmd; - Buffer reply(4); - writeU8(&reply[0], PACKET_TYPE_CONTROL); - writeU8(&reply[1], CONTROLTYPE_SET_PEER_ID); - writeU16(&reply[2], peer_id_new); - cmd.createPeer(peer_id_new,reply); - putCommand(std::move(cmd)); + { + Buffer reply(4); + writeU8(&reply[0], PACKET_TYPE_CONTROL); + writeU8(&reply[1], CONTROLTYPE_SET_PEER_ID); + writeU16(&reply[2], peer_id_new); + putCommand(ConnectionCommand::createPeer(peer_id_new, reply)); + } // Create peer addition event - ConnectionEvent e; - e.peerAdded(peer_id_new, sender); - putEvent(e); + putEvent(ConnectionEvent::peerAdded(peer_id_new, sender)); // We're now talking to a valid peer_id return peer_id_new; } -void Connection::PrintInfo(std::ostream &out) -{ - m_info_mutex.lock(); - out< ack(4); writeU8(&ack[0], PACKET_TYPE_CONTROL); writeU8(&ack[1], CONTROLTYPE_ACK); writeU16(&ack[2], seqnum); - c.ack(peer_id, channelnum, ack); - putCommand(std::move(c)); + putCommand(ConnectionCommand::ack(peer_id, channelnum, ack)); m_sendThread->Trigger(); } diff --git a/src/network/connection.h b/src/network/connection.h index ea74ffb1c..1afb4ae84 100644 --- a/src/network/connection.h +++ b/src/network/connection.h @@ -32,6 +32,95 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include +#define MAX_UDP_PEERS 65535 + +/* +=== NOTES === + +A packet is sent through a channel to a peer with a basic header: + Header (7 bytes): + [0] u32 protocol_id + [4] session_t sender_peer_id + [6] u8 channel +sender_peer_id: + Unique to each peer. + value 0 (PEER_ID_INEXISTENT) is reserved for making new connections + value 1 (PEER_ID_SERVER) is reserved for server + these constants are defined in constants.h +channel: + Channel numbers have no intrinsic meaning. Currently only 0, 1, 2 exist. +*/ +#define BASE_HEADER_SIZE 7 +#define CHANNEL_COUNT 3 + +/* +Packet types: + +CONTROL: This is a packet used by the protocol. +- When this is processed, nothing is handed to the user. + Header (2 byte): + [0] u8 type + [1] u8 controltype +controltype and data description: + CONTROLTYPE_ACK + [2] u16 seqnum + CONTROLTYPE_SET_PEER_ID + [2] session_t peer_id_new + CONTROLTYPE_PING + - There is no actual reply, but this can be sent in a reliable + packet to get a reply + CONTROLTYPE_DISCO +*/ +enum ControlType : u8 { + CONTROLTYPE_ACK = 0, + CONTROLTYPE_SET_PEER_ID = 1, + CONTROLTYPE_PING = 2, + CONTROLTYPE_DISCO = 3, +}; + +/* +ORIGINAL: This is a plain packet with no control and no error +checking at all. +- When this is processed, it is directly handed to the user. + Header (1 byte): + [0] u8 type +*/ +//#define TYPE_ORIGINAL 1 +#define ORIGINAL_HEADER_SIZE 1 + +/* +SPLIT: These are sequences of packets forming one bigger piece of +data. +- When processed and all the packet_nums 0...packet_count-1 are + present (this should be buffered), the resulting data shall be + directly handed to the user. +- If the data fails to come up in a reasonable time, the buffer shall + be silently discarded. +- These can be sent as-is or atop of a RELIABLE packet stream. + Header (7 bytes): + [0] u8 type + [1] u16 seqnum + [3] u16 chunk_count + [5] u16 chunk_num +*/ +//#define TYPE_SPLIT 2 + +/* +RELIABLE: Delivery of all RELIABLE packets shall be forced by ACKs, +and they shall be delivered in the same order as sent. This is done +with a buffer in the receiving and transmitting end. +- When this is processed, the contents of each packet is recursively + processed as packets. + Header (3 bytes): + [0] u8 type + [1] u16 seqnum + +*/ +//#define TYPE_RELIABLE 3 +#define RELIABLE_HEADER_SIZE 3 +#define SEQNUM_INITIAL 65500 +#define SEQNUM_MAX 65535 + class NetworkPacket; namespace con @@ -46,9 +135,13 @@ typedef enum MTProtocols { MTP_MINETEST_RELIABLE_UDP } MTProtocols; -#define MAX_UDP_PEERS 65535 - -#define SEQNUM_MAX 65535 +enum PacketType : u8 { + PACKET_TYPE_CONTROL = 0, + PACKET_TYPE_ORIGINAL = 1, + PACKET_TYPE_SPLIT = 2, + PACKET_TYPE_RELIABLE = 3, + PACKET_TYPE_MAX +}; inline bool seqnum_higher(u16 totest, u16 base) { @@ -85,24 +178,40 @@ static inline float CALC_DTIME(u64 lasttime, u64 curtime) return MYMAX(MYMIN(value,0.1),0.0); } -struct BufferedPacket -{ - BufferedPacket(u8 *a_data, u32 a_size): - data(a_data, a_size) - {} - BufferedPacket(u32 a_size): - data(a_size) - {} - Buffer data; // Data of the packet, including headers +/* + Struct for all kinds of packets. Includes following data: + BASE_HEADER + u8[] packet data (usually copied from SharedBuffer) +*/ +struct BufferedPacket { + BufferedPacket(u32 a_size) + { + m_data.resize(a_size); + data = &m_data[0]; + } + + DISABLE_CLASS_COPY(BufferedPacket) + + u16 getSeqnum() const; + + inline const size_t size() const { return m_data.size(); } + + u8 *data; // Direct memory access float time = 0.0f; // Seconds from buffering the packet or re-sending float totaltime = 0.0f; // Seconds from buffering the packet u64 absolute_send_time = -1; Address address; // Sender or destination unsigned int resend_count = 0; + +private: + std::vector m_data; // Data of the packet, including headers }; +typedef std::shared_ptr BufferedPacketPtr; + + // This adds the base headers to the data and makes a packet out of it -BufferedPacket makePacket(Address &address, const SharedBuffer &data, +BufferedPacketPtr makePacket(Address &address, const SharedBuffer &data, u32 protocol_id, session_t sender_peer_id, u8 channel); // Depending on size, make a TYPE_ORIGINAL or TYPE_SPLIT packet @@ -136,101 +245,12 @@ private: std::map> chunks; }; -/* -=== NOTES === - -A packet is sent through a channel to a peer with a basic header: - Header (7 bytes): - [0] u32 protocol_id - [4] session_t sender_peer_id - [6] u8 channel -sender_peer_id: - Unique to each peer. - value 0 (PEER_ID_INEXISTENT) is reserved for making new connections - value 1 (PEER_ID_SERVER) is reserved for server - these constants are defined in constants.h -channel: - Channel numbers have no intrinsic meaning. Currently only 0, 1, 2 exist. -*/ -#define BASE_HEADER_SIZE 7 -#define CHANNEL_COUNT 3 -/* -Packet types: - -CONTROL: This is a packet used by the protocol. -- When this is processed, nothing is handed to the user. - Header (2 byte): - [0] u8 type - [1] u8 controltype -controltype and data description: - CONTROLTYPE_ACK - [2] u16 seqnum - CONTROLTYPE_SET_PEER_ID - [2] session_t peer_id_new - CONTROLTYPE_PING - - There is no actual reply, but this can be sent in a reliable - packet to get a reply - CONTROLTYPE_DISCO -*/ -//#define TYPE_CONTROL 0 -#define CONTROLTYPE_ACK 0 -#define CONTROLTYPE_SET_PEER_ID 1 -#define CONTROLTYPE_PING 2 -#define CONTROLTYPE_DISCO 3 - -/* -ORIGINAL: This is a plain packet with no control and no error -checking at all. -- When this is processed, it is directly handed to the user. - Header (1 byte): - [0] u8 type -*/ -//#define TYPE_ORIGINAL 1 -#define ORIGINAL_HEADER_SIZE 1 -/* -SPLIT: These are sequences of packets forming one bigger piece of -data. -- When processed and all the packet_nums 0...packet_count-1 are - present (this should be buffered), the resulting data shall be - directly handed to the user. -- If the data fails to come up in a reasonable time, the buffer shall - be silently discarded. -- These can be sent as-is or atop of a RELIABLE packet stream. - Header (7 bytes): - [0] u8 type - [1] u16 seqnum - [3] u16 chunk_count - [5] u16 chunk_num -*/ -//#define TYPE_SPLIT 2 -/* -RELIABLE: Delivery of all RELIABLE packets shall be forced by ACKs, -and they shall be delivered in the same order as sent. This is done -with a buffer in the receiving and transmitting end. -- When this is processed, the contents of each packet is recursively - processed as packets. - Header (3 bytes): - [0] u8 type - [1] u16 seqnum - -*/ -//#define TYPE_RELIABLE 3 -#define RELIABLE_HEADER_SIZE 3 -#define SEQNUM_INITIAL 65500 - -enum PacketType: u8 { - PACKET_TYPE_CONTROL = 0, - PACKET_TYPE_ORIGINAL = 1, - PACKET_TYPE_SPLIT = 2, - PACKET_TYPE_RELIABLE = 3, - PACKET_TYPE_MAX -}; /* A buffer which stores reliable packets and sorts them internally for fast access to the smallest one. */ -typedef std::list::iterator RPBSearchResult; +typedef std::list::iterator RPBSearchResult; class ReliablePacketBuffer { @@ -239,12 +259,12 @@ public: bool getFirstSeqnum(u16& result); - BufferedPacket popFirst(); - BufferedPacket popSeqnum(u16 seqnum); - void insert(const BufferedPacket &p, u16 next_expected); + BufferedPacketPtr popFirst(); + BufferedPacketPtr popSeqnum(u16 seqnum); + void insert(BufferedPacketPtr &p_ptr, u16 next_expected); void incrementTimeouts(float dtime); - std::list getTimedOuts(float timeout, u32 max_packets); + std::list> getTimedOuts(float timeout, u32 max_packets); void print(); bool empty(); @@ -252,10 +272,9 @@ public: private: - RPBSearchResult findPacket(u16 seqnum); // does not perform locking - inline RPBSearchResult notFound() { return m_list.end(); } + RPBSearchResult findPacketNoLock(u16 seqnum); - std::list m_list; + std::list m_list; u16 m_oldest_non_answered_ack; @@ -274,7 +293,7 @@ public: Returns a reference counted buffer of length != 0 when a full split packet is constructed. If not, returns one of length 0. */ - SharedBuffer insert(const BufferedPacket &p, bool reliable); + SharedBuffer insert(BufferedPacketPtr &p_ptr, bool reliable); void removeUnreliableTimedOuts(float dtime, float timeout); @@ -285,25 +304,6 @@ private: std::mutex m_map_mutex; }; -struct OutgoingPacket -{ - session_t peer_id; - u8 channelnum; - SharedBuffer data; - bool reliable; - bool ack; - - OutgoingPacket(session_t peer_id_, u8 channelnum_, const SharedBuffer &data_, - bool reliable_,bool ack_=false): - peer_id(peer_id_), - channelnum(channelnum_), - data(data_), - reliable(reliable_), - ack(ack_) - { - } -}; - enum ConnectionCommandType{ CONNCMD_NONE, CONNCMD_SERVE, @@ -316,9 +316,13 @@ enum ConnectionCommandType{ CONCMD_CREATE_PEER }; +struct ConnectionCommand; +typedef std::shared_ptr ConnectionCommandPtr; + +// This is very similar to ConnectionEvent struct ConnectionCommand { - enum ConnectionCommandType type = CONNCMD_NONE; + const ConnectionCommandType type; Address address; session_t peer_id = PEER_ID_INEXISTENT; u8 channelnum = 0; @@ -326,48 +330,21 @@ struct ConnectionCommand bool reliable = false; bool raw = false; - ConnectionCommand() = default; - - void serve(Address address_) - { - type = CONNCMD_SERVE; - address = address_; - } - void connect(Address address_) - { - type = CONNCMD_CONNECT; - address = address_; - } - void disconnect() - { - type = CONNCMD_DISCONNECT; - } - void disconnect_peer(session_t peer_id_) - { - type = CONNCMD_DISCONNECT_PEER; - peer_id = peer_id_; - } + DISABLE_CLASS_COPY(ConnectionCommand); - void send(session_t peer_id_, u8 channelnum_, NetworkPacket *pkt, bool reliable_); + static ConnectionCommandPtr serve(Address address); + static ConnectionCommandPtr connect(Address address); + static ConnectionCommandPtr disconnect(); + static ConnectionCommandPtr disconnect_peer(session_t peer_id); + static ConnectionCommandPtr send(session_t peer_id, u8 channelnum, NetworkPacket *pkt, bool reliable); + static ConnectionCommandPtr ack(session_t peer_id, u8 channelnum, const Buffer &data); + static ConnectionCommandPtr createPeer(session_t peer_id, const Buffer &data); - void ack(session_t peer_id_, u8 channelnum_, const Buffer &data_) - { - type = CONCMD_ACK; - peer_id = peer_id_; - channelnum = channelnum_; - data = data_; - reliable = false; - } +private: + ConnectionCommand(ConnectionCommandType type_) : + type(type_) {} - void createPeer(session_t peer_id_, const Buffer &data_) - { - type = CONCMD_CREATE_PEER; - peer_id = peer_id_; - data = data_; - channelnum = 0; - reliable = true; - raw = true; - } + static ConnectionCommandPtr create(ConnectionCommandType type); }; /* maximum window size to use, 0xFFFF is theoretical maximum. don't think about @@ -402,10 +379,10 @@ public: ReliablePacketBuffer outgoing_reliables_sent; //queued reliable packets - std::queue queued_reliables; + std::queue queued_reliables; //queue commands prior splitting to packets - std::deque queued_commands; + std::deque queued_commands; IncomingSplitBuffer incoming_splits; @@ -514,7 +491,7 @@ class Peer { public: friend class PeerHelper; - Peer(Address address_,u16 id_,Connection* connection) : + Peer(Address address_,session_t id_,Connection* connection) : id(id_), m_connection(connection), address(address_), @@ -528,11 +505,11 @@ class Peer { }; // Unique id of the peer - u16 id; + const session_t id; void Drop(); - virtual void PutReliableSendCommand(ConnectionCommand &c, + virtual void PutReliableSendCommand(ConnectionCommandPtr &c, unsigned int max_packet_size) {}; virtual bool getAddress(MTProtocols type, Address& toset) = 0; @@ -549,7 +526,7 @@ class Peer { virtual u16 getNextSplitSequenceNumber(u8 channel) { return 0; }; virtual void setNextSplitSequenceNumber(u8 channel, u16 seqnum) {}; - virtual SharedBuffer addSplitPacket(u8 channel, const BufferedPacket &toadd, + virtual SharedBuffer addSplitPacket(u8 channel, BufferedPacketPtr &toadd, bool reliable) { errorstream << "Peer::addSplitPacket called," @@ -586,7 +563,7 @@ class Peer { bool IncUseCount(); void DecUseCount(); - std::mutex m_exclusive_access_mutex; + mutable std::mutex m_exclusive_access_mutex; bool m_pending_deletion = false; @@ -634,7 +611,7 @@ public: UDPPeer(u16 a_id, Address a_address, Connection* connection); virtual ~UDPPeer() = default; - void PutReliableSendCommand(ConnectionCommand &c, + void PutReliableSendCommand(ConnectionCommandPtr &c, unsigned int max_packet_size); bool getAddress(MTProtocols type, Address& toset); @@ -642,7 +619,7 @@ public: u16 getNextSplitSequenceNumber(u8 channel); void setNextSplitSequenceNumber(u8 channel, u16 seqnum); - SharedBuffer addSplitPacket(u8 channel, const BufferedPacket &toadd, + SharedBuffer addSplitPacket(u8 channel, BufferedPacketPtr &toadd, bool reliable); protected: @@ -671,7 +648,7 @@ private: float resend_timeout = 0.5; bool processReliableSendCommand( - ConnectionCommand &c, + ConnectionCommandPtr &c_ptr, unsigned int max_packet_size); }; @@ -679,7 +656,7 @@ private: Connection */ -enum ConnectionEventType{ +enum ConnectionEventType { CONNEVENT_NONE, CONNEVENT_DATA_RECEIVED, CONNEVENT_PEER_ADDED, @@ -687,56 +664,32 @@ enum ConnectionEventType{ CONNEVENT_BIND_FAILED, }; +struct ConnectionEvent; +typedef std::shared_ptr ConnectionEventPtr; + +// This is very similar to ConnectionCommand struct ConnectionEvent { - enum ConnectionEventType type = CONNEVENT_NONE; + const ConnectionEventType type; session_t peer_id = 0; Buffer data; bool timeout = false; Address address; - ConnectionEvent() = default; + // We don't want to copy "data" + DISABLE_CLASS_COPY(ConnectionEvent); - const char *describe() const - { - switch(type) { - case CONNEVENT_NONE: - return "CONNEVENT_NONE"; - case CONNEVENT_DATA_RECEIVED: - return "CONNEVENT_DATA_RECEIVED"; - case CONNEVENT_PEER_ADDED: - return "CONNEVENT_PEER_ADDED"; - case CONNEVENT_PEER_REMOVED: - return "CONNEVENT_PEER_REMOVED"; - case CONNEVENT_BIND_FAILED: - return "CONNEVENT_BIND_FAILED"; - } - return "Invalid ConnectionEvent"; - } + static ConnectionEventPtr create(ConnectionEventType type); + static ConnectionEventPtr dataReceived(session_t peer_id, const Buffer &data); + static ConnectionEventPtr peerAdded(session_t peer_id, Address address); + static ConnectionEventPtr peerRemoved(session_t peer_id, bool is_timeout, Address address); + static ConnectionEventPtr bindFailed(); - void dataReceived(session_t peer_id_, const Buffer &data_) - { - type = CONNEVENT_DATA_RECEIVED; - peer_id = peer_id_; - data = data_; - } - void peerAdded(session_t peer_id_, Address address_) - { - type = CONNEVENT_PEER_ADDED; - peer_id = peer_id_; - address = address_; - } - void peerRemoved(session_t peer_id_, bool timeout_, Address address_) - { - type = CONNEVENT_PEER_REMOVED; - peer_id = peer_id_; - timeout = timeout_; - address = address_; - } - void bindFailed() - { - type = CONNEVENT_BIND_FAILED; - } + const char *describe() const; + +private: + ConnectionEvent(ConnectionEventType type_) : + type(type_) {} }; class PeerHandler; @@ -752,10 +705,9 @@ public: ~Connection(); /* Interface */ - ConnectionEvent waitEvent(u32 timeout_ms); - // Warning: creates an unnecessary copy, prefer putCommand(T&&) if possible - void putCommand(const ConnectionCommand &c); - void putCommand(ConnectionCommand &&c); + ConnectionEventPtr waitEvent(u32 timeout_ms); + + void putCommand(ConnectionCommandPtr c); void SetTimeoutMs(u32 timeout) { m_bc_receive_timeout = timeout; } void Serve(Address bind_addr); @@ -785,8 +737,6 @@ protected: void sendAck(session_t peer_id, u8 channelnum, u16 seqnum); - void PrintInfo(std::ostream &out); - std::vector getPeerIDs() { MutexAutoLock peerlock(m_peers_mutex); @@ -795,13 +745,11 @@ protected: UDPSocket m_udpSocket; // Command queue: user -> SendThread - MutexedQueue m_command_queue; + MutexedQueue m_command_queue; bool Receive(NetworkPacket *pkt, u32 timeout); - // Warning: creates an unnecessary copy, prefer putEvent(T&&) if possible - void putEvent(const ConnectionEvent &e); - void putEvent(ConnectionEvent &&e); + void putEvent(ConnectionEventPtr e); void TriggerSend(); @@ -811,7 +759,7 @@ protected: } private: // Event queue: ReceiveThread -> user - MutexedQueue m_event_queue; + MutexedQueue m_event_queue; session_t m_peer_id = 0; u32 m_protocol_id; @@ -823,7 +771,7 @@ private: std::unique_ptr m_sendThread; std::unique_ptr m_receiveThread; - std::mutex m_info_mutex; + mutable std::mutex m_info_mutex; // Backwards compatibility PeerHandler *m_bc_peerhandler; diff --git a/src/network/connectionthreads.cpp b/src/network/connectionthreads.cpp index a306ced9b..dca065ae1 100644 --- a/src/network/connectionthreads.cpp +++ b/src/network/connectionthreads.cpp @@ -50,11 +50,11 @@ std::mutex log_conthread_mutex; #define WINDOW_SIZE 5 -static session_t readPeerId(u8 *packetdata) +static session_t readPeerId(const u8 *packetdata) { return readU16(&packetdata[4]); } -static u8 readChannel(u8 *packetdata) +static u8 readChannel(const u8 *packetdata) { return readU8(&packetdata[6]); } @@ -114,9 +114,9 @@ void *ConnectionSendThread::run() } /* translate commands to packets */ - ConnectionCommand c = m_connection->m_command_queue.pop_frontNoEx(0); - while (c.type != CONNCMD_NONE) { - if (c.reliable) + auto c = m_connection->m_command_queue.pop_frontNoEx(0); + while (c && c->type != CONNCMD_NONE) { + if (c->reliable) processReliableCommand(c); else processNonReliableCommand(c); @@ -227,21 +227,21 @@ void ConnectionSendThread::runTimeouts(float dtime) m_iteration_packets_avaialble -= timed_outs.size(); for (const auto &k : timed_outs) { - u8 channelnum = readChannel(*k.data); - u16 seqnum = readU16(&(k.data[BASE_HEADER_SIZE + 1])); + u8 channelnum = readChannel(k->data); + u16 seqnum = k->getSeqnum(); - channel.UpdateBytesLost(k.data.getSize()); + channel.UpdateBytesLost(k->size()); LOG(derr_con << m_connection->getDesc() << "RE-SENDING timed-out RELIABLE to " - << k.address.serializeString() + << k->address.serializeString() << "(t/o=" << resend_timeout << "): " - << "count=" << k.resend_count + << "count=" << k->resend_count << ", channel=" << ((int) channelnum & 0xff) << ", seqnum=" << seqnum << std::endl); - rawSend(k); + rawSend(k.get()); // do not handle rtt here as we can't decide if this packet was // lost or really takes more time to transmit @@ -274,25 +274,24 @@ void ConnectionSendThread::runTimeouts(float dtime) } } -void ConnectionSendThread::rawSend(const BufferedPacket &packet) +void ConnectionSendThread::rawSend(const BufferedPacket *p) { try { - m_connection->m_udpSocket.Send(packet.address, *packet.data, - packet.data.getSize()); + m_connection->m_udpSocket.Send(p->address, p->data, p->size()); LOG(dout_con << m_connection->getDesc() - << " rawSend: " << packet.data.getSize() + << " rawSend: " << p->size() << " bytes sent" << std::endl); } catch (SendFailedException &e) { LOG(derr_con << m_connection->getDesc() << "Connection::rawSend(): SendFailedException: " - << packet.address.serializeString() << std::endl); + << p->address.serializeString() << std::endl); } } -void ConnectionSendThread::sendAsPacketReliable(BufferedPacket &p, Channel *channel) +void ConnectionSendThread::sendAsPacketReliable(BufferedPacketPtr &p, Channel *channel) { try { - p.absolute_send_time = porting::getTimeMs(); + p->absolute_send_time = porting::getTimeMs(); // Buffer the packet channel->outgoing_reliables_sent.insert(p, (channel->readOutgoingSequenceNumber() - MAX_RELIABLE_WINDOW_SIZE) @@ -305,7 +304,7 @@ void ConnectionSendThread::sendAsPacketReliable(BufferedPacket &p, Channel *chan } // Send the packet - rawSend(p); + rawSend(p.get()); } bool ConnectionSendThread::rawSendAsPacket(session_t peer_id, u8 channelnum, @@ -321,11 +320,10 @@ bool ConnectionSendThread::rawSendAsPacket(session_t peer_id, u8 channelnum, Channel *channel = &(dynamic_cast(&peer)->channels[channelnum]); if (reliable) { - bool have_sequence_number_for_raw_packet = true; - u16 seqnum = - channel->getOutgoingSequenceNumber(have_sequence_number_for_raw_packet); + bool have_seqnum = false; + const u16 seqnum = channel->getOutgoingSequenceNumber(have_seqnum); - if (!have_sequence_number_for_raw_packet) + if (!have_seqnum) return false; SharedBuffer reliable = makeReliablePacket(data, seqnum); @@ -333,13 +331,12 @@ bool ConnectionSendThread::rawSendAsPacket(session_t peer_id, u8 channelnum, peer->getAddress(MTP_MINETEST_RELIABLE_UDP, peer_address); // Add base headers and make a packet - BufferedPacket p = con::makePacket(peer_address, reliable, + BufferedPacketPtr p = con::makePacket(peer_address, reliable, m_connection->GetProtocolID(), m_connection->GetPeerID(), channelnum); // first check if our send window is already maxed out - if (channel->outgoing_reliables_sent.size() - < channel->getWindowSize()) { + if (channel->outgoing_reliables_sent.size() < channel->getWindowSize()) { LOG(dout_con << m_connection->getDesc() << " INFO: sending a reliable packet to peer_id " << peer_id << " channel: " << (u32)channelnum @@ -352,19 +349,19 @@ bool ConnectionSendThread::rawSendAsPacket(session_t peer_id, u8 channelnum, << " INFO: queueing reliable packet for peer_id: " << peer_id << " channel: " << (u32)channelnum << " seqnum: " << seqnum << std::endl); - channel->queued_reliables.push(std::move(p)); + channel->queued_reliables.push(p); return false; } Address peer_address; if (peer->getAddress(MTP_UDP, peer_address)) { // Add base headers and make a packet - BufferedPacket p = con::makePacket(peer_address, data, + BufferedPacketPtr p = con::makePacket(peer_address, data, m_connection->GetProtocolID(), m_connection->GetPeerID(), channelnum); // Send the packet - rawSend(p); + rawSend(p.get()); return true; } @@ -374,11 +371,11 @@ bool ConnectionSendThread::rawSendAsPacket(session_t peer_id, u8 channelnum, return false; } -void ConnectionSendThread::processReliableCommand(ConnectionCommand &c) +void ConnectionSendThread::processReliableCommand(ConnectionCommandPtr &c) { - assert(c.reliable); // Pre-condition + assert(c->reliable); // Pre-condition - switch (c.type) { + switch (c->type) { case CONNCMD_NONE: LOG(dout_con << m_connection->getDesc() << "UDP processing reliable CONNCMD_NONE" << std::endl); @@ -399,7 +396,7 @@ void ConnectionSendThread::processReliableCommand(ConnectionCommand &c) case CONCMD_CREATE_PEER: LOG(dout_con << m_connection->getDesc() << "UDP processing reliable CONCMD_CREATE_PEER" << std::endl); - if (!rawSendAsPacket(c.peer_id, c.channelnum, c.data, c.reliable)) { + if (!rawSendAsPacket(c->peer_id, c->channelnum, c->data, c->reliable)) { /* put to queue if we couldn't send it immediately */ sendReliable(c); } @@ -412,13 +409,14 @@ void ConnectionSendThread::processReliableCommand(ConnectionCommand &c) FATAL_ERROR("Got command that shouldn't be reliable as reliable command"); default: LOG(dout_con << m_connection->getDesc() - << " Invalid reliable command type: " << c.type << std::endl); + << " Invalid reliable command type: " << c->type << std::endl); } } -void ConnectionSendThread::processNonReliableCommand(ConnectionCommand &c) +void ConnectionSendThread::processNonReliableCommand(ConnectionCommandPtr &c_ptr) { + const ConnectionCommand &c = *c_ptr; assert(!c.reliable); // Pre-condition switch (c.type) { @@ -480,9 +478,7 @@ void ConnectionSendThread::serve(Address bind_address) } catch (SocketException &e) { // Create event - ConnectionEvent ce; - ce.bindFailed(); - m_connection->putEvent(ce); + m_connection->putEvent(ConnectionEvent::bindFailed()); } } @@ -495,9 +491,7 @@ void ConnectionSendThread::connect(Address address) UDPPeer *peer = m_connection->createServerPeer(address); // Create event - ConnectionEvent e; - e.peerAdded(peer->id, peer->address); - m_connection->putEvent(e); + m_connection->putEvent(ConnectionEvent::peerAdded(peer->id, peer->address)); Address bind_addr; @@ -586,9 +580,9 @@ void ConnectionSendThread::send(session_t peer_id, u8 channelnum, } } -void ConnectionSendThread::sendReliable(ConnectionCommand &c) +void ConnectionSendThread::sendReliable(ConnectionCommandPtr &c) { - PeerHelper peer = m_connection->getPeerNoEx(c.peer_id); + PeerHelper peer = m_connection->getPeerNoEx(c->peer_id); if (!peer) return; @@ -604,7 +598,7 @@ void ConnectionSendThread::sendToAll(u8 channelnum, const SharedBuffer &data } } -void ConnectionSendThread::sendToAllReliable(ConnectionCommand &c) +void ConnectionSendThread::sendToAllReliable(ConnectionCommandPtr &c) { std::vector peerids = m_connection->getPeerIDs(); @@ -663,8 +657,12 @@ void ConnectionSendThread::sendPackets(float dtime) // first send queued reliable packets for all peers (if possible) for (unsigned int i = 0; i < CHANNEL_COUNT; i++) { Channel &channel = udpPeer->channels[i]; - u16 next_to_ack = 0; + // Reduces logging verbosity + if (channel.queued_reliables.empty()) + continue; + + u16 next_to_ack = 0; channel.outgoing_reliables_sent.getFirstSeqnum(next_to_ack); u16 next_to_receive = 0; channel.incoming_reliables.getFirstSeqnum(next_to_receive); @@ -694,13 +692,13 @@ void ConnectionSendThread::sendPackets(float dtime) channel.outgoing_reliables_sent.size() < channel.getWindowSize() && peer->m_increment_packets_remaining > 0) { - BufferedPacket p = std::move(channel.queued_reliables.front()); + BufferedPacketPtr p = channel.queued_reliables.front(); channel.queued_reliables.pop(); LOG(dout_con << m_connection->getDesc() << " INFO: sending a queued reliable packet " << " channel: " << i - << ", seqnum: " << readU16(&p.data[BASE_HEADER_SIZE + 1]) + << ", seqnum: " << p->getSeqnum() << std::endl); sendAsPacketReliable(p, &channel); @@ -881,17 +879,14 @@ void ConnectionReceiveThread::receive(SharedBuffer &packetdata, try { // First, see if there any buffered packets we can process now if (packet_queued) { - bool data_left = true; session_t peer_id; SharedBuffer resultdata; - while (data_left) { + while (true) { try { - data_left = getFromBuffers(peer_id, resultdata); - if (data_left) { - ConnectionEvent e; - e.dataReceived(peer_id, resultdata); - m_connection->putEvent(std::move(e)); - } + if (!getFromBuffers(peer_id, resultdata)) + break; + + m_connection->putEvent(ConnectionEvent::dataReceived(peer_id, resultdata)); } catch (ProcessedSilentlyException &e) { /* try reading again */ @@ -908,7 +903,7 @@ void ConnectionReceiveThread::receive(SharedBuffer &packetdata, return; if ((received_size < BASE_HEADER_SIZE) || - (readU32(&packetdata[0]) != m_connection->GetProtocolID())) { + (readU32(&packetdata[0]) != m_connection->GetProtocolID())) { LOG(derr_con << m_connection->getDesc() << "Receive(): Invalid incoming packet, " << "size: " << received_size @@ -999,9 +994,7 @@ void ConnectionReceiveThread::receive(SharedBuffer &packetdata, << ", channel: " << (u32)channelnum << ", returned " << resultdata.getSize() << " bytes" << std::endl); - ConnectionEvent e; - e.dataReceived(peer_id, resultdata); - m_connection->putEvent(std::move(e)); + m_connection->putEvent(ConnectionEvent::dataReceived(peer_id, resultdata)); } catch (ProcessedSilentlyException &e) { } @@ -1026,10 +1019,11 @@ bool ConnectionReceiveThread::getFromBuffers(session_t &peer_id, SharedBuffer(&peer) == 0) + UDPPeer *p = dynamic_cast(&peer); + if (!p) continue; - for (Channel &channel : (dynamic_cast(&peer))->channels) { + for (Channel &channel : p->channels) { if (checkIncomingBuffers(&channel, peer_id, dst)) { return true; } @@ -1042,32 +1036,34 @@ bool ConnectionReceiveThread::checkIncomingBuffers(Channel *channel, session_t &peer_id, SharedBuffer &dst) { u16 firstseqnum = 0; - if (channel->incoming_reliables.getFirstSeqnum(firstseqnum)) { - if (firstseqnum == channel->readNextIncomingSeqNum()) { - BufferedPacket p = channel->incoming_reliables.popFirst(); - peer_id = readPeerId(*p.data); - u8 channelnum = readChannel(*p.data); - u16 seqnum = readU16(&p.data[BASE_HEADER_SIZE + 1]); + if (!channel->incoming_reliables.getFirstSeqnum(firstseqnum)) + return false; - LOG(dout_con << m_connection->getDesc() - << "UNBUFFERING TYPE_RELIABLE" - << " seqnum=" << seqnum - << " peer_id=" << peer_id - << " channel=" << ((int) channelnum & 0xff) - << std::endl); + if (firstseqnum != channel->readNextIncomingSeqNum()) + return false; - channel->incNextIncomingSeqNum(); + BufferedPacketPtr p = channel->incoming_reliables.popFirst(); - u32 headers_size = BASE_HEADER_SIZE + RELIABLE_HEADER_SIZE; - // Get out the inside packet and re-process it - SharedBuffer payload(p.data.getSize() - headers_size); - memcpy(*payload, &p.data[headers_size], payload.getSize()); + peer_id = readPeerId(p->data); // Carried over to caller function + u8 channelnum = readChannel(p->data); + u16 seqnum = p->getSeqnum(); - dst = processPacket(channel, payload, peer_id, channelnum, true); - return true; - } - } - return false; + LOG(dout_con << m_connection->getDesc() + << "UNBUFFERING TYPE_RELIABLE" + << " seqnum=" << seqnum + << " peer_id=" << peer_id + << " channel=" << ((int) channelnum & 0xff) + << std::endl); + + channel->incNextIncomingSeqNum(); + + u32 headers_size = BASE_HEADER_SIZE + RELIABLE_HEADER_SIZE; + // Get out the inside packet and re-process it + SharedBuffer payload(p->size() - headers_size); + memcpy(*payload, &p->data[headers_size], payload.getSize()); + + dst = processPacket(channel, payload, peer_id, channelnum, true); + return true; } SharedBuffer ConnectionReceiveThread::processPacket(Channel *channel, @@ -1115,7 +1111,7 @@ SharedBuffer ConnectionReceiveThread::handlePacketType_Control(Channel *chan if (packetdata.getSize() < 2) throw InvalidIncomingDataException("packetdata.getSize() < 2"); - u8 controltype = readU8(&(packetdata[1])); + ControlType controltype = (ControlType)readU8(&(packetdata[1])); if (controltype == CONTROLTYPE_ACK) { assert(channel != NULL); @@ -1131,7 +1127,7 @@ SharedBuffer ConnectionReceiveThread::handlePacketType_Control(Channel *chan << seqnum << " ]" << std::endl); try { - BufferedPacket p = channel->outgoing_reliables_sent.popSeqnum(seqnum); + BufferedPacketPtr p = channel->outgoing_reliables_sent.popSeqnum(seqnum); // the rtt calculation will be a bit off for re-sent packets but that's okay { @@ -1140,14 +1136,14 @@ SharedBuffer ConnectionReceiveThread::handlePacketType_Control(Channel *chan // a overflow is quite unlikely but as it'd result in major // rtt miscalculation we handle it here - if (current_time > p.absolute_send_time) { - float rtt = (current_time - p.absolute_send_time) / 1000.0; + if (current_time > p->absolute_send_time) { + float rtt = (current_time - p->absolute_send_time) / 1000.0; // Let peer calculate stuff according to it // (avg_rtt and resend_timeout) dynamic_cast(peer)->reportRTT(rtt); - } else if (p.totaltime > 0) { - float rtt = p.totaltime; + } else if (p->totaltime > 0) { + float rtt = p->totaltime; // Let peer calculate stuff according to it // (avg_rtt and resend_timeout) @@ -1156,7 +1152,7 @@ SharedBuffer ConnectionReceiveThread::handlePacketType_Control(Channel *chan } // put bytes for max bandwidth calculation - channel->UpdateBytesSent(p.data.getSize(), 1); + channel->UpdateBytesSent(p->size(), 1); if (channel->outgoing_reliables_sent.size() == 0) m_connection->TriggerSend(); } catch (NotFoundException &e) { @@ -1204,7 +1200,7 @@ SharedBuffer ConnectionReceiveThread::handlePacketType_Control(Channel *chan throw ProcessedSilentlyException("Got a DISCO"); } else { LOG(derr_con << m_connection->getDesc() - << "INVALID TYPE_CONTROL: invalid controltype=" + << "INVALID controltype=" << ((int) controltype & 0xff) << std::endl); throw InvalidIncomingDataException("Invalid control type"); } @@ -1232,7 +1228,7 @@ SharedBuffer ConnectionReceiveThread::handlePacketType_Split(Channel *channe if (peer->getAddress(MTP_UDP, peer_address)) { // We have to create a packet again for buffering // This isn't actually too bad an idea. - BufferedPacket packet = makePacket(peer_address, + BufferedPacketPtr packet = con::makePacket(peer_address, packetdata, m_connection->GetProtocolID(), peer->id, @@ -1267,7 +1263,7 @@ SharedBuffer ConnectionReceiveThread::handlePacketType_Reliable(Channel *cha if (packetdata.getSize() < RELIABLE_HEADER_SIZE) throw InvalidIncomingDataException("packetdata.getSize() < RELIABLE_HEADER_SIZE"); - u16 seqnum = readU16(&packetdata[1]); + const u16 seqnum = readU16(&packetdata[1]); bool is_future_packet = false; bool is_old_packet = false; @@ -1311,7 +1307,7 @@ SharedBuffer ConnectionReceiveThread::handlePacketType_Reliable(Channel *cha // This one comes later, buffer it. // Actually we have to make a packet to buffer one. // Well, we have all the ingredients, so just do it. - BufferedPacket packet = con::makePacket( + BufferedPacketPtr packet = con::makePacket( peer_address, packetdata, m_connection->GetProtocolID(), @@ -1328,9 +1324,7 @@ SharedBuffer ConnectionReceiveThread::handlePacketType_Reliable(Channel *cha throw ProcessedQueued("Buffered future reliable packet"); } catch (AlreadyExistsException &e) { } catch (IncomingDataCorruption &e) { - ConnectionCommand discon; - discon.disconnect_peer(peer->id); - m_connection->putCommand(discon); + m_connection->putCommand(ConnectionCommand::disconnect_peer(peer->id)); LOG(derr_con << m_connection->getDesc() << "INVALID, TYPE_RELIABLE peer_id: " << peer->id @@ -1351,7 +1345,7 @@ SharedBuffer ConnectionReceiveThread::handlePacketType_Reliable(Channel *cha u16 queued_seqnum = 0; if (channel->incoming_reliables.getFirstSeqnum(queued_seqnum)) { if (queued_seqnum == seqnum) { - BufferedPacket queued_packet = channel->incoming_reliables.popFirst(); + BufferedPacketPtr queued_packet = channel->incoming_reliables.popFirst(); /** TODO find a way to verify the new against the old packet */ } } diff --git a/src/network/connectionthreads.h b/src/network/connectionthreads.h index 612407c3b..c2e2dae12 100644 --- a/src/network/connectionthreads.h +++ b/src/network/connectionthreads.h @@ -29,6 +29,25 @@ namespace con class Connection; +struct OutgoingPacket +{ + session_t peer_id; + u8 channelnum; + SharedBuffer data; + bool reliable; + bool ack; + + OutgoingPacket(session_t peer_id_, u8 channelnum_, const SharedBuffer &data_, + bool reliable_,bool ack_=false): + peer_id(peer_id_), + channelnum(channelnum_), + data(data_), + reliable(reliable_), + ack(ack_) + { + } +}; + class ConnectionSendThread : public Thread { @@ -51,27 +70,27 @@ public: private: void runTimeouts(float dtime); - void rawSend(const BufferedPacket &packet); + void rawSend(const BufferedPacket *p); bool rawSendAsPacket(session_t peer_id, u8 channelnum, const SharedBuffer &data, bool reliable); - void processReliableCommand(ConnectionCommand &c); - void processNonReliableCommand(ConnectionCommand &c); + void processReliableCommand(ConnectionCommandPtr &c); + void processNonReliableCommand(ConnectionCommandPtr &c); void serve(Address bind_address); void connect(Address address); void disconnect(); void disconnect_peer(session_t peer_id); void send(session_t peer_id, u8 channelnum, const SharedBuffer &data); - void sendReliable(ConnectionCommand &c); + void sendReliable(ConnectionCommandPtr &c); void sendToAll(u8 channelnum, const SharedBuffer &data); - void sendToAllReliable(ConnectionCommand &c); + void sendToAllReliable(ConnectionCommandPtr &c); void sendPackets(float dtime); void sendAsPacket(session_t peer_id, u8 channelnum, const SharedBuffer &data, bool ack = false); - void sendAsPacketReliable(BufferedPacket &p, Channel *channel); + void sendAsPacketReliable(BufferedPacketPtr &p, Channel *channel); bool packetsQueued(); diff --git a/src/unittest/test_connection.cpp b/src/unittest/test_connection.cpp index 23b7e9105..04fea90d6 100644 --- a/src/unittest/test_connection.cpp +++ b/src/unittest/test_connection.cpp @@ -124,7 +124,7 @@ void TestConnection::testHelpers() Address a(127,0,0,1, 10); const u16 seqnum = 34352; - con::BufferedPacket p1 = con::makePacket(a, data1, + con::BufferedPacketPtr p1 = con::makePacket(a, data1, proto_id, peer_id, channel); /* We should now have a packet with this data: @@ -135,10 +135,10 @@ void TestConnection::testHelpers() Data: [7] u8 data1[0] */ - UASSERT(readU32(&p1.data[0]) == proto_id); - UASSERT(readU16(&p1.data[4]) == peer_id); - UASSERT(readU8(&p1.data[6]) == channel); - UASSERT(readU8(&p1.data[7]) == data1[0]); + UASSERT(readU32(&p1->data[0]) == proto_id); + UASSERT(readU16(&p1->data[4]) == peer_id); + UASSERT(readU8(&p1->data[6]) == channel); + UASSERT(readU8(&p1->data[7]) == data1[0]); //infostream<<"initial data1[0]="<<((u32)data1[0]&0xff)< +#include // std::shared_ptr + + +template class ConstSharedPtr { +public: + ConstSharedPtr(T *ptr) : ptr(ptr) {} + ConstSharedPtr(const std::shared_ptr &ptr) : ptr(ptr) {} + + const T* get() const noexcept { return ptr.get(); } + const T& operator*() const noexcept { return *ptr.get(); } + const T* operator->() const noexcept { return ptr.get(); } + +private: + std::shared_ptr ptr; +}; template class Buffer @@ -40,17 +55,11 @@ public: else data = NULL; } - Buffer(const Buffer &buffer) - { - m_size = buffer.m_size; - if(m_size != 0) - { - data = new T[buffer.m_size]; - memcpy(data, buffer.data, buffer.m_size); - } - else - data = NULL; - } + + // Disable class copy + Buffer(const Buffer &) = delete; + Buffer &operator=(const Buffer &) = delete; + Buffer(Buffer &&buffer) { m_size = buffer.m_size; @@ -81,21 +90,6 @@ public: drop(); } - Buffer& operator=(const Buffer &buffer) - { - if(this == &buffer) - return *this; - drop(); - m_size = buffer.m_size; - if(m_size != 0) - { - data = new T[buffer.m_size]; - memcpy(data, buffer.data, buffer.m_size); - } - else - data = NULL; - return *this; - } Buffer& operator=(Buffer &&buffer) { if(this == &buffer) @@ -113,6 +107,18 @@ public: return *this; } + void copyTo(Buffer &buffer) const + { + buffer.drop(); + buffer.m_size = m_size; + if (m_size != 0) { + buffer.data = new T[m_size]; + memcpy(buffer.data, data, m_size); + } else { + buffer.data = nullptr; + } + } + T & operator[](unsigned int i) const { return data[i]; -- cgit v1.2.3 From 7f6306ca964ac5b9245c433e3b688c5d4ee08c35 Mon Sep 17 00:00:00 2001 From: JosiahWI <41302989+JosiahWI@users.noreply.github.com> Date: Tue, 28 Dec 2021 07:05:49 -0600 Subject: Restore GCC 5 compatibility (#11778) --- src/client/client.h | 2 +- src/network/clientpackethandler.cpp | 3 ++- src/util/png.cpp | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) (limited to 'src/util') diff --git a/src/client/client.h b/src/client/client.h index b92b456f4..bae40f389 100644 --- a/src/client/client.h +++ b/src/client/client.h @@ -548,7 +548,7 @@ private: // Set of media filenames pushed by server at runtime std::unordered_set m_media_pushed_files; // Pending downloads of dynamic media (key: token) - std::vector>> m_pending_media_downloads; + std::vector>> m_pending_media_downloads; // time_of_day speed approximation for old protocol bool m_time_of_day_set = false; diff --git a/src/network/clientpackethandler.cpp b/src/network/clientpackethandler.cpp index d20a80fb7..6aececa7f 100644 --- a/src/network/clientpackethandler.cpp +++ b/src/network/clientpackethandler.cpp @@ -43,6 +43,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "tileanimation.h" #include "gettext.h" #include "skyparams.h" +#include void Client::handleCommand_Deprecated(NetworkPacket* pkt) { @@ -1559,7 +1560,7 @@ void Client::handleCommand_MediaPush(NetworkPacket *pkt) m_media_pushed_files.insert(filename); // create a downloader for this file - auto downloader = new SingleMediaDownloader(cached); + auto downloader(std::make_shared(cached)); m_pending_media_downloads.emplace_back(token, downloader); downloader->addFile(filename, raw_hash); for (const auto &baseurl : m_remote_media_servers) diff --git a/src/util/png.cpp b/src/util/png.cpp index 7ac2e94a1..698cbc9a5 100755 --- a/src/util/png.cpp +++ b/src/util/png.cpp @@ -37,11 +37,11 @@ static void writeChunk(std::ostringstream &target, const std::string &chunk_str) std::string encodePNG(const u8 *data, u32 width, u32 height, s32 compression) { - auto file = std::ostringstream(std::ios::binary); + std::ostringstream file(std::ios::binary); file << "\x89PNG\r\n\x1a\n"; { - auto IHDR = std::ostringstream(std::ios::binary); + std::ostringstream IHDR(std::ios::binary); IHDR << "IHDR"; writeU32(IHDR, width); writeU32(IHDR, height); @@ -51,9 +51,9 @@ std::string encodePNG(const u8 *data, u32 width, u32 height, s32 compression) } { - auto IDAT = std::ostringstream(std::ios::binary); + std::ostringstream IDAT(std::ios::binary); IDAT << "IDAT"; - auto scanlines = std::ostringstream(std::ios::binary); + std::ostringstream scanlines(std::ios::binary); for(u32 i = 0; i < height; i++) { scanlines.write("\x00", 1); // Null predictor scanlines.write((const char*) data + width * 4 * i, width * 4); -- cgit v1.2.3 From 29d2b2ccd06cdd831a7fac66928bfa612003945c Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Sat, 1 Jan 2022 16:44:56 -0500 Subject: Print announce error response (#11878) Fix HTTPFetch caller and request ID to 64 bits Check that allocated caller ID is not DISCARD Print body if serverlist request returns error Don't print control characters from HTTP responses Document special HTTPFetch caller IDs Allow unicode to be printed --- src/client/clientmedia.h | 6 +++--- src/httpfetch.cpp | 51 ++++++++++++++++++++++++++---------------------- src/httpfetch.h | 27 +++++++++++++++---------- src/serverlist.cpp | 1 + src/util/string.cpp | 16 +++++++++++++++ src/util/string.h | 8 ++++++++ 6 files changed, 73 insertions(+), 36 deletions(-) (limited to 'src/util') diff --git a/src/client/clientmedia.h b/src/client/clientmedia.h index aa7b0f398..c297d737f 100644 --- a/src/client/clientmedia.h +++ b/src/client/clientmedia.h @@ -174,12 +174,12 @@ private: s32 m_uncached_received_count = 0; // Status of remote transfers - unsigned long m_httpfetch_caller; - unsigned long m_httpfetch_next_id = 0; + u64 m_httpfetch_caller; + u64 m_httpfetch_next_id = 0; s32 m_httpfetch_active = 0; s32 m_httpfetch_active_limit = 0; s32 m_outstanding_hash_sets = 0; - std::unordered_map m_remote_file_transfers; + std::unordered_map m_remote_file_transfers; // All files up to this name have either been received from a // remote server or failed on all remote servers, so those files diff --git a/src/httpfetch.cpp b/src/httpfetch.cpp index 1307bfec3..16f0791c9 100644 --- a/src/httpfetch.cpp +++ b/src/httpfetch.cpp @@ -38,7 +38,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "noise.h" static std::mutex g_httpfetch_mutex; -static std::unordered_map> +static std::unordered_map> g_httpfetch_results; static PcgRandom g_callerid_randomness; @@ -52,22 +52,21 @@ HTTPFetchRequest::HTTPFetchRequest() : static void httpfetch_deliver_result(const HTTPFetchResult &fetch_result) { - unsigned long caller = fetch_result.caller; + u64 caller = fetch_result.caller; if (caller != HTTPFETCH_DISCARD) { MutexAutoLock lock(g_httpfetch_mutex); g_httpfetch_results[caller].emplace(fetch_result); } } -static void httpfetch_request_clear(unsigned long caller); +static void httpfetch_request_clear(u64 caller); -unsigned long httpfetch_caller_alloc() +u64 httpfetch_caller_alloc() { MutexAutoLock lock(g_httpfetch_mutex); - // Check each caller ID except HTTPFETCH_DISCARD - const unsigned long discard = HTTPFETCH_DISCARD; - for (unsigned long caller = discard + 1; caller != discard; ++caller) { + // Check each caller ID except reserved ones + for (u64 caller = HTTPFETCH_CID_START; caller != 0; ++caller) { auto it = g_httpfetch_results.find(caller); if (it == g_httpfetch_results.end()) { verbosestream << "httpfetch_caller_alloc: allocating " @@ -79,18 +78,17 @@ unsigned long httpfetch_caller_alloc() } FATAL_ERROR("httpfetch_caller_alloc: ran out of caller IDs"); - return discard; } -unsigned long httpfetch_caller_alloc_secure() +u64 httpfetch_caller_alloc_secure() { MutexAutoLock lock(g_httpfetch_mutex); // Generate random caller IDs and make sure they're not - // already used or equal to HTTPFETCH_DISCARD + // already used or reserved. // Give up after 100 tries to prevent infinite loop - u8 tries = 100; - unsigned long caller; + size_t tries = 100; + u64 caller; do { caller = (((u64) g_callerid_randomness.next()) << 32) | @@ -100,7 +98,8 @@ unsigned long httpfetch_caller_alloc_secure() FATAL_ERROR("httpfetch_caller_alloc_secure: ran out of caller IDs"); return HTTPFETCH_DISCARD; } - } while (g_httpfetch_results.find(caller) != g_httpfetch_results.end()); + } while (caller >= HTTPFETCH_CID_START && + g_httpfetch_results.find(caller) != g_httpfetch_results.end()); verbosestream << "httpfetch_caller_alloc_secure: allocating " << caller << std::endl; @@ -110,7 +109,7 @@ unsigned long httpfetch_caller_alloc_secure() return caller; } -void httpfetch_caller_free(unsigned long caller) +void httpfetch_caller_free(u64 caller) { verbosestream<<"httpfetch_caller_free: freeing " <= 400) { + errorstream << "HTTPFetch for " << request.url + << " returned response code " << result.response_code << std::endl; + if (result.caller == HTTPFETCH_PRINT_ERR && !result.data.empty()) { + errorstream << "Response body:" << std::endl; + safe_print_string(errorstream, result.data); + errorstream << std::endl; + } } return &result; @@ -474,7 +479,7 @@ public: m_requests.push_back(req); } - void requestClear(unsigned long caller, Event *event) + void requestClear(u64 caller, Event *event) { Request req; req.type = RT_CLEAR; @@ -505,7 +510,7 @@ protected: } else if (req.type == RT_CLEAR) { - unsigned long caller = req.fetch_request.caller; + u64 caller = req.fetch_request.caller; // Abort all ongoing fetches for the caller for (std::vector::iterator @@ -778,7 +783,7 @@ void httpfetch_async(const HTTPFetchRequest &fetch_request) g_httpfetch_thread->start(); } -static void httpfetch_request_clear(unsigned long caller) +static void httpfetch_request_clear(u64 caller) { if (g_httpfetch_thread->isRunning()) { Event event; @@ -827,7 +832,7 @@ void httpfetch_async(const HTTPFetchRequest &fetch_request) httpfetch_deliver_result(fetch_result); } -static void httpfetch_request_clear(unsigned long caller) +static void httpfetch_request_clear(u64 caller) { } diff --git a/src/httpfetch.h b/src/httpfetch.h index 3b9f17f0a..a4901e63b 100644 --- a/src/httpfetch.h +++ b/src/httpfetch.h @@ -23,10 +23,17 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "util/string.h" #include "config.h" -// Can be used in place of "caller" in asynchronous transfers to discard result -// (used as default value of "caller") +// These can be used in place of "caller" in to specify special handling. +// Discard result (used as default value of "caller"). #define HTTPFETCH_DISCARD 0 +// Indicates that the result should not be discarded when performing a +// synchronous request (since a real caller ID is not needed for synchronous +// requests because the result does not have to be retrieved later). #define HTTPFETCH_SYNC 1 +// Print response body to console if the server returns an error code. +#define HTTPFETCH_PRINT_ERR 2 +// Start of regular allocated caller IDs. +#define HTTPFETCH_CID_START 3 // Methods enum HttpMethod : u8 @@ -43,11 +50,11 @@ struct HTTPFetchRequest // Identifies the caller (for asynchronous requests) // Ignored by httpfetch_sync - unsigned long caller = HTTPFETCH_DISCARD; + u64 caller = HTTPFETCH_DISCARD; // Some number that identifies the request // (when the same caller issues multiple httpfetch_async calls) - unsigned long request_id = 0; + u64 request_id = 0; // Timeout for the whole transfer, in milliseconds long timeout; @@ -85,8 +92,8 @@ struct HTTPFetchResult long response_code = 0; std::string data = ""; // The caller and request_id from the corresponding HTTPFetchRequest. - unsigned long caller = HTTPFETCH_DISCARD; - unsigned long request_id = 0; + u64 caller = HTTPFETCH_DISCARD; + u64 request_id = 0; HTTPFetchResult() = default; @@ -107,19 +114,19 @@ void httpfetch_async(const HTTPFetchRequest &fetch_request); // If any fetch for the given caller ID is complete, removes it from the // result queue, sets the fetch result and returns true. Otherwise returns false. -bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetch_result); +bool httpfetch_async_get(u64 caller, HTTPFetchResult &fetch_result); // Allocates a caller ID for httpfetch_async // Not required if you want to set caller = HTTPFETCH_DISCARD -unsigned long httpfetch_caller_alloc(); +u64 httpfetch_caller_alloc(); // Allocates a non-predictable caller ID for httpfetch_async -unsigned long httpfetch_caller_alloc_secure(); +u64 httpfetch_caller_alloc_secure(); // Frees a caller ID allocated with httpfetch_caller_alloc // Note: This can be expensive, because the httpfetch thread is told // to stop any ongoing fetches for the given caller. -void httpfetch_caller_free(unsigned long caller); +void httpfetch_caller_free(u64 caller); // Performs a synchronous HTTP request. This blocks and therefore should // only be used from background threads. diff --git a/src/serverlist.cpp b/src/serverlist.cpp index 3bcab3d58..29e3ac9a6 100644 --- a/src/serverlist.cpp +++ b/src/serverlist.cpp @@ -97,6 +97,7 @@ void sendAnnounce(AnnounceAction action, } HTTPFetchRequest fetch_request; + fetch_request.caller = HTTPFETCH_PRINT_ERR; fetch_request.url = g_settings->get("serverlist_url") + std::string("/announce"); fetch_request.method = HTTP_POST; fetch_request.fields["json"] = fastWriteJson(server); diff --git a/src/util/string.cpp b/src/util/string.cpp index 8be5e320a..bc4664997 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -887,3 +887,19 @@ std::string sanitizeDirName(const std::string &str, const std::string &optional_ return wide_to_utf8(safe_name); } + + +void safe_print_string(std::ostream &os, const std::string &str) +{ + std::ostream::fmtflags flags = os.flags(); + os << std::hex; + for (const char c : str) { + if (IS_ASCII_PRINTABLE_CHAR(c) || IS_UTF8_MULTB_START(c) || + IS_UTF8_MULTB_INNER(c) || c == '\n' || c == '\t') { + os << c; + } else { + os << '<' << std::setw(2) << (int)c << '>'; + } + } + os.setf(flags); +} diff --git a/src/util/string.h b/src/util/string.h index bca998f56..8a9e83f22 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -753,3 +753,11 @@ inline irr::core::stringw utf8_to_stringw(const std::string &input) * 2. Remove 'unsafe' characters from the name by replacing them with '_' */ std::string sanitizeDirName(const std::string &str, const std::string &optional_prefix); + +/** + * Prints a sanitized version of a string without control characters. + * '\t' and '\n' are allowed, as are UTF-8 control characters (e.g. RTL). + * ASCII control characters are replaced with their hex encoding in angle + * brackets (e.g. "a\x1eb" -> "a<1e>b"). + */ +void safe_print_string(std::ostream &os, const std::string &str); -- cgit v1.2.3 From 04bd253390cc6c67a555e4837e7e48d524fdf014 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Wed, 23 Feb 2022 20:02:58 +0100 Subject: Move the codebase to C++14 --- CMakeLists.txt | 3 ++- android/native/jni/Application.mk | 2 +- src/CMakeLists.txt | 6 +----- src/modchannels.cpp | 3 +-- src/server.cpp | 6 +++--- src/unittest/test_address.cpp | 2 +- src/unittest/test_eventmanager.cpp | 4 ++-- src/unittest/test_server_shutdown_state.cpp | 4 +--- src/util/metricsbackend.cpp | 3 +-- 9 files changed, 13 insertions(+), 20 deletions(-) (limited to 'src/util') diff --git a/CMakeLists.txt b/CMakeLists.txt index 1da83a99c..827191835 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,8 @@ endif() project(minetest) set(PROJECT_NAME_CAPITALIZED "Minetest") -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(GCC_MINIMUM_VERSION "5.1") set(CLANG_MINIMUM_VERSION "3.5") diff --git a/android/native/jni/Application.mk b/android/native/jni/Application.mk index e21bca61c..9d9596137 100644 --- a/android/native/jni/Application.mk +++ b/android/native/jni/Application.mk @@ -20,7 +20,7 @@ APP_CPPFLAGS := -g -Og -fno-omit-frame-pointer endif APP_CFLAGS := $(APP_CPPFLAGS) -Wno-inconsistent-missing-override -Wno-parentheses-equality -APP_CXXFLAGS := $(APP_CPPFLAGS) -fexceptions -frtti -std=gnu++17 +APP_CXXFLAGS := $(APP_CPPFLAGS) -fexceptions -frtti -std=gnu++14 APP_LDFLAGS := -Wl,--no-warn-mismatch,--gc-sections,--icf=safe ifeq ($(APP_ABI),arm64-v8a) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7f207244c..57baf20bd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -718,7 +718,6 @@ if(MSVC) endif() else() # GCC or compatible compilers such as Clang - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") if(WARN_ALL) set(RELEASE_WARNING_FLAGS "-Wall") else() @@ -751,6 +750,7 @@ else() if(MINGW) set(OTHER_FLAGS "${OTHER_FLAGS} -mthreads -fexceptions") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWIN32_LEAN_AND_MEAN") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -mwindows") endif() # Use a safe subset of flags to speed up math calculations: @@ -787,10 +787,6 @@ else() if(USE_GPROF) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pg") endif() - - if(MINGW) - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -mwindows") - endif() endif() diff --git a/src/modchannels.cpp b/src/modchannels.cpp index 301dcb092..9626e8e0c 100644 --- a/src/modchannels.cpp +++ b/src/modchannels.cpp @@ -88,8 +88,7 @@ bool ModChannelMgr::canWriteOnChannel(const std::string &channel) const void ModChannelMgr::registerChannel(const std::string &channel) { - m_registered_channels[channel] = - std::unique_ptr(new ModChannel(channel)); + m_registered_channels[channel] = std::make_unique(channel); } bool ModChannelMgr::setChannelState(const std::string &channel, ModChannelState state) diff --git a/src/server.cpp b/src/server.cpp index 685d8bb25..d9205c895 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -254,7 +254,7 @@ Server::Server( #if USE_PROMETHEUS m_metrics_backend = std::unique_ptr(createPrometheusMetricsBackend()); #else - m_metrics_backend = std::unique_ptr(new MetricsBackend()); + m_metrics_backend = std::make_unique(); #endif m_uptime_counter = m_metrics_backend->addCounter("minetest_core_server_uptime", "Server uptime (in seconds)"); @@ -406,7 +406,7 @@ void Server::init() m_mod_storage_database = openModStorageDatabase(m_path_world); m_mod_storage_database->beginSave(); - m_modmgr = std::unique_ptr(new ServerModManager(m_path_world)); + m_modmgr = std::make_unique(m_path_world); std::vector unsatisfied_mods = m_modmgr->getUnsatisfiedMods(); // complain about mods with unsatisfied dependencies if (!m_modmgr->isConsistent()) { @@ -426,7 +426,7 @@ void Server::init() m_script = new ServerScripting(this); // Must be created before mod loading because we have some inventory creation - m_inventory_mgr = std::unique_ptr(new ServerInventoryManager()); + m_inventory_mgr = std::make_unique(); m_script->loadMod(getBuiltinLuaPath() + DIR_DELIM "init.lua", BUILTIN_MOD_NAME); diff --git a/src/unittest/test_address.cpp b/src/unittest/test_address.cpp index 35d4effb6..f46135577 100644 --- a/src/unittest/test_address.cpp +++ b/src/unittest/test_address.cpp @@ -56,7 +56,7 @@ void TestAddress::testIsLocalhost() UASSERT(!Address(172, 45, 37, 68, 0).isLocalhost()); // v6 - std::unique_ptr ipv6Bytes(new IPv6AddressBytes()); + auto ipv6Bytes = std::make_unique(); std::vector ipv6RawAddr = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}; memcpy(ipv6Bytes->bytes, &ipv6RawAddr[0], 16); UASSERT(Address(ipv6Bytes.get(), 0).isLocalhost()) diff --git a/src/unittest/test_eventmanager.cpp b/src/unittest/test_eventmanager.cpp index bb0e59336..fec57f9fe 100644 --- a/src/unittest/test_eventmanager.cpp +++ b/src/unittest/test_eventmanager.cpp @@ -82,7 +82,7 @@ void TestEventManager::testDeregister() void TestEventManager::testRealEvent() { EventManager ev; - std::unique_ptr emt(new EventManagerTest()); + auto emt = std::make_unique(); ev.reg(MtEvent::PLAYER_REGAIN_GROUND, EventManagerTest::eventTest, emt.get()); // Put event & verify event value @@ -93,7 +93,7 @@ void TestEventManager::testRealEvent() void TestEventManager::testRealEventAfterDereg() { EventManager ev; - std::unique_ptr emt(new EventManagerTest()); + auto emt = std::make_unique(); ev.reg(MtEvent::PLAYER_REGAIN_GROUND, EventManagerTest::eventTest, emt.get()); // Put event & verify event value diff --git a/src/unittest/test_server_shutdown_state.cpp b/src/unittest/test_server_shutdown_state.cpp index fbb76ff6a..50305e725 100644 --- a/src/unittest/test_server_shutdown_state.cpp +++ b/src/unittest/test_server_shutdown_state.cpp @@ -26,12 +26,10 @@ with this program; if not, write to the Free Software Foundation, Inc., class FakeServer : public Server { public: - // clang-format off FakeServer() : Server("fakeworld", SubgameSpec("fakespec", "fakespec"), true, Address(), true, nullptr) { } - // clang-format on private: void SendChatMessage(session_t peer_id, const ChatMessage &message) @@ -95,7 +93,7 @@ void TestServerShutdownState::testTrigger() void TestServerShutdownState::testTick() { - std::unique_ptr fakeServer(new FakeServer()); + auto fakeServer = std::make_unique(); Server::ShutdownState ss; ss.trigger(28.0f, "testtrigger", true); ss.tick(0.0f, fakeServer.get()); diff --git a/src/util/metricsbackend.cpp b/src/util/metricsbackend.cpp index 4454557a3..c3b7def62 100644 --- a/src/util/metricsbackend.cpp +++ b/src/util/metricsbackend.cpp @@ -99,8 +99,7 @@ class PrometheusMetricsBackend : public MetricsBackend { public: PrometheusMetricsBackend(const std::string &addr) : - MetricsBackend(), m_exposer(std::unique_ptr( - new prometheus::Exposer(addr))), + MetricsBackend(), m_exposer(std::make_unique(addr)), m_registry(std::make_shared()) { m_exposer->RegisterCollectable(m_registry); -- cgit v1.2.3 From 598efbf7f9d75307e7ea48ed7ec0e9b560e69375 Mon Sep 17 00:00:00 2001 From: Daroc Alden Date: Wed, 9 Mar 2022 13:28:12 -0500 Subject: Fix memory leak from SpatialAreaStore (#12120) --- src/util/areastore.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'src/util') diff --git a/src/util/areastore.cpp b/src/util/areastore.cpp index 67bfef0c0..bf751476f 100644 --- a/src/util/areastore.cpp +++ b/src/util/areastore.cpp @@ -308,6 +308,7 @@ void SpatialAreaStore::getAreasInArea(std::vector *result, SpatialAreaStore::~SpatialAreaStore() { delete m_tree; + delete m_storagemanager; } SpatialAreaStore::SpatialAreaStore() -- cgit v1.2.3 From 5683bb76cc3a60d952c9f58c41adaa5f195dd3f4 Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Wed, 1 Dec 2021 18:30:40 -0500 Subject: Fix compiler warnings --- src/CMakeLists.txt | 6 ++++-- src/client/clientlauncher.cpp | 2 ++ src/client/shader.cpp | 10 ++++----- src/serialization.cpp | 14 ------------ src/server/player_sao.h | 48 ++++++++++++++++++++--------------------- src/terminal_chat_console.cpp | 3 ++- src/unittest/test_irrptr.cpp | 4 +++- src/unittest/test_voxelarea.cpp | 4 ++-- src/util/srp.cpp | 4 ++-- 9 files changed, 44 insertions(+), 51 deletions(-) (limited to 'src/util') diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ac4c1de73..2de68a8f0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -470,6 +470,9 @@ endif() include_directories( ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/script +) +include_directories(SYSTEM ${ZLIB_INCLUDE_DIR} ${ZSTD_INCLUDE_DIR} ${SQLITE3_INCLUDE_DIR} @@ -477,7 +480,6 @@ include_directories( ${GMP_INCLUDE_DIR} ${JSON_INCLUDE_DIR} ${LUA_BIT_INCLUDE_DIR} - ${PROJECT_SOURCE_DIR}/script ) if(USE_GETTEXT) @@ -485,7 +487,7 @@ if(USE_GETTEXT) endif() if(BUILD_CLIENT) - include_directories( + include_directories(SYSTEM ${FREETYPE_INCLUDE_DIRS} ${SOUND_INCLUDE_DIRS} ${X11_INCLUDE_DIR} diff --git a/src/client/clientlauncher.cpp b/src/client/clientlauncher.cpp index 063154316..54c561d11 100644 --- a/src/client/clientlauncher.cpp +++ b/src/client/clientlauncher.cpp @@ -564,6 +564,8 @@ void ClientLauncher::speed_tests() // volatile to avoid some potential compiler optimisations volatile static s16 temp16; volatile static f32 tempf; + // Silence compiler warning + (void)temp16; static v3f tempv3f1; static v3f tempv3f2; static std::string tempstring; diff --git a/src/client/shader.cpp b/src/client/shader.cpp index 1be9ef128..bbb872761 100644 --- a/src/client/shader.cpp +++ b/src/client/shader.cpp @@ -242,11 +242,6 @@ public: MainShaderConstantSetter() : m_world_view_proj("mWorldViewProj") , m_world("mWorld") -#if ENABLE_GLES - , m_world_view("mWorldView") - , m_texture("mTexture") - , m_normal("mNormal") -#endif , m_shadow_view_proj("m_ShadowViewProj") , m_light_direction("v_LightDirection") , m_texture_res("f_textureresolution") @@ -261,6 +256,11 @@ public: , m_perspective_bias1_pixel("xyPerspectiveBias1") , m_perspective_zbias_vertex("zPerspectiveBias") , m_perspective_zbias_pixel("zPerspectiveBias") +#if ENABLE_GLES + , m_world_view("mWorldView") + , m_texture("mTexture") + , m_normal("mNormal") +#endif {} ~MainShaderConstantSetter() = default; diff --git a/src/serialization.cpp b/src/serialization.cpp index d3009bc83..11164a0ed 100644 --- a/src/serialization.cpp +++ b/src/serialization.cpp @@ -108,7 +108,6 @@ void decompressZlib(std::istream &is, std::ostream &os, size_t limit) char output_buffer[bufsize]; int status = 0; int ret; - int bytes_read = 0; int bytes_written = 0; int input_buffer_len = 0; @@ -122,8 +121,6 @@ void decompressZlib(std::istream &is, std::ostream &os, size_t limit) z.avail_in = 0; - //dstream<<"initial fail="< &privs); diff --git a/src/terminal_chat_console.cpp b/src/terminal_chat_console.cpp index 9e3d33736..b12261c3b 100644 --- a/src/terminal_chat_console.cpp +++ b/src/terminal_chat_console.cpp @@ -17,6 +17,7 @@ with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ +#include #include "config.h" #if USE_CURSES #include "version.h" @@ -398,7 +399,7 @@ void TerminalChatConsole::step(int ch) minutes = (float)minutes / 1000 * 60; if (m_game_time) - printw(" | Game %d Time of day %02d:%02d ", + printw(" | Game %" PRIu64 " Time of day %02d:%02d ", m_game_time, hours, minutes); // draw text diff --git a/src/unittest/test_irrptr.cpp b/src/unittest/test_irrptr.cpp index 3484f1514..2fb7cfcd6 100644 --- a/src/unittest/test_irrptr.cpp +++ b/src/unittest/test_irrptr.cpp @@ -93,7 +93,9 @@ void TestIrrPtr::testRefCounting() #if defined(__clang__) #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wself-assign-overloaded" + #if __clang_major__ >= 7 + #pragma GCC diagnostic ignored "-Wself-assign-overloaded" + #endif #pragma GCC diagnostic ignored "-Wself-move" #endif diff --git a/src/unittest/test_voxelarea.cpp b/src/unittest/test_voxelarea.cpp index 9826d2ee7..1d72650d7 100644 --- a/src/unittest/test_voxelarea.cpp +++ b/src/unittest/test_voxelarea.cpp @@ -120,7 +120,7 @@ void TestVoxelArea::test_extent() VoxelArea v1(v3s16(-1337, -547, -789), v3s16(-147, 447, 669)); UASSERT(v1.getExtent() == v3s16(1191, 995, 1459)); - VoxelArea v2(v3s16(32493, -32507, 32753), v3s16(32508, -32492, 32768)); + VoxelArea v2(v3s16(32493, -32507, 32753), v3s16(32508, -32492, -32768)); UASSERT(v2.getExtent() == v3s16(16, 16, 16)); } @@ -129,7 +129,7 @@ void TestVoxelArea::test_volume() VoxelArea v1(v3s16(-1337, -547, -789), v3s16(-147, 447, 669)); UASSERTEQ(s32, v1.getVolume(), 1728980655); - VoxelArea v2(v3s16(32493, -32507, 32753), v3s16(32508, -32492, 32768)); + VoxelArea v2(v3s16(32493, -32507, 32753), v3s16(32508, -32492, -32768)); UASSERTEQ(s32, v2.getVolume(), 4096); } diff --git a/src/util/srp.cpp b/src/util/srp.cpp index ceb2fef9e..daa7f332b 100644 --- a/src/util/srp.cpp +++ b/src/util/srp.cpp @@ -354,7 +354,7 @@ static size_t hash_length(SRP_HashAlgorithm alg) case SRP_SHA384: return SHA384_DIGEST_LENGTH; case SRP_SHA512: return SHA512_DIGEST_LENGTH; */ - default: return -1; + default: return 0; }; } // clang-format on @@ -422,7 +422,7 @@ static SRP_Result H_nn( } static SRP_Result H_ns(mpz_t result, SRP_HashAlgorithm alg, const unsigned char *n, - size_t len_n, const unsigned char *bytes, uint32_t len_bytes) + size_t len_n, const unsigned char *bytes, size_t len_bytes) { unsigned char buff[SHA512_DIGEST_LENGTH]; size_t nbytes = len_n + len_bytes; -- cgit v1.2.3 From 7993909fabce4f796ca67b5a8139936667de25df Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Wed, 1 Dec 2021 18:54:12 -0500 Subject: Spacing fixes --- CMakeLists.txt | 2 +- client/shaders/nodes_shader/opengl_fragment.glsl | 2 +- client/shaders/nodes_shader/opengl_vertex.glsl | 4 +- clientmods/preview/mod.conf | 2 +- cmake/Modules/FindSQLite3.cmake | 2 +- cmake/Modules/FindVorbis.cmake | 17 ++++--- doc/lgpl-2.1.txt | 18 ++++---- minetest.conf.example | 58 ++++++++++++------------ src/CMakeLists.txt | 2 +- src/client/CMakeLists.txt | 4 +- src/client/clientmap.cpp | 4 +- src/client/imagefilters.cpp | 2 +- src/client/render/core.cpp | 2 +- src/clientiface.h | 2 +- src/collision.cpp | 4 +- src/config.h | 2 +- src/database/database-leveldb.cpp | 2 +- src/inventorymanager.cpp | 2 +- src/irrlicht_changes/CGUITTFont.cpp | 2 +- src/mapgen/dungeongen.cpp | 2 +- src/mapgen/mapgen_flat.cpp | 2 +- src/mapgen/mg_biome.cpp | 2 +- src/mapgen/mg_ore.cpp | 4 +- src/network/connection.h | 4 +- src/script/cpp_api/s_env.cpp | 4 +- src/script/lua_api/l_env.cpp | 4 +- src/script/lua_api/l_env.h | 2 +- src/serverenvironment.cpp | 2 +- src/util/ieee_float.cpp | 2 +- src/util/string.h | 2 +- util/generate-texture-normals.sh | 2 +- 31 files changed, 82 insertions(+), 83 deletions(-) (limited to 'src/util') diff --git a/CMakeLists.txt b/CMakeLists.txt index 827191835..5dfc857d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,7 +69,7 @@ if(NOT "${IRRLICHTMT_BUILD_DIR}" STREQUAL "") find_package(IrrlichtMt QUIET PATHS "${IRRLICHTMT_BUILD_DIR}" NO_DEFAULT_PATH -) + ) if(NOT TARGET IrrlichtMt::IrrlichtMt) # find_package() searches certain subdirectories. ${PATH}/cmake is not diff --git a/client/shaders/nodes_shader/opengl_fragment.glsl b/client/shaders/nodes_shader/opengl_fragment.glsl index 4d0d107d1..fea350788 100644 --- a/client/shaders/nodes_shader/opengl_fragment.glsl +++ b/client/shaders/nodes_shader/opengl_fragment.glsl @@ -557,6 +557,6 @@ void main(void) - fogShadingParameter * length(eyeVec) / fogDistance, 0.0, 1.0); col = mix(skyBgColor, col, clarity); col = vec4(col.rgb, base.a); - + gl_FragColor = col; } diff --git a/client/shaders/nodes_shader/opengl_vertex.glsl b/client/shaders/nodes_shader/opengl_vertex.glsl index 935fbf043..8c7e27459 100644 --- a/client/shaders/nodes_shader/opengl_vertex.glsl +++ b/client/shaders/nodes_shader/opengl_vertex.glsl @@ -199,13 +199,13 @@ void main(void) vec3 nNormal = normalize(vNormal); cosLight = dot(nNormal, -v_LightDirection); - // Calculate normal offset scale based on the texel size adjusted for + // Calculate normal offset scale based on the texel size adjusted for // curvature of the SM texture. This code must be change together with // getPerspectiveFactor or any light-space transformation. vec3 eyeToVertex = worldPosition - eyePosition + cameraOffset; // Distance from the vertex to the player float distanceToPlayer = length(eyeToVertex - v_LightDirection * dot(eyeToVertex, v_LightDirection)) / f_shadowfar; - // perspective factor estimation according to the + // perspective factor estimation according to the float perspectiveFactor = distanceToPlayer * xyPerspectiveBias0 + xyPerspectiveBias1; float texelSize = f_shadowfar * perspectiveFactor * perspectiveFactor / (f_textureresolution * xyPerspectiveBias1 - perspectiveFactor * xyPerspectiveBias0); diff --git a/clientmods/preview/mod.conf b/clientmods/preview/mod.conf index 4e56ec293..23a5c3e90 100644 --- a/clientmods/preview/mod.conf +++ b/clientmods/preview/mod.conf @@ -1 +1 @@ -name = preview +name = preview diff --git a/cmake/Modules/FindSQLite3.cmake b/cmake/Modules/FindSQLite3.cmake index b23553a80..8a66cb241 100644 --- a/cmake/Modules/FindSQLite3.cmake +++ b/cmake/Modules/FindSQLite3.cmake @@ -1,4 +1,4 @@ -mark_as_advanced(SQLITE3_LIBRARY SQLITE3_INCLUDE_DIR) +mark_as_advanced(SQLITE3_LIBRARY SQLITE3_INCLUDE_DIR) find_path(SQLITE3_INCLUDE_DIR sqlite3.h) diff --git a/cmake/Modules/FindVorbis.cmake b/cmake/Modules/FindVorbis.cmake index e5fe7f25e..222ddd9d5 100644 --- a/cmake/Modules/FindVorbis.cmake +++ b/cmake/Modules/FindVorbis.cmake @@ -29,18 +29,17 @@ else(NOT GP2XWIZ) find_package_handle_standard_args(Vorbis DEFAULT_MSG VORBIS_INCLUDE_DIR VORBIS_LIBRARY) endif(NOT GP2XWIZ) - + if(VORBIS_FOUND) - if(NOT GP2XWIZ) - set(VORBIS_LIBRARIES ${VORBISFILE_LIBRARY} ${VORBIS_LIBRARY} - ${OGG_LIBRARY}) - else(NOT GP2XWIZ) - set(VORBIS_LIBRARIES ${VORBIS_LIBRARY}) - endif(NOT GP2XWIZ) + if(NOT GP2XWIZ) + set(VORBIS_LIBRARIES ${VORBISFILE_LIBRARY} ${VORBIS_LIBRARY} + ${OGG_LIBRARY}) + else(NOT GP2XWIZ) + set(VORBIS_LIBRARIES ${VORBIS_LIBRARY}) + endif(NOT GP2XWIZ) else(VORBIS_FOUND) - set(VORBIS_LIBRARIES) + set(VORBIS_LIBRARIES) endif(VORBIS_FOUND) mark_as_advanced(OGG_INCLUDE_DIR VORBIS_INCLUDE_DIR) mark_as_advanced(OGG_LIBRARY VORBIS_LIBRARY VORBISFILE_LIBRARY) - diff --git a/doc/lgpl-2.1.txt b/doc/lgpl-2.1.txt index 4362b4915..e5ab03e12 100644 --- a/doc/lgpl-2.1.txt +++ b/doc/lgpl-2.1.txt @@ -55,7 +55,7 @@ modified by someone else and passed on, the recipients should know that what they have is not the original version, so that the original author's reputation will not be affected by problems that might be introduced by others. - + Finally, software patents pose a constant threat to the existence of any free program. We wish to make sure that a company cannot effectively restrict the users of a free program by obtaining a @@ -111,7 +111,7 @@ modification follow. Pay close attention to the difference between a "work based on the library" and a "work that uses the library". The former contains code derived from the library, whereas the latter must be combined with the library in order to run. - + GNU LESSER GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION @@ -158,7 +158,7 @@ Library. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. - + 2. You may modify your copy or copies of the Library or any portion of it, thus forming a work based on the Library, and copy and distribute such modifications or work under the terms of Section 1 @@ -216,7 +216,7 @@ instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify that version instead if you wish.) Do not make any other change in these notices. - + Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all subsequent copies and derivative works made from that copy. @@ -267,7 +267,7 @@ Library will still fall under Section 6.) distribute the object code for the work under the terms of Section 6. Any executables containing that work also fall under Section 6, whether or not they are linked directly with the Library itself. - + 6. As an exception to the Sections above, you may also combine or link a "work that uses the Library" with the Library to produce a work containing portions of the Library, and distribute that work @@ -329,7 +329,7 @@ restrictions of other proprietary libraries that do not normally accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you distribute. - + 7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library facilities not covered by this License, and distribute such a combined @@ -370,7 +370,7 @@ subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties with this License. - + 11. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or @@ -422,7 +422,7 @@ conditions either of that version or of any later version published by the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by the Free Software Foundation. - + 14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, write to the author to ask for permission. For software which is @@ -456,7 +456,7 @@ SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS - + How to Apply These Terms to Your New Libraries If you develop a new library, and you want it to be of the greatest diff --git a/minetest.conf.example b/minetest.conf.example index ed2ebc969..21aeb3546 100644 --- a/minetest.conf.example +++ b/minetest.conf.example @@ -1385,7 +1385,7 @@ # ask_reconnect_on_crash = false # From how far clients know about objects, stated in mapblocks (16 nodes). -# +# # Setting this larger than active_block_range will also cause the server # to maintain active objects up to this distance in the direction the # player is looking. (This can avoid mobs suddenly disappearing from view) @@ -1938,7 +1938,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -1951,7 +1951,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining giant caverns. @@ -1964,7 +1964,7 @@ # octaves = 5, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining terrain. @@ -1990,7 +1990,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Mapgen V6 @@ -2377,7 +2377,7 @@ # octaves = 5, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining structure of river canyon walls. @@ -2390,7 +2390,7 @@ # octaves = 4, # persistence = 0.75, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining structure of floatlands. @@ -2406,7 +2406,7 @@ # octaves = 4, # persistence = 0.75, # lacunarity = 1.618, -# flags = +# flags = # } # 3D noise defining giant caverns. @@ -2419,7 +2419,7 @@ # octaves = 5, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # First of two 3D noises that together define tunnels. @@ -2432,7 +2432,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -2445,7 +2445,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise that determines number of dungeons per mapchunk. @@ -2458,7 +2458,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Mapgen Carpathian @@ -2701,7 +2701,7 @@ # octaves = 5, # persistence = 0.55, # lacunarity = 2.0, -# flags = +# flags = # } # First of two 3D noises that together define tunnels. @@ -2714,7 +2714,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -2727,7 +2727,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining giant caverns. @@ -2740,7 +2740,7 @@ # octaves = 5, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise that determines number of dungeons per mapchunk. @@ -2753,7 +2753,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Mapgen Flat @@ -2875,7 +2875,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -2888,7 +2888,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise defining giant caverns. @@ -2901,7 +2901,7 @@ # octaves = 5, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise that determines number of dungeons per mapchunk. @@ -2914,7 +2914,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Mapgen Fractal @@ -3088,7 +3088,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -3101,7 +3101,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # 3D noise that determines number of dungeons per mapchunk. @@ -3114,7 +3114,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Mapgen Valleys @@ -3204,7 +3204,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # Second of two 3D noises that together define tunnels. @@ -3217,7 +3217,7 @@ # octaves = 3, # persistence = 0.5, # lacunarity = 2.0, -# flags = +# flags = # } # The depth of dirt or other biome filler node. @@ -3243,7 +3243,7 @@ # octaves = 6, # persistence = 0.63, # lacunarity = 2.0, -# flags = +# flags = # } # Defines large-scale river channel structure. @@ -3295,7 +3295,7 @@ # octaves = 6, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } # Amplifies the valleys. @@ -3334,7 +3334,7 @@ # octaves = 2, # persistence = 0.8, # lacunarity = 2.0, -# flags = +# flags = # } ## Advanced diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2de68a8f0..0323603fc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -711,7 +711,7 @@ else() # Move text segment below LuaJIT's 47-bit limit (see issue #9367) if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") # FreeBSD uses lld, and lld does not support -Ttext-segment, suggesting - # --image-base instead. Not sure if it's equivalent change for the purpose + # --image-base instead. Not sure if it's equivalent change for the purpose # but at least if fixes build on FreeBSD/aarch64 # XXX: the condition should also be changed to check for lld regardless of # os, bit CMake doesn't have anything like CMAKE_LINKER_IS_LLD yet diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 8d058852a..656ad45ce 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -60,7 +60,7 @@ set(client_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/wieldmesh.cpp ${CMAKE_CURRENT_SOURCE_DIR}/shadows/dynamicshadows.cpp ${CMAKE_CURRENT_SOURCE_DIR}/shadows/dynamicshadowsrender.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/shadows/shadowsshadercallbacks.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/shadows/shadowsScreenQuad.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/shadows/shadowsshadercallbacks.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/shadows/shadowsScreenQuad.cpp PARENT_SCOPE ) diff --git a/src/client/clientmap.cpp b/src/client/clientmap.cpp index 99ff0aefb..10967c0cb 100644 --- a/src/client/clientmap.cpp +++ b/src/client/clientmap.cpp @@ -847,12 +847,12 @@ void ClientMap::renderMapShadows(video::IVideoDriver *driver, vertex_count += buf->getIndexCount(); } - // restore the driver material state + // restore the driver material state video::SMaterial clean; clean.BlendOperation = video::EBO_ADD; driver->setMaterial(clean); // reset material to defaults driver->draw3DLine(v3f(), v3f(), video::SColor(0)); - + g_profiler->avg(prefix + "draw meshes [ms]", draw.stop(true)); g_profiler->avg(prefix + "vertices drawn [#]", vertex_count); g_profiler->avg(prefix + "drawcalls [#]", drawcall_count); diff --git a/src/client/imagefilters.cpp b/src/client/imagefilters.cpp index b62e336f7..c9d1504ad 100644 --- a/src/client/imagefilters.cpp +++ b/src/client/imagefilters.cpp @@ -124,7 +124,7 @@ void imageCleanTransparent(video::IImage *src, u32 threshold) // Ignore pixels we haven't processed if (!bitmap.get(sx, sy)) continue; - + // Add RGB values weighted by alpha IF the pixel is opaque, otherwise // use full weight since we want to propagate colors. video::SColor d = src->getPixel(sx, sy); diff --git a/src/client/render/core.cpp b/src/client/render/core.cpp index f151832f3..c67f297c4 100644 --- a/src/client/render/core.cpp +++ b/src/client/render/core.cpp @@ -103,7 +103,7 @@ void RenderingCore::drawHUD() if (show_hud) { if (draw_crosshair) hud->drawCrosshair(); - + hud->drawHotbar(client->getEnv().getLocalPlayer()->getWieldIndex()); hud->drawLuaElements(camera->getOffset()); camera->drawNametags(); diff --git a/src/clientiface.h b/src/clientiface.h index 1be9c972a..947952e82 100644 --- a/src/clientiface.h +++ b/src/clientiface.h @@ -341,7 +341,7 @@ public: u8 getMinor() const { return m_version_minor; } u8 getPatch() const { return m_version_patch; } const std::string &getFullVer() const { return m_full_version; } - + void setLangCode(const std::string &code) { m_lang_code = code; } const std::string &getLangCode() const { return m_lang_code; } diff --git a/src/collision.cpp b/src/collision.cpp index ccc3a058d..be135a225 100644 --- a/src/collision.cpp +++ b/src/collision.cpp @@ -153,7 +153,7 @@ CollisionAxis axisAlignedCollision( (std::max(movingbox.MaxEdge.Z + speed.Z * time, staticbox.MaxEdge.Z) - std::min(movingbox.MinEdge.Z + speed.Z * time, staticbox.MinEdge.Z) - relbox.MinEdge.Z < 0) - ) + ) return COLLISION_AXIS_X; } } else { @@ -180,7 +180,7 @@ CollisionAxis axisAlignedCollision( (std::max(movingbox.MaxEdge.Y + speed.Y * time, staticbox.MaxEdge.Y) - std::min(movingbox.MinEdge.Y + speed.Y * time, staticbox.MinEdge.Y) - relbox.MinEdge.Y < 0) - ) + ) return COLLISION_AXIS_Z; } } diff --git a/src/config.h b/src/config.h index 5e1164642..8d920b150 100644 --- a/src/config.h +++ b/src/config.h @@ -16,7 +16,7 @@ #define PROJECT_NAME_C "Minetest" #define STATIC_SHAREDIR "" #define VERSION_STRING STR(VERSION_MAJOR) "." STR(VERSION_MINOR) "." STR(VERSION_PATCH) STR(VERSION_EXTRA) -#ifdef NDEBUG + #ifdef NDEBUG #define BUILD_TYPE "Release" #else #define BUILD_TYPE "Debug" diff --git a/src/database/database-leveldb.cpp b/src/database/database-leveldb.cpp index 39f4c8442..6e59daab3 100644 --- a/src/database/database-leveldb.cpp +++ b/src/database/database-leveldb.cpp @@ -74,7 +74,7 @@ void Database_LevelDB::loadBlock(const v3s16 &pos, std::string *block) i64tos(getBlockAsInteger(pos)), block); if (!status.ok()) - block->clear(); + block->clear(); } bool Database_LevelDB::deleteBlock(const v3s16 &pos) diff --git a/src/inventorymanager.cpp b/src/inventorymanager.cpp index a159bf786..ecdb56a97 100644 --- a/src/inventorymanager.cpp +++ b/src/inventorymanager.cpp @@ -172,7 +172,7 @@ void IMoveAction::onPutAndOnTake(const ItemStack &src_item, ServerActiveObject * sa->player_inventory_OnPut(*this, src_item, player); else assert(false); - + if (from_inv.type == InventoryLocation::DETACHED) sa->detached_inventory_OnTake(*this, src_item, player); else if (from_inv.type == InventoryLocation::NODEMETA) diff --git a/src/irrlicht_changes/CGUITTFont.cpp b/src/irrlicht_changes/CGUITTFont.cpp index e785ea837..787f4cd5a 100644 --- a/src/irrlicht_changes/CGUITTFont.cpp +++ b/src/irrlicht_changes/CGUITTFont.cpp @@ -478,7 +478,7 @@ CGUITTGlyphPage* CGUITTFont::getLastGlyphPage() const CGUITTGlyphPage* CGUITTFont::createGlyphPage(const u8& pixel_mode) { CGUITTGlyphPage* page = 0; - + // Name of our page. io::path name("TTFontGlyphPage_"); name += tt_face->family_name; diff --git a/src/mapgen/dungeongen.cpp b/src/mapgen/dungeongen.cpp index acdb1a0f0..1d439abeb 100644 --- a/src/mapgen/dungeongen.cpp +++ b/src/mapgen/dungeongen.cpp @@ -71,7 +71,7 @@ DungeonGen::DungeonGen(const NodeDefManager *ndef, dp.num_dungeons = 1; dp.notifytype = GENNOTIFY_DUNGEON; - dp.np_alt_wall = + dp.np_alt_wall = NoiseParams(-0.4, 1.0, v3f(40.0, 40.0, 40.0), 32474, 6, 1.1, 2.0); } } diff --git a/src/mapgen/mapgen_flat.cpp b/src/mapgen/mapgen_flat.cpp index 342455029..6b249ea1f 100644 --- a/src/mapgen/mapgen_flat.cpp +++ b/src/mapgen/mapgen_flat.cpp @@ -177,7 +177,7 @@ void MapgenFlatParams::setDefaultSettings(Settings *settings) int MapgenFlat::getSpawnLevelAtPoint(v2s16 p) { s16 stone_level = ground_level; - float n_terrain = + float n_terrain = ((spflags & MGFLAT_LAKES) || (spflags & MGFLAT_HILLS)) ? NoisePerlin2D(&noise_terrain->np, p.X, p.Y, seed) : 0.0f; diff --git a/src/mapgen/mg_biome.cpp b/src/mapgen/mg_biome.cpp index f08cc190f..8b4c96cd5 100644 --- a/src/mapgen/mg_biome.cpp +++ b/src/mapgen/mg_biome.cpp @@ -273,7 +273,7 @@ Biome *BiomeGenOriginal::calcBiomeFromNoise(float heat, float humidity, v3s16 po pos.Y - biome_closest_blend->max_pos.Y) return biome_closest_blend; - return (biome_closest) ? biome_closest : (Biome *)m_bmgr->getRaw(BIOME_NONE); + return (biome_closest) ? biome_closest : (Biome *)m_bmgr->getRaw(BIOME_NONE); } diff --git a/src/mapgen/mg_ore.cpp b/src/mapgen/mg_ore.cpp index 5814f433a..4f0c35548 100644 --- a/src/mapgen/mg_ore.cpp +++ b/src/mapgen/mg_ore.cpp @@ -498,8 +498,8 @@ void OreVein::generate(MMVManip *vm, int mapseed, u32 blockseed, } // randval ranges from -1..1 - /* - Note: can generate values slightly larger than 1 + /* + Note: can generate values slightly larger than 1 but this can't be changed as mapgen must be deterministic accross versions. */ float randval = (float)pr.next() / float(pr.RANDOM_RANGE / 2) - 1.f; diff --git a/src/network/connection.h b/src/network/connection.h index 1afb4ae84..88e323bb1 100644 --- a/src/network/connection.h +++ b/src/network/connection.h @@ -752,8 +752,8 @@ protected: void putEvent(ConnectionEventPtr e); void TriggerSend(); - - bool ConnectedToServer() + + bool ConnectedToServer() { return getPeerNoEx(PEER_ID_SERVER) != nullptr; } diff --git a/src/script/cpp_api/s_env.cpp b/src/script/cpp_api/s_env.cpp index 874c37b6e..af68f689f 100644 --- a/src/script/cpp_api/s_env.cpp +++ b/src/script/cpp_api/s_env.cpp @@ -140,10 +140,10 @@ void ScriptApiEnv::initializeEnvironment(ServerEnvironment *env) bool simple_catch_up = true; getboolfield(L, current_abm, "catch_up", simple_catch_up); - + s16 min_y = INT16_MIN; getintfield(L, current_abm, "min_y", min_y); - + s16 max_y = INT16_MAX; getintfield(L, current_abm, "max_y", max_y); diff --git a/src/script/lua_api/l_env.cpp b/src/script/lua_api/l_env.cpp index 54821df24..7640f2782 100644 --- a/src/script/lua_api/l_env.cpp +++ b/src/script/lua_api/l_env.cpp @@ -757,7 +757,7 @@ int ModApiEnvMod::l_get_objects_in_area(lua_State *L) { GET_ENV_PTR; ScriptApiBase *script = getScriptApiBase(L); - + v3f minp = read_v3f(L, 1) * BS; v3f maxp = read_v3f(L, 2) * BS; aabb3f box(minp, maxp); @@ -1409,7 +1409,7 @@ int ModApiEnvMod::l_compare_block_status(lua_State *L) v3s16 nodepos = check_v3s16(L, 1); std::string condition_s = luaL_checkstring(L, 2); auto status = env->getBlockStatus(getNodeBlockPos(nodepos)); - + int condition_i = -1; if (!string_to_enum(es_BlockStatusType, condition_i, condition_s)) return 0; // Unsupported diff --git a/src/script/lua_api/l_env.h b/src/script/lua_api/l_env.h index 67c76faae..a7d406d2a 100644 --- a/src/script/lua_api/l_env.h +++ b/src/script/lua_api/l_env.h @@ -114,7 +114,7 @@ private: // get_objects_inside_radius(pos, radius) static int l_get_objects_inside_radius(lua_State *L); - + // get_objects_in_area(pos, minp, maxp) static int l_get_objects_in_area(lua_State *L); diff --git a/src/serverenvironment.cpp b/src/serverenvironment.cpp index f3711652c..34a1e33e5 100644 --- a/src/serverenvironment.cpp +++ b/src/serverenvironment.cpp @@ -892,7 +892,7 @@ public: for (ActiveABM &aabm : *m_aabms[c]) { if ((p.Y < aabm.min_y) || (p.Y > aabm.max_y)) continue; - + if (myrand() % aabm.chance != 0) continue; diff --git a/src/util/ieee_float.cpp b/src/util/ieee_float.cpp index 887258921..b73763c55 100644 --- a/src/util/ieee_float.cpp +++ b/src/util/ieee_float.cpp @@ -39,7 +39,7 @@ f32 u32Tof32Slow(u32 i) if (exp == 0xFF) { // Inf/NaN if (imant == 0) { - if (std::numeric_limits::has_infinity) + if (std::numeric_limits::has_infinity) return sign ? -std::numeric_limits::infinity() : std::numeric_limits::infinity(); return sign ? std::numeric_limits::max() : diff --git a/src/util/string.h b/src/util/string.h index 8a9e83f22..f4ca1a7de 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -410,7 +410,7 @@ DEFINE_STD_TOSTRING_FLOATINGPOINT(long double) template inline wstring to_wstring(T val) { - return utf8_to_wide(to_string(val)); + return utf8_to_wide(to_string(val)); } } #endif diff --git a/util/generate-texture-normals.sh b/util/generate-texture-normals.sh index 6279dfa69..b2fcbf77b 100755 --- a/util/generate-texture-normals.sh +++ b/util/generate-texture-normals.sh @@ -197,7 +197,7 @@ normalMap() (gimp-convert-rgb image) () ) - (plug-in-normalmap + (plug-in-normalmap RUN-NONINTERACTIVE image drawable -- cgit v1.2.3 From 00ebedad933244dbb866198b5769309ea011719c Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Sat, 29 Jan 2022 22:47:17 -0500 Subject: Add additional reserved directory names --- src/util/string.cpp | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) (limited to 'src/util') diff --git a/src/util/string.cpp b/src/util/string.cpp index bc4664997..689f58f7f 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -821,9 +821,11 @@ std::wstring translate_string(const std::wstring &s) #endif } -static const std::array disallowed_dir_names = { +static const std::array disallowed_dir_names = { // Problematic filenames from here: // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#file-and-directory-names + // Plus undocumented values from here: + // https://googleprojectzero.blogspot.com/2016/02/the-definitive-guide-on-win32-to-nt.html L"CON", L"PRN", L"AUX", @@ -837,6 +839,9 @@ static const std::array disallowed_dir_names = { L"COM7", L"COM8", L"COM9", + L"COM\u00B2", + L"COM\u00B3", + L"COM\u00B9", L"LPT1", L"LPT2", L"LPT3", @@ -846,6 +851,11 @@ static const std::array disallowed_dir_names = { L"LPT7", L"LPT8", L"LPT9", + L"LPT\u00B2", + L"LPT\u00B3", + L"LPT\u00B9", + L"CONIN$", + L"CONOUT$", }; /** @@ -853,6 +863,21 @@ static const std::array disallowed_dir_names = { */ static const std::wstring disallowed_path_chars = L"<>:\"/\\|?*."; + +/** + * @param str + * @return A copy of \p str with trailing whitespace removed. + */ +static std::wstring wrtrim(const std::wstring &str) +{ + size_t back = str.size(); + while (back > 0 && std::isspace(str[back - 1])) + --back; + + return str.substr(0, back); +} + + /** * Sanitize the name of a new directory. This consists of two stages: * 1. Check for 'reserved filenames' that can't be used on some filesystems @@ -863,8 +888,10 @@ std::string sanitizeDirName(const std::string &str, const std::string &optional_ { std::wstring safe_name = utf8_to_wide(str); + std::wstring dev_name = wrtrim(safe_name); + for (std::wstring disallowed_name : disallowed_dir_names) { - if (str_equal(safe_name, disallowed_name, true)) { + if (str_equal(dev_name, disallowed_name, true)) { safe_name = utf8_to_wide(optional_prefix) + safe_name; break; } -- cgit v1.2.3 From dae6fe91a1059751a0bde504cdf41a749234ce1a Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Mon, 31 Jan 2022 21:11:51 -0500 Subject: Update directory name sanitization Only ASCII spaces have to be handles specially, and leading spaces are also disallowed. --- src/unittest/test_utilities.cpp | 12 ++++++++---- src/util/string.cpp | 39 ++++++++++++++------------------------- src/util/string.h | 2 +- 3 files changed, 23 insertions(+), 30 deletions(-) (limited to 'src/util') diff --git a/src/unittest/test_utilities.cpp b/src/unittest/test_utilities.cpp index 228a9559f..10ea8d36a 100644 --- a/src/unittest/test_utilities.cpp +++ b/src/unittest/test_utilities.cpp @@ -636,8 +636,12 @@ void TestUtilities::testBase64() void TestUtilities::testSanitizeDirName() { - UASSERT(sanitizeDirName("a", "_") == "a"); - UASSERT(sanitizeDirName("COM1", "_") == "_COM1"); - UASSERT(sanitizeDirName("cOm\u00B2 .txt:a", "_") == "cOm\u00B2 _txt_a"); - UASSERT(sanitizeDirName("cOnIn$ ", "_") == "_cOnIn$ "); + UASSERT(sanitizeDirName("a", "~") == "a"); + UASSERT(sanitizeDirName(" ", "~") == "__"); + UASSERT(sanitizeDirName(" a ", "~") == "_a_"); + UASSERT(sanitizeDirName("COM1", "~") == "~COM1"); + UASSERT(sanitizeDirName("COM1", ":") == "_COM1"); + UASSERT(sanitizeDirName("cOm\u00B2", "~") == "~cOm\u00B2"); + UASSERT(sanitizeDirName("cOnIn$", "~") == "~cOnIn$"); + UASSERT(sanitizeDirName(" cOnIn$ ", "~") == "_cOnIn$_"); } diff --git a/src/util/string.cpp b/src/util/string.cpp index 689f58f7f..39cd44667 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -864,40 +864,29 @@ static const std::array disallowed_dir_names = { static const std::wstring disallowed_path_chars = L"<>:\"/\\|?*."; -/** - * @param str - * @return A copy of \p str with trailing whitespace removed. - */ -static std::wstring wrtrim(const std::wstring &str) -{ - size_t back = str.size(); - while (back > 0 && std::isspace(str[back - 1])) - --back; - - return str.substr(0, back); -} - - -/** - * Sanitize the name of a new directory. This consists of two stages: - * 1. Check for 'reserved filenames' that can't be used on some filesystems - * and add a prefix to them - * 2. Remove 'unsafe' characters from the name by replacing them with '_' - */ std::string sanitizeDirName(const std::string &str, const std::string &optional_prefix) { std::wstring safe_name = utf8_to_wide(str); - std::wstring dev_name = wrtrim(safe_name); - for (std::wstring disallowed_name : disallowed_dir_names) { - if (str_equal(dev_name, disallowed_name, true)) { + if (str_equal(safe_name, disallowed_name, true)) { safe_name = utf8_to_wide(optional_prefix) + safe_name; break; } } - for (unsigned long i = 0; i < safe_name.length(); i++) { + // Replace leading and trailing spaces with underscores. + size_t start = safe_name.find_first_not_of(L' '); + size_t end = safe_name.find_last_not_of(L' '); + if (start == std::wstring::npos || end == std::wstring::npos) + start = end = safe_name.size(); + for (size_t i = 0; i < start; i++) + safe_name[i] = L'_'; + for (size_t i = end + 1; i < safe_name.size(); i++) + safe_name[i] = L'_'; + + // Replace other disallowed characters with underscores + for (size_t i = 0; i < safe_name.length(); i++) { bool is_valid = true; // Unlikely, but control characters should always be blacklisted @@ -909,7 +898,7 @@ std::string sanitizeDirName(const std::string &str, const std::string &optional_ } if (!is_valid) - safe_name[i] = '_'; + safe_name[i] = L'_'; } return wide_to_utf8(safe_name); diff --git a/src/util/string.h b/src/util/string.h index f4ca1a7de..d8ec633ee 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -749,7 +749,7 @@ inline irr::core::stringw utf8_to_stringw(const std::string &input) /** * Sanitize the name of a new directory. This consists of two stages: * 1. Check for 'reserved filenames' that can't be used on some filesystems - * and prefix them + * and add a prefix to them * 2. Remove 'unsafe' characters from the name by replacing them with '_' */ std::string sanitizeDirName(const std::string &str, const std::string &optional_prefix); -- cgit v1.2.3 From f5e54cd39845aeeff50cdbebf625abf3c4a5b92d Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Sat, 29 Jan 2022 22:50:43 -0500 Subject: Fix OOB read in trim("") --- src/util/string.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/util') diff --git a/src/util/string.h b/src/util/string.h index d8ec633ee..aa4329f2f 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -295,11 +295,11 @@ inline std::string lowercase(const std::string &str) inline std::string trim(const std::string &str) { size_t front = 0; + size_t back = str.size(); - while (std::isspace(str[front])) + while (front < back && std::isspace(str[front])) ++front; - size_t back = str.size(); while (back > front && std::isspace(str[back - 1])) --back; -- cgit v1.2.3 From fccf1e2eac691443108de8f666d611327dbfcfb3 Mon Sep 17 00:00:00 2001 From: Lars Müller <34514239+appgurueu@users.noreply.github.com> Date: Wed, 27 Apr 2022 23:00:02 +0200 Subject: Support CSS Color Module Level 4 (#12204) --- doc/lua_api.txt | 2 +- src/util/string.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'src/util') diff --git a/doc/lua_api.txt b/doc/lua_api.txt index f53ab0ff7..f54672db7 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -3248,7 +3248,7 @@ Colors `#RRGGBBAA` defines a color in hexadecimal format and alpha channel. Named colors are also supported and are equivalent to -[CSS Color Module Level 3](https://www.w3.org/TR/css-color-3/). +[CSS Color Module Level 4](https://www.w3.org/TR/css-color-4/#named-color). To specify the value of the alpha channel, append `#A` or `#AA` to the end of the color name (e.g. `colorname#08`). diff --git a/src/util/string.cpp b/src/util/string.cpp index 39cd44667..b805b2f78 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -494,6 +494,7 @@ const static std::unordered_map s_named_colors = { {"plum", 0xdda0dd}, {"powderblue", 0xb0e0e6}, {"purple", 0x800080}, + {"rebeccapurple", 0x663399}, {"red", 0xff0000}, {"rosybrown", 0xbc8f8f}, {"royalblue", 0x4169e1}, -- cgit v1.2.3 From e7659883cc6fca343785da2a1af3890ae273abbf Mon Sep 17 00:00:00 2001 From: sfan5 Date: Mon, 2 May 2022 20:55:04 +0200 Subject: Async environment for mods to do concurrent tasks (#11131) --- builtin/async/game.lua | 46 ++ builtin/async/init.lua | 11 - builtin/async/mainmenu.lua | 9 + builtin/game/async.lua | 22 + builtin/game/init.lua | 1 + builtin/game/misc.lua | 29 ++ builtin/init.lua | 6 +- doc/lua_api.txt | 62 +++ games/devtest/mods/unittests/async_env.lua | 149 ++++++ games/devtest/mods/unittests/init.lua | 1 + games/devtest/mods/unittests/inside_async_env.lua | 15 + src/map.cpp | 33 ++ src/map.h | 17 +- src/script/common/CMakeLists.txt | 1 + src/script/common/c_internal.cpp | 14 + src/script/common/c_internal.h | 5 + src/script/common/c_packer.cpp | 583 ++++++++++++++++++++++ src/script/common/c_packer.h | 123 +++++ src/script/cpp_api/s_async.cpp | 183 +++++-- src/script/cpp_api/s_async.h | 54 +- src/script/lua_api/l_craft.cpp | 8 + src/script/lua_api/l_craft.h | 1 + src/script/lua_api/l_internal.h | 2 +- src/script/lua_api/l_item.cpp | 25 + src/script/lua_api/l_item.h | 7 +- src/script/lua_api/l_noise.cpp | 53 ++ src/script/lua_api/l_noise.h | 6 + src/script/lua_api/l_server.cpp | 85 ++++ src/script/lua_api/l_server.h | 10 + src/script/lua_api/l_util.cpp | 5 + src/script/lua_api/l_util.h | 2 - src/script/lua_api/l_vmanip.cpp | 33 ++ src/script/lua_api/l_vmanip.h | 3 + src/script/scripting_server.cpp | 67 ++- src/script/scripting_server.h | 17 + src/server.cpp | 4 + src/server.h | 8 +- src/serverenvironment.cpp | 2 + src/util/basic_macros.h | 8 +- 39 files changed, 1654 insertions(+), 56 deletions(-) create mode 100644 builtin/async/game.lua delete mode 100644 builtin/async/init.lua create mode 100644 builtin/async/mainmenu.lua create mode 100644 builtin/game/async.lua create mode 100644 games/devtest/mods/unittests/async_env.lua create mode 100644 games/devtest/mods/unittests/inside_async_env.lua create mode 100644 src/script/common/c_packer.cpp create mode 100644 src/script/common/c_packer.h (limited to 'src/util') diff --git a/builtin/async/game.lua b/builtin/async/game.lua new file mode 100644 index 000000000..212a33e17 --- /dev/null +++ b/builtin/async/game.lua @@ -0,0 +1,46 @@ +core.log("info", "Initializing asynchronous environment (game)") + +local function pack2(...) + return {n=select('#', ...), ...} +end + +-- Entrypoint to run async jobs, called by C++ +function core.job_processor(func, params) + local retval = pack2(func(unpack(params, 1, params.n))) + + return retval +end + +-- Import a bunch of individual files from builtin/game/ +local gamepath = core.get_builtin_path() .. "game" .. DIR_DELIM + +dofile(gamepath .. "constants.lua") +dofile(gamepath .. "item_s.lua") +dofile(gamepath .. "misc_s.lua") +dofile(gamepath .. "features.lua") +dofile(gamepath .. "voxelarea.lua") + +-- Transfer of globals +do + assert(core.transferred_globals) + local all = core.deserialize(core.transferred_globals, true) + core.transferred_globals = nil + + -- reassemble other tables + all.registered_nodes = {} + all.registered_craftitems = {} + all.registered_tools = {} + for k, v in pairs(all.registered_items) do + if v.type == "node" then + all.registered_nodes[k] = v + elseif v.type == "craftitem" then + all.registered_craftitems[k] = v + elseif v.type == "tool" then + all.registered_tools[k] = v + end + end + + for k, v in pairs(all) do + core[k] = v + end +end diff --git a/builtin/async/init.lua b/builtin/async/init.lua deleted file mode 100644 index 3803994d6..000000000 --- a/builtin/async/init.lua +++ /dev/null @@ -1,11 +0,0 @@ - -core.log("info", "Initializing Asynchronous environment") - -function core.job_processor(func, serialized_param) - local param = core.deserialize(serialized_param) - - local retval = core.serialize(func(param)) - - return retval or core.serialize(nil) -end - diff --git a/builtin/async/mainmenu.lua b/builtin/async/mainmenu.lua new file mode 100644 index 000000000..0e9c222d1 --- /dev/null +++ b/builtin/async/mainmenu.lua @@ -0,0 +1,9 @@ +core.log("info", "Initializing asynchronous environment") + +function core.job_processor(func, serialized_param) + local param = core.deserialize(serialized_param) + + local retval = core.serialize(func(param)) + + return retval or core.serialize(nil) +end diff --git a/builtin/game/async.lua b/builtin/game/async.lua new file mode 100644 index 000000000..469f179d7 --- /dev/null +++ b/builtin/game/async.lua @@ -0,0 +1,22 @@ + +core.async_jobs = {} + +function core.async_event_handler(jobid, retval) + local callback = core.async_jobs[jobid] + assert(type(callback) == "function") + callback(unpack(retval, 1, retval.n)) + core.async_jobs[jobid] = nil +end + +function core.handle_async(func, callback, ...) + assert(type(func) == "function" and type(callback) == "function", + "Invalid minetest.handle_async invocation") + local args = {n = select("#", ...), ...} + local mod_origin = core.get_last_run_mod() + + local jobid = core.do_async_callback(func, args, mod_origin) + core.async_jobs[jobid] = callback + + return true +end + diff --git a/builtin/game/init.lua b/builtin/game/init.lua index c5f08113b..68d6a10f8 100644 --- a/builtin/game/init.lua +++ b/builtin/game/init.lua @@ -34,5 +34,6 @@ dofile(gamepath .. "voxelarea.lua") dofile(gamepath .. "forceloading.lua") dofile(gamepath .. "statbars.lua") dofile(gamepath .. "knockback.lua") +dofile(gamepath .. "async.lua") profiler = nil diff --git a/builtin/game/misc.lua b/builtin/game/misc.lua index 18d5a7310..9f5e3312b 100644 --- a/builtin/game/misc.lua +++ b/builtin/game/misc.lua @@ -235,3 +235,32 @@ end -- Used for callback handling with dynamic_add_media core.dynamic_media_callbacks = {} + + +-- Transfer of certain globals into async environment +-- see builtin/async/game.lua for the other side + +local function copy_filtering(t, seen) + if type(t) == "userdata" or type(t) == "function" then + return true -- don't use nil so presence can still be detected + elseif type(t) ~= "table" then + return t + end + local n = {} + seen = seen or {} + seen[t] = n + for k, v in pairs(t) do + local k_ = seen[k] or copy_filtering(k, seen) + local v_ = seen[v] or copy_filtering(v, seen) + n[k_] = v_ + end + return n +end + +function core.get_globals_to_transfer() + local all = { + registered_items = copy_filtering(core.registered_items), + registered_aliases = core.registered_aliases, + } + return core.serialize(all) +end diff --git a/builtin/init.lua b/builtin/init.lua index 7a9b5c427..869136016 100644 --- a/builtin/init.lua +++ b/builtin/init.lua @@ -56,8 +56,10 @@ elseif INIT == "mainmenu" then if not custom_loaded then dofile(core.get_mainmenu_path() .. DIR_DELIM .. "init.lua") end -elseif INIT == "async" then - dofile(asyncpath .. "init.lua") +elseif INIT == "async" then + dofile(asyncpath .. "mainmenu.lua") +elseif INIT == "async_game" then + dofile(asyncpath .. "game.lua") elseif INIT == "client" then dofile(clientpath .. "init.lua") else diff --git a/doc/lua_api.txt b/doc/lua_api.txt index f54672db7..339ce8a27 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -5767,6 +5767,68 @@ Timing * `job:cancel()` * Cancels the job function from being called +Async environment +----------------- + +The engine allows you to submit jobs to be ran in an isolated environment +concurrently with normal server operation. +A job consists of a function to be ran in the async environment, any amount of +arguments (will be serialized) and a callback that will be called with the return +value of the job function once it is finished. + +The async environment does *not* have access to the map, entities, players or any +globals defined in the 'usual' environment. Consequently, functions like +`minetest.get_node()` or `minetest.get_player_by_name()` simply do not exist in it. + +Arguments and return values passed through this can contain certain userdata +objects that will be seamlessly copied (not shared) to the async environment. +This allows you easy interoperability for delegating work to jobs. + +* `minetest.handle_async(func, callback, ...)`: + * Queue the function `func` to be ran in an async environment. + Note that there are multiple persistent workers and any of them may + end up running a given job. The engine will scale the amount of + worker threads automatically. + * When `func` returns the callback is called (in the normal environment) + with all of the return values as arguments. + * Optional: Variable number of arguments that are passed to `func` +* `minetest.register_async_dofile(path)`: + * Register a path to a Lua file to be imported when an async environment + is initialized. You can use this to preload code which you can then call + later using `minetest.handle_async()`. + +### List of APIs available in an async environment + +Classes: +* `ItemStack` +* `PerlinNoise` +* `PerlinNoiseMap` +* `PseudoRandom` +* `PcgRandom` +* `SecureRandom` +* `VoxelArea` +* `VoxelManip` + * only if transferred into environment; can't read/write to map +* `Settings` + +Class instances that can be transferred between environments: +* `ItemStack` +* `PerlinNoise` +* `PerlinNoiseMap` +* `VoxelManip` + +Functions: +* Standalone helpers such as logging, filesystem, encoding, + hashing or compression APIs +* `minetest.request_insecure_environment` (same restrictions apply) + +Variables: +* `minetest.settings` +* `minetest.registered_items`, `registered_nodes`, `registered_tools`, + `registered_craftitems` and `registered_aliases` + * with all functions and userdata values replaced by `true`, calling any + callbacks here is obviously not possible + Server ------ diff --git a/games/devtest/mods/unittests/async_env.lua b/games/devtest/mods/unittests/async_env.lua new file mode 100644 index 000000000..aff1fc4d9 --- /dev/null +++ b/games/devtest/mods/unittests/async_env.lua @@ -0,0 +1,149 @@ +-- helper + +core.register_async_dofile(core.get_modpath(core.get_current_modname()) .. + DIR_DELIM .. "inside_async_env.lua") + +local function deepequal(a, b) + if type(a) == "function" then + return type(b) == "function" + elseif type(a) ~= "table" then + return a == b + elseif type(b) ~= "table" then + return false + end + for k, v in pairs(a) do + if not deepequal(v, b[k]) then + return false + end + end + for k, v in pairs(b) do + if not deepequal(a[k], v) then + return false + end + end + return true +end + +-- Object Passing / Serialization + +local test_object = { + name = "stairs:stair_glass", + type = "node", + groups = {oddly_breakable_by_hand = 3, cracky = 3, stair = 1}, + description = "Glass Stair", + sounds = { + dig = {name = "default_glass_footstep", gain = 0.5}, + footstep = {name = "default_glass_footstep", gain = 0.3}, + dug = {name = "default_break_glass", gain = 1} + }, + node_box = { + fixed = { + {-0.5, -0.5, -0.5, 0.5, 0, 0.5}, + {-0.5, 0, 0, 0.5, 0.5, 0.5} + }, + type = "fixed" + }, + tiles = { + {name = "stairs_glass_split.png", backface_culling = true}, + {name = "default_glass.png", backface_culling = true}, + {name = "stairs_glass_stairside.png^[transformFX", backface_culling = true} + }, + on_place = function(itemstack, placer) + return core.is_player(placer) + end, + sunlight_propagates = true, + is_ground_content = false, + light_source = 0, +} + +local function test_object_passing() + local tmp = core.serialize_roundtrip(test_object) + assert(deepequal(test_object, tmp)) + + -- Circular key, should error + tmp = {"foo", "bar"} + tmp[tmp] = true + assert(not pcall(core.serialize_roundtrip, tmp)) + + -- Circular value, should error + tmp = {"foo"} + tmp[2] = tmp + assert(not pcall(core.serialize_roundtrip, tmp)) +end +unittests.register("test_object_passing", test_object_passing) + +local function test_userdata_passing(_, pos) + -- basic userdata passing + local obj = table.copy(test_object.tiles[1]) + obj.test = ItemStack("default:cobble 99") + local tmp = core.serialize_roundtrip(obj) + assert(type(tmp.test) == "userdata") + assert(obj.test:to_string() == tmp.test:to_string()) + + -- object can't be passed, should error + obj = core.raycast(pos, pos) + assert(not pcall(core.serialize_roundtrip, obj)) + + -- VManip + local vm = core.get_voxel_manip(pos, pos) + local expect = vm:get_node_at(pos) + local vm2 = core.serialize_roundtrip(vm) + assert(deepequal(vm2:get_node_at(pos), expect)) +end +unittests.register("test_userdata_passing", test_userdata_passing, {map=true}) + +-- Asynchronous jobs + +local function test_handle_async(cb) + -- Basic test including mod name tracking and unittests.async_test() + -- which is defined inside_async_env.lua + local func = function(x) + return core.get_last_run_mod(), _VERSION, unittests[x]() + end + local expect = {core.get_last_run_mod(), _VERSION, true} + + core.handle_async(func, function(...) + if not deepequal(expect, {...}) then + cb("Values did not equal") + end + if core.get_last_run_mod() ~= expect[1] then + cb("Mod name not tracked correctly") + end + + -- Test passing of nil arguments and return values + core.handle_async(function(a, b) + return a, b + end, function(a, b) + if b ~= 123 then + cb("Argument went missing") + end + cb() + end, nil, 123) + end, "async_test") +end +unittests.register("test_handle_async", test_handle_async, {async=true}) + +local function test_userdata_passing2(cb, _, pos) + -- VManip: check transfer into other env + local vm = core.get_voxel_manip(pos, pos) + local expect = vm:get_node_at(pos) + + core.handle_async(function(vm_, pos_) + return vm_:get_node_at(pos_) + end, function(ret) + if not deepequal(expect, ret) then + cb("Node data mismatch (one-way)") + end + + -- VManip: test a roundtrip + core.handle_async(function(vm_) + return vm_ + end, function(vm2) + if not deepequal(expect, vm2:get_node_at(pos)) then + cb("Node data mismatch (roundtrip)") + end + cb() + end, vm) + end, vm, pos) +end +unittests.register("test_userdata_passing2", test_userdata_passing2, {map=true, async=true}) diff --git a/games/devtest/mods/unittests/init.lua b/games/devtest/mods/unittests/init.lua index 0754d507f..0608f2dd2 100644 --- a/games/devtest/mods/unittests/init.lua +++ b/games/devtest/mods/unittests/init.lua @@ -175,6 +175,7 @@ dofile(modpath .. "/misc.lua") dofile(modpath .. "/player.lua") dofile(modpath .. "/crafting.lua") dofile(modpath .. "/itemdescription.lua") +dofile(modpath .. "/async_env.lua") -------------- diff --git a/games/devtest/mods/unittests/inside_async_env.lua b/games/devtest/mods/unittests/inside_async_env.lua new file mode 100644 index 000000000..9774771f9 --- /dev/null +++ b/games/devtest/mods/unittests/inside_async_env.lua @@ -0,0 +1,15 @@ +unittests = {} + +core.log("info", "Hello World") + +function unittests.async_test() + assert(core == minetest) + -- stuff that should not be here + assert(not core.get_player_by_name) + assert(not core.set_node) + assert(not core.object_refs) + -- stuff that should be here + assert(ItemStack) + assert(core.registered_items[""]) + return true +end diff --git a/src/map.cpp b/src/map.cpp index 9c9324f5f..5153dcaa9 100644 --- a/src/map.cpp +++ b/src/map.cpp @@ -1896,6 +1896,7 @@ MMVManip::MMVManip(Map *map): VoxelManipulator(), m_map(map) { + assert(map); } void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max, @@ -1903,6 +1904,8 @@ void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max, { TimeTaker timer1("initialEmerge", &emerge_time); + assert(m_map); + // Units of these are MapBlocks v3s16 p_min = blockpos_min; v3s16 p_max = blockpos_max; @@ -1986,6 +1989,7 @@ void MMVManip::blitBackAll(std::map *modified_blocks, { if(m_area.getExtent() == v3s16(0,0,0)) return; + assert(m_map); /* Copy data of all blocks @@ -2006,4 +2010,33 @@ void MMVManip::blitBackAll(std::map *modified_blocks, } } +MMVManip *MMVManip::clone() const +{ + MMVManip *ret = new MMVManip(); + + const s32 size = m_area.getVolume(); + ret->m_area = m_area; + if (m_data) { + ret->m_data = new MapNode[size]; + memcpy(ret->m_data, m_data, size * sizeof(MapNode)); + } + if (m_flags) { + ret->m_flags = new u8[size]; + memcpy(ret->m_flags, m_flags, size * sizeof(u8)); + } + + ret->m_is_dirty = m_is_dirty; + // Even if the copy is disconnected from a map object keep the information + // needed to write it back to one + ret->m_loaded_blocks = m_loaded_blocks; + + return ret; +} + +void MMVManip::reparent(Map *map) +{ + assert(map && !m_map); + m_map = map; +} + //END diff --git a/src/map.h b/src/map.h index d8ed29106..21e3dbd6c 100644 --- a/src/map.h +++ b/src/map.h @@ -446,10 +446,25 @@ public: void blitBackAll(std::map * modified_blocks, bool overwrite_generated = true); + /* + Creates a copy of this VManip including contents, the copy will not be + associated with a Map. + */ + MMVManip *clone() const; + + // Reassociates a copied VManip to a map + void reparent(Map *map); + + // Is it impossible to call initialEmerge / blitBackAll? + inline bool isOrphan() const { return !m_map; } + bool m_is_dirty = false; protected: - Map *m_map; + MMVManip() {}; + + // may be null + Map *m_map = nullptr; /* key = blockpos value = flags describing the block diff --git a/src/script/common/CMakeLists.txt b/src/script/common/CMakeLists.txt index d07f6ab1b..3e84b46c7 100644 --- a/src/script/common/CMakeLists.txt +++ b/src/script/common/CMakeLists.txt @@ -3,6 +3,7 @@ set(common_SCRIPT_COMMON_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/c_converter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/c_types.cpp ${CMAKE_CURRENT_SOURCE_DIR}/c_internal.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/c_packer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helper.cpp PARENT_SCOPE) diff --git a/src/script/common/c_internal.cpp b/src/script/common/c_internal.cpp index df82dba14..ddd2d184c 100644 --- a/src/script/common/c_internal.cpp +++ b/src/script/common/c_internal.cpp @@ -166,3 +166,17 @@ void log_deprecated(lua_State *L, std::string message, int stack_depth) infostream << script_get_backtrace(L) << std::endl; } +void call_string_dump(lua_State *L, int idx) +{ + // Retrieve string.dump from insecure env to avoid it being tampered with + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_GLOBALS_BACKUP); + if (!lua_isnil(L, -1)) + lua_getfield(L, -1, "string"); + else + lua_getglobal(L, "string"); + lua_getfield(L, -1, "dump"); + lua_remove(L, -2); // remove _G + lua_remove(L, -2); // remove 'string' table + lua_pushvalue(L, idx); + lua_call(L, 1, 1); +} diff --git a/src/script/common/c_internal.h b/src/script/common/c_internal.h index c43db34aa..272a39941 100644 --- a/src/script/common/c_internal.h +++ b/src/script/common/c_internal.h @@ -56,6 +56,7 @@ extern "C" { #define CUSTOM_RIDX_BACKTRACE (CUSTOM_RIDX_BASE + 3) #define CUSTOM_RIDX_HTTP_API_LUA (CUSTOM_RIDX_BASE + 4) #define CUSTOM_RIDX_VECTOR_METATABLE (CUSTOM_RIDX_BASE + 5) +#define CUSTOM_RIDX_METATABLE_MAP (CUSTOM_RIDX_BASE + 6) // Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata @@ -139,3 +140,7 @@ DeprecatedHandlingMode get_deprecated_handling_mode(); * @param stack_depth How far on the stack to the first user function (ie: not builtin or core) */ void log_deprecated(lua_State *L, std::string message, int stack_depth = 1); + +// Safely call string.dump on a function value +// (does not pop, leaves one value on stack) +void call_string_dump(lua_State *L, int idx); diff --git a/src/script/common/c_packer.cpp b/src/script/common/c_packer.cpp new file mode 100644 index 000000000..fc5277330 --- /dev/null +++ b/src/script/common/c_packer.cpp @@ -0,0 +1,583 @@ +/* +Minetest +Copyright (C) 2022 sfan5 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#include +#include +#include +#include +#include +#include +#include "c_packer.h" +#include "c_internal.h" +#include "log.h" +#include "debug.h" +#include "threading/mutex_auto_lock.h" + +extern "C" { +#include +} + +// +// Helpers +// + +// convert negative index to absolute position on Lua stack +static inline int absidx(lua_State *L, int idx) +{ + assert(idx < 0); + return lua_gettop(L) + idx + 1; +} + +// does the type put anything into PackedInstr::sdata? +static inline bool uses_sdata(int type) +{ + switch (type) { + case LUA_TSTRING: + case LUA_TFUNCTION: + case LUA_TUSERDATA: + return true; + default: + return false; + } +} + +// does the type put anything into PackedInstr::? +static inline bool uses_union(int type) +{ + switch (type) { + case LUA_TNIL: + case LUA_TSTRING: + case LUA_TFUNCTION: + return false; + default: + return true; + } +} + +static inline bool can_set_into(int ktype, int vtype) +{ + switch (ktype) { + case LUA_TNUMBER: + return !uses_union(vtype); + case LUA_TSTRING: + return !uses_sdata(vtype); + default: + return false; + } +} + +// is the key suitable for use with set_into? +static inline bool suitable_key(lua_State *L, int idx) +{ + if (lua_type(L, idx) == LUA_TSTRING) { + // strings may not have a NULL byte (-> lua_setfield) + size_t len; + const char *str = lua_tolstring(L, idx, &len); + return strlen(str) == len; + } else { + assert(lua_type(L, idx) == LUA_TNUMBER); + // numbers must fit into an s32 and be integers (-> lua_rawseti) + lua_Number n = lua_tonumber(L, idx); + return std::floor(n) == n && n >= S32_MIN && n <= S32_MAX; + } +} + +namespace { + // checks if you left any values on the stack, for debugging + class StackChecker { + lua_State *L; + int top; + public: + StackChecker(lua_State *L) : L(L), top(lua_gettop(L)) {} + ~StackChecker() { + assert(lua_gettop(L) >= top); + if (lua_gettop(L) > top) { + rawstream << "Lua stack not cleaned up: " + << lua_gettop(L) << " != " << top + << " (false-positive if exception thrown)" << std::endl; + } + } + }; + + // Since an std::vector may reallocate, this is the only safe way to keep + // a reference to a particular element. + template + class VectorRef { + std::vector *vec; + size_t idx; + VectorRef(std::vector *vec, size_t idx) : vec(vec), idx(idx) {} + public: + static VectorRef front(std::vector &vec) { + return VectorRef(&vec, 0); + } + static VectorRef back(std::vector &vec) { + return VectorRef(&vec, vec.size() - 1); + } + T &operator*() { return (*vec)[idx]; } + T *operator->() { return &(*vec)[idx]; } + }; + + struct Packer { + PackInFunc fin; + PackOutFunc fout; + }; + + typedef std::pair PackerTuple; +} + +static inline auto emplace(PackedValue &pv, s16 type) +{ + pv.i.emplace_back(); + auto ref = VectorRef::back(pv.i); + ref->type = type; + // Initialize fields that may be left untouched + if (type == LUA_TTABLE) { + ref->uidata1 = 0; + ref->uidata2 = 0; + } else if (type == LUA_TUSERDATA) { + ref->ptrdata = nullptr; + } else if (type == INSTR_POP) { + ref->sidata2 = 0; + } + return ref; +} + +// +// Management of registered packers +// + +static std::unordered_map g_packers; +static std::mutex g_packers_lock; + +void script_register_packer(lua_State *L, const char *regname, + PackInFunc fin, PackOutFunc fout) +{ + // Store away callbacks + { + MutexAutoLock autolock(g_packers_lock); + auto it = g_packers.find(regname); + if (it == g_packers.end()) { + auto &ref = g_packers[regname]; + ref.fin = fin; + ref.fout = fout; + } else { + FATAL_ERROR_IF(it->second.fin != fin || it->second.fout != fout, + "Packer registered twice with mismatching callbacks"); + } + } + + // Save metatable so we can identify instances later + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + if (lua_isnil(L, -1)) { + lua_newtable(L); + lua_pushvalue(L, -1); + lua_rawseti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + } + + luaL_getmetatable(L, regname); + FATAL_ERROR_IF(lua_isnil(L, -1), "No metatable registered with that name"); + + // CUSTOM_RIDX_METATABLE_MAP contains { [metatable] = "regname", ... } + // check first + lua_pushstring(L, regname); + lua_rawget(L, -3); + if (!lua_isnil(L, -1)) { + FATAL_ERROR_IF(lua_topointer(L, -1) != lua_topointer(L, -2), + "Packer registered twice with inconsistent metatable"); + } + lua_pop(L, 1); + // then set + lua_pushstring(L, regname); + lua_rawset(L, -3); + + lua_pop(L, 1); +} + +static bool find_packer(const char *regname, PackerTuple &out) +{ + MutexAutoLock autolock(g_packers_lock); + auto it = g_packers.find(regname); + if (it == g_packers.end()) + return false; + // copy data for thread safety + out.first = it->first; + out.second = it->second; + return true; +} + +static bool find_packer(lua_State *L, int idx, PackerTuple &out) +{ +#ifndef NDEBUG + StackChecker checker(L); +#endif + + // retrieve metatable of the object + if (lua_getmetatable(L, idx) != 1) + return false; + + // use our global table to map it to the registry name + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP); + assert(lua_istable(L, -1)); + lua_pushvalue(L, -2); + lua_rawget(L, -2); + if (lua_isnil(L, -1)) { + lua_pop(L, 3); + return false; + } + + // load the associated data + bool found = find_packer(lua_tostring(L, -1), out); + FATAL_ERROR_IF(!found, "Inconsistent internal state"); + lua_pop(L, 3); + return true; +} + +// +// Packing implementation +// + +// recursively goes through the structure and ensures there are no circular references +static void pack_validate(lua_State *L, int idx, std::unordered_set &seen) +{ +#ifndef NDEBUG + StackChecker checker(L); + assert(idx > 0); +#endif + + if (lua_type(L, idx) != LUA_TTABLE) + return; + + const void *ptr = lua_topointer(L, idx); + assert(ptr); + + if (seen.find(ptr) != seen.end()) + throw LuaError("Circular references cannot be packed (yet)"); + seen.insert(ptr); + + lua_checkstack(L, 5); + lua_pushnil(L); + while (lua_next(L, idx) != 0) { + // key at -2, value at -1 + pack_validate(L, absidx(L, -2), seen); + pack_validate(L, absidx(L, -1), seen); + + lua_pop(L, 1); + } + + seen.erase(ptr); +} + +static VectorRef pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv) +{ +#ifndef NDEBUG + StackChecker checker(L); + assert(idx > 0); + assert(vidx > 0); +#endif + + switch (lua_type(L, idx)) { + case LUA_TNONE: + case LUA_TNIL: + return emplace(pv, LUA_TNIL); + case LUA_TBOOLEAN: { + auto r = emplace(pv, LUA_TBOOLEAN); + r->bdata = lua_toboolean(L, idx); + return r; + } + case LUA_TNUMBER: { + auto r = emplace(pv, LUA_TNUMBER); + r->ndata = lua_tonumber(L, idx); + return r; + } + case LUA_TSTRING: { + auto r = emplace(pv, LUA_TSTRING); + size_t len; + const char *str = lua_tolstring(L, idx, &len); + assert(str); + r->sdata.assign(str, len); + return r; + } + case LUA_TTABLE: + break; // execution continues + case LUA_TFUNCTION: { + auto r = emplace(pv, LUA_TFUNCTION); + call_string_dump(L, idx); + size_t len; + const char *str = lua_tolstring(L, -1, &len); + assert(str); + r->sdata.assign(str, len); + lua_pop(L, 1); + return r; + } + case LUA_TUSERDATA: { + PackerTuple ser; + if (!find_packer(L, idx, ser)) + throw LuaError("Cannot serialize unsupported userdata"); + pv.contains_userdata = true; + auto r = emplace(pv, LUA_TUSERDATA); + r->sdata = ser.first; + r->ptrdata = ser.second.fin(L, idx); + return r; + } + default: { + std::string err = "Cannot serialize type "; + err += lua_typename(L, lua_type(L, idx)); + throw LuaError(err); + } + } + + // LUA_TTABLE + lua_checkstack(L, 5); + + auto rtable = emplace(pv, LUA_TTABLE); + const int vi_table = vidx++; + + lua_pushnil(L); + while (lua_next(L, idx) != 0) { + // key at -2, value at -1 + const int ktype = lua_type(L, -2), vtype = lua_type(L, -1); + if (ktype == LUA_TNUMBER) + rtable->uidata1++; // narr + else + rtable->uidata2++; // nrec + + // check if we can use a shortcut + if (can_set_into(ktype, vtype) && suitable_key(L, -2)) { + // push only the value + auto rval = pack_inner(L, absidx(L, -1), vidx, pv); + rval->pop = vtype != LUA_TTABLE; + // and where to put it: + rval->set_into = vi_table; + if (ktype == LUA_TSTRING) + rval->sdata = lua_tostring(L, -2); + else + rval->sidata1 = lua_tointeger(L, -2); + // pop tables after the fact + if (!rval->pop) { + auto ri1 = emplace(pv, INSTR_POP); + ri1->sidata1 = vidx; + } + } else { + // push the key and value + pack_inner(L, absidx(L, -2), vidx, pv); + vidx++; + pack_inner(L, absidx(L, -1), vidx, pv); + vidx++; + // push an instruction to set them + auto ri1 = emplace(pv, INSTR_SETTABLE); + ri1->set_into = vi_table; + ri1->sidata1 = vidx - 2; + ri1->sidata2 = vidx - 1; + ri1->pop = true; + vidx -= 2; + } + + lua_pop(L, 1); + } + + assert(vidx == vi_table + 1); + return rtable; +} + +PackedValue *script_pack(lua_State *L, int idx) +{ + if (idx < 0) + idx = absidx(L, idx); + + std::unordered_set seen; + pack_validate(L, idx, seen); + assert(seen.size() == 0); + + // Actual serialization + PackedValue pv; + pack_inner(L, idx, 1, pv); + + return new PackedValue(std::move(pv)); +} + +// +// Unpacking implementation +// + +void script_unpack(lua_State *L, PackedValue *pv) +{ + const int top = lua_gettop(L); + int ctr = 0; + + for (auto &i : pv->i) { + // If leaving values on stack make sure there's space (every 5th iteration) + if (!i.pop && (ctr++) >= 5) { + lua_checkstack(L, 5); + ctr = 0; + } + + /* Instructions */ + switch (i.type) { + case INSTR_SETTABLE: + lua_pushvalue(L, top + i.sidata1); // key + lua_pushvalue(L, top + i.sidata2); // value + lua_rawset(L, top + i.set_into); + if (i.pop) { + if (i.sidata1 != i.sidata2) { + // removing moves indices so pop higher index first + lua_remove(L, top + std::max(i.sidata1, i.sidata2)); + lua_remove(L, top + std::min(i.sidata1, i.sidata2)); + } else { + lua_remove(L, top + i.sidata1); + } + } + continue; + case INSTR_POP: + lua_remove(L, top + i.sidata1); + if (i.sidata2 > 0) + lua_remove(L, top + i.sidata2); + continue; + default: + break; + } + + /* Lua types */ + switch (i.type) { + case LUA_TNIL: + lua_pushnil(L); + break; + case LUA_TBOOLEAN: + lua_pushboolean(L, i.bdata); + break; + case LUA_TNUMBER: + lua_pushnumber(L, i.ndata); + break; + case LUA_TSTRING: + lua_pushlstring(L, i.sdata.data(), i.sdata.size()); + break; + case LUA_TTABLE: + lua_createtable(L, i.uidata1, i.uidata2); + break; + case LUA_TFUNCTION: + luaL_loadbuffer(L, i.sdata.data(), i.sdata.size(), nullptr); + break; + case LUA_TUSERDATA: { + PackerTuple ser; + sanity_check(find_packer(i.sdata.c_str(), ser)); + ser.second.fout(L, i.ptrdata); + i.ptrdata = nullptr; // ownership taken by callback + break; + } + default: + assert(0); + break; + } + + if (i.set_into) { + if (!i.pop) + lua_pushvalue(L, -1); + if (uses_sdata(i.type)) + lua_rawseti(L, top + i.set_into, i.sidata1); + else + lua_setfield(L, top + i.set_into, i.sdata.c_str()); + } else { + if (i.pop) + lua_pop(L, 1); + } + } + + // as part of the unpacking process we take ownership of all userdata + pv->contains_userdata = false; + // leave exactly one value on the stack + lua_settop(L, top+1); +} + +// +// PackedValue +// + +PackedValue::~PackedValue() +{ + if (!contains_userdata) + return; + for (auto &i : this->i) { + if (i.type == LUA_TUSERDATA && i.ptrdata) { + PackerTuple ser; + if (find_packer(i.sdata.c_str(), ser)) { + // tell it to deallocate object + ser.second.fout(nullptr, i.ptrdata); + } else { + assert(false); + } + } + } +} + +// +// script_dump_packed +// + +#ifndef NDEBUG +void script_dump_packed(const PackedValue *val) +{ + printf("instruction stream: [\n"); + for (const auto &i : val->i) { + printf("\t("); + switch (i.type) { + case INSTR_SETTABLE: + printf("SETTABLE(%d, %d)", i.sidata1, i.sidata2); + break; + case INSTR_POP: + printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2); + break; + case LUA_TNIL: + printf("nil"); + break; + case LUA_TBOOLEAN: + printf(i.bdata ? "true" : "false"); + break; + case LUA_TNUMBER: + printf("%f", i.ndata); + break; + case LUA_TSTRING: + printf("\"%s\"", i.sdata.c_str()); + break; + case LUA_TTABLE: + printf("table(%d, %d)", i.uidata1, i.uidata2); + break; + case LUA_TFUNCTION: + printf("function(%d byte)", i.sdata.size()); + break; + case LUA_TUSERDATA: + printf("userdata %s %p", i.sdata.c_str(), i.ptrdata); + break; + default: + printf("!!UNKNOWN!!"); + break; + } + if (i.set_into) { + if (i.type >= 0 && uses_sdata(i.type)) + printf(", k=%d, into=%d", i.sidata1, i.set_into); + else if (i.type >= 0) + printf(", k=\"%s\", into=%d", i.sdata.c_str(), i.set_into); + else + printf(", into=%d", i.set_into); + } + if (i.pop) + printf(", pop"); + printf(")\n"); + } + printf("]\n"); +} +#endif diff --git a/src/script/common/c_packer.h b/src/script/common/c_packer.h new file mode 100644 index 000000000..8bccca98d --- /dev/null +++ b/src/script/common/c_packer.h @@ -0,0 +1,123 @@ +/* +Minetest +Copyright (C) 2022 sfan5 + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#pragma once + +#include +#include +#include "irrlichttypes.h" +#include "util/basic_macros.h" + +extern "C" { +#include +} + +/* + This file defines an in-memory representation of Lua objects including + support for functions and userdata. It it used to move data between Lua + states and cannot be used for persistence or network transfer. +*/ + +#define INSTR_SETTABLE (-10) +#define INSTR_POP (-11) + +/** + * Represents a single instruction that pushes a new value or works with existing ones. + */ +struct PackedInstr +{ + s16 type; // LUA_T* or INSTR_* + u16 set_into; // set into table on stack + bool pop; // remove from stack? + union { + bool bdata; // boolean: value + lua_Number ndata; // number: value + struct { + u16 uidata1, uidata2; // table: narr, nrec + }; + struct { + /* + SETTABLE: key index, value index + POP: indices to remove + otherwise w/ set_into: numeric key, - + */ + s32 sidata1, sidata2; + }; + void *ptrdata; // userdata: implementation defined + }; + /* + - string: value + - function: buffer + - w/ set_into: string key (no null bytes!) + - userdata: name in registry + */ + std::string sdata; + + PackedInstr() : type(0), set_into(0), pop(false) {} +}; + +/** + * A packed value can be a primitive like a string or number but also a table + * including all of its contents. It is made up of a linear stream of + * 'instructions' that build the final value when executed. + */ +struct PackedValue +{ + std::vector i; + // Indicates whether there are any userdata pointers that need to be deallocated + bool contains_userdata = false; + + PackedValue() = default; + ~PackedValue(); + + DISABLE_CLASS_COPY(PackedValue) + + ALLOW_CLASS_MOVE(PackedValue) +}; + +/* + * Packing callback: Turns a Lua value at given index into a void* + */ +typedef void *(*PackInFunc)(lua_State *L, int idx); +/* + * Unpacking callback: Turns a void* back into the Lua value (left on top of stack) + * + * Note that this function must take ownership of the pointer, so make sure + * to free or keep the memory. + * `L` can be nullptr to indicate that data should just be discarded. + */ +typedef void (*PackOutFunc)(lua_State *L, void *ptr); +/* + * Register a packable type with the name of its metatable. + * + * Even though the callbacks are global this must be called for every Lua state + * that supports objects of this type. + * This function is thread-safe. + */ +void script_register_packer(lua_State *L, const char *regname, + PackInFunc fin, PackOutFunc fout); + +// Pack a Lua value +PackedValue *script_pack(lua_State *L, int idx); +// Unpack a Lua value (left on top of stack) +// Note that this may modify the PackedValue, you can't reuse it! +void script_unpack(lua_State *L, PackedValue *val); + +// Dump contents of PackedValue to stdout for debugging +void script_dump_packed(const PackedValue *val); diff --git a/src/script/cpp_api/s_async.cpp b/src/script/cpp_api/s_async.cpp index dacdcd75a..42a794ceb 100644 --- a/src/script/cpp_api/s_async.cpp +++ b/src/script/cpp_api/s_async.cpp @@ -21,9 +21,9 @@ with this program; if not, write to the Free Software Foundation, Inc., #include extern "C" { -#include "lua.h" -#include "lauxlib.h" -#include "lualib.h" +#include +#include +#include } #include "server.h" @@ -32,6 +32,7 @@ extern "C" { #include "filesys.h" #include "porting.h" #include "common/c_internal.h" +#include "common/c_packer.h" #include "lua_api/l_base.h" /******************************************************************************/ @@ -76,19 +77,34 @@ void AsyncEngine::initialize(unsigned int numEngines) { initDone = true; - for (unsigned int i = 0; i < numEngines; i++) { - AsyncWorkerThread *toAdd = new AsyncWorkerThread(this, - std::string("AsyncWorker-") + itos(i)); - workerThreads.push_back(toAdd); - toAdd->start(); + if (numEngines == 0) { + // Leave one core for the main thread and one for whatever else + autoscaleMaxWorkers = Thread::getNumberOfProcessors(); + if (autoscaleMaxWorkers >= 2) + autoscaleMaxWorkers -= 2; + infostream << "AsyncEngine: using at most " << autoscaleMaxWorkers + << " threads with automatic scaling" << std::endl; + + addWorkerThread(); + } else { + for (unsigned int i = 0; i < numEngines; i++) + addWorkerThread(); } } +void AsyncEngine::addWorkerThread() +{ + AsyncWorkerThread *toAdd = new AsyncWorkerThread(this, + std::string("AsyncWorker-") + itos(workerThreads.size())); + workerThreads.push_back(toAdd); + toAdd->start(); +} + /******************************************************************************/ u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &¶ms, const std::string &mod_origin) { - jobQueueMutex.lock(); + MutexAutoLock autolock(jobQueueMutex); u32 jobId = jobIdCounter++; jobQueue.emplace_back(); @@ -99,7 +115,23 @@ u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &¶ms, to_add.mod_origin = mod_origin; jobQueueCounter.post(); - jobQueueMutex.unlock(); + return jobId; +} + +u32 AsyncEngine::queueAsyncJob(std::string &&func, PackedValue *params, + const std::string &mod_origin) +{ + MutexAutoLock autolock(jobQueueMutex); + u32 jobId = jobIdCounter++; + + jobQueue.emplace_back(); + auto &to_add = jobQueue.back(); + to_add.id = jobId; + to_add.function = std::move(func); + to_add.params_ext.reset(params); + to_add.mod_origin = mod_origin; + + jobQueueCounter.post(); return jobId; } @@ -131,6 +163,12 @@ void AsyncEngine::putJobResult(LuaJobInfo &&result) /******************************************************************************/ void AsyncEngine::step(lua_State *L) +{ + stepJobResults(L); + stepAutoscale(); +} + +void AsyncEngine::stepJobResults(lua_State *L) { int error_handler = PUSH_ERROR_HANDLER(L); lua_getglobal(L, "core"); @@ -148,7 +186,10 @@ void AsyncEngine::step(lua_State *L) luaL_checktype(L, -1, LUA_TFUNCTION); lua_pushinteger(L, j.id); - lua_pushlstring(L, j.result.data(), j.result.size()); + if (j.result_ext) + script_unpack(L, j.result_ext.get()); + else + lua_pushlstring(L, j.result.data(), j.result.size()); // Call handler const char *origin = j.mod_origin.empty() ? nullptr : j.mod_origin.c_str(); @@ -161,12 +202,71 @@ void AsyncEngine::step(lua_State *L) lua_pop(L, 2); // Pop core and error handler } +void AsyncEngine::stepAutoscale() +{ + if (workerThreads.size() >= autoscaleMaxWorkers) + return; + + MutexAutoLock autolock(jobQueueMutex); + + // 2) If the timer elapsed, check again + if (autoscaleTimer && porting::getTimeMs() >= autoscaleTimer) { + autoscaleTimer = 0; + // Determine overlap with previous snapshot + unsigned int n = 0; + for (const auto &it : jobQueue) + n += autoscaleSeenJobs.count(it.id); + autoscaleSeenJobs.clear(); + infostream << "AsyncEngine: " << n << " jobs were still waiting after 1s" << std::endl; + // Start this many new threads + while (workerThreads.size() < autoscaleMaxWorkers && n > 0) { + addWorkerThread(); + n--; + } + return; + } + + // 1) Check if there's anything in the queue + if (!autoscaleTimer && !jobQueue.empty()) { + // Take a snapshot of all jobs we have seen + for (const auto &it : jobQueue) + autoscaleSeenJobs.emplace(it.id); + // and set a timer for 1 second + autoscaleTimer = porting::getTimeMs() + 1000; + } +} + /******************************************************************************/ -void AsyncEngine::prepareEnvironment(lua_State* L, int top) +bool AsyncEngine::prepareEnvironment(lua_State* L, int top) { for (StateInitializer &stateInitializer : stateInitializers) { stateInitializer(L, top); } + + auto *script = ModApiBase::getScriptApiBase(L); + try { + script->loadMod(Server::getBuiltinLuaPath() + DIR_DELIM + "init.lua", + BUILTIN_MOD_NAME); + } catch (const ModError &e) { + errorstream << "Execution of async base environment failed: " + << e.what() << std::endl; + FATAL_ERROR("Execution of async base environment failed"); + } + + // Load per mod stuff + if (server) { + const auto &list = server->m_async_init_files; + try { + for (auto &it : list) + script->loadMod(it.second, it.first); + } catch (const ModError &e) { + errorstream << "Failed to load mod script inside async environment." << std::endl; + server->setAsyncFatalError(e.what()); + return false; + } + } + + return true; } /******************************************************************************/ @@ -178,15 +278,25 @@ AsyncWorkerThread::AsyncWorkerThread(AsyncEngine* jobDispatcher, { lua_State *L = getStack(); + if (jobDispatcher->server) { + setGameDef(jobDispatcher->server); + + if (g_settings->getBool("secure.enable_security")) + initializeSecurity(); + } + // Prepare job lua environment lua_getglobal(L, "core"); int top = lua_gettop(L); // Push builtin initialization type - lua_pushstring(L, "async"); + lua_pushstring(L, jobDispatcher->server ? "async_game" : "async"); lua_setglobal(L, "INIT"); - jobDispatcher->prepareEnvironment(L, top); + if (!jobDispatcher->prepareEnvironment(L, top)) { + // can't throw from here so we're stuck with this + isErrored = true; + } } /******************************************************************************/ @@ -198,19 +308,20 @@ AsyncWorkerThread::~AsyncWorkerThread() /******************************************************************************/ void* AsyncWorkerThread::run() { - lua_State *L = getStack(); + if (isErrored) + return nullptr; - try { - loadMod(getServer()->getBuiltinLuaPath() + DIR_DELIM + "init.lua", - BUILTIN_MOD_NAME); - } catch (const ModError &e) { - errorstream << "Execution of async base environment failed: " - << e.what() << std::endl; - FATAL_ERROR("Execution of async base environment failed"); - } + lua_State *L = getStack(); int error_handler = PUSH_ERROR_HANDLER(L); + auto report_error = [this] (const ModError &e) { + if (jobDispatcher->server) + jobDispatcher->server->setAsyncFatalError(e.what()); + else + errorstream << e.what() << std::endl; + }; + lua_getglobal(L, "core"); if (lua_isnil(L, -1)) { FATAL_ERROR("Unable to find core within async environment!"); @@ -223,6 +334,8 @@ void* AsyncWorkerThread::run() if (!jobDispatcher->getJob(&j) || stopRequested()) continue; + const bool use_ext = !!j.params_ext; + lua_getfield(L, -1, "job_processor"); if (lua_isnil(L, -1)) FATAL_ERROR("Unable to get async job processor!"); @@ -232,7 +345,10 @@ void* AsyncWorkerThread::run() errorstream << "ASYNC WORKER: Unable to deserialize function" << std::endl; lua_pushnil(L); } - lua_pushlstring(L, j.params.data(), j.params.size()); + if (use_ext) + script_unpack(L, j.params_ext.get()); + else + lua_pushlstring(L, j.params.data(), j.params.size()); // Call it setOriginDirect(j.mod_origin.empty() ? nullptr : j.mod_origin.c_str()); @@ -241,19 +357,28 @@ void* AsyncWorkerThread::run() try { scriptError(result, ""); } catch (const ModError &e) { - errorstream << e.what() << std::endl; + report_error(e); } } else { // Fetch result - size_t length; - const char *retval = lua_tolstring(L, -1, &length); - j.result.assign(retval, length); + if (use_ext) { + try { + j.result_ext.reset(script_pack(L, -1)); + } catch (const ModError &e) { + report_error(e); + result = LUA_ERRERR; + } + } else { + size_t length; + const char *retval = lua_tolstring(L, -1, &length); + j.result.assign(retval, length); + } } lua_pop(L, 1); // Pop retval // Put job result - if (!j.result.empty()) + if (result == 0) jobDispatcher->putJobResult(std::move(j)); } diff --git a/src/script/cpp_api/s_async.h b/src/script/cpp_api/s_async.h index 697cb0221..1e34e40ea 100644 --- a/src/script/cpp_api/s_async.h +++ b/src/script/cpp_api/s_async.h @@ -21,11 +21,15 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include +#include +#include +#include #include "threading/semaphore.h" #include "threading/thread.h" -#include "lua.h" +#include "common/c_packer.h" #include "cpp_api/s_base.h" +#include "cpp_api/s_security.h" // Forward declarations class AsyncEngine; @@ -42,8 +46,12 @@ struct LuaJobInfo std::string function; // Parameter to be passed to function (serialized) std::string params; + // Alternative parameters + std::unique_ptr params_ext; // Result of function call (serialized) std::string result; + // Alternative result + std::unique_ptr result_ext; // Name of the mod who invoked this call std::string mod_origin; // JobID used to identify a job and match it to callback @@ -51,7 +59,8 @@ struct LuaJobInfo }; // Asynchronous working environment -class AsyncWorkerThread : public Thread, virtual public ScriptApiBase { +class AsyncWorkerThread : public Thread, + virtual public ScriptApiBase, public ScriptApiSecurity { friend class AsyncEngine; public: virtual ~AsyncWorkerThread(); @@ -63,6 +72,7 @@ protected: private: AsyncEngine *jobDispatcher = nullptr; + bool isErrored = false; }; // Asynchornous thread and job management @@ -71,6 +81,7 @@ class AsyncEngine { typedef void (*StateInitializer)(lua_State *L, int top); public: AsyncEngine() = default; + AsyncEngine(Server *server) : server(server) {}; ~AsyncEngine(); /** @@ -81,7 +92,7 @@ public: /** * Create async engine tasks and lock function registration - * @param numEngines Number of async threads to be started + * @param numEngines Number of worker threads, 0 for automatic scaling */ void initialize(unsigned int numEngines); @@ -94,9 +105,17 @@ public: u32 queueAsyncJob(std::string &&func, std::string &¶ms, const std::string &mod_origin = ""); + /** + * Queue an async job + * @param func Serialized lua function + * @param params Serialized parameters (takes ownership!) + * @return ID of queued job + */ + u32 queueAsyncJob(std::string &&func, PackedValue *params, + const std::string &mod_origin = ""); + /** * Engine step to process finished jobs - * the engine step is one way to pass events back, PushFinishedJobs another * @param L The Lua stack */ void step(lua_State *L); @@ -116,19 +135,44 @@ protected: */ void putJobResult(LuaJobInfo &&result); + /** + * Start an additional worker thread + */ + void addWorkerThread(); + + /** + * Process finished jobs callbacks + */ + void stepJobResults(lua_State *L); + + /** + * Handle automatic scaling of worker threads + */ + void stepAutoscale(); + /** * Initialize environment with current registred functions * this function adds all functions registred by registerFunction to the * passed lua stack * @param L Lua stack to initialize * @param top Stack position + * @return false if a mod error ocurred */ - void prepareEnvironment(lua_State* L, int top); + bool prepareEnvironment(lua_State* L, int top); private: // Variable locking the engine against further modification bool initDone = false; + // Maximum number of worker threads for automatic scaling + // 0 if disabled + unsigned int autoscaleMaxWorkers = 0; + u64 autoscaleTimer = 0; + std::unordered_set autoscaleSeenJobs; + + // Only set for the server async environment (duh) + Server *server = nullptr; + // Internal store for registred state initializers std::vector stateInitializers; diff --git a/src/script/lua_api/l_craft.cpp b/src/script/lua_api/l_craft.cpp index c2c5a5551..137b210be 100644 --- a/src/script/lua_api/l_craft.cpp +++ b/src/script/lua_api/l_craft.cpp @@ -525,3 +525,11 @@ void ModApiCraft::Initialize(lua_State *L, int top) API_FCT(register_craft); API_FCT(clear_craft); } + +void ModApiCraft::InitializeAsync(lua_State *L, int top) +{ + // all read-only functions + API_FCT(get_all_craft_recipes); + API_FCT(get_craft_recipe); + API_FCT(get_craft_result); +} diff --git a/src/script/lua_api/l_craft.h b/src/script/lua_api/l_craft.h index 9002b23ef..5234af56f 100644 --- a/src/script/lua_api/l_craft.h +++ b/src/script/lua_api/l_craft.h @@ -45,4 +45,5 @@ private: public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_internal.h b/src/script/lua_api/l_internal.h index 672e535ca..de73ff42a 100644 --- a/src/script/lua_api/l_internal.h +++ b/src/script/lua_api/l_internal.h @@ -69,7 +69,7 @@ with this program; if not, write to the Free Software Foundation, Inc., // Retrieve Environment pointer as `env` (no map lock) #define GET_PLAIN_ENV_PTR_NO_MAP_LOCK \ - Environment *env = (Environment *)getEnv(L); \ + Environment *env = getEnv(L); \ if (env == NULL) \ return 0 diff --git a/src/script/lua_api/l_item.cpp b/src/script/lua_api/l_item.cpp index fc97a1736..b58b994d9 100644 --- a/src/script/lua_api/l_item.cpp +++ b/src/script/lua_api/l_item.cpp @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "itemdef.h" #include "nodedef.h" #include "server.h" @@ -441,6 +442,7 @@ int LuaItemStack::create_object(lua_State *L) lua_setmetatable(L, -2); return 1; } + // Not callable from Lua int LuaItemStack::create(lua_State *L, const ItemStack &item) { @@ -457,6 +459,20 @@ LuaItemStack *LuaItemStack::checkobject(lua_State *L, int narg) return *(LuaItemStack **)luaL_checkudata(L, narg, className); } +void *LuaItemStack::packIn(lua_State *L, int idx) +{ + LuaItemStack *o = checkobject(L, idx); + return new ItemStack(o->getItem()); +} + +void LuaItemStack::packOut(lua_State *L, void *ptr) +{ + ItemStack *stack = reinterpret_cast(ptr); + if (L) + create(L, *stack); + delete stack; +} + void LuaItemStack::Register(lua_State *L) { lua_newtable(L); @@ -488,6 +504,8 @@ void LuaItemStack::Register(lua_State *L) // Can be created from Lua (ItemStack(itemstack or itemstring or table or nil)) lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } const char LuaItemStack::className[] = "ItemStack"; @@ -673,3 +691,10 @@ void ModApiItemMod::Initialize(lua_State *L, int top) API_FCT(get_content_id); API_FCT(get_name_from_content_id); } + +void ModApiItemMod::InitializeAsync(lua_State *L, int top) +{ + // all read-only functions + API_FCT(get_content_id); + API_FCT(get_name_from_content_id); +} diff --git a/src/script/lua_api/l_item.h b/src/script/lua_api/l_item.h index 16878c101..180975061 100644 --- a/src/script/lua_api/l_item.h +++ b/src/script/lua_api/l_item.h @@ -141,8 +141,11 @@ public: // Not callable from Lua static int create(lua_State *L, const ItemStack &item); static LuaItemStack* checkobject(lua_State *L, int narg); - static void Register(lua_State *L); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + + static void Register(lua_State *L); }; class ModApiItemMod : public ModApiBase { @@ -152,6 +155,8 @@ private: static int l_register_alias_raw(lua_State *L); static int l_get_content_id(lua_State *L); static int l_get_name_from_content_id(lua_State *L); + public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_noise.cpp b/src/script/lua_api/l_noise.cpp index 0eee49b7d..5561eaebf 100644 --- a/src/script/lua_api/l_noise.cpp +++ b/src/script/lua_api/l_noise.cpp @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "log.h" #include "porting.h" #include "util/numeric.h" @@ -101,6 +102,25 @@ LuaPerlinNoise *LuaPerlinNoise::checkobject(lua_State *L, int narg) } +void *LuaPerlinNoise::packIn(lua_State *L, int idx) +{ + LuaPerlinNoise *o = checkobject(L, idx); + return new NoiseParams(o->np); +} + +void LuaPerlinNoise::packOut(lua_State *L, void *ptr) +{ + NoiseParams *np = reinterpret_cast(ptr); + if (L) { + LuaPerlinNoise *o = new LuaPerlinNoise(np); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); + } + delete np; +} + + void LuaPerlinNoise::Register(lua_State *L) { lua_newtable(L); @@ -126,6 +146,8 @@ void LuaPerlinNoise::Register(lua_State *L) lua_pop(L, 1); lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } @@ -357,6 +379,35 @@ LuaPerlinNoiseMap *LuaPerlinNoiseMap::checkobject(lua_State *L, int narg) } +struct NoiseMapParams { + NoiseParams np; + s32 seed; + v3s16 size; +}; + +void *LuaPerlinNoiseMap::packIn(lua_State *L, int idx) +{ + LuaPerlinNoiseMap *o = checkobject(L, idx); + NoiseMapParams *ret = new NoiseMapParams(); + ret->np = o->noise->np; + ret->seed = o->noise->seed; + ret->size = v3s16(o->noise->sx, o->noise->sy, o->noise->sz); + return ret; +} + +void LuaPerlinNoiseMap::packOut(lua_State *L, void *ptr) +{ + NoiseMapParams *p = reinterpret_cast(ptr); + if (L) { + LuaPerlinNoiseMap *o = new LuaPerlinNoiseMap(&p->np, p->seed, p->size); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); + } + delete p; +} + + void LuaPerlinNoiseMap::Register(lua_State *L) { lua_newtable(L); @@ -382,6 +433,8 @@ void LuaPerlinNoiseMap::Register(lua_State *L) lua_pop(L, 1); lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } diff --git a/src/script/lua_api/l_noise.h b/src/script/lua_api/l_noise.h index 29ab41a31..5d34a479b 100644 --- a/src/script/lua_api/l_noise.h +++ b/src/script/lua_api/l_noise.h @@ -52,6 +52,9 @@ public: static LuaPerlinNoise *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; @@ -91,6 +94,9 @@ public: static LuaPerlinNoiseMap *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index 42725e5d2..4b0b45887 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "common/c_packer.h" #include "cpp_api/s_base.h" #include "cpp_api/s_security.h" #include "scripting_server.h" @@ -526,6 +527,76 @@ int ModApiServer::l_notify_authentication_modified(lua_State *L) return 0; } +// do_async_callback(func, params, mod_origin) +int ModApiServer::l_do_async_callback(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + ServerScripting *script = getScriptApi(L); + + luaL_checktype(L, 1, LUA_TFUNCTION); + luaL_checktype(L, 2, LUA_TTABLE); + luaL_checktype(L, 3, LUA_TSTRING); + + call_string_dump(L, 1); + size_t func_length; + const char *serialized_func_raw = lua_tolstring(L, -1, &func_length); + + PackedValue *param = script_pack(L, 2); + + std::string mod_origin = readParam(L, 3); + + u32 jobId = script->queueAsync( + std::string(serialized_func_raw, func_length), + param, mod_origin); + + lua_settop(L, 0); + lua_pushinteger(L, jobId); + return 1; +} + +// register_async_dofile(path) +int ModApiServer::l_register_async_dofile(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + + std::string path = readParam(L, 1); + CHECK_SECURE_PATH(L, path.c_str(), false); + + // Find currently running mod name (only at init time) + lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME); + if (!lua_isstring(L, -1)) + return 0; + std::string modname = readParam(L, -1); + + getServer(L)->m_async_init_files.emplace_back(modname, path); + lua_pushboolean(L, true); + return 1; +} + +// serialize_roundtrip(value) +// Meant for unit testing the packer from Lua +int ModApiServer::l_serialize_roundtrip(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + + int top = lua_gettop(L); + auto *pv = script_pack(L, 1); + if (top != lua_gettop(L)) + throw LuaError("stack values leaked"); + +#ifndef NDEBUG + script_dump_packed(pv); +#endif + + top = lua_gettop(L); + script_unpack(L, pv); + delete pv; + if (top + 1 != lua_gettop(L)) + throw LuaError("stack values leaked"); + + return 1; +} + void ModApiServer::Initialize(lua_State *L, int top) { API_FCT(request_shutdown); @@ -559,4 +630,18 @@ void ModApiServer::Initialize(lua_State *L, int top) API_FCT(remove_player); API_FCT(unban_player_or_ip); API_FCT(notify_authentication_modified); + + API_FCT(do_async_callback); + API_FCT(register_async_dofile); + API_FCT(serialize_roundtrip); +} + +void ModApiServer::InitializeAsync(lua_State *L, int top) +{ + API_FCT(get_worldpath); + API_FCT(is_singleplayer); + + API_FCT(get_current_modname); + API_FCT(get_modpath); + API_FCT(get_modnames); } diff --git a/src/script/lua_api/l_server.h b/src/script/lua_api/l_server.h index f05c0b7c9..a4f38c34e 100644 --- a/src/script/lua_api/l_server.h +++ b/src/script/lua_api/l_server.h @@ -106,6 +106,16 @@ private: // notify_authentication_modified(name) static int l_notify_authentication_modified(lua_State *L); + // do_async_callback(func, params, mod_origin) + static int l_do_async_callback(lua_State *L); + + // register_async_dofile(path) + static int l_register_async_dofile(lua_State *L); + + // serialize_roundtrip(obj) + static int l_serialize_roundtrip(lua_State *L); + public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(lua_State *L, int top); }; diff --git a/src/script/lua_api/l_util.cpp b/src/script/lua_api/l_util.cpp index b04f26fda..97068ce4c 100644 --- a/src/script/lua_api/l_util.cpp +++ b/src/script/lua_api/l_util.cpp @@ -671,6 +671,9 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top) API_FCT(cpdir); API_FCT(mvdir); API_FCT(get_dir_list); + API_FCT(safe_file_write); + + API_FCT(request_insecure_environment); API_FCT(encode_base64); API_FCT(decode_base64); @@ -680,6 +683,8 @@ void ModApiUtil::InitializeAsync(lua_State *L, int top) API_FCT(colorspec_to_colorstring); API_FCT(colorspec_to_bytes); + API_FCT(encode_png); + API_FCT(get_last_run_mod); API_FCT(set_last_run_mod); diff --git a/src/script/lua_api/l_util.h b/src/script/lua_api/l_util.h index fcf8a1057..cc5563577 100644 --- a/src/script/lua_api/l_util.h +++ b/src/script/lua_api/l_util.h @@ -129,6 +129,4 @@ public: static void Initialize(lua_State *L, int top); static void InitializeAsync(lua_State *L, int top); static void InitializeClient(lua_State *L, int top); - - static void InitializeAsync(AsyncEngine &engine); }; diff --git a/src/script/lua_api/l_vmanip.cpp b/src/script/lua_api/l_vmanip.cpp index a3ece627c..6187a47db 100644 --- a/src/script/lua_api/l_vmanip.cpp +++ b/src/script/lua_api/l_vmanip.cpp @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_content.h" #include "common/c_converter.h" +#include "common/c_packer.h" #include "emerge.h" #include "environment.h" #include "map.h" @@ -45,6 +46,8 @@ int LuaVoxelManip::l_read_from_map(lua_State *L) LuaVoxelManip *o = checkobject(L, 1); MMVManip *vm = o->vm; + if (vm->isOrphan()) + return 0; v3s16 bp1 = getNodeBlockPos(check_v3s16(L, 2)); v3s16 bp2 = getNodeBlockPos(check_v3s16(L, 3)); @@ -429,6 +432,34 @@ LuaVoxelManip *LuaVoxelManip::checkobject(lua_State *L, int narg) return *(LuaVoxelManip **)ud; // unbox pointer } +void *LuaVoxelManip::packIn(lua_State *L, int idx) +{ + LuaVoxelManip *o = checkobject(L, idx); + + if (o->is_mapgen_vm) + throw LuaError("nope"); + return o->vm->clone(); +} + +void LuaVoxelManip::packOut(lua_State *L, void *ptr) +{ + MMVManip *vm = reinterpret_cast(ptr); + if (!L) { + delete vm; + return; + } + + // Associate vmanip with map if the Lua env has one + Environment *env = getEnv(L); + if (env) + vm->reparent(&(env->getMap())); + + LuaVoxelManip *o = new LuaVoxelManip(vm, false); + *(void **)(lua_newuserdata(L, sizeof(void *))) = o; + luaL_getmetatable(L, className); + lua_setmetatable(L, -2); +} + void LuaVoxelManip::Register(lua_State *L) { lua_newtable(L); @@ -455,6 +486,8 @@ void LuaVoxelManip::Register(lua_State *L) // Can be created from Lua (VoxelManip()) lua_register(L, className, create_object); + + script_register_packer(L, className, packIn, packOut); } const char LuaVoxelManip::className[] = "VoxelManip"; diff --git a/src/script/lua_api/l_vmanip.h b/src/script/lua_api/l_vmanip.h index 5113070dc..005133335 100644 --- a/src/script/lua_api/l_vmanip.h +++ b/src/script/lua_api/l_vmanip.h @@ -75,5 +75,8 @@ public: static LuaVoxelManip *checkobject(lua_State *L, int narg); + static void *packIn(lua_State *L, int idx); + static void packOut(lua_State *L, void *ptr); + static void Register(lua_State *L); }; diff --git a/src/script/scripting_server.cpp b/src/script/scripting_server.cpp index 85411ded4..5b99468dc 100644 --- a/src/script/scripting_server.cpp +++ b/src/script/scripting_server.cpp @@ -47,11 +47,12 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_storage.h" extern "C" { -#include "lualib.h" +#include } ServerScripting::ServerScripting(Server* server): - ScriptApiBase(ScriptingType::Server) + ScriptApiBase(ScriptingType::Server), + asyncEngine(server) { setGameDef(server); @@ -88,6 +89,47 @@ ServerScripting::ServerScripting(Server* server): infostream << "SCRIPTAPI: Initialized game modules" << std::endl; } +void ServerScripting::initAsync() +{ + // Save globals to transfer + { + lua_State *L = getStack(); + lua_getglobal(L, "core"); + luaL_checktype(L, -1, LUA_TTABLE); + lua_getfield(L, -1, "get_globals_to_transfer"); + lua_call(L, 0, 1); + luaL_checktype(L, -1, LUA_TSTRING); + getServer()->m_async_globals_data.set(readParam(L, -1)); + lua_pushnil(L); + lua_setfield(L, -3, "get_globals_to_transfer"); // unset function too + lua_pop(L, 2); // pop 'core', return value + } + + infostream << "SCRIPTAPI: Initializing async engine" << std::endl; + asyncEngine.registerStateInitializer(InitializeAsync); + asyncEngine.registerStateInitializer(ModApiUtil::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiCraft::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiItemMod::InitializeAsync); + asyncEngine.registerStateInitializer(ModApiServer::InitializeAsync); + // not added: ModApiMapgen is a minefield for thread safety + // not added: ModApiHttp async api can't really work together with our jobs + // not added: ModApiStorage is probably not thread safe(?) + + asyncEngine.initialize(0); +} + +void ServerScripting::stepAsync() +{ + asyncEngine.step(getStack()); +} + +u32 ServerScripting::queueAsync(std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin) +{ + return asyncEngine.queueAsyncJob(std::move(serialized_func), + param, mod_origin); +} + void ServerScripting::InitializeModApi(lua_State *L, int top) { // Register reference classes (userdata) @@ -125,3 +167,24 @@ void ServerScripting::InitializeModApi(lua_State *L, int top) ModApiStorage::Initialize(L, top); ModApiChannels::Initialize(L, top); } + +void ServerScripting::InitializeAsync(lua_State *L, int top) +{ + // classes + LuaItemStack::Register(L); + LuaPerlinNoise::Register(L); + LuaPerlinNoiseMap::Register(L); + LuaPseudoRandom::Register(L); + LuaPcgRandom::Register(L); + LuaSecureRandom::Register(L); + LuaVoxelManip::Register(L); + LuaSettings::Register(L); + + // globals data + lua_getglobal(L, "core"); + luaL_checktype(L, -1, LUA_TTABLE); + std::string s = ModApiBase::getServer(L)->m_async_globals_data.get(); + lua_pushlstring(L, s.c_str(), s.size()); + lua_setfield(L, -2, "transferred_globals"); + lua_pop(L, 1); // pop 'core' +} diff --git a/src/script/scripting_server.h b/src/script/scripting_server.h index bf06ab197..9803397c5 100644 --- a/src/script/scripting_server.h +++ b/src/script/scripting_server.h @@ -27,6 +27,9 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "cpp_api/s_player.h" #include "cpp_api/s_server.h" #include "cpp_api/s_security.h" +#include "cpp_api/s_async.h" + +struct PackedValue; /*****************************************************************************/ /* Scripting <-> Server Game Interface */ @@ -48,6 +51,20 @@ public: // use ScriptApiBase::loadMod() to load mods + // Initialize async engine, call this AFTER loading all mods + void initAsync(); + + // Global step handler to collect async results + void stepAsync(); + + // Pass job to async threads + u32 queueAsync(std::string &&serialized_func, + PackedValue *param, const std::string &mod_origin); + private: void InitializeModApi(lua_State *L, int top); + + static void InitializeAsync(lua_State *L, int top); + + AsyncEngine asyncEngine; }; diff --git a/src/server.cpp b/src/server.cpp index dec6cf44c..c9cd4e398 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -243,6 +243,7 @@ Server::Server( m_clients(m_con), m_admin_chat(iface), m_on_shutdown_errmsg(on_shutdown_errmsg), + m_async_globals_data(""), m_modchannel_mgr(new ModChannelMgr()) { if (m_path_world.empty()) @@ -480,6 +481,9 @@ void Server::init() // Give environment reference to scripting api m_script->initializeEnvironment(m_env); + // Do this after regular script init is done + m_script->initAsync(); + // Register us to receive map edit events servermap->addEventReceiver(this); diff --git a/src/server.h b/src/server.h index bd799c313..5090a3579 100644 --- a/src/server.h +++ b/src/server.h @@ -292,7 +292,7 @@ public: virtual const std::vector &getMods() const; virtual const ModSpec* getModSpec(const std::string &modname) const; - std::string getBuiltinLuaPath(); + static std::string getBuiltinLuaPath(); virtual std::string getWorldPath() const { return m_path_world; } inline bool isSingleplayer() const @@ -385,6 +385,12 @@ public: static bool migrateModStorageDatabase(const GameParams &game_params, const Settings &cmd_args); + // Lua files registered for init of async env, pair of modname + path + std::vector> m_async_init_files; + + // Serialized data transferred into async envs at init time + MutexedVariable m_async_globals_data; + // Bind address Address m_bind_addr; diff --git a/src/serverenvironment.cpp b/src/serverenvironment.cpp index f8d84604b..493210744 100644 --- a/src/serverenvironment.cpp +++ b/src/serverenvironment.cpp @@ -1481,6 +1481,8 @@ void ServerEnvironment::step(float dtime) */ m_script->environment_Step(dtime); + m_script->stepAsync(); + /* Step active objects */ diff --git a/src/util/basic_macros.h b/src/util/basic_macros.h index 334e342e0..3910c6185 100644 --- a/src/util/basic_macros.h +++ b/src/util/basic_macros.h @@ -29,13 +29,19 @@ with this program; if not, write to the Free Software Foundation, Inc., #define CONTAINS(c, v) (std::find((c).begin(), (c).end(), (v)) != (c).end()) // To disable copy constructors and assignment operations for some class -// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) as a private member. +// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) in the class definition. // Note this also disables copying for any classes derived from 'Foobar' as well // as classes having a 'Foobar' member. #define DISABLE_CLASS_COPY(C) \ C(const C &) = delete; \ C &operator=(const C &) = delete; +// If you have used DISABLE_CLASS_COPY with a class but still want to permit moving +// use this macro to add the default move constructors back. +#define ALLOW_CLASS_MOVE(C) \ + C(C &&other) = default; \ + C &operator=(C &&) = default; + #ifndef _MSC_VER #define UNUSED_ATTRIBUTE __attribute__ ((unused)) #else -- cgit v1.2.3 From 0704ca055059088bdd53e15be672e6b5663b8f50 Mon Sep 17 00:00:00 2001 From: paradust7 <102263465+paradust7@users.noreply.github.com> Date: Wed, 4 May 2022 11:55:01 -0700 Subject: Make logging cost free when there is no output target (#12247) The logging streams now do almost no work when there is no output target for them. For example, if LL_VERBOSE has no output targets, then `verbosestream << x` will return a StreamProxy with a null target. Any further `<<` operations applied to it will do nothing. --- builtin/settingtypes.txt | 5 +- minetest.conf.example | 5 +- src/client/game.cpp | 2 +- src/client/renderingengine.cpp | 2 +- src/log.cpp | 187 ++++++++++++----------------------- src/log.h | 202 +++++++++++++++++++++++++++++++++----- src/main.cpp | 16 ++- src/network/address.cpp | 8 +- src/network/address.h | 2 +- src/network/connection.cpp | 17 +--- src/network/connectionthreads.cpp | 10 +- src/network/socket.cpp | 4 +- src/server.cpp | 2 +- src/util/stream.h | 70 +++++++++++++ 14 files changed, 338 insertions(+), 194 deletions(-) create mode 100644 src/util/stream.h (limited to 'src/util') diff --git a/builtin/settingtypes.txt b/builtin/settingtypes.txt index a983a8f6b..e3589d76f 100644 --- a/builtin/settingtypes.txt +++ b/builtin/settingtypes.txt @@ -1468,7 +1468,8 @@ language (Language) enum ,be,bg,ca,cs,da,de,el,en,eo,es,et,eu,fi,fr,gd,gl,hu,i # - action # - info # - verbose -debug_log_level (Debug log level) enum action ,none,error,warning,action,info,verbose +# - trace +debug_log_level (Debug log level) enum action ,none,error,warning,action,info,verbose,trace # If the file size of debug.txt exceeds the number of megabytes specified in # this setting when it is opened, the file is moved to debug.txt.1, @@ -1477,7 +1478,7 @@ debug_log_level (Debug log level) enum action ,none,error,warning,action,info,ve debug_log_size_max (Debug log file size threshold) int 50 # Minimal level of logging to be written to chat. -chat_log_level (Chat log level) enum error ,none,error,warning,action,info,verbose +chat_log_level (Chat log level) enum error ,none,error,warning,action,info,verbose,trace # Enable IPv6 support (for both client and server). # Required for IPv6 connections to work at all. diff --git a/minetest.conf.example b/minetest.conf.example index 3cdb15026..68757680c 100644 --- a/minetest.conf.example +++ b/minetest.conf.example @@ -1768,7 +1768,8 @@ # - action # - info # - verbose -# type: enum values: , none, error, warning, action, info, verbose +# - trace +# type: enum values: , none, error, warning, action, info, verbose, trace # debug_log_level = action # If the file size of debug.txt exceeds the number of megabytes specified in @@ -1779,7 +1780,7 @@ # debug_log_size_max = 50 # Minimal level of logging to be written to chat. -# type: enum values: , none, error, warning, action, info, verbose +# type: enum values: , none, error, warning, action, info, verbose, trace # chat_log_level = error # Enable IPv6 support (for both client and server). diff --git a/src/client/game.cpp b/src/client/game.cpp index e0e60eb2c..1e34ca286 100644 --- a/src/client/game.cpp +++ b/src/client/game.cpp @@ -1491,7 +1491,7 @@ bool Game::connectToServer(const GameStartData &start_data, client->m_simple_singleplayer_mode = simple_singleplayer_mode; infostream << "Connecting to server at "; - connect_address.print(&infostream); + connect_address.print(infostream); infostream << std::endl; client->connect(connect_address, diff --git a/src/client/renderingengine.cpp b/src/client/renderingengine.cpp index 723865db4..455d5e538 100644 --- a/src/client/renderingengine.cpp +++ b/src/client/renderingengine.cpp @@ -116,7 +116,7 @@ RenderingEngine::RenderingEngine(IEventReceiver *receiver) } SIrrlichtCreationParameters params = SIrrlichtCreationParameters(); - if (g_logger.getTraceEnabled()) + if (tracestream) params.LoggingLevel = irr::ELL_DEBUG; params.DriverType = driverType; params.WindowSize = core::dimension2d(screen_w, screen_h); diff --git a/src/log.cpp b/src/log.cpp index 3c61414e9..51652fe0a 100644 --- a/src/log.cpp +++ b/src/log.cpp @@ -35,43 +35,30 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include -const int BUFFER_LENGTH = 256; - -class StringBuffer : public std::streambuf { +class LevelTarget : public LogTarget { public: - StringBuffer() { - buffer_index = 0; - } - - int overflow(int c); - virtual void flush(const std::string &buf) = 0; - std::streamsize xsputn(const char *s, std::streamsize n); - void push_back(char c); - -private: - char buffer[BUFFER_LENGTH]; - int buffer_index; -}; - - -class LogBuffer : public StringBuffer { -public: - LogBuffer(Logger &logger, LogLevel lev) : - logger(logger), - level(lev) + LevelTarget(Logger &logger, LogLevel level, bool raw = false) : + m_logger(logger), + m_level(level), + m_raw(raw) {} - void flush(const std::string &buffer); - -private: - Logger &logger; - LogLevel level; -}; + virtual bool hasOutput() override { + return m_logger.hasOutput(m_level); + } + virtual void log(const std::string &buf) override { + if (!m_raw) { + m_logger.log(m_level, buf); + } else { + m_logger.logRaw(m_level, buf); + } + } -class RawLogBuffer : public StringBuffer { -public: - void flush(const std::string &buffer); +private: + Logger &m_logger; + LogLevel m_level; + bool m_raw; }; //// @@ -80,31 +67,33 @@ public: Logger g_logger; +#ifdef __ANDROID__ +AndroidLogOutput stdout_output; +AndroidLogOutput stderr_output; +#else StreamLogOutput stdout_output(std::cout); StreamLogOutput stderr_output(std::cerr); -std::ostream null_stream(NULL); - -RawLogBuffer raw_buf; - -LogBuffer none_buf(g_logger, LL_NONE); -LogBuffer error_buf(g_logger, LL_ERROR); -LogBuffer warning_buf(g_logger, LL_WARNING); -LogBuffer action_buf(g_logger, LL_ACTION); -LogBuffer info_buf(g_logger, LL_INFO); -LogBuffer verbose_buf(g_logger, LL_VERBOSE); - -// Connection -std::ostream *dout_con_ptr = &null_stream; -std::ostream *derr_con_ptr = &verbosestream; - -// Common streams -std::ostream rawstream(&raw_buf); -std::ostream dstream(&none_buf); -std::ostream errorstream(&error_buf); -std::ostream warningstream(&warning_buf); -std::ostream actionstream(&action_buf); -std::ostream infostream(&info_buf); -std::ostream verbosestream(&verbose_buf); +#endif + +LevelTarget none_target_raw(g_logger, LL_NONE, true); +LevelTarget none_target(g_logger, LL_NONE); +LevelTarget error_target(g_logger, LL_ERROR); +LevelTarget warning_target(g_logger, LL_WARNING); +LevelTarget action_target(g_logger, LL_ACTION); +LevelTarget info_target(g_logger, LL_INFO); +LevelTarget verbose_target(g_logger, LL_VERBOSE); +LevelTarget trace_target(g_logger, LL_TRACE); + +thread_local LogStream dstream(none_target); +thread_local LogStream rawstream(none_target_raw); +thread_local LogStream errorstream(error_target); +thread_local LogStream warningstream(warning_target); +thread_local LogStream actionstream(action_target); +thread_local LogStream infostream(info_target); +thread_local LogStream verbosestream(verbose_target); +thread_local LogStream tracestream(trace_target); +thread_local LogStream derr_con(verbose_target); +thread_local LogStream dout_con(trace_target); // Android #ifdef __ANDROID__ @@ -118,29 +107,15 @@ static unsigned int g_level_to_android[] = { //ANDROID_LOG_INFO, ANDROID_LOG_DEBUG, // LL_INFO ANDROID_LOG_VERBOSE, // LL_VERBOSE + ANDROID_LOG_VERBOSE, // LL_TRACE }; -class AndroidSystemLogOutput : public ICombinedLogOutput { - public: - AndroidSystemLogOutput() - { - g_logger.addOutput(this); - } - ~AndroidSystemLogOutput() - { - g_logger.removeOutput(this); - } - void logRaw(LogLevel lev, const std::string &line) - { - STATIC_ASSERT(ARRLEN(g_level_to_android) == LL_MAX, - mismatch_between_android_and_internal_loglevels); - __android_log_print(g_level_to_android[lev], - PROJECT_NAME_C, "%s", line.c_str()); - } -}; - -AndroidSystemLogOutput g_android_log_output; - +void AndroidLogOutput::logRaw(LogLevel lev, const std::string &line) { + STATIC_ASSERT(ARRLEN(g_level_to_android) == LL_MAX, + mismatch_between_android_and_internal_loglevels); + __android_log_print(g_level_to_android[lev], + PROJECT_NAME_C, "%s", line.c_str()); +} #endif /////////////////////////////////////////////////////////////////////////////// @@ -164,6 +139,8 @@ LogLevel Logger::stringToLevel(const std::string &name) return LL_INFO; else if (name == "verbose") return LL_VERBOSE; + else if (name == "trace") + return LL_TRACE; else return LL_MAX; } @@ -176,21 +153,26 @@ void Logger::addOutput(ILogOutput *out) void Logger::addOutput(ILogOutput *out, LogLevel lev) { m_outputs[lev].push_back(out); + m_has_outputs[lev] = true; } void Logger::addOutputMasked(ILogOutput *out, LogLevelMask mask) { for (size_t i = 0; i < LL_MAX; i++) { - if (mask & LOGLEVEL_TO_MASKLEVEL(i)) + if (mask & LOGLEVEL_TO_MASKLEVEL(i)) { m_outputs[i].push_back(out); + m_has_outputs[i] = true; + } } } void Logger::addOutputMaxLevel(ILogOutput *out, LogLevel lev) { assert(lev < LL_MAX); - for (size_t i = 0; i <= lev; i++) + for (size_t i = 0; i <= lev; i++) { m_outputs[i].push_back(out); + m_has_outputs[i] = true; + } } LogLevelMask Logger::removeOutput(ILogOutput *out) @@ -203,6 +185,7 @@ LogLevelMask Logger::removeOutput(ILogOutput *out) if (it != m_outputs[i].end()) { ret_mask |= LOGLEVEL_TO_MASKLEVEL(i); m_outputs[i].erase(it); + m_has_outputs[i] = !m_outputs[i].empty(); } } return ret_mask; @@ -236,6 +219,7 @@ const std::string Logger::getLevelLabel(LogLevel lev) "ACTION", "INFO", "VERBOSE", + "TRACE", }; assert(lev < LL_MAX && lev >= 0); STATIC_ASSERT(ARRLEN(names) == LL_MAX, @@ -297,7 +281,6 @@ void Logger::logToOutputs(LogLevel lev, const std::string &combined, m_outputs[lev][i]->log(lev, combined, time, thread_name, payload_text); } - //// //// *LogOutput methods //// @@ -349,6 +332,7 @@ void StreamLogOutput::logRaw(LogLevel lev, const std::string &line) m_stream << "\033[37m"; break; case LL_VERBOSE: + case LL_TRACE: // verbose is darker than info m_stream << "\033[2m"; break; @@ -396,6 +380,7 @@ void LogOutputBuffer::logRaw(LogLevel lev, const std::string &line) color = "\x1b(c@#BBB)"; break; case LL_VERBOSE: // dark grey + case LL_TRACE: color = "\x1b(c@#888)"; break; default: break; @@ -404,47 +389,3 @@ void LogOutputBuffer::logRaw(LogLevel lev, const std::string &line) m_buffer.push(color.append(line)); } - -//// -//// *Buffer methods -//// - -int StringBuffer::overflow(int c) -{ - push_back(c); - return c; -} - - -std::streamsize StringBuffer::xsputn(const char *s, std::streamsize n) -{ - for (int i = 0; i < n; ++i) - push_back(s[i]); - return n; -} - -void StringBuffer::push_back(char c) -{ - if (c == '\n' || c == '\r') { - if (buffer_index) - flush(std::string(buffer, buffer_index)); - buffer_index = 0; - } else { - buffer[buffer_index++] = c; - if (buffer_index >= BUFFER_LENGTH) { - flush(std::string(buffer, buffer_index)); - buffer_index = 0; - } - } -} - - -void LogBuffer::flush(const std::string &buffer) -{ - logger.log(level, buffer); -} - -void RawLogBuffer::flush(const std::string &buffer) -{ - g_logger.logRaw(LL_NONE, buffer); -} diff --git a/src/log.h b/src/log.h index 6ed6b1fb7..5c6ba7d64 100644 --- a/src/log.h +++ b/src/log.h @@ -19,6 +19,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #pragma once +#include #include #include #include @@ -28,6 +29,8 @@ with this program; if not, write to the Free Software Foundation, Inc., #if !defined(_WIN32) // POSIX #include #endif +#include "util/basic_macros.h" +#include "util/stream.h" #include "irrlichttypes.h" class ILogOutput; @@ -39,6 +42,7 @@ enum LogLevel { LL_ACTION, // In-game actions LL_INFO, LL_VERBOSE, + LL_TRACE, LL_MAX, }; @@ -67,12 +71,13 @@ public: // Logs without a prefix void logRaw(LogLevel lev, const std::string &text); - void setTraceEnabled(bool enable) { m_trace_enabled = enable; } - bool getTraceEnabled() { return m_trace_enabled; } - static LogLevel stringToLevel(const std::string &name); static const std::string getLevelLabel(LogLevel lev); + bool hasOutput(LogLevel level) { + return m_has_outputs[level].load(std::memory_order_relaxed); + } + static LogColor color_mode; private: @@ -84,6 +89,7 @@ private: const std::string getThreadName(); std::vector m_outputs[LL_MAX]; + std::atomic m_has_outputs[LL_MAX]; // Should implement atomic loads and stores (even though it's only // written to when one thread has access currently). @@ -91,7 +97,6 @@ private: volatile bool m_silenced_levels[LL_MAX]; std::map m_thread_names; mutable std::mutex m_mutex; - bool m_trace_enabled; }; class ILogOutput { @@ -185,35 +190,178 @@ private: Logger &m_logger; }; +#ifdef __ANDROID__ +class AndroidLogOutput : public ICombinedLogOutput { +public: + void logRaw(LogLevel lev, const std::string &line); +}; +#endif -extern StreamLogOutput stdout_output; -extern StreamLogOutput stderr_output; -extern std::ostream null_stream; +/* + * LogTarget + * + * This is the interface that sits between the LogStreams and the global logger. + * Primarily used to route streams to log levels, but could also enable other + * custom behavior. + * + */ +class LogTarget { +public: + // Must be thread-safe. These can be called from any thread. + virtual bool hasOutput() = 0; + virtual void log(const std::string &buf) = 0; +}; -extern std::ostream *dout_con_ptr; -extern std::ostream *derr_con_ptr; -extern std::ostream *derr_server_ptr; -extern Logger g_logger; +/* + * StreamProxy + * + * An ostream-like object that can proxy to a real ostream or do nothing, + * depending on how it is configured. See LogStream below. + * + */ +class StreamProxy { +public: + StreamProxy(std::ostream *os) : m_os(os) { } + + template + StreamProxy& operator<<(T&& arg) { + if (m_os) { + *m_os << std::forward(arg); + } + return *this; + } -// Writes directly to all LL_NONE log outputs for g_logger with no prefix. -extern std::ostream rawstream; + StreamProxy& operator<<(std::ostream& (*manip)(std::ostream&)) { + if (m_os) { + *m_os << manip; + } + return *this; + } -extern std::ostream errorstream; -extern std::ostream warningstream; -extern std::ostream actionstream; -extern std::ostream infostream; -extern std::ostream verbosestream; -extern std::ostream dstream; +private: + std::ostream *m_os; +}; -#define TRACEDO(x) do { \ - if (g_logger.getTraceEnabled()) { \ - x; \ - } \ -} while (0) -#define TRACESTREAM(x) TRACEDO(verbosestream x) +/* + * LogStream + * + * The public interface for log streams (infostream, verbosestream, etc). + * + * LogStream minimizes the work done when a given stream is off. (meaning + * it has no output targets, so it goes to /dev/null) + * + * For example, consider: + * + * verbosestream << "hello world" << 123 << std::endl; + * + * The compiler evaluates this as: + * + * (((verbosestream << "hello world") << 123) << std::endl) + * ^ ^ + * + * If `verbosestream` is on, the innermost expression (marked by ^) will return + * a StreamProxy that forwards to a real ostream, that feeds into the logger. + * However, if `verbosestream` is off, it will return a StreamProxy that does + * nothing on all later operations. Specifically, CPU time won't be wasted + * writing "hello world" and 123 into a buffer, or formatting the log entry. + * + * It is also possible to directly check if the stream is on/off: + * + * if (verbosestream) { + * auto data = ComputeExpensiveDataForTheLog(); + * verbosestream << data << endl; + * } + * +*/ -#define dout_con (*dout_con_ptr) -#define derr_con (*derr_con_ptr) +class LogStream { +public: + LogStream() = delete; + DISABLE_CLASS_COPY(LogStream); + + LogStream(LogTarget &target) : + m_target(target), + m_buffer(std::bind(&LogStream::internalFlush, this, std::placeholders::_1)), + m_dummy_buffer(), + m_stream(&m_buffer), + m_dummy_stream(&m_dummy_buffer), + m_proxy(&m_stream), + m_dummy_proxy(nullptr) { } + + template + StreamProxy& operator<<(T&& arg) { + StreamProxy& sp = m_target.hasOutput() ? m_proxy : m_dummy_proxy; + sp << std::forward(arg); + return sp; + } + StreamProxy& operator<<(std::ostream& (*manip)(std::ostream&)) { + StreamProxy& sp = m_target.hasOutput() ? m_proxy : m_dummy_proxy; + sp << manip; + return sp; + } + + operator bool() { + return m_target.hasOutput(); + } + + void internalFlush(const std::string &buf) { + m_target.log(buf); + } + + operator std::ostream&() { + return m_target.hasOutput() ? m_stream : m_dummy_stream; + } + +private: + // 10 streams per thread x (256 + overhead) ~ 3K per thread + static const int BUFFER_LENGTH = 256; + LogTarget &m_target; + StringStreamBuffer m_buffer; + DummyStreamBuffer m_dummy_buffer; + std::ostream m_stream; + std::ostream m_dummy_stream; + StreamProxy m_proxy; + StreamProxy m_dummy_proxy; + +}; + +#ifdef __ANDROID__ +extern AndroidLogOutput stdout_output; +extern AndroidLogOutput stderr_output; +#else +extern StreamLogOutput stdout_output; +extern StreamLogOutput stderr_output; +#endif + +extern Logger g_logger; + +/* + * By making the streams thread_local, each thread has its own + * private buffer. Two or more threads can write to the same stream + * simultaneously (lock-free), and there won't be any interference. + * + * The finished lines are sent to a LogTarget which is a global (not thread-local) + * object, and from there relayed to g_logger. The final writes are serialized + * by the mutex in g_logger. +*/ + +extern thread_local LogStream dstream; +extern thread_local LogStream rawstream; // Writes directly to all LL_NONE log outputs with no prefix. +extern thread_local LogStream errorstream; +extern thread_local LogStream warningstream; +extern thread_local LogStream actionstream; +extern thread_local LogStream infostream; +extern thread_local LogStream verbosestream; +extern thread_local LogStream tracestream; +// TODO: Search/replace these with verbose/tracestream +extern thread_local LogStream derr_con; +extern thread_local LogStream dout_con; + +#define TRACESTREAM(x) do { \ + if (tracestream) { \ + tracestream x; \ + } \ +} while (0) diff --git a/src/main.cpp b/src/main.cpp index 5ea212d8a..ddd725134 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -453,14 +453,6 @@ static bool setup_log_params(const Settings &cmd_args) } } - // If trace is enabled, enable logging of certain things - if (cmd_args.getFlag("trace")) { - dstream << _("Enabling trace level debug output") << std::endl; - g_logger.setTraceEnabled(true); - dout_con_ptr = &verbosestream; // This is somewhat old - socket_enable_debug_output = true; // Sockets doesn't use log.h - } - // In certain cases, output info level on stderr if (cmd_args.getFlag("info") || cmd_args.getFlag("verbose") || cmd_args.getFlag("trace") || cmd_args.getFlag("speedtests")) @@ -470,6 +462,12 @@ static bool setup_log_params(const Settings &cmd_args) if (cmd_args.getFlag("verbose") || cmd_args.getFlag("trace")) g_logger.addOutput(&stderr_output, LL_VERBOSE); + if (cmd_args.getFlag("trace")) { + dstream << _("Enabling trace level debug output") << std::endl; + g_logger.addOutput(&stderr_output, LL_TRACE); + socket_enable_debug_output = true; + } + return true; } @@ -599,7 +597,7 @@ static void init_log_streams(const Settings &cmd_args) warningstream << "Deprecated use of debug_log_level with an " "integer value; please update your configuration." << std::endl; static const char *lev_name[] = - {"", "error", "action", "info", "verbose"}; + {"", "error", "action", "info", "verbose", "trace"}; int lev_i = atoi(conf_loglev.c_str()); if (lev_i < 0 || lev_i >= (int)ARRLEN(lev_name)) { warningstream << "Supplied invalid debug_log_level!" diff --git a/src/network/address.cpp b/src/network/address.cpp index 90e561802..cf2e6208d 100644 --- a/src/network/address.cpp +++ b/src/network/address.cpp @@ -230,14 +230,14 @@ void Address::setPort(u16 port) m_port = port; } -void Address::print(std::ostream *s) const +void Address::print(std::ostream& s) const { if (m_addr_family == AF_INET6) - *s << "[" << serializeString() << "]:" << m_port; + s << "[" << serializeString() << "]:" << m_port; else if (m_addr_family == AF_INET) - *s << serializeString() << ":" << m_port; + s << serializeString() << ":" << m_port; else - *s << "(undefined)"; + s << "(undefined)"; } bool Address::isLocalhost() const diff --git a/src/network/address.h b/src/network/address.h index c2f5f2eef..692bf82c5 100644 --- a/src/network/address.h +++ b/src/network/address.h @@ -59,7 +59,7 @@ public: int getFamily() const { return m_addr_family; } bool isIPv6() const { return m_addr_family == AF_INET6; } bool isZero() const; - void print(std::ostream *s) const; + void print(std::ostream &s) const; std::string serializeString() const; bool isLocalhost() const; diff --git a/src/network/connection.cpp b/src/network/connection.cpp index 2d3cf6e88..6fb676f25 100644 --- a/src/network/connection.cpp +++ b/src/network/connection.cpp @@ -41,25 +41,14 @@ namespace con /* defines used for debugging and profiling */ /******************************************************************************/ #ifdef NDEBUG - #define LOG(a) a #define PROFILE(a) #else - #if 0 - /* this mutex is used to achieve log message consistency */ - std::mutex log_message_mutex; - #define LOG(a) \ - { \ - MutexAutoLock loglock(log_message_mutex); \ - a; \ - } - #else - // Prevent deadlocks until a solution is found after 5.2.0 (TODO) - #define LOG(a) a - #endif - #define PROFILE(a) a #endif +// TODO: Clean this up. +#define LOG(a) a + #define PING_TIMEOUT 5.0 u16 BufferedPacket::getSeqnum() const diff --git a/src/network/connectionthreads.cpp b/src/network/connectionthreads.cpp index dca065ae1..90936b43d 100644 --- a/src/network/connectionthreads.cpp +++ b/src/network/connectionthreads.cpp @@ -32,22 +32,18 @@ namespace con /* defines used for debugging and profiling */ /******************************************************************************/ #ifdef NDEBUG -#define LOG(a) a #define PROFILE(a) #undef DEBUG_CONNECTION_KBPS #else /* this mutex is used to achieve log message consistency */ -std::mutex log_conthread_mutex; -#define LOG(a) \ - { \ - MutexAutoLock loglock(log_conthread_mutex); \ - a; \ - } #define PROFILE(a) a //#define DEBUG_CONNECTION_KBPS #undef DEBUG_CONNECTION_KBPS #endif +// TODO: Clean this up. +#define LOG(a) a + #define WINDOW_SIZE 5 static session_t readPeerId(const u8 *packetdata) diff --git a/src/network/socket.cpp b/src/network/socket.cpp index 97a5f19f7..df15c89ba 100644 --- a/src/network/socket.cpp +++ b/src/network/socket.cpp @@ -198,7 +198,7 @@ void UDPSocket::Send(const Address &destination, const void *data, int size) if (socket_enable_debug_output) { // Print packet destination and size dstream << (int)m_handle << " -> "; - destination.print(&dstream); + destination.print(dstream); dstream << ", size=" << size; // Print packet contents @@ -295,7 +295,7 @@ int UDPSocket::Receive(Address &sender, void *data, int size) if (socket_enable_debug_output) { // Print packet sender and size dstream << (int)m_handle << " <- "; - sender.print(&dstream); + sender.print(dstream); dstream << ", size=" << received; // Print packet contents diff --git a/src/server.cpp b/src/server.cpp index c9cd4e398..6a4349ba5 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -525,7 +525,7 @@ void Server::start() actionstream << "World at [" << m_path_world << "]" << std::endl; actionstream << "Server for gameid=\"" << m_gamespec.id << "\" listening on "; - m_bind_addr.print(&actionstream); + m_bind_addr.print(actionstream); actionstream << "." << std::endl; } diff --git a/src/util/stream.h b/src/util/stream.h new file mode 100644 index 000000000..2e61b46d2 --- /dev/null +++ b/src/util/stream.h @@ -0,0 +1,70 @@ +/* +Minetest +Copyright (C) 2022 Minetest Authors + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#pragma once + +#include +#include +#include + +template > +class StringStreamBuffer : public std::streambuf { +public: + StringStreamBuffer(Emitter emitter) : m_emitter(emitter) { + buffer_index = 0; + } + + int overflow(int c) { + push_back(c); + return c; + } + + void push_back(char c) { + if (c == '\n' || c == '\r') { + if (buffer_index) + m_emitter(std::string(buffer, buffer_index)); + buffer_index = 0; + } else { + buffer[buffer_index++] = c; + if (buffer_index >= BufferLength) { + m_emitter(std::string(buffer, buffer_index)); + buffer_index = 0; + } + } + } + + std::streamsize xsputn(const char *s, std::streamsize n) { + for (int i = 0; i < n; ++i) + push_back(s[i]); + return n; + } +private: + Emitter m_emitter; + char buffer[BufferLength]; + int buffer_index; +}; + +class DummyStreamBuffer : public std::streambuf { + int overflow(int c) { + return c; + } + std::streamsize xsputn(const char *s, std::streamsize n) { + return n; + } +}; -- cgit v1.2.3 From f5a8593b11382b70ee969dc93b71f34fb0cad5df Mon Sep 17 00:00:00 2001 From: sfan5 Date: Mon, 9 May 2022 21:20:58 +0200 Subject: Add more Prometheus metrics (#12274) --- src/emerge.cpp | 25 +++++++++++- src/emerge.h | 16 +++++++- src/map.cpp | 28 ++++++++++++-- src/map.h | 10 +++++ src/server.cpp | 40 ++++++++++++------- src/server.h | 4 +- src/serverenvironment.cpp | 25 +++++++++++- src/serverenvironment.h | 18 ++++++--- src/util/metricsbackend.cpp | 93 ++++++++++++++++++++++++++++++++++++++------- src/util/metricsbackend.h | 84 ++++------------------------------------ 10 files changed, 224 insertions(+), 119 deletions(-) (limited to 'src/util') diff --git a/src/emerge.cpp b/src/emerge.cpp index 3760b24e6..5c1569f04 100644 --- a/src/emerge.cpp +++ b/src/emerge.cpp @@ -61,7 +61,9 @@ public: void cancelPendingItems(); - static void runCompletionCallbacks( +protected: + + void runCompletionCallbacks( const v3s16 &pos, EmergeAction action, const EmergeCallbackList &callbacks); @@ -138,7 +140,7 @@ EmergeParams::EmergeParams(EmergeManager *parent, const BiomeGen *biomegen, //// EmergeManager //// -EmergeManager::EmergeManager(Server *server) +EmergeManager::EmergeManager(Server *server, MetricsBackend *mb) { this->ndef = server->getNodeDefManager(); this->biomemgr = new BiomeManager(server); @@ -156,6 +158,17 @@ EmergeManager::EmergeManager(Server *server) enable_mapgen_debug_info = g_settings->getBool("enable_mapgen_debug_info"); + STATIC_ASSERT(ARRLEN(emergeActionStrs) == ARRLEN(m_completed_emerge_counter), + enum_size_mismatches); + for (u32 i = 0; i < ARRLEN(m_completed_emerge_counter); i++) { + std::string help_str("Number of completed emerges with status "); + help_str.append(emergeActionStrs[i]); + m_completed_emerge_counter[i] = mb->addCounter( + "minetest_emerge_completed", help_str, + {{"status", emergeActionStrs[i]}} + ); + } + s16 nthreads = 1; g_settings->getS16NoEx("num_emerge_threads", nthreads); // If automatic, leave a proc for the main thread and one for @@ -489,6 +502,12 @@ EmergeThread *EmergeManager::getOptimalThread() return m_threads[index]; } +void EmergeManager::reportCompletedEmerge(EmergeAction action) +{ + assert((int)action < ARRLEN(m_completed_emerge_counter)); + m_completed_emerge_counter[(int)action]->increment(); +} + //// //// EmergeThread @@ -540,6 +559,8 @@ void EmergeThread::cancelPendingItems() void EmergeThread::runCompletionCallbacks(const v3s16 &pos, EmergeAction action, const EmergeCallbackList &callbacks) { + m_emerge->reportCompletedEmerge(action); + for (size_t i = 0; i != callbacks.size(); i++) { EmergeCompletionCallback callback; void *param; diff --git a/src/emerge.h b/src/emerge.h index 61e7bda63..1bac4b708 100644 --- a/src/emerge.h +++ b/src/emerge.h @@ -24,6 +24,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "network/networkprotocol.h" #include "irr_v3d.h" #include "util/container.h" +#include "util/metricsbackend.h" #include "mapgen/mapgen.h" // for MapgenParams #include "map.h" @@ -69,6 +70,14 @@ enum EmergeAction { EMERGE_GENERATED, }; +const static std::string emergeActionStrs[] = { + "cancelled", + "errored", + "from_memory", + "from_disk", + "generated", +}; + // Callback typedef void (*EmergeCompletionCallback)( v3s16 blockpos, EmergeAction action, void *param); @@ -138,7 +147,7 @@ public: MapSettingsManager *map_settings_mgr; // Methods - EmergeManager(Server *server); + EmergeManager(Server *server, MetricsBackend *mb); ~EmergeManager(); DISABLE_CLASS_COPY(EmergeManager); @@ -197,6 +206,9 @@ private: u32 m_qlimit_diskonly; u32 m_qlimit_generate; + // Emerge metrics + MetricCounterPtr m_completed_emerge_counter[5]; + // Managers of various map generation-related components // Note that each Mapgen gets a copy(!) of these to work with BiomeGen *biomegen; @@ -218,5 +230,7 @@ private: bool popBlockEmergeData(v3s16 pos, BlockEmergeData *bedata); + void reportCompletedEmerge(EmergeAction action); + friend class EmergeThread; }; diff --git a/src/map.cpp b/src/map.cpp index 5153dcaa9..ce69accb5 100644 --- a/src/map.cpp +++ b/src/map.cpp @@ -336,6 +336,7 @@ void Map::timerUpdate(float dtime, float unload_timeout, u32 max_loaded_blocks, u32 saved_blocks_count = 0; u32 block_count_all = 0; + const auto start_time = porting::getTimeUs(); beginSave(); // If there is no practical limit, we spare creation of mapblock_queue @@ -377,6 +378,7 @@ void Map::timerUpdate(float dtime, float unload_timeout, u32 max_loaded_blocks, } } + // Delete sector if we emptied it if (all_blocks_deleted) { sector_deletion_queue.push_back(sector_it.first); } @@ -395,6 +397,7 @@ void Map::timerUpdate(float dtime, float unload_timeout, u32 max_loaded_blocks, } } block_count_all = mapblock_queue.size(); + // Delete old blocks, and blocks over the limit from the memory while (!mapblock_queue.empty() && (mapblock_queue.size() > max_loaded_blocks || mapblock_queue.top().block->getUsageTimer() > unload_timeout)) { @@ -425,6 +428,7 @@ void Map::timerUpdate(float dtime, float unload_timeout, u32 max_loaded_blocks, deleted_blocks_count++; block_count_all--; } + // Delete empty sectors for (auto §or_it : m_sectors) { if (sector_it.second->empty()) { @@ -432,7 +436,11 @@ void Map::timerUpdate(float dtime, float unload_timeout, u32 max_loaded_blocks, } } } + endSave(); + const auto end_time = porting::getTimeUs(); + + reportMetrics(end_time - start_time, saved_blocks_count, block_count_all); // Finally delete the empty sectors deleteSectors(sector_deletion_queue); @@ -1218,7 +1226,12 @@ ServerMap::ServerMap(const std::string &savedir, IGameDef *gamedef, m_savedir = savedir; m_map_saving_enabled = false; - m_save_time_counter = mb->addCounter("minetest_core_map_save_time", "Map save time (in nanoseconds)"); + m_save_time_counter = mb->addCounter( + "minetest_map_save_time", "Time spent saving blocks (in microseconds)"); + m_save_count_counter = mb->addCounter( + "minetest_map_saved_blocks", "Number of blocks saved"); + m_loaded_blocks_gauge = mb->addGauge( + "minetest_map_loaded_blocks", "Number of loaded blocks"); m_map_compression_level = rangelim(g_settings->getS16("map_compression_level_disk"), -1, 9); @@ -1594,6 +1607,13 @@ void ServerMap::updateVManip(v3s16 pos) vm->m_is_dirty = true; } +void ServerMap::reportMetrics(u64 save_time_us, u32 saved_blocks, u32 all_blocks) +{ + m_loaded_blocks_gauge->set(all_blocks); + m_save_time_counter->increment(save_time_us); + m_save_count_counter->increment(saved_blocks); +} + void ServerMap::save(ModifiedState save_level) { if (!m_map_saving_enabled) { @@ -1601,7 +1621,7 @@ void ServerMap::save(ModifiedState save_level) return; } - u64 start_time = porting::getTimeNs(); + const auto start_time = porting::getTimeUs(); if(save_level == MOD_STATE_CLEAN) infostream<<"ServerMap: Saving whole map, this can take time." @@ -1662,8 +1682,8 @@ void ServerMap::save(ModifiedState save_level) modprofiler.print(infostream); } - auto end_time = porting::getTimeNs(); - m_save_time_counter->increment(end_time - start_time); + const auto end_time = porting::getTimeUs(); + reportMetrics(end_time - start_time, block_count, block_count_all); } void ServerMap::listAllLoadableBlocks(std::vector &dst) diff --git a/src/map.h b/src/map.h index 21e3dbd6c..1e5499586 100644 --- a/src/map.h +++ b/src/map.h @@ -275,6 +275,9 @@ protected: // This stores the properties of the nodes on the map. const NodeDefManager *m_nodedef; + // Can be implemented by child class + virtual void reportMetrics(u64 save_time_us, u32 saved_blocks, u32 all_blocks) {} + bool determineAdditionalOcclusionCheck(const v3s16 &pos_camera, const core::aabbox3d &block_bounds, v3s16 &check); bool isOccluded(const v3s16 &pos_camera, const v3s16 &pos_target, @@ -392,6 +395,10 @@ public: MapSettingsManager settings_mgr; +protected: + + void reportMetrics(u64 save_time_us, u32 saved_blocks, u32 all_blocks) override; + private: friend class LuaVoxelManip; @@ -420,7 +427,10 @@ private: MapDatabase *dbase = nullptr; MapDatabase *dbase_ro = nullptr; + // Map metrics + MetricGaugePtr m_loaded_blocks_gauge; MetricCounterPtr m_save_time_counter; + MetricCounterPtr m_save_count_counter; }; diff --git a/src/server.cpp b/src/server.cpp index d85a2e95a..ebe1d1f6b 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -269,9 +269,15 @@ Server::Server( "minetest_core_latency", "Latency value (in seconds)"); - m_aom_buffer_counter = m_metrics_backend->addCounter( - "minetest_core_aom_generated_count", - "Number of active object messages generated"); + + const std::string aom_types[] = {"reliable", "unreliable"}; + for (u32 i = 0; i < ARRLEN(aom_types); i++) { + std::string help_str("Number of active object messages generated ("); + help_str.append(aom_types[i]).append(")"); + m_aom_buffer_counter[i] = m_metrics_backend->addCounter( + "minetest_core_aom_generated_count", help_str, + {{"type", aom_types[i]}}); + } m_packet_recv_counter = m_metrics_backend->addCounter( "minetest_core_server_packet_recv", @@ -281,6 +287,10 @@ Server::Server( "minetest_core_server_packet_recv_processed", "Valid received packets processed"); + m_map_edit_event_counter = m_metrics_backend->addCounter( + "minetest_core_map_edit_events", + "Number of map edit events"); + m_lag_gauge->set(g_settings->getFloat("dedicated_server_step")); } @@ -397,7 +407,7 @@ void Server::init() } // Create emerge manager - m_emerge = new EmergeManager(this); + m_emerge = new EmergeManager(this, m_metrics_backend.get()); // Create ban manager std::string ban_path = m_path_world + DIR_DELIM "ipban.txt"; @@ -462,7 +472,8 @@ void Server::init() // Initialize Environment m_startup_server_map = nullptr; // Ownership moved to ServerEnvironment - m_env = new ServerEnvironment(servermap, m_script, this, m_path_world); + m_env = new ServerEnvironment(servermap, m_script, this, + m_path_world, m_metrics_backend.get()); m_inventory_mgr->setEnv(m_env); m_clients.setEnv(m_env); @@ -623,6 +634,7 @@ void Server::AsyncRunStep(bool initial_step) max_lag = dtime; } m_env->reportMaxLagEstimate(max_lag); + // Step environment m_env->step(dtime); } @@ -773,10 +785,14 @@ void Server::AsyncRunStep(bool initial_step) // Get active object messages from environment ActiveObjectMessage aom(0); - u32 aom_count = 0; + u32 count_reliable = 0, count_unreliable = 0; for(;;) { if (!m_env->getActiveObjectMessage(&aom)) break; + if (aom.reliable) + count_reliable++; + else + count_unreliable++; std::vector* message_list = nullptr; auto n = buffered_messages.find(aom.id); @@ -787,10 +803,10 @@ void Server::AsyncRunStep(bool initial_step) message_list = n->second; } message_list->push_back(std::move(aom)); - aom_count++; } - m_aom_buffer_counter->increment(aom_count); + m_aom_buffer_counter[0]->increment(count_reliable); + m_aom_buffer_counter[1]->increment(count_unreliable); { ClientInterface::AutoLock clientlock(m_clients); @@ -864,15 +880,13 @@ void Server::AsyncRunStep(bool initial_step) // We will be accessing the environment MutexAutoLock lock(m_env_mutex); - // Don't send too many at a time - //u32 count = 0; - - // Single change sending is disabled if queue size is not small + // Single change sending is disabled if queue size is big bool disable_single_change_sending = false; if(m_unsent_map_edit_queue.size() >= 4) disable_single_change_sending = true; - int event_count = m_unsent_map_edit_queue.size(); + const auto event_count = m_unsent_map_edit_queue.size(); + m_map_edit_event_counter->increment(event_count); // We'll log the amount of each Profiler prof; diff --git a/src/server.h b/src/server.h index b1b9675d9..ecba30b95 100644 --- a/src/server.h +++ b/src/server.h @@ -724,11 +724,11 @@ private: MetricCounterPtr m_uptime_counter; MetricGaugePtr m_player_gauge; MetricGaugePtr m_timeofday_gauge; - // current server step lag MetricGaugePtr m_lag_gauge; - MetricCounterPtr m_aom_buffer_counter; + MetricCounterPtr m_aom_buffer_counter[2]; // [0] = rel, [1] = unrel MetricCounterPtr m_packet_recv_counter; MetricCounterPtr m_packet_recv_processed_counter; + MetricCounterPtr m_map_edit_event_counter; }; /* diff --git a/src/serverenvironment.cpp b/src/serverenvironment.cpp index 493210744..630c111c5 100644 --- a/src/serverenvironment.cpp +++ b/src/serverenvironment.cpp @@ -393,7 +393,7 @@ static std::random_device seed; ServerEnvironment::ServerEnvironment(ServerMap *map, ServerScripting *scriptIface, Server *server, - const std::string &path_world): + const std::string &path_world, MetricsBackend *mb): Environment(server), m_map(map), m_script(scriptIface), @@ -460,6 +460,15 @@ ServerEnvironment::ServerEnvironment(ServerMap *map, m_player_database = openPlayerDatabase(player_backend_name, path_world, conf); m_auth_database = openAuthDatabase(auth_backend_name, path_world, conf); + + m_step_time_counter = mb->addCounter( + "minetest_env_step_time", "Time spent in environment step (in microseconds)"); + + m_active_block_gauge = mb->addGauge( + "minetest_env_active_blocks", "Number of active blocks"); + + m_active_object_gauge = mb->addGauge( + "minetest_env_active_objects", "Number of active objects"); } ServerEnvironment::~ServerEnvironment() @@ -1283,6 +1292,8 @@ void ServerEnvironment::clearObjects(ClearObjectsMode mode) void ServerEnvironment::step(float dtime) { ScopeProfiler sp2(g_profiler, "ServerEnv::step()", SPT_AVG); + const auto start_time = porting::getTimeUs(); + /* Step time of day */ stepTimeOfDay(dtime); @@ -1352,6 +1363,8 @@ void ServerEnvironment::step(float dtime) m_active_blocks.update(players, active_block_range, active_object_range, blocks_removed, blocks_added); + m_active_block_gauge->set(m_active_blocks.size()); + /* Handle removed blocks */ @@ -1497,9 +1510,12 @@ void ServerEnvironment::step(float dtime) send_recommended = true; } - auto cb_state = [this, dtime, send_recommended] (ServerActiveObject *obj) { + u32 object_count = 0; + + auto cb_state = [&] (ServerActiveObject *obj) { if (obj->isGone()) return; + object_count++; // Step object obj->step(dtime, send_recommended); @@ -1507,6 +1523,8 @@ void ServerEnvironment::step(float dtime) obj->dumpAOMessagesToQueue(m_active_object_messages); }; m_ao_manager.step(dtime, cb_state); + + m_active_object_gauge->set(object_count); } /* @@ -1548,6 +1566,9 @@ void ServerEnvironment::step(float dtime) // Send outdated detached inventories m_server->sendDetachedInventories(PEER_ID_INEXISTENT, true); + + const auto end_time = porting::getTimeUs(); + m_step_time_counter->increment(end_time - start_time); } ServerEnvironment::BlockStatus ServerEnvironment::getBlockStatus(v3s16 blockpos) diff --git a/src/serverenvironment.h b/src/serverenvironment.h index 8733c2dd2..5dc329a60 100644 --- a/src/serverenvironment.h +++ b/src/serverenvironment.h @@ -25,6 +25,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "settings.h" #include "server/activeobjectmgr.h" #include "util/numeric.h" +#include "util/metricsbackend.h" #include #include @@ -167,19 +168,21 @@ public: std::set &blocks_removed, std::set &blocks_added); - bool contains(v3s16 p){ + bool contains(v3s16 p) const { return (m_list.find(p) != m_list.end()); } - void clear(){ + auto size() const { + return m_list.size(); + } + + void clear() { m_list.clear(); } std::set m_list; std::set m_abm_list; std::set m_forceloaded_list; - -private: }; /* @@ -198,7 +201,7 @@ class ServerEnvironment : public Environment { public: ServerEnvironment(ServerMap *map, ServerScripting *scriptIface, - Server *server, const std::string &path_world); + Server *server, const std::string &path_world, MetricsBackend *mb); ~ServerEnvironment(); Map & getMap(); @@ -487,5 +490,10 @@ private: std::unordered_map m_particle_spawners; std::unordered_map m_particle_spawner_attachments; + // Environment metrics + MetricCounterPtr m_step_time_counter; + MetricGaugePtr m_active_block_gauge; + MetricGaugePtr m_active_object_gauge; + ServerActiveObject* createSAO(ActiveObjectType type, v3f pos, const std::string &data); }; diff --git a/src/util/metricsbackend.cpp b/src/util/metricsbackend.cpp index c3b7def62..63b49ac0a 100644 --- a/src/util/metricsbackend.cpp +++ b/src/util/metricsbackend.cpp @@ -18,6 +18,7 @@ with this program; if not, write to the Free Software Foundation, Inc., */ #include "metricsbackend.h" +#include "util/thread.h" #if USE_PROMETHEUS #include #include @@ -27,18 +28,78 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "settings.h" #endif +/* Plain implementation */ + +class SimpleMetricCounter : public MetricCounter +{ +public: + SimpleMetricCounter() : MetricCounter(), m_counter(0.0) {} + + virtual ~SimpleMetricCounter() {} + + void increment(double number) override + { + MutexAutoLock lock(m_mutex); + m_counter += number; + } + double get() const override + { + MutexAutoLock lock(m_mutex); + return m_counter; + } + +private: + mutable std::mutex m_mutex; + double m_counter; +}; + +class SimpleMetricGauge : public MetricGauge +{ +public: + SimpleMetricGauge() : MetricGauge(), m_gauge(0.0) {} + + virtual ~SimpleMetricGauge() {} + + void increment(double number) override + { + MutexAutoLock lock(m_mutex); + m_gauge += number; + } + void decrement(double number) override + { + MutexAutoLock lock(m_mutex); + m_gauge -= number; + } + void set(double number) override + { + MutexAutoLock lock(m_mutex); + m_gauge = number; + } + double get() const override + { + MutexAutoLock lock(m_mutex); + return m_gauge; + } + +private: + mutable std::mutex m_mutex; + double m_gauge; +}; + MetricCounterPtr MetricsBackend::addCounter( - const std::string &name, const std::string &help_str) + const std::string &name, const std::string &help_str, Labels labels) { - return std::make_shared(name, help_str); + return std::make_shared(); } MetricGaugePtr MetricsBackend::addGauge( - const std::string &name, const std::string &help_str) + const std::string &name, const std::string &help_str, Labels labels) { - return std::make_shared(name, help_str); + return std::make_shared(); } +/* Prometheus backend */ + #if USE_PROMETHEUS class PrometheusMetricCounter : public MetricCounter @@ -47,13 +108,14 @@ public: PrometheusMetricCounter() = delete; PrometheusMetricCounter(const std::string &name, const std::string &help_str, + MetricsBackend::Labels labels, std::shared_ptr registry) : MetricCounter(), m_family(prometheus::BuildCounter() .Name(name) .Help(help_str) .Register(*registry)), - m_counter(m_family.Add({})) + m_counter(m_family.Add(labels)) { } @@ -73,13 +135,14 @@ public: PrometheusMetricGauge() = delete; PrometheusMetricGauge(const std::string &name, const std::string &help_str, + MetricsBackend::Labels labels, std::shared_ptr registry) : MetricGauge(), m_family(prometheus::BuildGauge() .Name(name) .Help(help_str) .Register(*registry)), - m_gauge(m_family.Add({})) + m_gauge(m_family.Add(labels)) { } @@ -107,10 +170,12 @@ public: virtual ~PrometheusMetricsBackend() {} - virtual MetricCounterPtr addCounter( - const std::string &name, const std::string &help_str); - virtual MetricGaugePtr addGauge( - const std::string &name, const std::string &help_str); + MetricCounterPtr addCounter( + const std::string &name, const std::string &help_str, + Labels labels = {}) override; + MetricGaugePtr addGauge( + const std::string &name, const std::string &help_str, + Labels labels = {}) override; private: std::unique_ptr m_exposer; @@ -118,15 +183,15 @@ private: }; MetricCounterPtr PrometheusMetricsBackend::addCounter( - const std::string &name, const std::string &help_str) + const std::string &name, const std::string &help_str, Labels labels) { - return std::make_shared(name, help_str, m_registry); + return std::make_shared(name, help_str, labels, m_registry); } MetricGaugePtr PrometheusMetricsBackend::addGauge( - const std::string &name, const std::string &help_str) + const std::string &name, const std::string &help_str, Labels labels) { - return std::make_shared(name, help_str, m_registry); + return std::make_shared(name, help_str, labels, m_registry); } MetricsBackend *createPrometheusMetricsBackend() diff --git a/src/util/metricsbackend.h b/src/util/metricsbackend.h index c37306392..644c73325 100644 --- a/src/util/metricsbackend.h +++ b/src/util/metricsbackend.h @@ -19,8 +19,9 @@ with this program; if not, write to the Free Software Foundation, Inc., #pragma once #include +#include +#include #include "config.h" -#include "util/thread.h" class MetricCounter { @@ -35,38 +36,6 @@ public: typedef std::shared_ptr MetricCounterPtr; -class SimpleMetricCounter : public MetricCounter -{ -public: - SimpleMetricCounter() = delete; - - virtual ~SimpleMetricCounter() {} - - SimpleMetricCounter(const std::string &name, const std::string &help_str) : - MetricCounter(), m_name(name), m_help_str(help_str), - m_counter(0.0) - { - } - - virtual void increment(double number) - { - MutexAutoLock lock(m_mutex); - m_counter += number; - } - virtual double get() const - { - MutexAutoLock lock(m_mutex); - return m_counter; - } - -private: - std::string m_name; - std::string m_help_str; - - mutable std::mutex m_mutex; - double m_counter; -}; - class MetricGauge { public: @@ -81,47 +50,6 @@ public: typedef std::shared_ptr MetricGaugePtr; -class SimpleMetricGauge : public MetricGauge -{ -public: - SimpleMetricGauge() = delete; - - SimpleMetricGauge(const std::string &name, const std::string &help_str) : - MetricGauge(), m_name(name), m_help_str(help_str), m_gauge(0.0) - { - } - - virtual ~SimpleMetricGauge() {} - - virtual void increment(double number) - { - MutexAutoLock lock(m_mutex); - m_gauge += number; - } - virtual void decrement(double number) - { - MutexAutoLock lock(m_mutex); - m_gauge -= number; - } - virtual void set(double number) - { - MutexAutoLock lock(m_mutex); - m_gauge = number; - } - virtual double get() const - { - MutexAutoLock lock(m_mutex); - return m_gauge; - } - -private: - std::string m_name; - std::string m_help_str; - - mutable std::mutex m_mutex; - double m_gauge; -}; - class MetricsBackend { public: @@ -129,10 +57,14 @@ public: virtual ~MetricsBackend() {} + typedef std::initializer_list> Labels; + virtual MetricCounterPtr addCounter( - const std::string &name, const std::string &help_str); + const std::string &name, const std::string &help_str, + Labels labels = {}); virtual MetricGaugePtr addGauge( - const std::string &name, const std::string &help_str); + const std::string &name, const std::string &help_str, + Labels labels = {}); }; #if USE_PROMETHEUS -- cgit v1.2.3