From c47313db65f968559711ac1b505ef341a9872017 Mon Sep 17 00:00:00 2001 From: Liso Date: Sun, 6 Jun 2021 18:51:21 +0200 Subject: Shadow mapping render pass (#11244) Co-authored-by: x2048 --- src/defaultsettings.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'src/defaultsettings.cpp') diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp index 871290944..38cade80b 100644 --- a/src/defaultsettings.cpp +++ b/src/defaultsettings.cpp @@ -262,6 +262,18 @@ void set_default_settings() settings->setDefault("enable_waving_leaves", "false"); settings->setDefault("enable_waving_plants", "false"); + // Effects Shadows + settings->setDefault("enable_dynamic_shadows", "false"); + settings->setDefault("shadow_strength", "0.2"); + settings->setDefault("shadow_map_max_distance", "200.0"); + settings->setDefault("shadow_map_texture_size", "2048"); + settings->setDefault("shadow_map_texture_32bit", "true"); + settings->setDefault("shadow_map_color", "false"); + settings->setDefault("shadow_filters", "1"); + settings->setDefault("shadow_poisson_filter", "true"); + settings->setDefault("shadow_update_time", "0.2"); + settings->setDefault("shadow_soft_radius", "1.0"); + settings->setDefault("shadow_sky_body_orbit_tilt", "0.0"); // Input settings->setDefault("invert_mouse", "false"); -- cgit v1.2.3 From edf098db637bf74d4675ce900dc9c0b8ee528a03 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Tue, 15 Jun 2021 19:11:56 +0200 Subject: Drop --videomodes, fullscreen_bpp and high_precision_fpu settings These have been pointless for a while. --- builtin/settingtypes.txt | 10 +---- src/client/clientlauncher.cpp | 6 --- src/client/clientlauncher.h | 1 - src/client/renderingengine.cpp | 78 +-------------------------------------- src/client/renderingengine.h | 1 - src/defaultsettings.cpp | 2 - src/main.cpp | 2 - src/script/lua_api/l_mainmenu.cpp | 23 ------------ src/script/lua_api/l_mainmenu.h | 2 - 9 files changed, 3 insertions(+), 122 deletions(-) (limited to 'src/defaultsettings.cpp') diff --git a/builtin/settingtypes.txt b/builtin/settingtypes.txt index ae421de2c..57857cabb 100644 --- a/builtin/settingtypes.txt +++ b/builtin/settingtypes.txt @@ -660,10 +660,10 @@ viewing_range (Viewing range) int 190 20 4000 # 0.1 = Default, 0.25 = Good value for weaker tablets. near_plane (Near plane) float 0.1 0 0.25 -# Width component of the initial window size. +# Width component of the initial window size. Ignored in fullscreen mode. screen_w (Screen width) int 1024 1 -# Height component of the initial window size. +# Height component of the initial window size. Ignored in fullscreen mode. screen_h (Screen height) int 600 1 # Save window size automatically when modified. @@ -672,9 +672,6 @@ autosave_screensize (Autosave screen size) bool true # Fullscreen mode. fullscreen (Full screen) bool false -# Bits per pixel (aka color depth) in fullscreen mode. -fullscreen_bpp (Full screen BPP) int 24 - # Vertical screen synchronization. vsync (VSync) bool false @@ -1477,9 +1474,6 @@ curl_parallel_limit (cURL parallel limit) int 8 # Maximum time a file download (e.g. a mod download) may take, stated in milliseconds. curl_file_download_timeout (cURL file download timeout) int 300000 -# Makes DirectX work with LuaJIT. Disable if it causes troubles. -high_precision_fpu (High-precision FPU) bool true - # Replaces the default main menu with a custom one. main_menu_script (Main menu script) string diff --git a/src/client/clientlauncher.cpp b/src/client/clientlauncher.cpp index dbf1d1cd1..13e7aefcf 100644 --- a/src/client/clientlauncher.cpp +++ b/src/client/clientlauncher.cpp @@ -99,10 +99,6 @@ bool ClientLauncher::run(GameStartData &start_data, const Settings &cmd_args) init_args(start_data, cmd_args); - // List video modes if requested - if (list_video_modes) - return m_rendering_engine->print_video_modes(); - #if USE_SOUND if (g_settings->getBool("enable_sound")) g_sound_manager_singleton = createSoundManagerSingleton(); @@ -336,8 +332,6 @@ void ClientLauncher::init_args(GameStartData &start_data, const Settings &cmd_ar if (cmd_args.exists("name")) start_data.name = cmd_args.get("name"); - list_video_modes = cmd_args.getFlag("videomodes"); - random_input = g_settings->getBool("random_input") || cmd_args.getFlag("random-input"); } diff --git a/src/client/clientlauncher.h b/src/client/clientlauncher.h index 79727e1fe..d1fd9a258 100644 --- a/src/client/clientlauncher.h +++ b/src/client/clientlauncher.h @@ -46,7 +46,6 @@ private: void speed_tests(); - bool list_video_modes = false; bool skip_main_menu = false; bool random_input = false; RenderingEngine *m_rendering_engine = nullptr; diff --git a/src/client/renderingengine.cpp b/src/client/renderingengine.cpp index 4f96a6e37..9015fb82a 100644 --- a/src/client/renderingengine.cpp +++ b/src/client/renderingengine.cpp @@ -92,7 +92,6 @@ RenderingEngine::RenderingEngine(IEventReceiver *receiver) // bpp, fsaa, vsync bool vsync = g_settings->getBool("vsync"); - u16 bits = g_settings->getU16("fullscreen_bpp"); u16 fsaa = g_settings->getU16("fsaa"); // stereo buffer required for pageflip stereo @@ -122,15 +121,13 @@ RenderingEngine::RenderingEngine(IEventReceiver *receiver) params.LoggingLevel = irr::ELL_DEBUG; params.DriverType = driverType; params.WindowSize = core::dimension2d(screen_w, screen_h); - params.Bits = bits; params.AntiAlias = fsaa; params.Fullscreen = fullscreen; params.Stencilbuffer = false; params.Stereobuffer = stereo_buffer; params.Vsync = vsync; params.EventReceiver = receiver; - params.HighPrecisionFPU = g_settings->getBool("high_precision_fpu"); - params.ZBufferBits = 24; + params.HighPrecisionFPU = true; #ifdef __ANDROID__ params.PrivateData = porting::app_global; #endif @@ -171,60 +168,6 @@ void RenderingEngine::setResizable(bool resize) m_device->setResizable(resize); } -bool RenderingEngine::print_video_modes() -{ - IrrlichtDevice *nulldevice; - - bool vsync = g_settings->getBool("vsync"); - u16 fsaa = g_settings->getU16("fsaa"); - MyEventReceiver *receiver = new MyEventReceiver(); - - SIrrlichtCreationParameters params = SIrrlichtCreationParameters(); - params.DriverType = video::EDT_NULL; - params.WindowSize = core::dimension2d(640, 480); - params.Bits = 24; - params.AntiAlias = fsaa; - params.Fullscreen = false; - params.Stencilbuffer = false; - params.Vsync = vsync; - params.EventReceiver = receiver; - params.HighPrecisionFPU = g_settings->getBool("high_precision_fpu"); - - nulldevice = createDeviceEx(params); - - if (!nulldevice) { - delete receiver; - return false; - } - - std::cout << _("Available video modes (WxHxD):") << std::endl; - - video::IVideoModeList *videomode_list = nulldevice->getVideoModeList(); - - if (videomode_list != NULL) { - s32 videomode_count = videomode_list->getVideoModeCount(); - core::dimension2d videomode_res; - s32 videomode_depth; - for (s32 i = 0; i < videomode_count; ++i) { - videomode_res = videomode_list->getVideoModeResolution(i); - videomode_depth = videomode_list->getVideoModeDepth(i); - std::cout << videomode_res.Width << "x" << videomode_res.Height - << "x" << videomode_depth << std::endl; - } - - std::cout << _("Active video mode (WxHxD):") << std::endl; - videomode_res = videomode_list->getDesktopResolution(); - videomode_depth = videomode_list->getDesktopDepth(); - std::cout << videomode_res.Width << "x" << videomode_res.Height << "x" - << videomode_depth << std::endl; - } - - nulldevice->drop(); - delete receiver; - - return videomode_list != NULL; -} - void RenderingEngine::removeMesh(const scene::IMesh* mesh) { m_device->getSceneManager()->getMeshCache()->removeMesh(mesh); @@ -582,25 +525,6 @@ void RenderingEngine::draw_menu_scene(gui::IGUIEnvironment *guienv, get_video_driver()->endScene(); } -std::vector> RenderingEngine::getSupportedVideoModes() -{ - IrrlichtDevice *nulldevice = createDevice(video::EDT_NULL); - sanity_check(nulldevice); - - std::vector> mlist; - video::IVideoModeList *modelist = nulldevice->getVideoModeList(); - - s32 num_modes = modelist->getVideoModeCount(); - for (s32 i = 0; i != num_modes; i++) { - core::dimension2d mode_res = modelist->getVideoModeResolution(i); - u32 mode_depth = (u32)modelist->getVideoModeDepth(i); - mlist.emplace_back(mode_res.Width, mode_res.Height, mode_depth); - } - - nulldevice->drop(); - return mlist; -} - std::vector RenderingEngine::getSupportedVideoDrivers() { std::vector drivers; diff --git a/src/client/renderingengine.h b/src/client/renderingengine.h index 6d42221d6..5299222e2 100644 --- a/src/client/renderingengine.h +++ b/src/client/renderingengine.h @@ -124,7 +124,6 @@ public: return s_singleton->core->get_shadow_renderer(); return nullptr; } - static std::vector> getSupportedVideoModes(); static std::vector getSupportedVideoDrivers(); private: diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp index 38cade80b..0895bf898 100644 --- a/src/defaultsettings.cpp +++ b/src/defaultsettings.cpp @@ -177,7 +177,6 @@ void set_default_settings() settings->setDefault("screen_h", "600"); settings->setDefault("autosave_screensize", "true"); settings->setDefault("fullscreen", "false"); - settings->setDefault("fullscreen_bpp", "24"); settings->setDefault("vsync", "false"); settings->setDefault("fov", "72"); settings->setDefault("leaves_style", "fancy"); @@ -454,7 +453,6 @@ void set_default_settings() settings->setDefault("server_name", ""); settings->setDefault("server_description", ""); - settings->setDefault("high_precision_fpu", "true"); settings->setDefault("enable_console", "false"); settings->setDefault("screen_dpi", "72"); diff --git a/src/main.cpp b/src/main.cpp index 7f96836b5..4a69f83b5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -304,8 +304,6 @@ static void set_allowed_options(OptionList *allowed_options) allowed_options->insert(std::make_pair("terminal", ValueSpec(VALUETYPE_FLAG, _("Feature an interactive terminal (Only works when using minetestserver or with --server)")))); #ifndef SERVER - allowed_options->insert(std::make_pair("videomodes", ValueSpec(VALUETYPE_FLAG, - _("Show available video modes")))); allowed_options->insert(std::make_pair("speedtests", ValueSpec(VALUETYPE_FLAG, _("Run speed tests")))); allowed_options->insert(std::make_pair("address", ValueSpec(VALUETYPE_STRING, diff --git a/src/script/lua_api/l_mainmenu.cpp b/src/script/lua_api/l_mainmenu.cpp index ee3e72dea..788557460 100644 --- a/src/script/lua_api/l_mainmenu.cpp +++ b/src/script/lua_api/l_mainmenu.cpp @@ -752,28 +752,6 @@ int ModApiMainMenu::l_get_video_drivers(lua_State *L) return 1; } -/******************************************************************************/ -int ModApiMainMenu::l_get_video_modes(lua_State *L) -{ - std::vector > videomodes - = RenderingEngine::getSupportedVideoModes(); - - lua_newtable(L); - for (u32 i = 0; i != videomodes.size(); i++) { - lua_newtable(L); - lua_pushnumber(L, videomodes[i].X); - lua_setfield(L, -2, "w"); - lua_pushnumber(L, videomodes[i].Y); - lua_setfield(L, -2, "h"); - lua_pushnumber(L, videomodes[i].Z); - lua_setfield(L, -2, "depth"); - - lua_rawseti(L, -2, i + 1); - } - - return 1; -} - /******************************************************************************/ int ModApiMainMenu::l_gettext(lua_State *L) { @@ -895,7 +873,6 @@ void ModApiMainMenu::Initialize(lua_State *L, int top) API_FCT(download_file); API_FCT(gettext); API_FCT(get_video_drivers); - API_FCT(get_video_modes); API_FCT(get_screen_info); API_FCT(get_min_supp_proto); API_FCT(get_max_supp_proto); diff --git a/src/script/lua_api/l_mainmenu.h b/src/script/lua_api/l_mainmenu.h index 33ac9e721..ec2d20da2 100644 --- a/src/script/lua_api/l_mainmenu.h +++ b/src/script/lua_api/l_mainmenu.h @@ -140,8 +140,6 @@ private: static int l_get_video_drivers(lua_State *L); - static int l_get_video_modes(lua_State *L); - //version compatibility static int l_get_min_supp_proto(lua_State *L); -- cgit v1.2.3 From 1805775f3d54043c3b1e75e47b9b85e3b12bab00 Mon Sep 17 00:00:00 2001 From: pecksin <78765996+pecksin@users.noreply.github.com> Date: Sun, 20 Jun 2021 11:20:24 -0400 Subject: Make chat web links clickable (#11092) If enabled in minetest.conf, provides colored, clickable (middle-mouse or ctrl-left-mouse) weblinks in chat output, to open the OS' default web browser. --- builtin/settingtypes.txt | 6 ++ minetest.conf.example | 8 +++ po/minetest.pot | 9 +++ src/chat.cpp | 133 +++++++++++++++++++++++++++++++++++---------- src/chat.h | 8 +++ src/defaultsettings.cpp | 2 + src/gui/guiChatConsole.cpp | 98 ++++++++++++++++++++++++++++++++- src/gui/guiChatConsole.h | 8 +++ 8 files changed, 242 insertions(+), 30 deletions(-) (limited to 'src/defaultsettings.cpp') diff --git a/builtin/settingtypes.txt b/builtin/settingtypes.txt index 57857cabb..fd7d8b9b9 100644 --- a/builtin/settingtypes.txt +++ b/builtin/settingtypes.txt @@ -973,6 +973,12 @@ mute_sound (Mute sound) bool false [Client] +# Clickable weblinks (middle-click or ctrl-left-click) enabled in chat console output. +clickable_chat_weblinks (Chat weblinks) bool false + +# Optional override for chat weblink color. +chat_weblink_color (Weblink color) string + [*Network] # Address to connect to. diff --git a/minetest.conf.example b/minetest.conf.example index 718cb0c75..b252f4f70 100644 --- a/minetest.conf.example +++ b/minetest.conf.example @@ -1155,6 +1155,14 @@ # Client # +# If enabled, http links in chat can be middle-clicked or ctrl-left-clicked to open the link in the OS's default web browser. +# type: bool +# clickable_chat_weblinks = false + +# If clickable_chat_weblinks is enabled, specify the color (as 24-bit hexadecimal) of weblinks in chat. +# type: string +# chat_weblink_color = #8888FF + ## Network # Address to connect to. diff --git a/po/minetest.pot b/po/minetest.pot index 4ed1e2434..53b706f5f 100644 --- a/po/minetest.pot +++ b/po/minetest.pot @@ -6551,3 +6551,12 @@ msgid "" "be queued.\n" "This should be lower than curl_parallel_limit." msgstr "" + +#: src/gui/guiChatConsole.cpp +msgid "Opening webpage" +msgstr "" + +#: src/gui/guiChatConsole.cpp +msgid "Failed to open webpage" +msgstr "" + diff --git a/src/chat.cpp b/src/chat.cpp index c9317a079..e44d73ac0 100644 --- a/src/chat.cpp +++ b/src/chat.cpp @@ -35,6 +35,17 @@ ChatBuffer::ChatBuffer(u32 scrollback): if (m_scrollback == 0) m_scrollback = 1; m_empty_formatted_line.first = true; + + m_cache_clickable_chat_weblinks = false; + // Curses mode cannot access g_settings here + if (g_settings != nullptr) { + m_cache_clickable_chat_weblinks = g_settings->getBool("clickable_chat_weblinks"); + if (m_cache_clickable_chat_weblinks) { + std::string colorval = g_settings->get("chat_weblink_color"); + parseColorString(colorval, m_cache_chat_weblink_color, false, 255); + m_cache_chat_weblink_color.setAlpha(255); + } + } } void ChatBuffer::addLine(const std::wstring &name, const std::wstring &text) @@ -263,78 +274,144 @@ u32 ChatBuffer::formatChatLine(const ChatLine& line, u32 cols, //EnrichedString line_text(line.text); next_line.first = true; - bool text_processing = false; + // Set/use forced newline after the last frag in each line + bool mark_newline = false; // Produce fragments and layout them into lines - while (!next_frags.empty() || in_pos < line.text.size()) - { + while (!next_frags.empty() || in_pos < line.text.size()) { + mark_newline = false; // now using this to USE line-end frag + // Layout fragments into lines - while (!next_frags.empty()) - { + while (!next_frags.empty()) { ChatFormattedFragment& frag = next_frags[0]; - if (frag.text.size() <= cols - out_column) - { + + // Force newline after this frag, if marked + if (frag.column == INT_MAX) + mark_newline = true; + + if (frag.text.size() <= cols - out_column) { // Fragment fits into current line frag.column = out_column; next_line.fragments.push_back(frag); out_column += frag.text.size(); next_frags.erase(next_frags.begin()); - } - else - { + } else { // Fragment does not fit into current line // So split it up temp_frag.text = frag.text.substr(0, cols - out_column); temp_frag.column = out_column; - //temp_frag.bold = frag.bold; + temp_frag.weblink = frag.weblink; + next_line.fragments.push_back(temp_frag); frag.text = frag.text.substr(cols - out_column); + frag.column = 0; out_column = cols; } - if (out_column == cols || text_processing) - { + + if (out_column == cols || mark_newline) { // End the current line destination.push_back(next_line); num_added++; next_line.fragments.clear(); next_line.first = false; - out_column = text_processing ? hanging_indentation : 0; + out_column = hanging_indentation; + mark_newline = false; } } - // Produce fragment - if (in_pos < line.text.size()) - { - u32 remaining_in_input = line.text.size() - in_pos; - u32 remaining_in_output = cols - out_column; + // Produce fragment(s) for next formatted line + if (!(in_pos < line.text.size())) + continue; + const std::wstring &linestring = line.text.getString(); + u32 remaining_in_output = cols - out_column; + size_t http_pos = std::wstring::npos; + mark_newline = false; // now using this to SET line-end frag + + // Construct all frags for next output line + while (!mark_newline) { // Determine a fragment length <= the minimum of // remaining_in_{in,out}put. Try to end the fragment // on a word boundary. - u32 frag_length = 1, space_pos = 0; + u32 frag_length = 0, space_pos = 0; + u32 remaining_in_input = line.text.size() - in_pos; + + if (m_cache_clickable_chat_weblinks) { + // Note: unsigned(-1) on fail + http_pos = linestring.find(L"https://", in_pos); + if (http_pos == std::wstring::npos) + http_pos = linestring.find(L"http://", in_pos); + if (http_pos != std::wstring::npos) + http_pos -= in_pos; + } + while (frag_length < remaining_in_input && - frag_length < remaining_in_output) - { - if (iswspace(line.text.getString()[in_pos + frag_length])) + frag_length < remaining_in_output) { + if (iswspace(linestring[in_pos + frag_length])) space_pos = frag_length; ++frag_length; } + + if (http_pos >= remaining_in_output) { + // Http not in range, grab until space or EOL, halt as normal. + // Note this works because (http_pos = npos) is unsigned(-1) + + mark_newline = true; + } else if (http_pos == 0) { + // At http, grab ALL until FIRST whitespace or end marker. loop. + // If at end of string, next loop will be empty string to mark end of weblink. + + frag_length = 6; // Frag is at least "http://" + + // Chars to mark end of weblink + // TODO? replace this with a safer (slower) regex whitelist? + static const std::wstring delim_chars = L"\'\");,"; + wchar_t tempchar = linestring[in_pos+frag_length]; + while (frag_length < remaining_in_input && + !iswspace(tempchar) && + delim_chars.find(tempchar) == std::wstring::npos) { + ++frag_length; + tempchar = linestring[in_pos+frag_length]; + } + + space_pos = frag_length - 1; + // This frag may need to be force-split. That's ok, urls aren't "words" + if (frag_length >= remaining_in_output) { + mark_newline = true; + } + } else { + // Http in range, grab until http, loop + + space_pos = http_pos - 1; + frag_length = http_pos; + } + + // Include trailing space in current frag if (space_pos != 0 && frag_length < remaining_in_input) frag_length = space_pos + 1; temp_frag.text = line.text.substr(in_pos, frag_length); - temp_frag.column = 0; - //temp_frag.bold = 0; + // A hack so this frag remembers mark_newline for the layout phase + temp_frag.column = mark_newline ? INT_MAX : 0; + + if (http_pos == 0) { + // Discard color stuff from the source frag + temp_frag.text = EnrichedString(temp_frag.text.getString()); + temp_frag.text.setDefaultColor(m_cache_chat_weblink_color); + // Set weblink in the frag meta + temp_frag.weblink = wide_to_utf8(temp_frag.text.getString()); + } else { + temp_frag.weblink.clear(); + } next_frags.push_back(temp_frag); in_pos += frag_length; - text_processing = true; + remaining_in_output -= std::min(frag_length, remaining_in_output); } } // End the last line - if (num_added == 0 || !next_line.fragments.empty()) - { + if (num_added == 0 || !next_line.fragments.empty()) { destination.push_back(next_line); num_added++; } diff --git a/src/chat.h b/src/chat.h index 0b98e4d3c..aabb0821e 100644 --- a/src/chat.h +++ b/src/chat.h @@ -57,6 +57,8 @@ struct ChatFormattedFragment EnrichedString text; // starting column u32 column; + // web link is empty for most frags + std::string weblink; // formatting //u8 bold:1; }; @@ -118,6 +120,7 @@ public: std::vector& destination) const; void resize(u32 scrollback); + protected: s32 getTopScrollPos() const; s32 getBottomScrollPos() const; @@ -138,6 +141,11 @@ private: std::vector m_formatted; // Empty formatted line, for error returns ChatFormattedLine m_empty_formatted_line; + + // Enable clickable chat weblinks + bool m_cache_clickable_chat_weblinks; + // Color of clickable chat weblinks + irr::video::SColor m_cache_chat_weblink_color; }; class ChatPrompt diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp index 0895bf898..6791fccf5 100644 --- a/src/defaultsettings.cpp +++ b/src/defaultsettings.cpp @@ -65,6 +65,8 @@ void set_default_settings() settings->setDefault("max_out_chat_queue_size", "20"); settings->setDefault("pause_on_lost_focus", "false"); settings->setDefault("enable_register_confirmation", "true"); + settings->setDefault("clickable_chat_weblinks", "false"); + settings->setDefault("chat_weblink_color", "#8888FF"); // Keymap settings->setDefault("remote_port", "30000"); diff --git a/src/gui/guiChatConsole.cpp b/src/gui/guiChatConsole.cpp index baaaea5e8..85617d862 100644 --- a/src/gui/guiChatConsole.cpp +++ b/src/gui/guiChatConsole.cpp @@ -41,6 +41,10 @@ inline u32 clamp_u8(s32 value) return (u32) MYMIN(MYMAX(value, 0), 255); } +inline bool isInCtrlKeys(const irr::EKEY_CODE& kc) +{ + return kc == KEY_LCONTROL || kc == KEY_RCONTROL || kc == KEY_CONTROL; +} GUIChatConsole::GUIChatConsole( gui::IGUIEnvironment* env, @@ -91,6 +95,10 @@ GUIChatConsole::GUIChatConsole( // set default cursor options setCursor(true, true, 2.0, 0.1); + + // track ctrl keys for mouse event + m_is_ctrl_down = false; + m_cache_clickable_chat_weblinks = g_settings->getBool("clickable_chat_weblinks"); } GUIChatConsole::~GUIChatConsole() @@ -405,8 +413,21 @@ bool GUIChatConsole::OnEvent(const SEvent& event) ChatPrompt &prompt = m_chat_backend->getPrompt(); - if(event.EventType == EET_KEY_INPUT_EVENT && event.KeyInput.PressedDown) + if (event.EventType == EET_KEY_INPUT_EVENT && !event.KeyInput.PressedDown) + { + // CTRL up + if (isInCtrlKeys(event.KeyInput.Key)) + { + m_is_ctrl_down = false; + } + } + else if(event.EventType == EET_KEY_INPUT_EVENT && event.KeyInput.PressedDown) { + // CTRL down + if (isInCtrlKeys(event.KeyInput.Key)) { + m_is_ctrl_down = true; + } + // Key input if (KeyPress(event.KeyInput) == getKeySetting("keymap_console")) { closeConsole(); @@ -613,11 +634,24 @@ bool GUIChatConsole::OnEvent(const SEvent& event) } else if(event.EventType == EET_MOUSE_INPUT_EVENT) { - if(event.MouseInput.Event == EMIE_MOUSE_WHEEL) + if (event.MouseInput.Event == EMIE_MOUSE_WHEEL) { s32 rows = myround(-3.0 * event.MouseInput.Wheel); m_chat_backend->scroll(rows); } + // Middle click or ctrl-click opens weblink, if enabled in config + else if(m_cache_clickable_chat_weblinks && ( + event.MouseInput.Event == EMIE_MMOUSE_PRESSED_DOWN || + (event.MouseInput.Event == EMIE_LMOUSE_PRESSED_DOWN && m_is_ctrl_down) + )) + { + // If clicked within console output region + if (event.MouseInput.Y / m_fontsize.Y < (m_height / m_fontsize.Y) - 1 ) + { + // Translate pixel position to font position + middleClick(event.MouseInput.X / m_fontsize.X, event.MouseInput.Y / m_fontsize.Y); + } + } } #if (IRRLICHT_VERSION_MT_REVISION >= 2) else if(event.EventType == EET_STRING_INPUT_EVENT) @@ -640,3 +674,63 @@ void GUIChatConsole::setVisible(bool visible) } } +void GUIChatConsole::middleClick(s32 col, s32 row) +{ + // Prevent accidental rapid clicking + static u64 s_oldtime = 0; + u64 newtime = porting::getTimeMs(); + + // 0.6 seconds should suffice + if (newtime - s_oldtime < 600) + return; + s_oldtime = newtime; + + const std::vector & + frags = m_chat_backend->getConsoleBuffer().getFormattedLine(row).fragments; + std::string weblink = ""; // from frag meta + + // Identify targetted fragment, if exists + int indx = frags.size() - 1; + if (indx < 0) { + // Invalid row, frags is empty + return; + } + // Scan from right to left, offset by 1 font space because left margin + while (indx > -1 && (u32)col < frags[indx].column + 1) { + --indx; + } + if (indx > -1) { + weblink = frags[indx].weblink; + // Note if(indx < 0) then a frag somehow had a corrupt column field + } + + /* + // Debug help. Please keep this in case adjustments are made later. + std::string ws; + ws = "Middleclick: (" + std::to_string(col) + ',' + std::to_string(row) + ')' + " frags:"; + // show all frags () for the clicked row + for (u32 i=0;iaddUnparsedMessage(utf8_to_wide(msg.str())); + } +} diff --git a/src/gui/guiChatConsole.h b/src/gui/guiChatConsole.h index 1152f2b2d..32628f0d8 100644 --- a/src/gui/guiChatConsole.h +++ b/src/gui/guiChatConsole.h @@ -84,6 +84,9 @@ private: void drawText(); void drawPrompt(); + // If clicked fragment has a web url, send it to the system default web browser + void middleClick(s32 col, s32 row); + private: ChatBackend* m_chat_backend; Client* m_client; @@ -126,4 +129,9 @@ private: // font gui::IGUIFont *m_font = nullptr; v2u32 m_fontsize; + + // Enable clickable chat weblinks + bool m_cache_clickable_chat_weblinks; + // Track if a ctrl key is currently held down + bool m_is_ctrl_down; }; -- cgit v1.2.3 From bf3acbf388406f736286d990adb5f35a9023c390 Mon Sep 17 00:00:00 2001 From: x2048 Date: Sun, 25 Jul 2021 12:36:23 +0200 Subject: Distribute shadow map update over multiple frames to reduce stutter (#11422) Reduces stutter and freezes when playing. * Maintains double SM and SM Color textures * Light frustum update triggers incremental generation of shadow map into secondary 'future' textures. * Every incremental update renders a portion of the shadow draw list (split equally). * After defined number of frames (currently, 4), 'future' and 'current' textures are swapped, and DirectionalLight 'commits' the new frustum to use when rendering shadows on screen. Co-authored-by: sfan5 --- builtin/settingtypes.txt | 10 +- client/shaders/nodes_shader/opengl_fragment.glsl | 10 +- src/client/clientmap.cpp | 25 +++- src/client/clientmap.h | 2 +- src/client/game.cpp | 17 +-- src/client/shadows/dynamicshadows.cpp | 60 +++++++-- src/client/shadows/dynamicshadows.h | 9 +- src/client/shadows/dynamicshadowsrender.cpp | 156 ++++++++++++++++++----- src/client/shadows/dynamicshadowsrender.h | 7 +- src/defaultsettings.cpp | 2 +- 10 files changed, 225 insertions(+), 73 deletions(-) (limited to 'src/defaultsettings.cpp') diff --git a/builtin/settingtypes.txt b/builtin/settingtypes.txt index 17843fac8..420e9d49c 100644 --- a/builtin/settingtypes.txt +++ b/builtin/settingtypes.txt @@ -618,11 +618,11 @@ shadow_filters (Shadow filter quality) enum 1 0,1,2 # On true translucent nodes cast colored shadows. This is expensive. shadow_map_color (Colored shadows) bool false - -# Set the shadow update time, in seconds. -# Lower value means shadows and map updates faster, but it consumes more resources. -# Minimum value: 0.001; maximum value: 0.2 -shadow_update_time (Map update time) float 0.2 0.001 0.2 +# Spread a complete update of shadow map over given amount of frames. +# Higher values might make shadows laggy, lower values +# will consume more resources. +# Minimum value: 1; maximum value: 16 +shadow_update_frames (Map shadows update frames) int 8 1 16 # Set the soft shadow radius size. # Lower values mean sharper shadows, bigger values mean softer shadows. diff --git a/client/shaders/nodes_shader/opengl_fragment.glsl b/client/shaders/nodes_shader/opengl_fragment.glsl index 64a88ebbb..f85ca7b48 100644 --- a/client/shaders/nodes_shader/opengl_fragment.glsl +++ b/client/shaders/nodes_shader/opengl_fragment.glsl @@ -197,7 +197,7 @@ float getPenumbraRadius(sampler2D shadowsampler, vec2 smTexCoord, float realDist float pointDepth; float maxRadius = SOFTSHADOWRADIUS * 5.0 * multiplier; - float bound = clamp(PCFBOUND * (1 - baseLength), 0.5, PCFBOUND); + float bound = clamp(PCFBOUND * (1 - baseLength), 0.0, PCFBOUND); int n = 0; for (y = -bound; y <= bound; y += 1.0) @@ -304,7 +304,7 @@ vec4 getShadowColor(sampler2D shadowsampler, vec2 smTexCoord, float realDistance float perspectiveFactor; float texture_size = 1.0 / (f_textureresolution * 0.5); - int samples = int(clamp(PCFSAMPLES * (1 - baseLength) * (1 - baseLength), 1, PCFSAMPLES)); + int samples = int(clamp(PCFSAMPLES * (1 - baseLength) * (1 - baseLength), PCFSAMPLES / 4, PCFSAMPLES)); int init_offset = int(floor(mod(((smTexCoord.x * 34.0) + 1.0) * smTexCoord.y, 64.0-samples))); int end_offset = int(samples) + init_offset; @@ -334,7 +334,7 @@ float getShadow(sampler2D shadowsampler, vec2 smTexCoord, float realDistance) float perspectiveFactor; float texture_size = 1.0 / (f_textureresolution * 0.5); - int samples = int(clamp(PCFSAMPLES * (1 - baseLength) * (1 - baseLength), 1, PCFSAMPLES)); + int samples = int(clamp(PCFSAMPLES * (1 - baseLength) * (1 - baseLength), PCFSAMPLES / 4, PCFSAMPLES)); int init_offset = int(floor(mod(((smTexCoord.x * 34.0) + 1.0) * smTexCoord.y, 64.0-samples))); int end_offset = int(samples) + init_offset; @@ -370,7 +370,7 @@ vec4 getShadowColor(sampler2D shadowsampler, vec2 smTexCoord, float realDistance float texture_size = 1.0 / (f_textureresolution * 0.5); float y, x; - float bound = clamp(PCFBOUND * (1 - baseLength), 0.5, PCFBOUND); + float bound = clamp(PCFBOUND * (1 - baseLength), PCFBOUND / 2, PCFBOUND); int n = 0; // basic PCF filter @@ -402,7 +402,7 @@ float getShadow(sampler2D shadowsampler, vec2 smTexCoord, float realDistance) float texture_size = 1.0 / (f_textureresolution * 0.5); float y, x; - float bound = clamp(PCFBOUND * (1 - baseLength), 0.5, PCFBOUND); + float bound = clamp(PCFBOUND * (1 - baseLength), PCFBOUND / 2, PCFBOUND); int n = 0; // basic PCF filter diff --git a/src/client/clientmap.cpp b/src/client/clientmap.cpp index 8b09eade1..77f3b9fe8 100644 --- a/src/client/clientmap.cpp +++ b/src/client/clientmap.cpp @@ -636,7 +636,7 @@ void ClientMap::PrintInfo(std::ostream &out) } void ClientMap::renderMapShadows(video::IVideoDriver *driver, - const video::SMaterial &material, s32 pass) + const video::SMaterial &material, s32 pass, int frame, int total_frames) { bool is_transparent_pass = pass != scene::ESNRP_SOLID; std::string prefix; @@ -650,7 +650,23 @@ void ClientMap::renderMapShadows(video::IVideoDriver *driver, MeshBufListList drawbufs; + int count = 0; + int low_bound = is_transparent_pass ? 0 : m_drawlist_shadow.size() / total_frames * frame; + int high_bound = is_transparent_pass ? m_drawlist_shadow.size() : m_drawlist_shadow.size() / total_frames * (frame + 1); + + // transparent pass should be rendered in one go + if (is_transparent_pass && frame != total_frames - 1) { + return; + } + for (auto &i : m_drawlist_shadow) { + // only process specific part of the list & break early + ++count; + if (count <= low_bound) + continue; + if (count > high_bound) + break; + v3s16 block_pos = i.first; MapBlock *block = i.second; @@ -705,6 +721,7 @@ void ClientMap::renderMapShadows(video::IVideoDriver *driver, local_material.MaterialType = material.MaterialType; local_material.BackfaceCulling = material.BackfaceCulling; local_material.FrontfaceCulling = material.FrontfaceCulling; + local_material.BlendOperation = material.BlendOperation; local_material.Lighting = false; driver->setMaterial(local_material); @@ -720,6 +737,12 @@ void ClientMap::renderMapShadows(video::IVideoDriver *driver, } } + // restore the driver material state + video::SMaterial clean; + clean.BlendOperation = video::EBO_ADD; + driver->setMaterial(clean); // reset material to defaults + driver->draw3DLine(v3f(), v3f(), video::SColor(0)); + g_profiler->avg(prefix + "draw meshes [ms]", draw.stop(true)); g_profiler->avg(prefix + "vertices drawn [#]", vertex_count); g_profiler->avg(prefix + "drawcalls [#]", drawcall_count); diff --git a/src/client/clientmap.h b/src/client/clientmap.h index 93ade4c15..97ce8d355 100644 --- a/src/client/clientmap.h +++ b/src/client/clientmap.h @@ -125,7 +125,7 @@ public: void renderMap(video::IVideoDriver* driver, s32 pass); void renderMapShadows(video::IVideoDriver *driver, - const video::SMaterial &material, s32 pass); + const video::SMaterial &material, s32 pass, int frame, int total_frames); int getBackgroundBrightness(float max_d, u32 daylight_factor, int oldvalue, bool *sunlight_seen_result); diff --git a/src/client/game.cpp b/src/client/game.cpp index 85dd8f4bb..6fc57c8cc 100644 --- a/src/client/game.cpp +++ b/src/client/game.cpp @@ -609,7 +609,6 @@ struct GameRunData { float jump_timer; float damage_flash; float update_draw_list_timer; - float update_shadows_timer; f32 fog_range; @@ -3881,10 +3880,8 @@ void Game::updateFrame(ProfilerGraph *graph, RunStats *stats, f32 dtime, changed much */ runData.update_draw_list_timer += dtime; - runData.update_shadows_timer += dtime; float update_draw_list_delta = 0.2f; - bool draw_list_updated = false; v3f camera_direction = camera->getDirection(); if (runData.update_draw_list_timer >= update_draw_list_delta @@ -3894,18 +3891,10 @@ void Game::updateFrame(ProfilerGraph *graph, RunStats *stats, f32 dtime, runData.update_draw_list_timer = 0; client->getEnv().getClientMap().updateDrawList(); runData.update_draw_list_last_cam_dir = camera_direction; - draw_list_updated = true; } - if (ShadowRenderer *shadow = RenderingEngine::get_shadow_renderer()) { - update_draw_list_delta = shadow->getUpdateDelta(); - - if (m_camera_offset_changed || - (runData.update_shadows_timer > update_draw_list_delta && - (!draw_list_updated || shadow->getDirectionalLightCount() == 0))) { - runData.update_shadows_timer = 0; - updateShadows(); - } + if (RenderingEngine::get_shadow_renderer()) { + updateShadows(); } m_game_ui->update(*stats, client, draw_control, cam, runData.pointed_old, gui_chat_console, dtime); @@ -4062,7 +4051,7 @@ void Game::updateShadows() shadow->getDirectionalLight().setDirection(sun_pos); shadow->setTimeOfDay(in_timeofday); - shadow->getDirectionalLight().update_frustum(camera, client); + shadow->getDirectionalLight().update_frustum(camera, client, m_camera_offset_changed); } /**************************************************************************** diff --git a/src/client/shadows/dynamicshadows.cpp b/src/client/shadows/dynamicshadows.cpp index 17b711a61..0c7eea0e7 100644 --- a/src/client/shadows/dynamicshadows.cpp +++ b/src/client/shadows/dynamicshadows.cpp @@ -38,8 +38,8 @@ void DirectionalLight::createSplitMatrices(const Camera *cam) float tanFovX = tanf(cam->getFovX() * 0.5f); // adjusted frustum boundaries - float sfNear = shadow_frustum.zNear; - float sfFar = adjustDist(shadow_frustum.zFar, cam->getFovY()); + float sfNear = future_frustum.zNear; + float sfFar = adjustDist(future_frustum.zFar, cam->getFovY()); // adjusted camera positions v3f camPos2 = cam->getPosition(); @@ -87,14 +87,15 @@ void DirectionalLight::createSplitMatrices(const Camera *cam) v3f eye_displacement = direction * vvolume; // we must compute the viewmat with the position - the camera offset - // but the shadow_frustum position must be the actual world position + // but the future_frustum position must be the actual world position v3f eye = frustumCenter - eye_displacement; - shadow_frustum.position = world_center - eye_displacement; - shadow_frustum.length = vvolume; - shadow_frustum.ViewMat.buildCameraLookAtMatrixLH(eye, frustumCenter, v3f(0.0f, 1.0f, 0.0f)); - shadow_frustum.ProjOrthMat.buildProjectionMatrixOrthoLH(shadow_frustum.length, - shadow_frustum.length, -shadow_frustum.length, - shadow_frustum.length,false); + future_frustum.position = world_center - eye_displacement; + future_frustum.length = vvolume; + future_frustum.ViewMat.buildCameraLookAtMatrixLH(eye, frustumCenter, v3f(0.0f, 1.0f, 0.0f)); + future_frustum.ProjOrthMat.buildProjectionMatrixOrthoLH(future_frustum.length, + future_frustum.length, -future_frustum.length, + future_frustum.length,false); + future_frustum.camera_offset = cam->getOffset(); } DirectionalLight::DirectionalLight(const u32 shadowMapResolution, @@ -104,23 +105,44 @@ DirectionalLight::DirectionalLight(const u32 shadowMapResolution, farPlane(farValue), mapRes(shadowMapResolution), pos(position) {} -void DirectionalLight::update_frustum(const Camera *cam, Client *client) +void DirectionalLight::update_frustum(const Camera *cam, Client *client, bool force) { - should_update_map_shadow = true; + if (dirty && !force) + return; + float zNear = cam->getCameraNode()->getNearValue(); float zFar = getMaxFarValue(); /////////////////////////////////// // update splits near and fars - shadow_frustum.zNear = zNear; - shadow_frustum.zFar = zFar; + future_frustum.zNear = zNear; + future_frustum.zFar = zFar; // update shadow frustum createSplitMatrices(cam); // get the draw list for shadows client->getEnv().getClientMap().updateDrawListShadow( - getPosition(), getDirection(), shadow_frustum.length); + getPosition(), getDirection(), future_frustum.length); should_update_map_shadow = true; + dirty = true; + + // when camera offset changes, adjust the current frustum view matrix to avoid flicker + v3s16 cam_offset = cam->getOffset(); + if (cam_offset != shadow_frustum.camera_offset) { + v3f rotated_offset; + shadow_frustum.ViewMat.rotateVect(rotated_offset, intToFloat(cam_offset - shadow_frustum.camera_offset, BS)); + shadow_frustum.ViewMat.setTranslation(shadow_frustum.ViewMat.getTranslation() + rotated_offset); + shadow_frustum.camera_offset = cam_offset; + } +} + +void DirectionalLight::commitFrustum() +{ + if (!dirty) + return; + + shadow_frustum = future_frustum; + dirty = false; } void DirectionalLight::setDirection(v3f dir) @@ -144,6 +166,16 @@ const m4f &DirectionalLight::getProjectionMatrix() const return shadow_frustum.ProjOrthMat; } +const m4f &DirectionalLight::getFutureViewMatrix() const +{ + return future_frustum.ViewMat; +} + +const m4f &DirectionalLight::getFutureProjectionMatrix() const +{ + return future_frustum.ProjOrthMat; +} + m4f DirectionalLight::getViewProjMatrix() { return shadow_frustum.ProjOrthMat * shadow_frustum.ViewMat; diff --git a/src/client/shadows/dynamicshadows.h b/src/client/shadows/dynamicshadows.h index a53612f7c..d8be66be8 100644 --- a/src/client/shadows/dynamicshadows.h +++ b/src/client/shadows/dynamicshadows.h @@ -34,6 +34,7 @@ struct shadowFrustum core::matrix4 ProjOrthMat; core::matrix4 ViewMat; v3f position; + v3s16 camera_offset; }; class DirectionalLight @@ -47,7 +48,7 @@ public: //DISABLE_CLASS_COPY(DirectionalLight) - void update_frustum(const Camera *cam, Client *client); + void update_frustum(const Camera *cam, Client *client, bool force = false); // when set direction is updated to negative normalized(direction) void setDirection(v3f dir); @@ -59,6 +60,8 @@ public: /// Gets the light's matrices. const core::matrix4 &getViewMatrix() const; const core::matrix4 &getProjectionMatrix() const; + const core::matrix4 &getFutureViewMatrix() const; + const core::matrix4 &getFutureProjectionMatrix() const; core::matrix4 getViewProjMatrix(); /// Gets the light's far value. @@ -88,6 +91,8 @@ public: bool should_update_map_shadow{true}; + void commitFrustum(); + private: void createSplitMatrices(const Camera *cam); @@ -99,4 +104,6 @@ private: v3f pos; v3f direction{0}; shadowFrustum shadow_frustum; + shadowFrustum future_frustum; + bool dirty{false}; }; diff --git a/src/client/shadows/dynamicshadowsrender.cpp b/src/client/shadows/dynamicshadowsrender.cpp index 135c9f895..350586225 100644 --- a/src/client/shadows/dynamicshadowsrender.cpp +++ b/src/client/shadows/dynamicshadowsrender.cpp @@ -27,10 +27,11 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "client/shader.h" #include "client/client.h" #include "client/clientmap.h" +#include "profiler.h" ShadowRenderer::ShadowRenderer(IrrlichtDevice *device, Client *client) : m_device(device), m_smgr(device->getSceneManager()), - m_driver(device->getVideoDriver()), m_client(client) + m_driver(device->getVideoDriver()), m_client(client), m_current_frame(0) { m_shadows_enabled = true; @@ -43,7 +44,7 @@ ShadowRenderer::ShadowRenderer(IrrlichtDevice *device, Client *client) : m_shadow_map_texture_32bit = g_settings->getBool("shadow_map_texture_32bit"); m_shadow_map_colored = g_settings->getBool("shadow_map_color"); m_shadow_samples = g_settings->getS32("shadow_filters"); - m_update_delta = g_settings->getFloat("shadow_update_time"); + m_map_shadow_update_frames = g_settings->getS16("shadow_update_frames"); } ShadowRenderer::~ShadowRenderer() @@ -66,6 +67,9 @@ ShadowRenderer::~ShadowRenderer() if (shadowMapClientMap) m_driver->removeTexture(shadowMapClientMap); + + if (shadowMapClientMapFuture) + m_driver->removeTexture(shadowMapClientMapFuture); } void ShadowRenderer::initialize() @@ -93,11 +97,6 @@ void ShadowRenderer::initialize() } -float ShadowRenderer::getUpdateDelta() const -{ - return m_update_delta; -} - size_t ShadowRenderer::addDirectionalLight() { m_light_list.emplace_back(m_shadow_map_texture_size, @@ -152,10 +151,9 @@ void ShadowRenderer::setClearColor(video::SColor ClearColor) m_clear_color = ClearColor; } -void ShadowRenderer::update(video::ITexture *outputTarget) +void ShadowRenderer::updateSMTextures() { if (!m_shadows_enabled || m_smgr->getActiveCamera() == nullptr) { - m_smgr->drawAll(); return; } @@ -174,6 +172,13 @@ void ShadowRenderer::update(video::ITexture *outputTarget) true); } + if (!shadowMapClientMapFuture && m_map_shadow_update_frames > 1) { + shadowMapClientMapFuture = getSMTexture( + std::string("shadow_clientmap_bb_") + itos(m_shadow_map_texture_size), + m_shadow_map_colored ? m_texture_format_color : m_texture_format, + true); + } + if (m_shadow_map_colored && !shadowMapTextureColors) { shadowMapTextureColors = getSMTexture( std::string("shadow_colored_") + itos(m_shadow_map_texture_size), @@ -201,7 +206,22 @@ void ShadowRenderer::update(video::ITexture *outputTarget) } if (!m_shadow_node_array.empty() && !m_light_list.empty()) { - // for every directional light: + bool reset_sm_texture = false; + + // detect if SM should be regenerated + for (DirectionalLight &light : m_light_list) { + if (light.should_update_map_shadow) { + light.should_update_map_shadow = false; + m_current_frame = 0; + reset_sm_texture = true; + } + } + + video::ITexture* shadowMapTargetTexture = shadowMapClientMapFuture; + if (shadowMapTargetTexture == nullptr) + shadowMapTargetTexture = shadowMapClientMap; + + // Update SM incrementally: for (DirectionalLight &light : m_light_list) { // Static shader values. m_shadow_depth_cb->MapRes = (f32)m_shadow_map_texture_size; @@ -212,22 +232,60 @@ void ShadowRenderer::update(video::ITexture *outputTarget) // Depth texture is available in irrlicth maybe we // should put some gl* fn here - if (light.should_update_map_shadow) { - light.should_update_map_shadow = false; - m_driver->setRenderTarget(shadowMapClientMap, true, true, + if (m_current_frame < m_map_shadow_update_frames) { + m_driver->setRenderTarget(shadowMapTargetTexture, reset_sm_texture, true, video::SColor(255, 255, 255, 255)); - renderShadowMap(shadowMapClientMap, light); - - if (m_shadow_map_colored) { - m_driver->setRenderTarget(shadowMapTextureColors, - true, false, video::SColor(255, 255, 255, 255)); + renderShadowMap(shadowMapTargetTexture, light); + + // Render transparent part in one pass. + // This is also handled in ClientMap. + if (m_current_frame == m_map_shadow_update_frames - 1) { + if (m_shadow_map_colored) { + m_driver->setRenderTarget(shadowMapTextureColors, + true, false, video::SColor(255, 255, 255, 255)); + } + renderShadowMap(shadowMapTextureColors, light, + scene::ESNRP_TRANSPARENT); } - renderShadowMap(shadowMapTextureColors, light, - scene::ESNRP_TRANSPARENT); m_driver->setRenderTarget(0, false, false); } + reset_sm_texture = false; + } // end for lights + + // move to the next section + if (m_current_frame <= m_map_shadow_update_frames) + ++m_current_frame; + + // pass finished, swap textures and commit light changes + if (m_current_frame == m_map_shadow_update_frames) { + if (shadowMapClientMapFuture != nullptr) + std::swap(shadowMapClientMapFuture, shadowMapClientMap); + + // Let all lights know that maps are updated + for (DirectionalLight &light : m_light_list) + light.commitFrustum(); + } + } +} + +void ShadowRenderer::update(video::ITexture *outputTarget) +{ + if (!m_shadows_enabled || m_smgr->getActiveCamera() == nullptr) { + m_smgr->drawAll(); + return; + } + + updateSMTextures(); + + if (!m_shadow_node_array.empty() && !m_light_list.empty()) { + + for (DirectionalLight &light : m_light_list) { + // Static shader values. + m_shadow_depth_cb->MapRes = (f32)m_shadow_map_texture_size; + m_shadow_depth_cb->MaxFar = (f32)m_shadow_map_max_distance * BS; + // render shadows for the n0n-map objects. m_driver->setRenderTarget(shadowMapTextureDynamicObjects, true, true, video::SColor(255, 255, 255, 255)); @@ -299,8 +357,8 @@ video::ITexture *ShadowRenderer::getSMTexture(const std::string &shadow_map_name void ShadowRenderer::renderShadowMap(video::ITexture *target, DirectionalLight &light, scene::E_SCENE_NODE_RENDER_PASS pass) { - m_driver->setTransform(video::ETS_VIEW, light.getViewMatrix()); - m_driver->setTransform(video::ETS_PROJECTION, light.getProjectionMatrix()); + m_driver->setTransform(video::ETS_VIEW, light.getFutureViewMatrix()); + m_driver->setTransform(video::ETS_PROJECTION, light.getFutureProjectionMatrix()); // Operate on the client map for (const auto &shadow_node : m_shadow_node_array) { @@ -322,10 +380,13 @@ void ShadowRenderer::renderShadowMap(video::ITexture *target, //material.PolygonOffsetDepthBias = 1.0f/4.0f; //material.PolygonOffsetSlopeScale = -1.f; - if (m_shadow_map_colored && pass != scene::ESNRP_SOLID) + if (m_shadow_map_colored && pass != scene::ESNRP_SOLID) { material.MaterialType = (video::E_MATERIAL_TYPE) depth_shader_trans; - else + } + else { material.MaterialType = (video::E_MATERIAL_TYPE) depth_shader; + material.BlendOperation = video::EBO_MIN; + } // FIXME: I don't think this is needed here map_node->OnAnimate(m_device->getTimer()->getTime()); @@ -333,7 +394,7 @@ void ShadowRenderer::renderShadowMap(video::ITexture *target, m_driver->setTransform(video::ETS_WORLD, map_node->getAbsoluteTransformation()); - map_node->renderMapShadows(m_driver, material, pass); + map_node->renderMapShadows(m_driver, material, pass, m_current_frame, m_map_shadow_update_frames); break; } } @@ -354,8 +415,10 @@ void ShadowRenderer::renderShadowObjects( u32 n_node_materials = shadow_node.node->getMaterialCount(); std::vector BufferMaterialList; std::vector> BufferMaterialCullingList; + std::vector BufferBlendOperationList; BufferMaterialList.reserve(n_node_materials); BufferMaterialCullingList.reserve(n_node_materials); + BufferBlendOperationList.reserve(n_node_materials); // backup materialtype for each material // (aka shader) @@ -365,12 +428,11 @@ void ShadowRenderer::renderShadowObjects( BufferMaterialList.push_back(current_mat.MaterialType); current_mat.MaterialType = - (video::E_MATERIAL_TYPE)depth_shader; - - current_mat.setTexture(3, shadowMapTextureFinal); + (video::E_MATERIAL_TYPE)depth_shader_entities; BufferMaterialCullingList.emplace_back( (bool)current_mat.BackfaceCulling, (bool)current_mat.FrontfaceCulling); + BufferBlendOperationList.push_back(current_mat.BlendOperation); current_mat.BackfaceCulling = true; current_mat.FrontfaceCulling = false; @@ -393,6 +455,7 @@ void ShadowRenderer::renderShadowObjects( current_mat.BackfaceCulling = BufferMaterialCullingList[m].first; current_mat.FrontfaceCulling = BufferMaterialCullingList[m].second; + current_mat.BlendOperation = BufferBlendOperationList[m]; } } // end for caster shadow nodes @@ -433,7 +496,7 @@ void ShadowRenderer::createShaders() readShaderFile(depth_shader_vs).c_str(), "vertexMain", video::EVST_VS_1_1, readShaderFile(depth_shader_fs).c_str(), "pixelMain", - video::EPST_PS_1_2, m_shadow_depth_cb); + video::EPST_PS_1_2, m_shadow_depth_cb, video::EMT_ONETEXTURE_BLEND); if (depth_shader == -1) { // upsi, something went wrong loading shader. @@ -449,6 +512,41 @@ void ShadowRenderer::createShaders() m_driver->getMaterialRenderer(depth_shader)->grab(); } + // This creates a clone of depth_shader with base material set to EMT_SOLID, + // because entities won't render shadows with base material EMP_ONETEXTURE_BLEND + if (depth_shader_entities == -1) { + std::string depth_shader_vs = getShaderPath("shadow_shaders", "pass1_vertex.glsl"); + if (depth_shader_vs.empty()) { + m_shadows_enabled = false; + errorstream << "Error shadow mapping vs shader not found." << std::endl; + return; + } + std::string depth_shader_fs = getShaderPath("shadow_shaders", "pass1_fragment.glsl"); + if (depth_shader_fs.empty()) { + m_shadows_enabled = false; + errorstream << "Error shadow mapping fs shader not found." << std::endl; + return; + } + + depth_shader_entities = gpu->addHighLevelShaderMaterial( + readShaderFile(depth_shader_vs).c_str(), "vertexMain", + video::EVST_VS_1_1, + readShaderFile(depth_shader_fs).c_str(), "pixelMain", + video::EPST_PS_1_2, m_shadow_depth_cb); + + if (depth_shader_entities == -1) { + // upsi, something went wrong loading shader. + m_shadows_enabled = false; + errorstream << "Error compiling shadow mapping shader (dynamic)." << std::endl; + return; + } + + // HACK, TODO: investigate this better + // Grab the material renderer once more so minetest doesn't crash + // on exit + m_driver->getMaterialRenderer(depth_shader_entities)->grab(); + } + if (mixcsm_shader == -1) { std::string depth_shader_vs = getShaderPath("shadow_shaders", "pass2_vertex.glsl"); if (depth_shader_vs.empty()) { diff --git a/src/client/shadows/dynamicshadowsrender.h b/src/client/shadows/dynamicshadowsrender.h index e633bd4f7..52b24a18f 100644 --- a/src/client/shadows/dynamicshadowsrender.h +++ b/src/client/shadows/dynamicshadowsrender.h @@ -64,7 +64,6 @@ public: size_t getDirectionalLightCount() const; f32 getMaxShadowFar() const; - float getUpdateDelta() const; /// Adds a shadow to the scene node. /// The shadow mode can be ESM_BOTH, or ESM_RECEIVE. /// ESM_BOTH casts and receives shadows @@ -101,6 +100,7 @@ private: scene::ESNRP_SOLID); void renderShadowObjects(video::ITexture *target, DirectionalLight &light); void mixShadowsQuad(); + void updateSMTextures(); // a bunch of variables IrrlichtDevice *m_device{nullptr}; @@ -108,6 +108,7 @@ private: video::IVideoDriver *m_driver{nullptr}; Client *m_client{nullptr}; video::ITexture *shadowMapClientMap{nullptr}; + video::ITexture *shadowMapClientMapFuture{nullptr}; video::ITexture *shadowMapTextureFinal{nullptr}; video::ITexture *shadowMapTextureDynamicObjects{nullptr}; video::ITexture *shadowMapTextureColors{nullptr}; @@ -120,11 +121,12 @@ private: float m_shadow_map_max_distance; float m_shadow_map_texture_size; float m_time_day{0.0f}; - float m_update_delta; int m_shadow_samples; bool m_shadow_map_texture_32bit; bool m_shadows_enabled; bool m_shadow_map_colored; + u8 m_map_shadow_update_frames; /* Use this number of frames to update map shaodw */ + u8 m_current_frame{0}; /* Current frame */ video::ECOLOR_FORMAT m_texture_format{video::ECOLOR_FORMAT::ECF_R16F}; video::ECOLOR_FORMAT m_texture_format_color{video::ECOLOR_FORMAT::ECF_R16G16}; @@ -135,6 +137,7 @@ private: std::string readShaderFile(const std::string &path); s32 depth_shader{-1}; + s32 depth_shader_entities{-1}; s32 depth_shader_trans{-1}; s32 mixcsm_shader{-1}; diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp index 6791fccf5..faf839b3a 100644 --- a/src/defaultsettings.cpp +++ b/src/defaultsettings.cpp @@ -272,7 +272,7 @@ void set_default_settings() settings->setDefault("shadow_map_color", "false"); settings->setDefault("shadow_filters", "1"); settings->setDefault("shadow_poisson_filter", "true"); - settings->setDefault("shadow_update_time", "0.2"); + settings->setDefault("shadow_update_frames", "8"); settings->setDefault("shadow_soft_radius", "1.0"); settings->setDefault("shadow_sky_body_orbit_tilt", "0.0"); -- cgit v1.2.3 From d1624a552151bcb152b7abf63df6501b63458d78 Mon Sep 17 00:00:00 2001 From: lhofhansl Date: Tue, 31 Aug 2021 17:32:31 -0700 Subject: Switch MapBlock compression to zstd (#10788) * Add zstd support. * Rearrange serialization order * Compress entire mapblock Co-authored-by: sfan5 --- .github/workflows/build.yml | 2 +- .github/workflows/macos.yml | 2 +- .gitlab-ci.yml | 4 +- Dockerfile | 2 +- android/native/jni/Android.mk | 10 ++- builtin/settingtypes.txt | 16 ++-- cmake/Modules/FindZstd.cmake | 9 +++ doc/world_format.txt | 91 ++++++++++++--------- misc/debpkg-control | 2 +- src/CMakeLists.txt | 11 +++ src/defaultsettings.cpp | 4 +- src/main.cpp | 73 ++++++++++++++++- src/mapblock.cpp | 166 ++++++++++++++++++++++++++------------ src/mapblock.h | 2 +- src/mapgen/mg_schematic.cpp | 9 ++- src/mapgen/mg_schematic.h | 1 + src/mapnode.cpp | 30 +++---- src/mapnode.h | 5 +- src/serialization.cpp | 135 +++++++++++++++++++++++++++++-- src/serialization.h | 14 +++- src/unittest/test_compression.cpp | 42 +++++++++- util/buildbot/buildwin32.sh | 6 ++ util/buildbot/buildwin64.sh | 6 ++ util/ci/common.sh | 2 +- 24 files changed, 493 insertions(+), 151 deletions(-) create mode 100644 cmake/Modules/FindZstd.cmake (limited to 'src/defaultsettings.cpp') diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d268aa0cb..98b1ffe8a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -227,7 +227,7 @@ jobs: env: VCPKG_VERSION: 0bf3923f9fab4001c00f0f429682a0853b5749e0 # 2020.11 - vcpkg_packages: irrlicht zlib curl[winssl] openal-soft libvorbis libogg sqlite3 freetype luajit + vcpkg_packages: irrlicht zlib zstd curl[winssl] openal-soft libvorbis libogg sqlite3 freetype luajit strategy: fail-fast: false matrix: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 3ec157d0e..d97cff1aa 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -34,7 +34,7 @@ jobs: - uses: actions/checkout@v2 - name: Install deps run: | - pkgs=(cmake freetype gettext gmp hiredis jpeg jsoncpp leveldb libogg libpng libvorbis luajit) + pkgs=(cmake freetype gettext gmp hiredis jpeg jsoncpp leveldb libogg libpng libvorbis luajit zstd) brew update brew install ${pkgs[@]} brew unlink $(brew ls --formula) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cabce627f..252ed8a5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,7 @@ variables: stage: build before_script: - apt-get update - - apt-get -y install build-essential git cmake libpng-dev libjpeg-dev libxxf86vm-dev libgl1-mesa-dev libsqlite3-dev libleveldb-dev libogg-dev libvorbis-dev libopenal-dev libcurl4-gnutls-dev libfreetype6-dev zlib1g-dev libgmp-dev libjsoncpp-dev + - apt-get -y install build-essential git cmake libpng-dev libjpeg-dev libxxf86vm-dev libgl1-mesa-dev libsqlite3-dev libleveldb-dev libogg-dev libvorbis-dev libopenal-dev libcurl4-gnutls-dev libfreetype6-dev zlib1g-dev libgmp-dev libjsoncpp-dev libzstd-dev script: - git clone https://github.com/minetest/irrlicht -b $IRRLICHT_TAG lib/irrlichtmt - mkdir cmakebuild @@ -187,7 +187,7 @@ build:fedora-28: extends: .build_template image: fedora:28 before_script: - - dnf -y install make git gcc gcc-c++ kernel-devel cmake libjpeg-devel libpng-devel libcurl-devel openal-soft-devel libvorbis-devel libXxf86vm-devel libogg-devel freetype-devel mesa-libGL-devel zlib-devel jsoncpp-devel gmp-devel sqlite-devel luajit-devel leveldb-devel ncurses-devel spatialindex-devel + - dnf -y install make git gcc gcc-c++ kernel-devel cmake libjpeg-devel libpng-devel libcurl-devel openal-soft-devel libvorbis-devel libXxf86vm-devel libogg-devel freetype-devel mesa-libGL-devel zlib-devel jsoncpp-devel gmp-devel sqlite-devel luajit-devel leveldb-devel ncurses-devel spatialindex-devel libzstd-devel ## ## MinGW for Windows diff --git a/Dockerfile b/Dockerfile index 8843e4bbc..481dab237 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ COPY textures /usr/src/minetest/textures WORKDIR /usr/src/minetest -RUN apk add --no-cache git build-base cmake sqlite-dev curl-dev zlib-dev \ +RUN apk add --no-cache git build-base cmake sqlite-dev curl-dev zlib-dev zstd-dev \ gmp-dev jsoncpp-dev postgresql-dev ninja luajit-dev ca-certificates && \ git clone --depth=1 -b ${MINETEST_GAME_VERSION} https://github.com/minetest/minetest_game.git ./games/minetest_game && \ rm -fr ./games/minetest_game/.git diff --git a/android/native/jni/Android.mk b/android/native/jni/Android.mk index f92ac1d60..26e9b058b 100644 --- a/android/native/jni/Android.mk +++ b/android/native/jni/Android.mk @@ -57,6 +57,11 @@ LOCAL_MODULE := Vorbis LOCAL_SRC_FILES := deps/Android/Vorbis/${NDK_TOOLCHAIN_VERSION}/$(APP_ABI)/libvorbis.a include $(PREBUILT_STATIC_LIBRARY) +include $(CLEAR_VARS) +LOCAL_MODULE := Zstd +LOCAL_SRC_FILES := deps/Android/Zstd/${NDK_TOOLCHAIN_VERSION}/$(APP_ABI)/libzstd.a +include $(PREBUILT_STATIC_LIBRARY) + include $(CLEAR_VARS) LOCAL_MODULE := Minetest @@ -101,7 +106,8 @@ LOCAL_C_INCLUDES := \ deps/Android/LuaJIT/src \ deps/Android/OpenAL-Soft/include \ deps/Android/sqlite \ - deps/Android/Vorbis/include + deps/Android/Vorbis/include \ + deps/Android/Zstd/include LOCAL_SRC_FILES := \ $(wildcard ../../src/client/*.cpp) \ @@ -201,7 +207,7 @@ LOCAL_SRC_FILES += \ # SQLite3 LOCAL_SRC_FILES += deps/Android/sqlite/sqlite3.c -LOCAL_STATIC_LIBRARIES += Curl Freetype Irrlicht OpenAL mbedTLS mbedx509 mbedcrypto Vorbis LuaJIT GetText android_native_app_glue $(PROFILER_LIBS) #LevelDB +LOCAL_STATIC_LIBRARIES += Curl Freetype Irrlicht OpenAL mbedTLS mbedx509 mbedcrypto Vorbis LuaJIT GetText Zstd android_native_app_glue $(PROFILER_LIBS) #LevelDB LOCAL_LDLIBS := -lEGL -lGLESv1_CM -lGLESv2 -landroid -lOpenSLES diff --git a/builtin/settingtypes.txt b/builtin/settingtypes.txt index 25a51b888..43e70e052 100644 --- a/builtin/settingtypes.txt +++ b/builtin/settingtypes.txt @@ -1100,11 +1100,10 @@ full_block_send_enable_min_time_from_building (Delay in sending blocks after bui # client number. max_packets_per_iteration (Max. packets per iteration) int 1024 -# ZLib compression level to use when sending mapblocks to the client. -# -1 - Zlib's default compression level -# 0 - no compresson, fastest +# Compression level to use when sending mapblocks to the client. +# -1 - use default compression level +# 0 - least compresson, fastest # 9 - best compression, slowest -# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method) map_compression_level_net (Map Compression Level for Network Transfer) int -1 -1 9 [*Game] @@ -1303,12 +1302,11 @@ max_objects_per_block (Maximum objects per block) int 64 # See https://www.sqlite.org/pragma.html#pragma_synchronous sqlite_synchronous (Synchronous SQLite) enum 2 0,1,2 -# ZLib compression level to use when saving mapblocks to disk. -# -1 - Zlib's default compression level -# 0 - no compresson, fastest +# Compression level to use when saving mapblocks to disk. +# -1 - use default compression level +# 0 - least compresson, fastest # 9 - best compression, slowest -# (levels 1-3 use Zlib's "fast" method, 4-9 use the normal method) -map_compression_level_disk (Map Compression Level for Disk Storage) int 3 -1 9 +map_compression_level_disk (Map Compression Level for Disk Storage) int -1 -1 9 # Length of a server tick and the interval at which objects are generally updated over # network. diff --git a/cmake/Modules/FindZstd.cmake b/cmake/Modules/FindZstd.cmake new file mode 100644 index 000000000..461e4d5a6 --- /dev/null +++ b/cmake/Modules/FindZstd.cmake @@ -0,0 +1,9 @@ +mark_as_advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR) + +find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) + +find_library(ZSTD_LIBRARY NAMES zstd) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Zstd DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR) + diff --git a/doc/world_format.txt b/doc/world_format.txt index a8a9e463e..eb1d7f728 100644 --- a/doc/world_format.txt +++ b/doc/world_format.txt @@ -1,5 +1,5 @@ ============================= -Minetest World Format 22...27 +Minetest World Format 22...29 ============================= This applies to a world format carrying the block serialization version @@ -8,6 +8,7 @@ This applies to a world format carrying the block serialization version - 0.4.0 (23) - 24 was never released as stable and existed for ~2 days - 27 was added in 0.4.15-dev +- 29 was added in 5.5.0-dev The block serialization version does not fully specify every aspect of this format; if compliance with this format is to be checked, it needs to be @@ -281,6 +282,8 @@ MapBlock serialization format NOTE: Byte order is MSB first (big-endian). NOTE: Zlib data is in such a format that Python's zlib at least can directly decompress. +NOTE: Since version 29 zstd is used instead of zlib. In addition the entire + block is first serialized and then compressed (except the version byte). u8 version - map format version number, see serialisation.h for the latest number @@ -324,6 +327,20 @@ u16 lighting_complete then Minetest will correct lighting in the day light bank when the block at (1, 0, 0) is also loaded. +if map format version >= 29: + u32 timestamp + - Timestamp when last saved, as seconds from starting the game. + - 0xffffffff = invalid/unknown timestamp, nothing should be done with the time + difference when loaded + + u16 num_name_id_mappings + foreach num_name_id_mappings + u16 id + u16 name_len + u8[name_len] name +if map format version < 29: + -- Nothing right here, timpstamp and node id mappings are serialized later + u8 content_width - Number of bytes in the content (param0) fields of nodes if map format version <= 23: @@ -335,7 +352,7 @@ u8 params_width - Number of bytes used for parameters per node - Always 2 -zlib-compressed node data: +node data (zlib-compressed if version < 29): if content_width == 1: - content: u8[4096]: param0 fields @@ -348,31 +365,31 @@ if content_width == 2: u8[4096]: param2 fields - The location of a node in each of those arrays is (z*16*16 + y*16 + x). -zlib-compressed node metadata list +node metadata list (zlib-compressed if version < 29): - content: -if map format version <= 22: - u16 version (=1) - u16 count of metadata - foreach count: - u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X) - u16 type_id - u16 content_size - u8[content_size] content of metadata. Format depends on type_id, see below. -if map format version >= 23: - u8 version -- Note: type was u16 for map format version <= 22 - -- = 1 for map format version < 28 - -- = 2 since map format version 28 - u16 count of metadata - foreach count: - u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X) - u32 num_vars - foreach num_vars: - u16 key_len - u8[key_len] key - u32 val_len - u8[val_len] value - u8 is_private -- only for version >= 2. 0 = not private, 1 = private - serialized inventory + if map format version <= 22: + u16 version (=1) + u16 count of metadata + foreach count: + u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X) + u16 type_id + u16 content_size + u8[content_size] content of metadata. Format depends on type_id, see below. + if map format version >= 23: + u8 version -- Note: type was u16 for map format version <= 22 + -- = 1 for map format version < 28 + -- = 2 since map format version 28 + u16 count of metadata + foreach count: + u16 position (p.Z*MAP_BLOCKSIZE*MAP_BLOCKSIZE + p.Y*MAP_BLOCKSIZE + p.X) + u32 num_vars + foreach num_vars: + u16 key_len + u8[key_len] key + u32 val_len + u8[val_len] value + u8 is_private -- only for version >= 2. 0 = not private, 1 = private + serialized inventory - Node timers if map format version == 23: @@ -403,20 +420,18 @@ foreach static_object_count: u16 data_size u8[data_size] data -u32 timestamp -- Timestamp when last saved, as seconds from starting the game. -- 0xffffffff = invalid/unknown timestamp, nothing should be done with the time - difference when loaded - -u8 name-id-mapping version -- Always 0 +if map format version < 29: + u32 timestamp + - Same meaning as the timestamp further up -u16 num_name_id_mappings + u8 name-id-mapping version + - Always 0 -foreach num_name_id_mappings - u16 id - u16 name_len - u8[name_len] name + u16 num_name_id_mappings + foreach num_name_id_mappings + u16 id + u16 name_len + u8[name_len] name - Node timers if map format version == 25: diff --git a/misc/debpkg-control b/misc/debpkg-control index 7c0134bb0..e867f3eb9 100644 --- a/misc/debpkg-control +++ b/misc/debpkg-control @@ -3,7 +3,7 @@ Priority: extra Standards-Version: 3.6.2 Package: minetest-staging Version: 5.4.0-DATEPLACEHOLDER -Depends: libc6, libcurl3-gnutls, libfreetype6, libgl1, JPEG_PLACEHOLDER, JSONCPP_PLACEHOLDER, LEVELDB_PLACEHOLDER, libopenal1, libpng16-16, libsqlite3-0, libstdc++6, libvorbisfile3, libx11-6, libxxf86vm1, zlib1g +Depends: libc6, libcurl3-gnutls, libfreetype6, libgl1, JPEG_PLACEHOLDER, JSONCPP_PLACEHOLDER, LEVELDB_PLACEHOLDER, libopenal1, libpng16-16, libsqlite3-0, libstdc++6, libvorbisfile3, libx11-6, libxxf86vm1, libzstd1, zlib1g Maintainer: Loic Blot Homepage: https://www.minetest.net/ Vcs-Git: https://github.com/minetest/minetest.git diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7a5e48b49..addb0af3f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -271,9 +271,13 @@ if(WIN32) find_path(ZLIB_INCLUDE_DIR "zlib.h" DOC "Zlib include directory") find_library(ZLIB_LIBRARIES "zlib" DOC "Path to zlib library") + find_path(ZSTD_INCLUDE_DIR "zstd.h" DOC "Zstd include directory") + find_library(ZSTD_LIBRARY "zstd" DOC "Path to zstd library") + # Dll's are automatically copied to the output directory by vcpkg when VCPKG_APPLOCAL_DEPS=ON if(NOT VCPKG_APPLOCAL_DEPS) find_file(ZLIB_DLL NAMES "zlib.dll" "zlib1.dll" DOC "Path to zlib.dll for installation (optional)") + find_file(ZSTD_DLL NAMES "zstd.dll" DOC "Path to zstd.dll for installation (optional)") if(ENABLE_SOUND) set(OPENAL_DLL "" CACHE FILEPATH "Path to OpenAL32.dll for installation (optional)") set(OGG_DLL "" CACHE FILEPATH "Path to libogg.dll for installation (optional)") @@ -296,6 +300,7 @@ else() endif() find_package(ZLIB REQUIRED) + find_package(Zstd REQUIRED) set(PLATFORM_LIBS -lpthread ${CMAKE_DL_LIBS}) if(APPLE) set(PLATFORM_LIBS "-framework CoreFoundation" ${PLATFORM_LIBS}) @@ -486,6 +491,7 @@ include_directories( ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} ${ZLIB_INCLUDE_DIR} + ${ZSTD_INCLUDE_DIR} ${SOUND_INCLUDE_DIRS} ${SQLITE3_INCLUDE_DIR} ${LUA_INCLUDE_DIR} @@ -521,6 +527,7 @@ if(BUILD_CLIENT) ${PROJECT_NAME} ${ZLIB_LIBRARIES} IrrlichtMt::IrrlichtMt + ${ZSTD_LIBRARY} ${X11_LIBRARIES} ${SOUND_LIBRARIES} ${SQLITE3_LIBRARY} @@ -605,6 +612,7 @@ if(BUILD_SERVER) target_link_libraries( ${PROJECT_NAME}server ${ZLIB_LIBRARIES} + ${ZSTD_LIBRARY} ${SQLITE3_LIBRARY} ${JSON_LIBRARY} ${LUA_LIBRARY} @@ -821,6 +829,9 @@ if(WIN32) if(ZLIB_DLL) install(FILES ${ZLIB_DLL} DESTINATION ${BINDIR}) endif() + if(ZSTD_DLL) + install(FILES ${ZSTD_DLL} DESTINATION ${BINDIR}) + endif() if(BUILD_CLIENT AND FREETYPE_DLL) install(FILES ${FREETYPE_DLL} DESTINATION ${BINDIR}) endif() diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp index faf839b3a..2cb345ba7 100644 --- a/src/defaultsettings.cpp +++ b/src/defaultsettings.cpp @@ -398,7 +398,7 @@ void set_default_settings() settings->setDefault("chat_message_limit_per_10sec", "8.0"); settings->setDefault("chat_message_limit_trigger_kick", "50"); settings->setDefault("sqlite_synchronous", "2"); - settings->setDefault("map_compression_level_disk", "3"); + settings->setDefault("map_compression_level_disk", "-1"); settings->setDefault("map_compression_level_net", "-1"); settings->setDefault("full_block_send_enable_min_time_from_building", "2.0"); settings->setDefault("dedicated_server_step", "0.09"); @@ -484,7 +484,7 @@ void set_default_settings() settings->setDefault("max_objects_per_block", "20"); settings->setDefault("sqlite_synchronous", "1"); settings->setDefault("map_compression_level_disk", "-1"); - settings->setDefault("map_compression_level_net", "3"); + settings->setDefault("map_compression_level_net", "-1"); settings->setDefault("server_map_save_interval", "15"); settings->setDefault("client_mapblock_limit", "1000"); settings->setDefault("active_block_range", "2"); diff --git a/src/main.cpp b/src/main.cpp index ffbdb7b5b..543b70333 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -38,6 +38,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "player.h" #include "porting.h" #include "network/socket.h" +#include "mapblock.h" #if USE_CURSES #include "terminal_chat_console.h" #endif @@ -111,6 +112,7 @@ static bool determine_subgame(GameParams *game_params); static bool run_dedicated_server(const GameParams &game_params, const Settings &cmd_args); static bool migrate_map_database(const GameParams &game_params, const Settings &cmd_args); +static bool recompress_map_database(const GameParams &game_params, const Settings &cmd_args, const Address &addr); /**********************************************************************/ @@ -302,6 +304,8 @@ static void set_allowed_options(OptionList *allowed_options) _("Migrate from current auth backend to another (Only works when using minetestserver or with --server)")))); allowed_options->insert(std::make_pair("terminal", ValueSpec(VALUETYPE_FLAG, _("Feature an interactive terminal (Only works when using minetestserver or with --server)")))); + allowed_options->insert(std::make_pair("recompress", ValueSpec(VALUETYPE_FLAG, + _("Recompress the blocks of the given map database.")))); #ifndef SERVER allowed_options->insert(std::make_pair("speedtests", ValueSpec(VALUETYPE_FLAG, _("Run speed tests")))); @@ -875,7 +879,7 @@ static bool run_dedicated_server(const GameParams &game_params, const Settings & return false; } - // Database migration + // Database migration/compression if (cmd_args.exists("migrate")) return migrate_map_database(game_params, cmd_args); @@ -885,6 +889,9 @@ static bool run_dedicated_server(const GameParams &game_params, const Settings & if (cmd_args.exists("migrate-auth")) return ServerEnvironment::migrateAuthDatabase(game_params, cmd_args); + if (cmd_args.getFlag("recompress")) + return recompress_map_database(game_params, cmd_args, bind_addr); + if (cmd_args.exists("terminal")) { #if USE_CURSES bool name_ok = true; @@ -1034,3 +1041,67 @@ static bool migrate_map_database(const GameParams &game_params, const Settings & return true; } + +static bool recompress_map_database(const GameParams &game_params, const Settings &cmd_args, const Address &addr) +{ + Settings world_mt; + const std::string world_mt_path = game_params.world_path + DIR_DELIM + "world.mt"; + + if (!world_mt.readConfigFile(world_mt_path.c_str())) { + errorstream << "Cannot read world.mt at " << world_mt_path << std::endl; + return false; + } + const std::string &backend = world_mt.get("backend"); + Server server(game_params.world_path, game_params.game_spec, false, addr, false); + MapDatabase *db = ServerMap::createDatabase(backend, game_params.world_path, world_mt); + + u32 count = 0; + u64 last_update_time = 0; + bool &kill = *porting::signal_handler_killstatus(); + const u8 serialize_as_ver = SER_FMT_VER_HIGHEST_WRITE; + + // This is ok because the server doesn't actually run + std::vector blocks; + db->listAllLoadableBlocks(blocks); + db->beginSave(); + std::istringstream iss(std::ios_base::binary); + std::ostringstream oss(std::ios_base::binary); + for (auto it = blocks.begin(); it != blocks.end(); ++it) { + if (kill) return false; + + std::string data; + db->loadBlock(*it, &data); + if (data.empty()) { + errorstream << "Failed to load block " << PP(*it) << std::endl; + return false; + } + + iss.str(data); + iss.clear(); + + MapBlock mb(nullptr, v3s16(0,0,0), &server); + u8 ver = readU8(iss); + mb.deSerialize(iss, ver, true); + + oss.str(""); + oss.clear(); + writeU8(oss, serialize_as_ver); + mb.serialize(oss, serialize_as_ver, true, -1); + + db->saveBlock(*it, oss.str()); + + count++; + if (count % 0xFF == 0 && porting::getTimeS() - last_update_time >= 1) { + std::cerr << " Recompressed " << count << " blocks, " + << (100.0f * count / blocks.size()) << "% completed.\r"; + db->endSave(); + db->beginSave(); + last_update_time = porting::getTimeS(); + } + } + std::cerr << std::endl; + db->endSave(); + + actionstream << "Done, " << count << " blocks were recompressed." << std::endl; + return true; +} diff --git a/src/mapblock.cpp b/src/mapblock.cpp index 0ca71e643..4958d3a65 100644 --- a/src/mapblock.cpp +++ b/src/mapblock.cpp @@ -355,7 +355,7 @@ static void correctBlockNodeIds(const NameIdMapping *nimap, MapNode *nodes, } } -void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compression_level) +void MapBlock::serialize(std::ostream &os_compressed, u8 version, bool disk, int compression_level) { if(!ser_ver_supported(version)) throw VersionMismatchException("ERROR: MapBlock format not supported"); @@ -365,6 +365,9 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compressio FATAL_ERROR_IF(version < SER_FMT_VER_LOWEST_WRITE, "Serialisation version error"); + std::ostringstream os_raw(std::ios_base::binary); + std::ostream &os = version >= 29 ? os_raw : os_compressed; + // First byte u8 flags = 0; if(is_underground) @@ -382,37 +385,52 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compressio Bulk node data */ NameIdMapping nimap; - if(disk) + SharedBuffer buf; + const u8 content_width = 2; + const u8 params_width = 2; + if(disk) { MapNode *tmp_nodes = new MapNode[nodecount]; - for(u32 i=0; indef()); - u8 content_width = 2; - u8 params_width = 2; - writeU8(os, content_width); - writeU8(os, params_width); - MapNode::serializeBulk(os, version, tmp_nodes, nodecount, - content_width, params_width, compression_level); + buf = MapNode::serializeBulk(version, tmp_nodes, nodecount, + content_width, params_width); delete[] tmp_nodes; + + // write timestamp and node/id mapping first + if (version >= 29) { + writeU32(os, getTimestamp()); + + nimap.serialize(os); + } } else { - u8 content_width = 2; - u8 params_width = 2; - writeU8(os, content_width); - writeU8(os, params_width); - MapNode::serializeBulk(os, version, data, nodecount, - content_width, params_width, compression_level); + buf = MapNode::serializeBulk(version, data, nodecount, + content_width, params_width); + } + + writeU8(os, content_width); + writeU8(os, params_width); + if (version >= 29) { + os.write(reinterpret_cast(*buf), buf.getSize()); + } else { + // prior to 29 node data was compressed individually + compress(buf, os, version, compression_level); } /* Node metadata */ - std::ostringstream oss(std::ios_base::binary); - m_node_metadata.serialize(oss, version, disk); - compressZlib(oss.str(), os, compression_level); + if (version >= 29) { + m_node_metadata.serialize(os, version, disk); + } else { + // use os_raw from above to avoid allocating another stream object + m_node_metadata.serialize(os_raw, version, disk); + // prior to 29 node data was compressed individually + compress(os_raw.str(), os, version, compression_level); + } /* Data that goes to disk, but not the network @@ -427,17 +445,24 @@ void MapBlock::serialize(std::ostream &os, u8 version, bool disk, int compressio // Static objects m_static_objects.serialize(os); - // Timestamp - writeU32(os, getTimestamp()); + if(version < 29){ + // Timestamp + writeU32(os, getTimestamp()); - // Write block-specific node definition id mapping - nimap.serialize(os); + // Write block-specific node definition id mapping + nimap.serialize(os); + } if(version >= 25){ // Node timers m_node_timers.serialize(os, version); } } + + if (version >= 29) { + // now compress the whole thing + compress(os_raw.str(), os_compressed, version, compression_level); + } } void MapBlock::serializeNetworkSpecific(std::ostream &os) @@ -449,7 +474,7 @@ void MapBlock::serializeNetworkSpecific(std::ostream &os) writeU8(os, 2); // version } -void MapBlock::deSerialize(std::istream &is, u8 version, bool disk) +void MapBlock::deSerialize(std::istream &in_compressed, u8 version, bool disk) { if(!ser_ver_supported(version)) throw VersionMismatchException("ERROR: MapBlock format not supported"); @@ -460,10 +485,16 @@ void MapBlock::deSerialize(std::istream &is, u8 version, bool disk) if(version <= 21) { - deSerialize_pre22(is, version, disk); + deSerialize_pre22(in_compressed, version, disk); return; } + // Decompress the whole block (version >= 29) + std::stringstream in_raw(std::ios_base::binary | std::ios_base::in | std::ios_base::out); + if (version >= 29) + decompress(in_compressed, in_raw, version); + std::istream &is = version >= 29 ? in_raw : in_compressed; + u8 flags = readU8(is); is_underground = (flags & 0x01) != 0; m_day_night_differs = (flags & 0x02) != 0; @@ -473,9 +504,20 @@ void MapBlock::deSerialize(std::istream &is, u8 version, bool disk) m_lighting_complete = readU16(is); m_generated = (flags & 0x08) == 0; - /* - Bulk node data - */ + NameIdMapping nimap; + if (disk && version >= 29) { + // Timestamp + TRACESTREAM(<<"MapBlock::deSerialize "<= 29) { + MapNode::deSerializeBulk(is, version, data, nodecount, content_width, params_width); + } else { + // use in_raw from above to avoid allocating another stream object + decompress(is, in_raw, version); + MapNode::deSerializeBulk(in_raw, version, data, nodecount, + content_width, params_width); + } /* NodeMetadata */ TRACESTREAM(<<"MapBlock::deSerialize "<= 23) - m_node_metadata.deSerialize(iss, m_gamedef->idef()); - else - content_nodemeta_deserialize_legacy(iss, - &m_node_metadata, &m_node_timers, - m_gamedef->idef()); - } catch(SerializationError &e) { - warningstream<<"MapBlock::deSerialize(): Ignoring an error" - <<" while deserializing node metadata at (" - <= 29) { + m_node_metadata.deSerialize(is, m_gamedef->idef()); + } else { + try { + // reuse in_raw + in_raw.str(""); + in_raw.clear(); + decompress(is, in_raw, version); + if (version >= 23) + m_node_metadata.deSerialize(in_raw, m_gamedef->idef()); + else + content_nodemeta_deserialize_legacy(in_raw, + &m_node_metadata, &m_node_timers, + m_gamedef->idef()); + } catch(SerializationError &e) { + warningstream<<"MapBlock::deSerialize(): Ignoring an error" + <<" while deserializing node metadata at (" + <= 25){ diff --git a/src/mapblock.h b/src/mapblock.h index 2e3eb0d76..8de631a29 100644 --- a/src/mapblock.h +++ b/src/mapblock.h @@ -473,7 +473,7 @@ public: // These don't write or read version by itself // Set disk to true for on-disk format, false for over-the-network format // Precondition: version >= SER_FMT_VER_LOWEST_WRITE - void serialize(std::ostream &os, u8 version, bool disk, int compression_level); + void serialize(std::ostream &result, u8 version, bool disk, int compression_level); // If disk == true: In addition to doing other things, will add // unknown blocks from id-name mapping to wndef void deSerialize(std::istream &is, u8 version, bool disk); diff --git a/src/mapgen/mg_schematic.cpp b/src/mapgen/mg_schematic.cpp index 848a43626..b9ba70302 100644 --- a/src/mapgen/mg_schematic.cpp +++ b/src/mapgen/mg_schematic.cpp @@ -339,7 +339,9 @@ bool Schematic::deserializeFromMts(std::istream *is) delete []schemdata; schemdata = new MapNode[nodecount]; - MapNode::deSerializeBulk(ss, SER_FMT_VER_HIGHEST_READ, schemdata, + std::stringstream d_ss(std::ios_base::binary | std::ios_base::in | std::ios_base::out); + decompress(ss, d_ss, MTSCHEM_MAPNODE_SER_FMT_VER); + MapNode::deSerializeBulk(d_ss, MTSCHEM_MAPNODE_SER_FMT_VER, schemdata, nodecount, 2, 2); // Fix probability values for nodes that were ignore; removed in v2 @@ -384,8 +386,9 @@ bool Schematic::serializeToMts(std::ostream *os) const } // compressed bulk node data - MapNode::serializeBulk(ss, SER_FMT_VER_HIGHEST_WRITE, - schemdata, size.X * size.Y * size.Z, 2, 2, -1); + SharedBuffer buf = MapNode::serializeBulk(MTSCHEM_MAPNODE_SER_FMT_VER, + schemdata, size.X * size.Y * size.Z, 2, 2); + compress(buf, ss, MTSCHEM_MAPNODE_SER_FMT_VER); return true; } diff --git a/src/mapgen/mg_schematic.h b/src/mapgen/mg_schematic.h index 5f64ea280..9189bb3a7 100644 --- a/src/mapgen/mg_schematic.h +++ b/src/mapgen/mg_schematic.h @@ -70,6 +70,7 @@ class Server; #define MTSCHEM_FILE_SIGNATURE 0x4d54534d // 'MTSM' #define MTSCHEM_FILE_VER_HIGHEST_READ 4 #define MTSCHEM_FILE_VER_HIGHEST_WRITE 4 +#define MTSCHEM_MAPNODE_SER_FMT_VER 28 // Fixed serialization version for schematics since these still need to use Zlib #define MTSCHEM_PROB_MASK 0x7F diff --git a/src/mapnode.cpp b/src/mapnode.cpp index f212ea8c9..73bd620fb 100644 --- a/src/mapnode.cpp +++ b/src/mapnode.cpp @@ -730,9 +730,10 @@ void MapNode::deSerialize(u8 *source, u8 version) } } } -void MapNode::serializeBulk(std::ostream &os, int version, + +SharedBuffer MapNode::serializeBulk(int version, const MapNode *nodes, u32 nodecount, - u8 content_width, u8 params_width, int compression_level) + u8 content_width, u8 params_width) { if (!ser_ver_supported(version)) throw VersionMismatchException("ERROR: MapNode format not supported"); @@ -746,8 +747,7 @@ void MapNode::serializeBulk(std::ostream &os, int version, throw SerializationError("MapNode::serializeBulk: serialization to " "version < 24 not possible"); - size_t databuf_size = nodecount * (content_width + params_width); - u8 *databuf = new u8[databuf_size]; + SharedBuffer databuf(nodecount * (content_width + params_width)); u32 start1 = content_width * nodecount; u32 start2 = (content_width + 1) * nodecount; @@ -758,14 +758,7 @@ void MapNode::serializeBulk(std::ostream &os, int version, writeU8(&databuf[start1 + i], nodes[i].param1); writeU8(&databuf[start2 + i], nodes[i].param2); } - - /* - Compress data to output stream - */ - - compressZlib(databuf, databuf_size, os, compression_level); - - delete [] databuf; + return databuf; } // Deserialize bulk node data @@ -781,15 +774,10 @@ void MapNode::deSerializeBulk(std::istream &is, int version, || params_width != 2) FATAL_ERROR("Deserialize bulk node data error"); - // Uncompress or read data - u32 len = nodecount * (content_width + params_width); - std::ostringstream os(std::ios_base::binary); - decompressZlib(is, os); - std::string s = os.str(); - if(s.size() != len) - throw SerializationError("deSerializeBulkNodes: " - "decompress resulted in invalid size"); - const u8 *databuf = reinterpret_cast(s.c_str()); + // read data + const u32 len = nodecount * (content_width + params_width); + Buffer databuf(len); + is.read(reinterpret_cast(*databuf), len); // Deserialize content if(content_width == 1) diff --git a/src/mapnode.h b/src/mapnode.h index 28ff9e43d..afd3a96be 100644 --- a/src/mapnode.h +++ b/src/mapnode.h @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "irrlichttypes_bloated.h" #include "light.h" +#include "util/pointer.h" #include #include @@ -293,9 +294,9 @@ struct MapNode // content_width = the number of bytes of content per node // params_width = the number of bytes of params per node // compressed = true to zlib-compress output - static void serializeBulk(std::ostream &os, int version, + static SharedBuffer serializeBulk(int version, const MapNode *nodes, u32 nodecount, - u8 content_width, u8 params_width, int compression_level); + u8 content_width, u8 params_width); static void deSerializeBulk(std::istream &is, int version, MapNode *nodes, u32 nodecount, u8 content_width, u8 params_width); diff --git a/src/serialization.cpp b/src/serialization.cpp index 310604f54..b6ce3b37f 100644 --- a/src/serialization.cpp +++ b/src/serialization.cpp @@ -21,7 +21,8 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "util/serialize.h" -#include "zlib.h" +#include +#include /* report a zlib or i/o error */ void zerr(int ret) @@ -197,27 +198,133 @@ void decompressZlib(std::istream &is, std::ostream &os, size_t limit) inflateEnd(&z); } -void compress(const SharedBuffer &data, std::ostream &os, u8 version) +struct ZSTD_Deleter { + void operator() (ZSTD_CStream* cstream) { + ZSTD_freeCStream(cstream); + } + + void operator() (ZSTD_DStream* dstream) { + ZSTD_freeDStream(dstream); + } +}; + +void compressZstd(const u8 *data, size_t data_size, std::ostream &os, int level) +{ + // reusing the context is recommended for performance + // it will destroyed when the thread ends + thread_local std::unique_ptr stream(ZSTD_createCStream()); + + ZSTD_initCStream(stream.get(), level); + + const size_t bufsize = 16384; + char output_buffer[bufsize]; + + ZSTD_inBuffer input = { data, data_size, 0 }; + ZSTD_outBuffer output = { output_buffer, bufsize, 0 }; + + while (input.pos < input.size) { + size_t ret = ZSTD_compressStream(stream.get(), &output, &input); + if (ZSTD_isError(ret)) { + dstream << ZSTD_getErrorName(ret) << std::endl; + throw SerializationError("compressZstd: failed"); + } + if (output.pos) { + os.write(output_buffer, output.pos); + output.pos = 0; + } + } + + size_t ret; + do { + ret = ZSTD_endStream(stream.get(), &output); + if (ZSTD_isError(ret)) { + dstream << ZSTD_getErrorName(ret) << std::endl; + throw SerializationError("compressZstd: failed"); + } + if (output.pos) { + os.write(output_buffer, output.pos); + output.pos = 0; + } + } while (ret != 0); + +} + +void compressZstd(const std::string &data, std::ostream &os, int level) { + compressZstd((u8*)data.c_str(), data.size(), os, level); +} + +void decompressZstd(std::istream &is, std::ostream &os) +{ + // reusing the context is recommended for performance + // it will destroyed when the thread ends + thread_local std::unique_ptr stream(ZSTD_createDStream()); + + ZSTD_initDStream(stream.get()); + + const size_t bufsize = 16384; + char output_buffer[bufsize]; + char input_buffer[bufsize]; + + ZSTD_outBuffer output = { output_buffer, bufsize, 0 }; + ZSTD_inBuffer input = { input_buffer, 0, 0 }; + size_t ret; + do + { + if (input.size == input.pos) { + is.read(input_buffer, bufsize); + input.size = is.gcount(); + input.pos = 0; + } + + ret = ZSTD_decompressStream(stream.get(), &output, &input); + if (ZSTD_isError(ret)) { + dstream << ZSTD_getErrorName(ret) << std::endl; + throw SerializationError("decompressZstd: failed"); + } + if (output.pos) { + os.write(output_buffer, output.pos); + output.pos = 0; + } + } while (ret != 0); + + // Unget all the data that ZSTD_decompressStream didn't take + is.clear(); // Just in case EOF is set + for (u32 i = 0; i < input.size - input.pos; i++) { + is.unget(); + if (is.fail() || is.bad()) + throw SerializationError("decompressZstd: unget failed"); + } +} + +void compress(u8 *data, u32 size, std::ostream &os, u8 version, int level) +{ + if(version >= 29) + { + // map the zlib levels [0,9] to [1,10]. -1 becomes 0 which indicates the default (currently 3) + compressZstd(data, size, os, level + 1); + return; + } + if(version >= 11) { - compressZlib(*data ,data.getSize(), os); + compressZlib(data, size, os, level); return; } - if(data.getSize() == 0) + if(size == 0) return; // Write length (u32) u8 tmp[4]; - writeU32(tmp, data.getSize()); + writeU32(tmp, size); os.write((char*)tmp, 4); // We will be writing 8-bit pairs of more_count and byte u8 more_count = 0; u8 current_byte = data[0]; - for(u32 i=1; i &data, std::ostream &os, u8 version) os.write((char*)¤t_byte, 1); } +void compress(const SharedBuffer &data, std::ostream &os, u8 version, int level) +{ + compress(*data, data.getSize(), os, version, level); +} + +void compress(const std::string &data, std::ostream &os, u8 version, int level) +{ + compress((u8*)data.c_str(), data.size(), os, version, level); +} + void decompress(std::istream &is, std::ostream &os, u8 version) { + if(version >= 29) + { + decompressZstd(is, os); + return; + } + if(version >= 11) { decompressZlib(is, os); diff --git a/src/serialization.h b/src/serialization.h index f399983c4..e83a8c179 100644 --- a/src/serialization.h +++ b/src/serialization.h @@ -63,13 +63,14 @@ with this program; if not, write to the Free Software Foundation, Inc., 26: Never written; read the same as 25 27: Added light spreading flags to blocks 28: Added "private" flag to NodeMetadata + 29: Switched compression to zstd, a bit of reorganization */ // This represents an uninitialized or invalid format #define SER_FMT_VER_INVALID 255 // Highest supported serialization version -#define SER_FMT_VER_HIGHEST_READ 28 +#define SER_FMT_VER_HIGHEST_READ 29 // Saved on disk version -#define SER_FMT_VER_HIGHEST_WRITE 28 +#define SER_FMT_VER_HIGHEST_WRITE 29 // Lowest supported serialization version #define SER_FMT_VER_LOWEST_READ 0 // Lowest serialization version for writing @@ -89,7 +90,12 @@ void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level void compressZlib(const std::string &data, std::ostream &os, int level = -1); void decompressZlib(std::istream &is, std::ostream &os, size_t limit = 0); +void compressZstd(const u8 *data, size_t data_size, std::ostream &os, int level = 0); +void compressZstd(const std::string &data, std::ostream &os, int level = 0); +void decompressZstd(std::istream &is, std::ostream &os); + // These choose between zlib and a self-made one according to version -void compress(const SharedBuffer &data, std::ostream &os, u8 version); -//void compress(const std::string &data, std::ostream &os, u8 version); +void compress(const SharedBuffer &data, std::ostream &os, u8 version, int level = -1); +void compress(const std::string &data, std::ostream &os, u8 version, int level = -1); +void compress(u8 *data, u32 size, std::ostream &os, u8 version, int level = -1); void decompress(std::istream &is, std::ostream &os, u8 version); diff --git a/src/unittest/test_compression.cpp b/src/unittest/test_compression.cpp index dfcadd4b2..a96282f58 100644 --- a/src/unittest/test_compression.cpp +++ b/src/unittest/test_compression.cpp @@ -37,6 +37,7 @@ public: void testRLECompression(); void testZlibCompression(); void testZlibLargeData(); + void testZstdLargeData(); void testZlibLimit(); void _testZlibLimit(u32 size, u32 limit); }; @@ -48,6 +49,7 @@ void TestCompression::runTests(IGameDef *gamedef) TEST(testRLECompression); TEST(testZlibCompression); TEST(testZlibLargeData); + TEST(testZstdLargeData); TEST(testZlibLimit); } @@ -111,7 +113,7 @@ void TestCompression::testZlibCompression() fromdata[3]=1; std::ostringstream os(std::ios_base::binary); - compress(fromdata, os, SER_FMT_VER_HIGHEST_READ); + compressZlib(*fromdata, fromdata.getSize(), os); std::string str_out = os.str(); @@ -124,7 +126,7 @@ void TestCompression::testZlibCompression() std::istringstream is(str_out, std::ios_base::binary); std::ostringstream os2(std::ios_base::binary); - decompress(is, os2, SER_FMT_VER_HIGHEST_READ); + decompressZlib(is, os2); std::string str_out2 = os2.str(); infostream << "decompress: "; @@ -174,6 +176,42 @@ void TestCompression::testZlibLargeData() } } +void TestCompression::testZstdLargeData() +{ + infostream << "Test: Testing zstd wrappers with a large amount " + "of pseudorandom data" << std::endl; + + u32 size = 500000; + infostream << "Test: Input size of large compressZstd is " + << size << std::endl; + + std::string data_in; + data_in.resize(size); + PseudoRandom pseudorandom(9420); + for (u32 i = 0; i < size; i++) + data_in[i] = pseudorandom.range(0, 255); + + std::ostringstream os_compressed(std::ios::binary); + compressZstd(data_in, os_compressed, 0); + infostream << "Test: Output size of large compressZstd is " + << os_compressed.str().size()<