From 2628316842e9c3fc1c54c6d57505c7851fa83490 Mon Sep 17 00:00:00 2001 From: nia <29542929+alarixnia@users.noreply.github.com> Date: Sun, 19 Sep 2021 18:23:52 +0000 Subject: Fix src/util/string.cpp on NetBSD - iconv() prototype changed from traditional Unix defintion to POSIX definition in 9.99.x. - wchar_t is not a valid character set for iconv. Share code with Android for using UTF-32. --- src/util/string.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'src/util/string.cpp') diff --git a/src/util/string.cpp b/src/util/string.cpp index eec5ab4cd..8be5e320a 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -39,8 +39,13 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #endif -#if defined(_ICONV_H_) && (defined(__FreeBSD__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) || defined(__DragonFly__)) +#ifdef __NetBSD__ + #include + #if __NetBSD_Version__ <= 999001500 + #define BSD_ICONV_USED + #endif +#elif defined(_ICONV_H_) && (defined(__FreeBSD__) || defined(__OpenBSD__) || \ + defined(__DragonFly__)) #define BSD_ICONV_USED #endif @@ -79,6 +84,14 @@ static bool convert(const char *to, const char *from, char *outbuf, #ifdef __ANDROID__ // On Android iconv disagrees how big a wchar_t is for whatever reason const char *DEFAULT_ENCODING = "UTF-32LE"; +#elif defined(__NetBSD__) + // NetBSD does not allow "WCHAR_T" as a charset input to iconv. + #include + #if BYTE_ORDER == BIG_ENDIAN + const char *DEFAULT_ENCODING = "UTF-32BE"; + #else + const char *DEFAULT_ENCODING = "UTF-32LE"; + #endif #else const char *DEFAULT_ENCODING = "WCHAR_T"; #endif @@ -94,7 +107,7 @@ std::wstring utf8_to_wide(const std::string &input) std::wstring out; out.resize(outbuf_size / sizeof(wchar_t)); -#ifdef __ANDROID__ +#if defined(__ANDROID__) || defined(__NetBSD__) SANITY_CHECK(sizeof(wchar_t) == 4); #endif -- cgit v1.2.3 From 29d2b2ccd06cdd831a7fac66928bfa612003945c Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Sat, 1 Jan 2022 16:44:56 -0500 Subject: Print announce error response (#11878) Fix HTTPFetch caller and request ID to 64 bits Check that allocated caller ID is not DISCARD Print body if serverlist request returns error Don't print control characters from HTTP responses Document special HTTPFetch caller IDs Allow unicode to be printed --- src/client/clientmedia.h | 6 +++--- src/httpfetch.cpp | 51 ++++++++++++++++++++++++++---------------------- src/httpfetch.h | 27 +++++++++++++++---------- src/serverlist.cpp | 1 + src/util/string.cpp | 16 +++++++++++++++ src/util/string.h | 8 ++++++++ 6 files changed, 73 insertions(+), 36 deletions(-) (limited to 'src/util/string.cpp') diff --git a/src/client/clientmedia.h b/src/client/clientmedia.h index aa7b0f398..c297d737f 100644 --- a/src/client/clientmedia.h +++ b/src/client/clientmedia.h @@ -174,12 +174,12 @@ private: s32 m_uncached_received_count = 0; // Status of remote transfers - unsigned long m_httpfetch_caller; - unsigned long m_httpfetch_next_id = 0; + u64 m_httpfetch_caller; + u64 m_httpfetch_next_id = 0; s32 m_httpfetch_active = 0; s32 m_httpfetch_active_limit = 0; s32 m_outstanding_hash_sets = 0; - std::unordered_map m_remote_file_transfers; + std::unordered_map m_remote_file_transfers; // All files up to this name have either been received from a // remote server or failed on all remote servers, so those files diff --git a/src/httpfetch.cpp b/src/httpfetch.cpp index 1307bfec3..16f0791c9 100644 --- a/src/httpfetch.cpp +++ b/src/httpfetch.cpp @@ -38,7 +38,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "noise.h" static std::mutex g_httpfetch_mutex; -static std::unordered_map> +static std::unordered_map> g_httpfetch_results; static PcgRandom g_callerid_randomness; @@ -52,22 +52,21 @@ HTTPFetchRequest::HTTPFetchRequest() : static void httpfetch_deliver_result(const HTTPFetchResult &fetch_result) { - unsigned long caller = fetch_result.caller; + u64 caller = fetch_result.caller; if (caller != HTTPFETCH_DISCARD) { MutexAutoLock lock(g_httpfetch_mutex); g_httpfetch_results[caller].emplace(fetch_result); } } -static void httpfetch_request_clear(unsigned long caller); +static void httpfetch_request_clear(u64 caller); -unsigned long httpfetch_caller_alloc() +u64 httpfetch_caller_alloc() { MutexAutoLock lock(g_httpfetch_mutex); - // Check each caller ID except HTTPFETCH_DISCARD - const unsigned long discard = HTTPFETCH_DISCARD; - for (unsigned long caller = discard + 1; caller != discard; ++caller) { + // Check each caller ID except reserved ones + for (u64 caller = HTTPFETCH_CID_START; caller != 0; ++caller) { auto it = g_httpfetch_results.find(caller); if (it == g_httpfetch_results.end()) { verbosestream << "httpfetch_caller_alloc: allocating " @@ -79,18 +78,17 @@ unsigned long httpfetch_caller_alloc() } FATAL_ERROR("httpfetch_caller_alloc: ran out of caller IDs"); - return discard; } -unsigned long httpfetch_caller_alloc_secure() +u64 httpfetch_caller_alloc_secure() { MutexAutoLock lock(g_httpfetch_mutex); // Generate random caller IDs and make sure they're not - // already used or equal to HTTPFETCH_DISCARD + // already used or reserved. // Give up after 100 tries to prevent infinite loop - u8 tries = 100; - unsigned long caller; + size_t tries = 100; + u64 caller; do { caller = (((u64) g_callerid_randomness.next()) << 32) | @@ -100,7 +98,8 @@ unsigned long httpfetch_caller_alloc_secure() FATAL_ERROR("httpfetch_caller_alloc_secure: ran out of caller IDs"); return HTTPFETCH_DISCARD; } - } while (g_httpfetch_results.find(caller) != g_httpfetch_results.end()); + } while (caller >= HTTPFETCH_CID_START && + g_httpfetch_results.find(caller) != g_httpfetch_results.end()); verbosestream << "httpfetch_caller_alloc_secure: allocating " << caller << std::endl; @@ -110,7 +109,7 @@ unsigned long httpfetch_caller_alloc_secure() return caller; } -void httpfetch_caller_free(unsigned long caller) +void httpfetch_caller_free(u64 caller) { verbosestream<<"httpfetch_caller_free: freeing " <= 400) { + errorstream << "HTTPFetch for " << request.url + << " returned response code " << result.response_code << std::endl; + if (result.caller == HTTPFETCH_PRINT_ERR && !result.data.empty()) { + errorstream << "Response body:" << std::endl; + safe_print_string(errorstream, result.data); + errorstream << std::endl; + } } return &result; @@ -474,7 +479,7 @@ public: m_requests.push_back(req); } - void requestClear(unsigned long caller, Event *event) + void requestClear(u64 caller, Event *event) { Request req; req.type = RT_CLEAR; @@ -505,7 +510,7 @@ protected: } else if (req.type == RT_CLEAR) { - unsigned long caller = req.fetch_request.caller; + u64 caller = req.fetch_request.caller; // Abort all ongoing fetches for the caller for (std::vector::iterator @@ -778,7 +783,7 @@ void httpfetch_async(const HTTPFetchRequest &fetch_request) g_httpfetch_thread->start(); } -static void httpfetch_request_clear(unsigned long caller) +static void httpfetch_request_clear(u64 caller) { if (g_httpfetch_thread->isRunning()) { Event event; @@ -827,7 +832,7 @@ void httpfetch_async(const HTTPFetchRequest &fetch_request) httpfetch_deliver_result(fetch_result); } -static void httpfetch_request_clear(unsigned long caller) +static void httpfetch_request_clear(u64 caller) { } diff --git a/src/httpfetch.h b/src/httpfetch.h index 3b9f17f0a..a4901e63b 100644 --- a/src/httpfetch.h +++ b/src/httpfetch.h @@ -23,10 +23,17 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "util/string.h" #include "config.h" -// Can be used in place of "caller" in asynchronous transfers to discard result -// (used as default value of "caller") +// These can be used in place of "caller" in to specify special handling. +// Discard result (used as default value of "caller"). #define HTTPFETCH_DISCARD 0 +// Indicates that the result should not be discarded when performing a +// synchronous request (since a real caller ID is not needed for synchronous +// requests because the result does not have to be retrieved later). #define HTTPFETCH_SYNC 1 +// Print response body to console if the server returns an error code. +#define HTTPFETCH_PRINT_ERR 2 +// Start of regular allocated caller IDs. +#define HTTPFETCH_CID_START 3 // Methods enum HttpMethod : u8 @@ -43,11 +50,11 @@ struct HTTPFetchRequest // Identifies the caller (for asynchronous requests) // Ignored by httpfetch_sync - unsigned long caller = HTTPFETCH_DISCARD; + u64 caller = HTTPFETCH_DISCARD; // Some number that identifies the request // (when the same caller issues multiple httpfetch_async calls) - unsigned long request_id = 0; + u64 request_id = 0; // Timeout for the whole transfer, in milliseconds long timeout; @@ -85,8 +92,8 @@ struct HTTPFetchResult long response_code = 0; std::string data = ""; // The caller and request_id from the corresponding HTTPFetchRequest. - unsigned long caller = HTTPFETCH_DISCARD; - unsigned long request_id = 0; + u64 caller = HTTPFETCH_DISCARD; + u64 request_id = 0; HTTPFetchResult() = default; @@ -107,19 +114,19 @@ void httpfetch_async(const HTTPFetchRequest &fetch_request); // If any fetch for the given caller ID is complete, removes it from the // result queue, sets the fetch result and returns true. Otherwise returns false. -bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetch_result); +bool httpfetch_async_get(u64 caller, HTTPFetchResult &fetch_result); // Allocates a caller ID for httpfetch_async // Not required if you want to set caller = HTTPFETCH_DISCARD -unsigned long httpfetch_caller_alloc(); +u64 httpfetch_caller_alloc(); // Allocates a non-predictable caller ID for httpfetch_async -unsigned long httpfetch_caller_alloc_secure(); +u64 httpfetch_caller_alloc_secure(); // Frees a caller ID allocated with httpfetch_caller_alloc // Note: This can be expensive, because the httpfetch thread is told // to stop any ongoing fetches for the given caller. -void httpfetch_caller_free(unsigned long caller); +void httpfetch_caller_free(u64 caller); // Performs a synchronous HTTP request. This blocks and therefore should // only be used from background threads. diff --git a/src/serverlist.cpp b/src/serverlist.cpp index 3bcab3d58..29e3ac9a6 100644 --- a/src/serverlist.cpp +++ b/src/serverlist.cpp @@ -97,6 +97,7 @@ void sendAnnounce(AnnounceAction action, } HTTPFetchRequest fetch_request; + fetch_request.caller = HTTPFETCH_PRINT_ERR; fetch_request.url = g_settings->get("serverlist_url") + std::string("/announce"); fetch_request.method = HTTP_POST; fetch_request.fields["json"] = fastWriteJson(server); diff --git a/src/util/string.cpp b/src/util/string.cpp index 8be5e320a..bc4664997 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -887,3 +887,19 @@ std::string sanitizeDirName(const std::string &str, const std::string &optional_ return wide_to_utf8(safe_name); } + + +void safe_print_string(std::ostream &os, const std::string &str) +{ + std::ostream::fmtflags flags = os.flags(); + os << std::hex; + for (const char c : str) { + if (IS_ASCII_PRINTABLE_CHAR(c) || IS_UTF8_MULTB_START(c) || + IS_UTF8_MULTB_INNER(c) || c == '\n' || c == '\t') { + os << c; + } else { + os << '<' << std::setw(2) << (int)c << '>'; + } + } + os.setf(flags); +} diff --git a/src/util/string.h b/src/util/string.h index bca998f56..8a9e83f22 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -753,3 +753,11 @@ inline irr::core::stringw utf8_to_stringw(const std::string &input) * 2. Remove 'unsafe' characters from the name by replacing them with '_' */ std::string sanitizeDirName(const std::string &str, const std::string &optional_prefix); + +/** + * Prints a sanitized version of a string without control characters. + * '\t' and '\n' are allowed, as are UTF-8 control characters (e.g. RTL). + * ASCII control characters are replaced with their hex encoding in angle + * brackets (e.g. "a\x1eb" -> "a<1e>b"). + */ +void safe_print_string(std::ostream &os, const std::string &str); -- cgit v1.2.3 From 00ebedad933244dbb866198b5769309ea011719c Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Sat, 29 Jan 2022 22:47:17 -0500 Subject: Add additional reserved directory names --- src/util/string.cpp | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) (limited to 'src/util/string.cpp') diff --git a/src/util/string.cpp b/src/util/string.cpp index bc4664997..689f58f7f 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -821,9 +821,11 @@ std::wstring translate_string(const std::wstring &s) #endif } -static const std::array disallowed_dir_names = { +static const std::array disallowed_dir_names = { // Problematic filenames from here: // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#file-and-directory-names + // Plus undocumented values from here: + // https://googleprojectzero.blogspot.com/2016/02/the-definitive-guide-on-win32-to-nt.html L"CON", L"PRN", L"AUX", @@ -837,6 +839,9 @@ static const std::array disallowed_dir_names = { L"COM7", L"COM8", L"COM9", + L"COM\u00B2", + L"COM\u00B3", + L"COM\u00B9", L"LPT1", L"LPT2", L"LPT3", @@ -846,6 +851,11 @@ static const std::array disallowed_dir_names = { L"LPT7", L"LPT8", L"LPT9", + L"LPT\u00B2", + L"LPT\u00B3", + L"LPT\u00B9", + L"CONIN$", + L"CONOUT$", }; /** @@ -853,6 +863,21 @@ static const std::array disallowed_dir_names = { */ static const std::wstring disallowed_path_chars = L"<>:\"/\\|?*."; + +/** + * @param str + * @return A copy of \p str with trailing whitespace removed. + */ +static std::wstring wrtrim(const std::wstring &str) +{ + size_t back = str.size(); + while (back > 0 && std::isspace(str[back - 1])) + --back; + + return str.substr(0, back); +} + + /** * Sanitize the name of a new directory. This consists of two stages: * 1. Check for 'reserved filenames' that can't be used on some filesystems @@ -863,8 +888,10 @@ std::string sanitizeDirName(const std::string &str, const std::string &optional_ { std::wstring safe_name = utf8_to_wide(str); + std::wstring dev_name = wrtrim(safe_name); + for (std::wstring disallowed_name : disallowed_dir_names) { - if (str_equal(safe_name, disallowed_name, true)) { + if (str_equal(dev_name, disallowed_name, true)) { safe_name = utf8_to_wide(optional_prefix) + safe_name; break; } -- cgit v1.2.3 From dae6fe91a1059751a0bde504cdf41a749234ce1a Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Mon, 31 Jan 2022 21:11:51 -0500 Subject: Update directory name sanitization Only ASCII spaces have to be handles specially, and leading spaces are also disallowed. --- src/unittest/test_utilities.cpp | 12 ++++++++---- src/util/string.cpp | 39 ++++++++++++++------------------------- src/util/string.h | 2 +- 3 files changed, 23 insertions(+), 30 deletions(-) (limited to 'src/util/string.cpp') diff --git a/src/unittest/test_utilities.cpp b/src/unittest/test_utilities.cpp index 228a9559f..10ea8d36a 100644 --- a/src/unittest/test_utilities.cpp +++ b/src/unittest/test_utilities.cpp @@ -636,8 +636,12 @@ void TestUtilities::testBase64() void TestUtilities::testSanitizeDirName() { - UASSERT(sanitizeDirName("a", "_") == "a"); - UASSERT(sanitizeDirName("COM1", "_") == "_COM1"); - UASSERT(sanitizeDirName("cOm\u00B2 .txt:a", "_") == "cOm\u00B2 _txt_a"); - UASSERT(sanitizeDirName("cOnIn$ ", "_") == "_cOnIn$ "); + UASSERT(sanitizeDirName("a", "~") == "a"); + UASSERT(sanitizeDirName(" ", "~") == "__"); + UASSERT(sanitizeDirName(" a ", "~") == "_a_"); + UASSERT(sanitizeDirName("COM1", "~") == "~COM1"); + UASSERT(sanitizeDirName("COM1", ":") == "_COM1"); + UASSERT(sanitizeDirName("cOm\u00B2", "~") == "~cOm\u00B2"); + UASSERT(sanitizeDirName("cOnIn$", "~") == "~cOnIn$"); + UASSERT(sanitizeDirName(" cOnIn$ ", "~") == "_cOnIn$_"); } diff --git a/src/util/string.cpp b/src/util/string.cpp index 689f58f7f..39cd44667 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -864,40 +864,29 @@ static const std::array disallowed_dir_names = { static const std::wstring disallowed_path_chars = L"<>:\"/\\|?*."; -/** - * @param str - * @return A copy of \p str with trailing whitespace removed. - */ -static std::wstring wrtrim(const std::wstring &str) -{ - size_t back = str.size(); - while (back > 0 && std::isspace(str[back - 1])) - --back; - - return str.substr(0, back); -} - - -/** - * Sanitize the name of a new directory. This consists of two stages: - * 1. Check for 'reserved filenames' that can't be used on some filesystems - * and add a prefix to them - * 2. Remove 'unsafe' characters from the name by replacing them with '_' - */ std::string sanitizeDirName(const std::string &str, const std::string &optional_prefix) { std::wstring safe_name = utf8_to_wide(str); - std::wstring dev_name = wrtrim(safe_name); - for (std::wstring disallowed_name : disallowed_dir_names) { - if (str_equal(dev_name, disallowed_name, true)) { + if (str_equal(safe_name, disallowed_name, true)) { safe_name = utf8_to_wide(optional_prefix) + safe_name; break; } } - for (unsigned long i = 0; i < safe_name.length(); i++) { + // Replace leading and trailing spaces with underscores. + size_t start = safe_name.find_first_not_of(L' '); + size_t end = safe_name.find_last_not_of(L' '); + if (start == std::wstring::npos || end == std::wstring::npos) + start = end = safe_name.size(); + for (size_t i = 0; i < start; i++) + safe_name[i] = L'_'; + for (size_t i = end + 1; i < safe_name.size(); i++) + safe_name[i] = L'_'; + + // Replace other disallowed characters with underscores + for (size_t i = 0; i < safe_name.length(); i++) { bool is_valid = true; // Unlikely, but control characters should always be blacklisted @@ -909,7 +898,7 @@ std::string sanitizeDirName(const std::string &str, const std::string &optional_ } if (!is_valid) - safe_name[i] = '_'; + safe_name[i] = L'_'; } return wide_to_utf8(safe_name); diff --git a/src/util/string.h b/src/util/string.h index f4ca1a7de..d8ec633ee 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -749,7 +749,7 @@ inline irr::core::stringw utf8_to_stringw(const std::string &input) /** * Sanitize the name of a new directory. This consists of two stages: * 1. Check for 'reserved filenames' that can't be used on some filesystems - * and prefix them + * and add a prefix to them * 2. Remove 'unsafe' characters from the name by replacing them with '_' */ std::string sanitizeDirName(const std::string &str, const std::string &optional_prefix); -- cgit v1.2.3 From fccf1e2eac691443108de8f666d611327dbfcfb3 Mon Sep 17 00:00:00 2001 From: Lars Müller <34514239+appgurueu@users.noreply.github.com> Date: Wed, 27 Apr 2022 23:00:02 +0200 Subject: Support CSS Color Module Level 4 (#12204) --- doc/lua_api.txt | 2 +- src/util/string.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'src/util/string.cpp') diff --git a/doc/lua_api.txt b/doc/lua_api.txt index f53ab0ff7..f54672db7 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -3248,7 +3248,7 @@ Colors `#RRGGBBAA` defines a color in hexadecimal format and alpha channel. Named colors are also supported and are equivalent to -[CSS Color Module Level 3](https://www.w3.org/TR/css-color-3/). +[CSS Color Module Level 4](https://www.w3.org/TR/css-color-4/#named-color). To specify the value of the alpha channel, append `#A` or `#AA` to the end of the color name (e.g. `colorname#08`). diff --git a/src/util/string.cpp b/src/util/string.cpp index 39cd44667..b805b2f78 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -494,6 +494,7 @@ const static std::unordered_map s_named_colors = { {"plum", 0xdda0dd}, {"powderblue", 0xb0e0e6}, {"purple", 0x800080}, + {"rebeccapurple", 0x663399}, {"red", 0xff0000}, {"rosybrown", 0xbc8f8f}, {"royalblue", 0x4169e1}, -- cgit v1.2.3