Merge branch 'master' of https://github.com/minetest/minetest

17 files changed, 340 insertions, 66 deletions

diff --git a/src/script/cpp_api/s_async.cpp b/src/script/cpp_api/s_async.cpp
index dacdcd75a..42a794ceb 100644
--- a/src/script/cpp_api/s_async.cpp
+++ b/src/script/cpp_api/s_async.cpp
@@ -21,9 +21,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include <cstdlib>
 
 extern "C" {
-#include "lua.h"
-#include "lauxlib.h"
-#include "lualib.h"
+#include <lua.h>
+#include <lauxlib.h>
+#include <lualib.h>
 }
 
 #include "server.h"
@@ -32,6 +32,7 @@ extern "C" {
 #include "filesys.h"
 #include "porting.h"
 #include "common/c_internal.h"
+#include "common/c_packer.h"
 #include "lua_api/l_base.h"
 
 /******************************************************************************/
@@ -76,19 +77,34 @@ void AsyncEngine::initialize(unsigned int numEngines)
 {
 	initDone = true;
 
-	for (unsigned int i = 0; i < numEngines; i++) {
-		AsyncWorkerThread *toAdd = new AsyncWorkerThread(this,
-			std::string("AsyncWorker-") + itos(i));
-		workerThreads.push_back(toAdd);
-		toAdd->start();
+	if (numEngines == 0) {
+		// Leave one core for the main thread and one for whatever else
+		autoscaleMaxWorkers = Thread::getNumberOfProcessors();
+		if (autoscaleMaxWorkers >= 2)
+			autoscaleMaxWorkers -= 2;
+		infostream << "AsyncEngine: using at most " << autoscaleMaxWorkers
+			<< " threads with automatic scaling" << std::endl;
+
+		addWorkerThread();
+	} else {
+		for (unsigned int i = 0; i < numEngines; i++)
+			addWorkerThread();
 	}
 }
 
+void AsyncEngine::addWorkerThread()
+{
+	AsyncWorkerThread *toAdd = new AsyncWorkerThread(this,
+		std::string("AsyncWorker-") + itos(workerThreads.size()));
+	workerThreads.push_back(toAdd);
+	toAdd->start();
+}
+
 /******************************************************************************/
 u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &&params,
 		const std::string &mod_origin)
 {
-	jobQueueMutex.lock();
+	MutexAutoLock autolock(jobQueueMutex);
 	u32 jobId = jobIdCounter++;
 
 	jobQueue.emplace_back();
@@ -99,7 +115,23 @@ u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &&params,
 	to_add.mod_origin = mod_origin;
 
 	jobQueueCounter.post();
-	jobQueueMutex.unlock();
+	return jobId;
+}
+
+u32 AsyncEngine::queueAsyncJob(std::string &&func, PackedValue *params,
+		const std::string &mod_origin)
+{
+	MutexAutoLock autolock(jobQueueMutex);
+	u32 jobId = jobIdCounter++;
+
+	jobQueue.emplace_back();
+	auto &to_add = jobQueue.back();
+	to_add.id = jobId;
+	to_add.function = std::move(func);
+	to_add.params_ext.reset(params);
+	to_add.mod_origin = mod_origin;
+
+	jobQueueCounter.post();
 	return jobId;
 }
 
@@ -132,6 +164,12 @@ void AsyncEngine::putJobResult(LuaJobInfo &&result)
 /******************************************************************************/
 void AsyncEngine::step(lua_State *L)
 {
+	stepJobResults(L);
+	stepAutoscale();
+}
+
+void AsyncEngine::stepJobResults(lua_State *L)
+{
 	int error_handler = PUSH_ERROR_HANDLER(L);
 	lua_getglobal(L, "core");
 
@@ -148,7 +186,10 @@ void AsyncEngine::step(lua_State *L)
 		luaL_checktype(L, -1, LUA_TFUNCTION);
 
 		lua_pushinteger(L, j.id);
-		lua_pushlstring(L, j.result.data(), j.result.size());
+		if (j.result_ext)
+			script_unpack(L, j.result_ext.get());
+		else
+			lua_pushlstring(L, j.result.data(), j.result.size());
 
 		// Call handler
 		const char *origin = j.mod_origin.empty() ? nullptr : j.mod_origin.c_str();
@@ -161,12 +202,71 @@ void AsyncEngine::step(lua_State *L)
 	lua_pop(L, 2); // Pop core and error handler
 }
 
+void AsyncEngine::stepAutoscale()
+{
+	if (workerThreads.size() >= autoscaleMaxWorkers)
+		return;
+
+	MutexAutoLock autolock(jobQueueMutex);
+
+	// 2) If the timer elapsed, check again
+	if (autoscaleTimer && porting::getTimeMs() >= autoscaleTimer) {
+		autoscaleTimer = 0;
+		// Determine overlap with previous snapshot
+		unsigned int n = 0;
+		for (const auto &it : jobQueue)
+			n += autoscaleSeenJobs.count(it.id);
+		autoscaleSeenJobs.clear();
+		infostream << "AsyncEngine: " << n << " jobs were still waiting after 1s" << std::endl;
+		// Start this many new threads
+		while (workerThreads.size() < autoscaleMaxWorkers && n > 0) {
+			addWorkerThread();
+			n--;
+		}
+		return;
+	}
+
+	// 1) Check if there's anything in the queue
+	if (!autoscaleTimer && !jobQueue.empty()) {
+		// Take a snapshot of all jobs we have seen
+		for (const auto &it : jobQueue)
+			autoscaleSeenJobs.emplace(it.id);
+		// and set a timer for 1 second
+		autoscaleTimer = porting::getTimeMs() + 1000;
+	}
+}
+
 /******************************************************************************/
-void AsyncEngine::prepareEnvironment(lua_State* L, int top)
+bool AsyncEngine::prepareEnvironment(lua_State* L, int top)
 {
 	for (StateInitializer &stateInitializer : stateInitializers) {
 		stateInitializer(L, top);
 	}
+
+	auto *script = ModApiBase::getScriptApiBase(L);
+	try {
+		script->loadMod(Server::getBuiltinLuaPath() + DIR_DELIM + "init.lua",
+			BUILTIN_MOD_NAME);
+	} catch (const ModError &e) {
+		errorstream << "Execution of async base environment failed: "
+			<< e.what() << std::endl;
+		FATAL_ERROR("Execution of async base environment failed");
+	}
+
+	// Load per mod stuff
+	if (server) {
+		const auto &list = server->m_async_init_files;
+		try {
+			for (auto &it : list)
+				script->loadMod(it.second, it.first);
+		} catch (const ModError &e) {
+			errorstream << "Failed to load mod script inside async environment." << std::endl;
+			server->setAsyncFatalError(e.what());
+			return false;
+		}
+	}
+
+	return true;
 }
 
 /******************************************************************************/
@@ -178,15 +278,25 @@ AsyncWorkerThread::AsyncWorkerThread(AsyncEngine* jobDispatcher,
 {
 	lua_State *L = getStack();
 
+	if (jobDispatcher->server) {
+		setGameDef(jobDispatcher->server);
+
+		if (g_settings->getBool("secure.enable_security"))
+			initializeSecurity();
+	}
+
 	// Prepare job lua environment
 	lua_getglobal(L, "core");
 	int top = lua_gettop(L);
 
 	// Push builtin initialization type
-	lua_pushstring(L, "async");
+	lua_pushstring(L, jobDispatcher->server ? "async_game" : "async");
 	lua_setglobal(L, "INIT");
 
-	jobDispatcher->prepareEnvironment(L, top);
+	if (!jobDispatcher->prepareEnvironment(L, top)) {
+		// can't throw from here so we're stuck with this
+		isErrored = true;
+	}
 }
 
 /******************************************************************************/
@@ -198,19 +308,20 @@ AsyncWorkerThread::~AsyncWorkerThread()
 /******************************************************************************/
 void* AsyncWorkerThread::run()
 {
-	lua_State *L = getStack();
+	if (isErrored)
+		return nullptr;
 
-	try {
-		loadMod(getServer()->getBuiltinLuaPath() + DIR_DELIM + "init.lua",
-			BUILTIN_MOD_NAME);
-	} catch (const ModError &e) {
-		errorstream << "Execution of async base environment failed: "
-			<< e.what() << std::endl;
-		FATAL_ERROR("Execution of async base environment failed");
-	}
+	lua_State *L = getStack();
 
 	int error_handler = PUSH_ERROR_HANDLER(L);
 
+	auto report_error = [this] (const ModError &e) {
+		if (jobDispatcher->server)
+			jobDispatcher->server->setAsyncFatalError(e.what());
+		else
+			errorstream << e.what() << std::endl;
+	};
+
 	lua_getglobal(L, "core");
 	if (lua_isnil(L, -1)) {
 		FATAL_ERROR("Unable to find core within async environment!");
@@ -223,6 +334,8 @@ void* AsyncWorkerThread::run()
 		if (!jobDispatcher->getJob(&j) || stopRequested())
 			continue;
 
+		const bool use_ext = !!j.params_ext;
+
 		lua_getfield(L, -1, "job_processor");
 		if (lua_isnil(L, -1))
 			FATAL_ERROR("Unable to get async job processor!");
@@ -232,7 +345,10 @@ void* AsyncWorkerThread::run()
 			errorstream << "ASYNC WORKER: Unable to deserialize function" << std::endl;
 			lua_pushnil(L);
 		}
-		lua_pushlstring(L, j.params.data(), j.params.size());
+		if (use_ext)
+			script_unpack(L, j.params_ext.get());
+		else
+			lua_pushlstring(L, j.params.data(), j.params.size());
 
 		// Call it
 		setOriginDirect(j.mod_origin.empty() ? nullptr : j.mod_origin.c_str());
@@ -241,19 +357,28 @@ void* AsyncWorkerThread::run()
 			try {
 				scriptError(result, "<async>");
 			} catch (const ModError &e) {
-				errorstream << e.what() << std::endl;
+				report_error(e);
 			}
 		} else {
 			// Fetch result
-			size_t length;
-			const char *retval = lua_tolstring(L, -1, &length);
-			j.result.assign(retval, length);
+			if (use_ext) {
+				try {
+					j.result_ext.reset(script_pack(L, -1));
+				} catch (const ModError &e) {
+					report_error(e);
+					result = LUA_ERRERR;
+				}
+			} else {
+				size_t length;
+				const char *retval = lua_tolstring(L, -1, &length);
+				j.result.assign(retval, length);
+			}
 		}
 
 		lua_pop(L, 1);  // Pop retval
 
 		// Put job result
-		if (!j.result.empty())
+		if (result == 0)
 			jobDispatcher->putJobResult(std::move(j));
 	}
 
diff --git a/src/script/cpp_api/s_async.h b/src/script/cpp_api/s_async.h
index 697cb0221..1e34e40ea 100644
--- a/src/script/cpp_api/s_async.h
+++ b/src/script/cpp_api/s_async.h
@@ -21,11 +21,15 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include <vector>
 #include <deque>
+#include <unordered_set>
+#include <memory>
 
+#include <lua.h>
 #include "threading/semaphore.h"
 #include "threading/thread.h"
-#include "lua.h"
+#include "common/c_packer.h"
 #include "cpp_api/s_base.h"
+#include "cpp_api/s_security.h"
 
 // Forward declarations
 class AsyncEngine;
@@ -42,8 +46,12 @@ struct LuaJobInfo
 	std::string function;
 	// Parameter to be passed to function (serialized)
 	std::string params;
+	// Alternative parameters
+	std::unique_ptr<PackedValue> params_ext;
 	// Result of function call (serialized)
 	std::string result;
+	// Alternative result
+	std::unique_ptr<PackedValue> result_ext;
 	// Name of the mod who invoked this call
 	std::string mod_origin;
 	// JobID used to identify a job and match it to callback
@@ -51,7 +59,8 @@ struct LuaJobInfo
 };
 
 // Asynchronous working environment
-class AsyncWorkerThread : public Thread, virtual public ScriptApiBase {
+class AsyncWorkerThread : public Thread,
+	virtual public ScriptApiBase, public ScriptApiSecurity {
 	friend class AsyncEngine;
 public:
 	virtual ~AsyncWorkerThread();
@@ -63,6 +72,7 @@ protected:
 
 private:
 	AsyncEngine *jobDispatcher = nullptr;
+	bool isErrored = false;
 };
 
 // Asynchornous thread and job management
@@ -71,6 +81,7 @@ class AsyncEngine {
 	typedef void (*StateInitializer)(lua_State *L, int top);
 public:
 	AsyncEngine() = default;
+	AsyncEngine(Server *server) : server(server) {};
 	~AsyncEngine();
 
 	/**
@@ -81,7 +92,7 @@ public:
 
 	/**
 	 * Create async engine tasks and lock function registration
-	 * @param numEngines Number of async threads to be started
+	 * @param numEngines Number of worker threads, 0 for automatic scaling
 	 */
 	void initialize(unsigned int numEngines);
 
@@ -95,8 +106,16 @@ public:
 			const std::string &mod_origin = "");
 
 	/**
+	 * Queue an async job
+	 * @param func Serialized lua function
+	 * @param params Serialized parameters (takes ownership!)
+	 * @return ID of queued job
+	 */
+	u32 queueAsyncJob(std::string &&func, PackedValue *params,
+			const std::string &mod_origin = "");
+
+	/**
 	 * Engine step to process finished jobs
-	 *   the engine step is one way to pass events back, PushFinishedJobs another
 	 * @param L The Lua stack
 	 */
 	void step(lua_State *L);
@@ -117,18 +136,43 @@ protected:
 	void putJobResult(LuaJobInfo &&result);
 
 	/**
+	 * Start an additional worker thread
+	 */
+	void addWorkerThread();
+
+	/**
+	 * Process finished jobs callbacks
+	 */
+	void stepJobResults(lua_State *L);
+
+	/**
+	 * Handle automatic scaling of worker threads
+	 */
+	void stepAutoscale();
+
+	/**
 	 * Initialize environment with current registred functions
 	 *  this function adds all functions registred by registerFunction to the
 	 *  passed lua stack
 	 * @param L Lua stack to initialize
 	 * @param top Stack position
+	 * @return false if a mod error ocurred
 	 */
-	void prepareEnvironment(lua_State* L, int top);
+	bool prepareEnvironment(lua_State* L, int top);
 
 private:
 	// Variable locking the engine against further modification
 	bool initDone = false;
 
+	// Maximum number of worker threads for automatic scaling
+	// 0 if disabled
+	unsigned int autoscaleMaxWorkers = 0;
+	u64 autoscaleTimer = 0;
+	std::unordered_set<u32> autoscaleSeenJobs;
+
+	// Only set for the server async environment (duh)
+	Server *server = nullptr;
+
 	// Internal store for registred state initializers
 	std::vector<StateInitializer> stateInitializers;
 
diff --git a/src/script/cpp_api/s_base.cpp b/src/script/cpp_api/s_base.cpp
index 27d730b1d..ae4a16771 100644
--- a/src/script/cpp_api/s_base.cpp
+++ b/src/script/cpp_api/s_base.cpp
@@ -38,6 +38,8 @@ extern "C" {
 #include "lualib.h"
 #if USE_LUAJIT
 	#include "luajit.h"
+#else
+	#include "bit.h"
 #endif
 }
 
@@ -89,6 +91,11 @@ ScriptApiBase::ScriptApiBase(ScriptingType type):
 	else*/
 		luaL_openlibs(m_luastack);
 
+	// Load bit library
+	lua_pushcfunction(m_luastack, luaopen_bit);
+	lua_pushstring(m_luastack, LUA_BITLIBNAME);
+	lua_call(m_luastack, 1, 0);
+
 	// Make the ScriptApiBase* accessible to ModApiBase
 #if INDIRECT_SCRIPTAPI_RIDX
 	*(void **)(lua_newuserdata(m_luastack, sizeof(void *))) = this;
@@ -115,6 +122,14 @@ ScriptApiBase::ScriptApiBase(ScriptingType type):
 	lua_newtable(m_luastack);
 	lua_setglobal(m_luastack, "core");
 
+	// vector.metatable is stored in the registry for quick access from C++.
+	lua_newtable(m_luastack);
+	lua_rawseti(m_luastack, LUA_REGISTRYINDEX, CUSTOM_RIDX_VECTOR_METATABLE);
+	lua_newtable(m_luastack);
+	lua_rawgeti(m_luastack, LUA_REGISTRYINDEX, CUSTOM_RIDX_VECTOR_METATABLE);
+	lua_setfield(m_luastack, -2, "metatable");
+	lua_setglobal(m_luastack, "vector");
+
 	if (m_type == ScriptingType::Client)
 		lua_pushstring(m_luastack, "/");
 	else
diff --git a/src/script/cpp_api/s_base.h b/src/script/cpp_api/s_base.h
index e49745f4e..19ae8783b 100644
--- a/src/script/cpp_api/s_base.h
+++ b/src/script/cpp_api/s_base.h
@@ -128,6 +128,15 @@ protected:
 	friend class ModApiEnvMod;
 	friend class LuaVoxelManip;
 
+	/*
+		Subtle edge case with coroutines: If for whatever reason you have a
+		method in a subclass that's called from existing lua_CFunction
+		(any of the l_*.cpp files) then make it static and take the lua_State*
+		as an argument. This is REQUIRED because getStack() will not return the
+		correct state if called inside coroutines.
+
+		Also note that src/script/common/ is the better place for such helpers.
+	*/
 	lua_State* getStack()
 		{ return m_luastack; }
 
diff --git a/src/script/cpp_api/s_client.cpp b/src/script/cpp_api/s_client.cpp
index 5d20f547d..e54a1361d 100644
--- a/src/script/cpp_api/s_client.cpp
+++ b/src/script/cpp_api/s_client.cpp
@@ -399,7 +399,7 @@ bool ScriptApiClient::on_inventory_open(Inventory *inventory)
 	lua_getglobal(L, "core");
 	lua_getfield(L, -1, "registered_on_inventory_open");
 
-	push_inventory(L, inventory);
+	push_inventory_lists(L, *inventory);
 
 	try {
 		runCallbacks(1, RUN_CALLBACKS_MODE_OR);
diff --git a/src/script/cpp_api/s_entity.cpp b/src/script/cpp_api/s_entity.cpp
index 746f7013e..06337b9e8 100644
--- a/src/script/cpp_api/s_entity.cpp
+++ b/src/script/cpp_api/s_entity.cpp
@@ -240,7 +240,7 @@ void ScriptApiEntity::luaentity_Step(u16 id, float dtime,
 //                       tool_capabilities, direction, damage)
 bool ScriptApiEntity::luaentity_Punch(u16 id,
 		ServerActiveObject *puncher, float time_from_last_punch,
-		const ToolCapabilities *toolcap, v3f dir, s16 damage)
+		const ToolCapabilities *toolcap, v3f dir, s32 damage)
 {
 	SCRIPTAPI_PRECHECKHEADER
 
diff --git a/src/script/cpp_api/s_entity.h b/src/script/cpp_api/s_entity.h
index b52f6e447..7658ae922 100644
--- a/src/script/cpp_api/s_entity.h
+++ b/src/script/cpp_api/s_entity.h
@@ -42,7 +42,7 @@ public:
 		const collisionMoveResult *moveresult);
 	bool luaentity_Punch(u16 id,
 			ServerActiveObject *puncher, float time_from_last_punch,
-			const ToolCapabilities *toolcap, v3f dir, s16 damage);
+			const ToolCapabilities *toolcap, v3f dir, s32 damage);
 	bool luaentity_on_death(u16 id, ServerActiveObject *killer);
 	void luaentity_Rightclick(u16 id, ServerActiveObject *clicker);
 	void luaentity_on_attach_child(u16 id, ServerActiveObject *child);
diff --git a/src/script/cpp_api/s_env.cpp b/src/script/cpp_api/s_env.cpp
index 874c37b6e..af68f689f 100644
--- a/src/script/cpp_api/s_env.cpp
+++ b/src/script/cpp_api/s_env.cpp
@@ -140,10 +140,10 @@ void ScriptApiEnv::initializeEnvironment(ServerEnvironment *env)
 
 		bool simple_catch_up = true;
 		getboolfield(L, current_abm, "catch_up", simple_catch_up);
-		
+
 		s16 min_y = INT16_MIN;
 		getintfield(L, current_abm, "min_y", min_y);
-		
+
 		s16 max_y = INT16_MAX;
 		getintfield(L, current_abm, "max_y", max_y);
 
diff --git a/src/script/cpp_api/s_item.cpp b/src/script/cpp_api/s_item.cpp
index 48dce14f3..b1916070e 100644
--- a/src/script/cpp_api/s_item.cpp
+++ b/src/script/cpp_api/s_item.cpp
@@ -59,13 +59,14 @@ bool ScriptApiItem::item_OnDrop(ItemStack &item,
 	return true;
 }
 
-bool ScriptApiItem::item_OnPlace(ItemStack &item,
+bool ScriptApiItem::item_OnPlace(Optional<ItemStack> &ret_item,
 		ServerActiveObject *placer, const PointedThing &pointed)
 {
 	SCRIPTAPI_PRECHECKHEADER
 
 	int error_handler = PUSH_ERROR_HANDLER(L);
 
+	const ItemStack &item = *ret_item;
 	// Push callback function on stack
 	if (!getItemCallback(item.name.c_str(), "on_place"))
 		return false;
@@ -82,22 +83,25 @@ bool ScriptApiItem::item_OnPlace(ItemStack &item,
 	PCALL_RES(lua_pcall(L, 3, 1, error_handler));
 	if (!lua_isnil(L, -1)) {
 		try {
-			item = read_item(L, -1, getServer()->idef());
+			ret_item = read_item(L, -1, getServer()->idef());
 		} catch (LuaError &e) {
 			throw WRAP_LUAERROR(e, "item=" + item.name);
 		}
+	} else {
+		ret_item = nullopt;
 	}
 	lua_pop(L, 2);  // Pop item and error handler
 	return true;
 }
 
-bool ScriptApiItem::item_OnUse(ItemStack &item,
+bool ScriptApiItem::item_OnUse(Optional<ItemStack> &ret_item,
 		ServerActiveObject *user, const PointedThing &pointed)
 {
 	SCRIPTAPI_PRECHECKHEADER
 
 	int error_handler = PUSH_ERROR_HANDLER(L);
 
+	const ItemStack &item = *ret_item;
 	// Push callback function on stack
 	if (!getItemCallback(item.name.c_str(), "on_use"))
 		return false;
@@ -109,22 +113,25 @@ bool ScriptApiItem::item_OnUse(ItemStack &item,
 	PCALL_RES(lua_pcall(L, 3, 1, error_handler));
 	if(!lua_isnil(L, -1)) {
 		try {
-			item = read_item(L, -1, getServer()->idef());
+			ret_item = read_item(L, -1, getServer()->idef());
 		} catch (LuaError &e) {
 			throw WRAP_LUAERROR(e, "item=" + item.name);
 		}
+	} else {
+		ret_item = nullopt;
 	}
 	lua_pop(L, 2);  // Pop item and error handler
 	return true;
 }
 
-bool ScriptApiItem::item_OnSecondaryUse(ItemStack &item,
+bool ScriptApiItem::item_OnSecondaryUse(Optional<ItemStack> &ret_item,
 		ServerActiveObject *user, const PointedThing &pointed)
 {
 	SCRIPTAPI_PRECHECKHEADER
 
 	int error_handler = PUSH_ERROR_HANDLER(L);
 
+	const ItemStack &item = *ret_item;
 	if (!getItemCallback(item.name.c_str(), "on_secondary_use"))
 		return false;
 
@@ -134,10 +141,12 @@ bool ScriptApiItem::item_OnSecondaryUse(ItemStack &item,
 	PCALL_RES(lua_pcall(L, 3, 1, error_handler));
 	if (!lua_isnil(L, -1)) {
 		try {
-			item = read_item(L, -1, getServer()->idef());
+			ret_item = read_item(L, -1, getServer()->idef());
 		} catch (LuaError &e) {
 			throw WRAP_LUAERROR(e, "item=" + item.name);
 		}
+	} else {
+		ret_item = nullopt;
 	}
 	lua_pop(L, 2);  // Pop item and error handler
 	return true;
diff --git a/src/script/cpp_api/s_item.h b/src/script/cpp_api/s_item.h
index 25a3501f9..5015d8bd4 100644
--- a/src/script/cpp_api/s_item.h
+++ b/src/script/cpp_api/s_item.h
@@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include "cpp_api/s_base.h"
 #include "irr_v3d.h"
+#include "util/Optional.h"
 
 struct PointedThing;
 struct ItemStack;
@@ -35,13 +36,20 @@ class ScriptApiItem
 : virtual public ScriptApiBase
 {
 public:
+	/*
+	 * Functions with Optional<ItemStack> are for callbacks where Lua may
+	 * want to prevent the engine from modifying the inventory after it's done.
+	 * This has a longer backstory where on_use may need to empty the player's
+	 * inventory without the engine interfering (see issue #6546).
+	 */
+
 	bool item_OnDrop(ItemStack &item,
 			ServerActiveObject *dropper, v3f pos);
-	bool item_OnPlace(ItemStack &item,
+	bool item_OnPlace(Optional<ItemStack> &item,
 			ServerActiveObject *placer, const PointedThing &pointed);
-	bool item_OnUse(ItemStack &item,
+	bool item_OnUse(Optional<ItemStack> &item,
 			ServerActiveObject *user, const PointedThing &pointed);
-	bool item_OnSecondaryUse(ItemStack &item,
+	bool item_OnSecondaryUse(Optional<ItemStack> &item,
 			ServerActiveObject *user, const PointedThing &pointed);
 	bool item_OnCraft(ItemStack &item, ServerActiveObject *user,
 			const InventoryList *old_craft_grid, const InventoryLocation &craft_inv);
diff --git a/src/script/cpp_api/s_node.h b/src/script/cpp_api/s_node.h
index 3f771c838..3c6a8445b 100644
--- a/src/script/cpp_api/s_node.h
+++ b/src/script/cpp_api/s_node.h
@@ -53,6 +53,7 @@ public:
 	static struct EnumString es_ContentParamType[];
 	static struct EnumString es_ContentParamType2[];
 	static struct EnumString es_LiquidType[];
+	static struct EnumString es_LiquidMoveType[];
 	static struct EnumString es_NodeBoxType[];
 	static struct EnumString es_TextureAlphaMode[];
 };
diff --git a/src/script/cpp_api/s_player.cpp b/src/script/cpp_api/s_player.cpp
index d3e6138dc..22b24f363 100644
--- a/src/script/cpp_api/s_player.cpp
+++ b/src/script/cpp_api/s_player.cpp
@@ -60,7 +60,7 @@ bool ScriptApiPlayer::on_punchplayer(ServerActiveObject *player,
 		float time_from_last_punch,
 		const ToolCapabilities *toolcap,
 		v3f dir,
-		s16 damage)
+		s32 damage)
 {
 	SCRIPTAPI_PRECHECKHEADER
 	// Get core.registered_on_punchplayers
diff --git a/src/script/cpp_api/s_player.h b/src/script/cpp_api/s_player.h
index c0f141862..e866aee46 100644
--- a/src/script/cpp_api/s_player.h
+++ b/src/script/cpp_api/s_player.h
@@ -46,7 +46,7 @@ public:
 	void on_cheat(ServerActiveObject *player, const std::string &cheat_type);
 	bool on_punchplayer(ServerActiveObject *player, ServerActiveObject *hitter,
 			float time_from_last_punch, const ToolCapabilities *toolcap,
-			v3f dir, s16 damage);
+			v3f dir, s32 damage);
 	void on_rightclickplayer(ServerActiveObject *player, ServerActiveObject *clicker);
 	s32 on_player_hpchange(ServerActiveObject *player, s32 hp_change,
 			const PlayerHPChangeReason &reason);
diff --git a/src/script/cpp_api/s_security.cpp b/src/script/cpp_api/s_security.cpp
index bd9c80e15..76509038f 100644
--- a/src/script/cpp_api/s_security.cpp
+++ b/src/script/cpp_api/s_security.cpp
@@ -27,6 +27,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 
 #include <cerrno>
 #include <string>
+#include <algorithm>
 #include <iostream>
 
 
@@ -97,6 +98,7 @@ void ScriptApiSecurity::initializeSecurity()
 		"type",
 		"unpack",
 		"_VERSION",
+		"vector",
 		"xpcall",
 	};
 	static const char *whitelist_tables[] = {
@@ -106,6 +108,7 @@ void ScriptApiSecurity::initializeSecurity()
 		"string",
 		"table",
 		"math",
+		"bit"
 	};
 	static const char *io_whitelist[] = {
 		"open",
@@ -120,21 +123,17 @@ void ScriptApiSecurity::initializeSecurity()
 		"date",
 		"difftime",
 		"getenv",
-		"setlocale",
 		"time",
-		"tmpname",
 	};
 	static const char *debug_whitelist[] = {
 		"gethook",
 		"traceback",
 		"getinfo",
 		"getmetatable",
-		"setupvalue",
 		"setmetatable",
 		"upvalueid",
 		"sethook",
 		"debug",
-		"setlocal",
 	};
 	static const char *package_whitelist[] = {
 		"config",
@@ -220,6 +219,7 @@ void ScriptApiSecurity::initializeSecurity()
 	// And replace unsafe ones
 	SECURE_API(os, remove);
 	SECURE_API(os, rename);
+	SECURE_API(os, setlocale);
 
 	lua_setglobal(L, "os");
 	lua_pop(L, 1);  // Pop old OS
@@ -251,6 +251,15 @@ void ScriptApiSecurity::initializeSecurity()
 	lua_pop(L, 1);  // Pop old jit
 #endif
 
+	// Get rid of 'core' in the old globals, we don't want anyone thinking it's
+	// safe or even usable.
+	lua_pushnil(L);
+	lua_setfield(L, old_globals, "core");
+
+	// 'vector' as well.
+	lua_pushnil(L);
+	lua_setfield(L, old_globals, "vector");
+
 	lua_pop(L, 1); // Pop globals_backup
 
 
@@ -286,19 +295,21 @@ void ScriptApiSecurity::initializeSecurityClient()
 		"rawset",
 		"select",
 		"setfenv",
-		// getmetatable can be used to escape the sandbox
+		// getmetatable can be used to escape the sandbox <- ???
 		"setmetatable",
 		"tonumber",
 		"tostring",
 		"type",
 		"unpack",
 		"_VERSION",
+		"vector",
 		"xpcall",
 		// Completely safe libraries
 		"coroutine",
 		"string",
 		"table",
 		"math",
+		"bit",
 	};
 	static const char *os_whitelist[] = {
 		"clock",
@@ -307,7 +318,7 @@ void ScriptApiSecurity::initializeSecurityClient()
 		"time"
 	};
 	static const char *debug_whitelist[] = {
-		"getinfo",
+		"getinfo", // used by builtin and unset before mods load
 		"traceback"
 	};
 #if USE_LUAJIT
@@ -607,6 +618,38 @@ bool ScriptApiSecurity::checkPath(lua_State *L, const char *path,
 	return false;
 }
 
+bool ScriptApiSecurity::checkWhitelisted(lua_State *L, const std::string &setting)
+{
+	assert(str_starts_with(setting, "secure."));
+
+	// We have to make sure that this function is being called directly by
+	// a mod, otherwise a malicious mod could override this function and
+	// steal its return value.
+	lua_Debug info;
+
+	// Make sure there's only one item below this function on the stack...
+	if (lua_getstack(L, 2, &info))
+		return false;
+	FATAL_ERROR_IF(!lua_getstack(L, 1, &info), "lua_getstack() failed");
+	FATAL_ERROR_IF(!lua_getinfo(L, "S", &info), "lua_getinfo() failed");
+
+	// ...and that that item is the main file scope.
+	if (strcmp(info.what, "main") != 0)
+		return false;
+
+	// Mod must be listed in secure.http_mods or secure.trusted_mods
+	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME);
+	if (!lua_isstring(L, -1))
+		return false;
+	std::string mod_name = readParam<std::string>(L, -1);
+
+	std::string value = g_settings->get(setting);
+	value.erase(std::remove(value.begin(), value.end(), ' '), value.end());
+	auto mod_list = str_split(value, ',');
+
+	return CONTAINS(mod_list, mod_name);
+}
+
 
 int ScriptApiSecurity::sl_g_dofile(lua_State *L)
 {
@@ -838,3 +881,20 @@ int ScriptApiSecurity::sl_os_remove(lua_State *L)
 	return 2;
 }
 
+
+int ScriptApiSecurity::sl_os_setlocale(lua_State *L)
+{
+	const bool cat = lua_gettop(L) > 1;
+	// Don't allow changes
+	if (!lua_isnoneornil(L, 1)) {
+		lua_pushnil(L);
+		return 1;
+	}
+
+	push_original(L, "os", "setlocale");
+	lua_pushnil(L);
+	if (cat)
+		lua_pushvalue(L, 2);
+	lua_call(L, cat ? 2 : 1, 1);
+	return 1;
+}
diff --git a/src/script/cpp_api/s_security.h b/src/script/cpp_api/s_security.h
index 73e763548..880ce1638 100644
--- a/src/script/cpp_api/s_security.h
+++ b/src/script/cpp_api/s_security.h
@@ -40,11 +40,6 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 class ScriptApiSecurity : virtual public ScriptApiBase
 {
 public:
-	int getThread(lua_State *L);
-	// creates an empty Lua environment
-	void createEmptyEnv(lua_State *L);
-	// sets the enviroment to the table thats on top of the stack
-	void setLuaEnv(lua_State *L, int thread);
 	// Sets up security on the ScriptApi's Lua state
 	void initializeSecurity();
 	void initializeSecurityClient();
@@ -57,8 +52,17 @@ public:
 	// Checks if mods are allowed to read (and optionally write) to the path
 	static bool checkPath(lua_State *L, const char *path, bool write_required,
 			bool *write_allowed=NULL);
+	// Check if mod is whitelisted in the given setting
+	// This additionally checks that the mod's main file scope is executing.
+	static bool checkWhitelisted(lua_State *L, const std::string &setting);
 
 private:
+	int getThread(lua_State *L);
+	// sets the enviroment to the table thats on top of the stack
+	void setLuaEnv(lua_State *L, int thread);
+	// creates an empty Lua environment
+	void createEmptyEnv(lua_State *L);
+
 	// Syntax: "sl_" <Library name or 'g' (global)> '_' <Function name>
 	// (sl stands for Secure Lua)
 
@@ -75,4 +79,5 @@ private:
 
 	static int sl_os_rename(lua_State *L);
 	static int sl_os_remove(lua_State *L);
+	static int sl_os_setlocale(lua_State *L);
 };
diff --git a/src/script/cpp_api/s_server.cpp b/src/script/cpp_api/s_server.cpp
index 6ddb2630d..c255b0c71 100644
--- a/src/script/cpp_api/s_server.cpp
+++ b/src/script/cpp_api/s_server.cpp
@@ -198,10 +198,8 @@ std::string ScriptApiServer::formatChatMessage(const std::string &name,
 	return ret;
 }
 
-u32 ScriptApiServer::allocateDynamicMediaCallback(int f_idx)
+u32 ScriptApiServer::allocateDynamicMediaCallback(lua_State *L, int f_idx)
 {
-	lua_State *L = getStack();
-
 	if (f_idx < 0)
 		f_idx = lua_gettop(L) + f_idx + 1;
 
@@ -235,7 +233,7 @@ u32 ScriptApiServer::allocateDynamicMediaCallback(int f_idx)
 
 void ScriptApiServer::freeDynamicMediaCallback(u32 token)
 {
-	lua_State *L = getStack();
+	SCRIPTAPI_PRECHECKHEADER
 
 	verbosestream << "freeDynamicMediaCallback(" << token << ")" << std::endl;
 
diff --git a/src/script/cpp_api/s_server.h b/src/script/cpp_api/s_server.h
index c5c3d5596..58c8c0e48 100644
--- a/src/script/cpp_api/s_server.h
+++ b/src/script/cpp_api/s_server.h
@@ -51,7 +51,7 @@ public:
 		const std::string &password);
 
 	/* dynamic media handling */
-	u32 allocateDynamicMediaCallback(int f_idx);
+	static u32 allocateDynamicMediaCallback(lua_State *L, int f_idx);
 	void freeDynamicMediaCallback(u32 token);
 	void on_dynamic_media_added(u32 token, const char *playername);