From 27a485a472706bededf38f0de2630688739ca0b2 Mon Sep 17 00:00:00 2001 From: Loïc Blot Date: Mon, 20 Apr 2020 20:18:40 +0200 Subject: Replace travis with github actions (#9641) * Move outside of travis to Github actions This will permit to have better integrated CI workflow than the previous travis one. --- util/ci/build.sh | 8 + util/ci/clang-format-whitelist.txt | 500 +++++++++++++++++++++++++++++++++++++ util/ci/clang-tidy.sh | 18 ++ util/ci/common.sh | 45 ++++ util/ci/lint.sh | 43 ++++ util/ci/run-clang-tidy.py | 321 ++++++++++++++++++++++++ 6 files changed, 935 insertions(+) create mode 100755 util/ci/build.sh create mode 100644 util/ci/clang-format-whitelist.txt create mode 100755 util/ci/clang-tidy.sh create mode 100644 util/ci/common.sh create mode 100644 util/ci/lint.sh create mode 100755 util/ci/run-clang-tidy.py (limited to 'util/ci') diff --git a/util/ci/build.sh b/util/ci/build.sh new file mode 100755 index 000000000..59069b00a --- /dev/null +++ b/util/ci/build.sh @@ -0,0 +1,8 @@ +#! /bin/bash -eu + +mkdir cmakebuild +cd cmakebuild +cmake -DCMAKE_BUILD_TYPE=Debug \ + -DRUN_IN_PLACE=TRUE -DENABLE_GETTEXT=TRUE \ + -DBUILD_SERVER=TRUE ${CMAKE_FLAGS} .. +make -j2 diff --git a/util/ci/clang-format-whitelist.txt b/util/ci/clang-format-whitelist.txt new file mode 100644 index 000000000..3334257ae --- /dev/null +++ b/util/ci/clang-format-whitelist.txt @@ -0,0 +1,500 @@ +src/activeobject.h +src/ban.cpp +src/camera.cpp +src/camera.h +src/chat.cpp +src/chat.h +src/chat_interface.h +src/client/clientlauncher.cpp +src/client/clientlauncher.h +src/client/sound_openal.cpp +src/client.cpp +src/clientenvironment.cpp +src/clientenvironment.h +src/client/gameui.cpp +src/client.h +src/client/hud.cpp +src/client/hud.h +src/clientiface.cpp +src/clientiface.h +src/client/joystick_controller.cpp +src/client/joystick_controller.h +src/clientmap.cpp +src/clientmap.h +src/clientmedia.cpp +src/clientmedia.h +src/clientobject.cpp +src/clientobject.h +src/client/render/core.cpp +src/client/renderingengine.cpp +src/client/render/interlaced.cpp +src/client/render/plain.cpp +src/client/render/sidebyside.cpp +src/client/render/stereo.cpp +src/client/tile.cpp +src/client/tile.h +src/client/fontengine.h +src/client/clientenvironment.cpp +src/client/mapblock_mesh.cpp +src/client/sound_openal.h +src/client/clouds.cpp +src/client/fontengine.cpp +src/client/camera.h +src/client/hud.cpp +src/client/clientmap.cpp +src/client/sound_openal.cpp +src/client/minimap.h +src/client/content_cao.cpp +src/client/localplayer.h +src/client/mapblock_mesh.h +src/client/mesh.cpp +src/client/sound.cpp +src/client/guiscalingfilter.cpp +src/client/content_cso.cpp +src/client/gameui.cpp +src/client/wieldmesh.cpp +src/client/clientmedia.h +src/client/game.cpp +src/client/keys.h +src/client/client.h +src/client/shader.cpp +src/client/clientmap.h +src/client/inputhandler.h +src/client/content_mapblock.h +src/client/game.h +src/client/mesh.h +src/client/camera.cpp +src/client/sky.h +src/client/mesh_generator_thread.cpp +src/client/guiscalingfilter.h +src/client/clientobject.cpp +src/client/tile.cpp +src/client/hud.h +src/client/inputhandler.cpp +src/client/clientevent.h +src/client/gameui.h +src/client/content_cso.h +src/client/sky.cpp +src/client/localplayer.cpp +src/client/content_mapblock.cpp +src/client/clientobject.h +src/client/filecache.cpp +src/client/particles.h +src/client/clientenvironment.h +src/client/imagefilters.h +src/client/renderingengine.cpp +src/client/tile.h +src/client/clientmedia.cpp +src/client/event_manager.h +src/client/joystick_controller.h +src/client/clouds.h +src/client/clientlauncher.h +src/client/content_cao.h +src/client/minimap.cpp +src/client/sound.h +src/client/keycode.cpp +src/client/particles.cpp +src/client/joystick_controller.cpp +src/client/keycode.h +src/client/wieldmesh.h +src/client/filecache.h +src/client/shader.h +src/client/mesh_generator_thread.h +src/client/renderingengine.h +src/client/client.cpp +src/client/imagefilters.cpp +src/client/clientlauncher.cpp +src/clouds.cpp +src/clouds.h +src/collision.cpp +src/collision.h +src/config.h +src/content_cao.cpp +src/content_cao.h +src/content_cso.cpp +src/content_cso.h +src/content_mapblock.cpp +src/content_mapblock.h +src/content_mapnode.cpp +src/content_nodemeta.cpp +src/content_nodemeta.h +src/convert_json.cpp +src/convert_json.h +src/craftdef.cpp +src/craftdef.h +src/database/database.cpp +src/database/database-dummy.cpp +src/database/database-files.cpp +src/database/database-leveldb.cpp +src/database/database-postgresql.cpp +src/database/database-postgresql.h +src/database/database-redis.cpp +src/database/database-sqlite3.cpp +src/database/database-sqlite3.h +src/daynightratio.h +src/debug.cpp +src/debug.h +src/defaultsettings.cpp +src/emerge.cpp +src/emerge.h +src/environment.cpp +src/exceptions.h +src/face_position_cache.cpp +src/face_position_cache.h +src/filecache.cpp +src/filesys.cpp +src/filesys.h +src/fontengine.cpp +src/fontengine.h +src/game.cpp +src/gamedef.h +src/game.h +src/gettext.cpp +src/gettext.h +src/gui/guiAnimatedImage.cpp +src/gui/guiAnimatedImage.h +src/gui/guiBackgroundImage.cpp +src/gui/guiBackgroundImage.h +src/gui/guiBox.cpp +src/gui/guiBox.h +src/gui/guiButton.cpp +src/gui/guiButton.h +src/gui/guiButtonImage.cpp +src/gui/guiButtonImage.h +src/gui/guiButtonItemImage.cpp +src/gui/guiButtonItemImage.h +src/gui/guiChatConsole.cpp +src/gui/guiChatConsole.h +src/gui/guiConfirmRegistration.cpp +src/gui/guiEditBoxWithScrollbar.cpp +src/gui/guiEditBoxWithScrollbar.h +src/gui/guiEngine.cpp +src/gui/guiEngine.h +src/gui/guiFormSpecMenu.cpp +src/gui/guiFormSpecMenu.h +src/gui/guiKeyChangeMenu.cpp +src/gui/guiHyperText.cpp +src/gui/guiHyperText.h +src/gui/guiInventoryList.cpp +src/gui/guiInventoryList.h +src/gui/guiItemImage.cpp +src/gui/guiItemImage.h +src/gui/guiMainMenu.h +src/gui/guiPasswordChange.cpp +src/gui/guiPathSelectMenu.cpp +src/gui/guiPathSelectMenu.h +src/gui/guiScrollBar.cpp +src/gui/guiSkin.cpp +src/gui/guiSkin.h +src/gui/guiTable.cpp +src/gui/guiTable.h +src/gui/guiVolumeChange.cpp +src/gui/guiVolumeChange.h +src/gui/intlGUIEditBox.cpp +src/gui/intlGUIEditBox.h +src/gui/mainmenumanager.h +src/gui/modalMenu.h +src/guiscalingfilter.cpp +src/guiscalingfilter.h +src/gui/StyleSpec.h +src/gui/touchscreengui.cpp +src/httpfetch.cpp +src/hud.cpp +src/hud.h +src/imagefilters.cpp +src/imagefilters.h +src/inventory.cpp +src/inventory.h +src/inventorymanager.cpp +src/inventorymanager.h +src/irrlicht_changes/CGUITTFont.cpp +src/irrlicht_changes/CGUITTFont.h +src/irrlicht_changes/irrUString.h +src/irrlicht_changes/static_text.cpp +src/irrlicht_changes/static_text.h +src/irrlichttypes.h +src/itemdef.cpp +src/itemdef.h +src/itemstackmetadata.cpp +src/keycode.cpp +src/light.cpp +src/localplayer.cpp +src/log.cpp +src/log.h +src/main.cpp +src/mapblock.cpp +src/mapblock.h +src/mapblock_mesh.cpp +src/mapblock_mesh.h +src/map.cpp +src/mapgen/cavegen.cpp +src/mapgen/cavegen.h +src/mapgen/dungeongen.cpp +src/mapgen/dungeongen.h +src/mapgen/mapgen.cpp +src/mapgen/mapgen.h +src/mapgen/mapgen_carpathian.cpp +src/mapgen/mapgen_carpathian.h +src/mapgen/mapgen_flat.cpp +src/mapgen/mapgen_flat.h +src/mapgen/mapgen_fractal.cpp +src/mapgen/mapgen_fractal.h +src/mapgen/mapgen_singlenode.cpp +src/mapgen/mapgen_singlenode.h +src/mapgen/mapgen_v5.cpp +src/mapgen/mapgen_v5.h +src/mapgen/mapgen_v6.cpp +src/mapgen/mapgen_v6.h +src/mapgen/mapgen_v7.cpp +src/mapgen/mapgen_v7.h +src/mapgen/mapgen_valleys.cpp +src/mapgen/mapgen_valleys.h +src/mapgen/mg_biome.cpp +src/mapgen/mg_biome.h +src/mapgen/mg_decoration.cpp +src/mapgen/mg_decoration.h +src/mapgen/mg_ore.cpp +src/mapgen/mg_ore.h +src/mapgen/mg_schematic.cpp +src/mapgen/mg_schematic.h +src/mapgen/treegen.cpp +src/mapgen/treegen.h +src/map.h +src/mapnode.cpp +src/mapnode.h +src/mapsector.cpp +src/mapsector.h +src/map_settings_manager.cpp +src/map_settings_manager.h +src/mesh.cpp +src/mesh_generator_thread.cpp +src/mesh.h +src/metadata.h +src/minimap.cpp +src/minimap.h +src/mods.cpp +src/mods.h +src/network/address.cpp +src/network/clientopcodes.cpp +src/network/clientopcodes.h +src/network/clientpackethandler.cpp +src/network/connection.cpp +src/network/connection.h +src/network/connectionthreads.cpp +src/network/networkpacket.cpp +src/network/networkprotocol.h +src/network/serveropcodes.cpp +src/network/serveropcodes.h +src/network/serverpackethandler.cpp +src/nodedef.cpp +src/nodedef.h +src/nodemetadata.cpp +src/nodemetadata.h +src/nodetimer.cpp +src/nodetimer.h +src/noise.cpp +src/noise.h +src/objdef.cpp +src/objdef.h +src/object_properties.cpp +src/object_properties.h +src/particles.cpp +src/particles.h +src/pathfinder.cpp +src/pathfinder.h +src/player.cpp +src/player.h +src/porting_android.cpp +src/porting_android.h +src/porting.cpp +src/porting.h +src/profiler.h +src/raycast.cpp +src/raycast.h +src/reflowscan.cpp +src/reflowscan.h +src/remoteplayer.cpp +src/rollback.cpp +src/rollback.h +src/rollback_interface.cpp +src/rollback_interface.h +src/script/common/c_content.cpp +src/script/common/c_content.h +src/script/common/c_converter.cpp +src/script/common/c_converter.h +src/script/common/c_internal.cpp +src/script/common/c_internal.h +src/script/common/c_types.cpp +src/script/common/c_types.h +src/script/cpp_api/s_async.cpp +src/script/cpp_api/s_async.h +src/script/cpp_api/s_base.cpp +src/script/cpp_api/s_base.h +src/script/cpp_api/s_client.cpp +src/script/cpp_api/s_entity.cpp +src/script/cpp_api/s_entity.h +src/script/cpp_api/s_env.cpp +src/script/cpp_api/s_env.h +src/script/cpp_api/s_internal.h +src/script/cpp_api/s_inventory.cpp +src/script/cpp_api/s_inventory.h +src/script/cpp_api/s_item.cpp +src/script/cpp_api/s_item.h +src/script/cpp_api/s_mainmenu.h +src/script/cpp_api/s_node.cpp +src/script/cpp_api/s_node.h +src/script/cpp_api/s_nodemeta.cpp +src/script/cpp_api/s_nodemeta.h +src/script/cpp_api/s_player.cpp +src/script/cpp_api/s_player.h +src/script/cpp_api/s_security.cpp +src/script/cpp_api/s_security.h +src/script/cpp_api/s_server.cpp +src/script/cpp_api/s_server.h +src/script/lua_api/l_areastore.cpp +src/script/lua_api/l_base.cpp +src/script/lua_api/l_base.h +src/script/lua_api/l_client.cpp +src/script/lua_api/l_craft.cpp +src/script/lua_api/l_craft.h +src/script/lua_api/l_env.cpp +src/script/lua_api/l_env.h +src/script/lua_api/l_http.cpp +src/script/lua_api/l_http.h +src/script/lua_api/l_internal.h +src/script/lua_api/l_inventory.cpp +src/script/lua_api/l_inventory.h +src/script/lua_api/l_item.cpp +src/script/lua_api/l_item.h +src/script/lua_api/l_itemstackmeta.cpp +src/script/lua_api/l_itemstackmeta.h +src/script/lua_api/l_localplayer.cpp +src/script/lua_api/l_mainmenu.cpp +src/script/lua_api/l_mainmenu.h +src/script/lua_api/l_mapgen.cpp +src/script/lua_api/l_mapgen.h +src/script/lua_api/l_metadata.cpp +src/script/lua_api/l_minimap.cpp +src/script/lua_api/l_nodemeta.cpp +src/script/lua_api/l_nodemeta.h +src/script/lua_api/l_nodetimer.cpp +src/script/lua_api/l_noise.cpp +src/script/lua_api/l_object.cpp +src/script/lua_api/l_object.h +src/script/lua_api/l_particles.cpp +src/script/lua_api/l_particles.h +src/script/lua_api/l_particles_local.cpp +src/script/lua_api/l_rollback.cpp +src/script/lua_api/l_rollback.h +src/script/lua_api/l_server.cpp +src/script/lua_api/l_settings.cpp +src/script/lua_api/l_sound.cpp +src/script/lua_api/l_storage.cpp +src/script/lua_api/l_util.cpp +src/script/lua_api/l_vmanip.cpp +src/script/scripting_client.cpp +src/script/scripting_client.h +src/script/scripting_mainmenu.cpp +src/script/scripting_mainmenu.h +src/script/scripting_server.cpp +src/script/scripting_server.h +src/serialization.cpp +src/serialization.h +src/server.cpp +src/serverenvironment.cpp +src/serverenvironment.h +src/server.h +src/serverlist.cpp +src/serverlist.h +src/server/luaentity_sao.cpp +src/server/player_sao.cpp +src/server/serveractiveobject.cpp +src/server/serveractiveobject.h +src/settings.cpp +src/settings.h +src/settings_translation_file.cpp +src/shader.cpp +src/shader.h +src/sky.cpp +src/sound.cpp +src/staticobject.cpp +src/staticobject.h +src/subgame.cpp +src/subgame.h +src/terminal_chat_console.cpp +src/terminal_chat_console.h +src/texture_override.cpp +src/threading/atomic.h +src/threading/event.cpp +src/threading/mutex_auto_lock.h +src/threading/mutex.cpp +src/threading/mutex.h +src/threading/semaphore.cpp +src/threading/thread.cpp +src/threading/thread.h +src/threads.h +src/tileanimation.cpp +src/tileanimation.h +src/tool.cpp +src/tool.h +src/translation.cpp +src/unittest/test_areastore.cpp +src/unittest/test_collision.cpp +src/unittest/test_compression.cpp +src/unittest/test_connection.cpp +src/unittest/test.cpp +src/unittest/test_filepath.cpp +src/unittest/test.h +src/unittest/test_inventory.cpp +src/unittest/test_keycode.cpp +src/unittest/test_map_settings_manager.cpp +src/unittest/test_noderesolver.cpp +src/unittest/test_noise.cpp +src/unittest/test_random.cpp +src/unittest/test_schematic.cpp +src/unittest/test_serialization.cpp +src/unittest/test_settings.cpp +src/unittest/test_socket.cpp +src/unittest/test_threading.cpp +src/unittest/test_utilities.cpp +src/unittest/test_voxelalgorithms.cpp +src/unittest/test_voxelmanipulator.cpp +src/util/areastore.cpp +src/util/areastore.h +src/util/auth.cpp +src/util/auth.h +src/util/base64.cpp +src/util/base64.h +src/util/basic_macros.h +src/util/container.h +src/util/directiontables.cpp +src/util/directiontables.h +src/util/enriched_string.cpp +src/util/enriched_string.h +src/util/md32_common.h +src/util/numeric.cpp +src/util/numeric.h +src/util/pointedthing.cpp +src/util/pointedthing.h +src/util/pointer.h +src/util/quicktune.h +src/util/quicktune_shortcutter.h +src/util/quicktune.cpp +src/util/serialize.cpp +src/util/serialize.h +src/util/sha1.cpp +src/util/srp.cpp +src/util/srp.h +src/util/strfnd.h +src/util/string.cpp +src/util/string.h +src/util/thread.h +src/util/timetaker.cpp +src/util/timetaker.h +src/version.cpp +src/version.h +src/voxelalgorithms.cpp +src/voxelalgorithms.h +src/voxel.cpp +src/voxel.h +src/wieldmesh.cpp diff --git a/util/ci/clang-tidy.sh b/util/ci/clang-tidy.sh new file mode 100755 index 000000000..d048f54ee --- /dev/null +++ b/util/ci/clang-tidy.sh @@ -0,0 +1,18 @@ +#! /bin/bash -eu + +mkdir -p cmakebuild +cd cmakebuild +cmake -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DRUN_IN_PLACE=TRUE \ + -DENABLE_GETTEXT=TRUE \ + -DENABLE_SOUND=FALSE \ + -DBUILD_SERVER=TRUE .. +make GenerateVersion + +cd .. + +./util/ci/run-clang-tidy.py \ + -clang-tidy-binary=clang-tidy-9 -p cmakebuild \ + -quiet -config="$(cat .clang-tidy)" \ + 'src/.*' diff --git a/util/ci/common.sh b/util/ci/common.sh new file mode 100644 index 000000000..5a4f78457 --- /dev/null +++ b/util/ci/common.sh @@ -0,0 +1,45 @@ +#!/bin/bash -e + +set_linux_compiler_env() { + if [[ "${COMPILER}" == "gcc-6" ]]; then + export CC=gcc-6 + export CXX=g++-6 + elif [[ "${COMPILER}" == "gcc-8" ]]; then + export CC=gcc-8 + export CXX=g++-8 + elif [[ "${COMPILER}" == "clang-3.9" ]]; then + export CC=clang-3.9 + export CXX=clang++-3.9 + elif [[ "${COMPILER}" == "clang-9" ]]; then + export CC=clang-9 + export CXX=clang++-9 + fi +} + +# Linux build only +install_linux_deps() { + local pkgs=(libirrlicht-dev cmake libbz2-dev libpng-dev \ + libjpeg-dev libxxf86vm-dev libgl1-mesa-dev libsqlite3-dev \ + libhiredis-dev libogg-dev libgmp-dev libvorbis-dev libopenal-dev \ + gettext libpq-dev postgresql-server-dev-all libleveldb-dev \ + libcurl4-openssl-dev) + # for better coverage, build some jobs with luajit + if [ -n "$WITH_LUAJIT" ]; then + pkgs+=(libluajit-5.1-dev) + fi + + sudo apt-get update + sudo apt-get install -y --no-install-recommends ${pkgs[@]} +} + +# Mac OSX build only +install_macosx_deps() { + brew update + brew install freetype gettext hiredis irrlicht leveldb libogg libvorbis luajit + if brew ls | grep -q jpeg; then + brew upgrade jpeg + else + brew install jpeg + fi + #brew upgrade postgresql +} diff --git a/util/ci/lint.sh b/util/ci/lint.sh new file mode 100644 index 000000000..395445ca7 --- /dev/null +++ b/util/ci/lint.sh @@ -0,0 +1,43 @@ +#! /bin/bash +function perform_lint() { + echo "Performing LINT..." + if [ -z "${CLANG_FORMAT}" ]; then + CLANG_FORMAT=clang-format + fi + echo "LINT: Using binary $CLANG_FORMAT" + CLANG_FORMAT_WHITELIST="util/ci/clang-format-whitelist.txt" + + files_to_lint="$(find src/ -name '*.cpp' -or -name '*.h')" + + local errorcount=0 + local fail=0 + for f in ${files_to_lint}; do + d=$(diff -u "$f" <(${CLANG_FORMAT} "$f") || true) + + if ! [ -z "$d" ]; then + whitelisted=$(awk '$1 == "'$f'" { print 1 }' "$CLANG_FORMAT_WHITELIST") + + # If file is not whitelisted, mark a failure + if [ -z "${whitelisted}" ]; then + errorcount=$((errorcount+1)) + + printf "The file %s is not compliant with the coding style" "$f" + if [ ${errorcount} -gt 50 ]; then + printf "\nToo many errors encountered previously, this diff is hidden.\n" + else + printf ":\n%s\n" "$d" + fi + + fail=1 + fi + fi + done + + if [ "$fail" = 1 ]; then + echo "LINT reports failure." + exit 1 + fi + + echo "LINT OK" +} + diff --git a/util/ci/run-clang-tidy.py b/util/ci/run-clang-tidy.py new file mode 100755 index 000000000..6ad0ff24f --- /dev/null +++ b/util/ci/run-clang-tidy.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python +# +#===- run-clang-tidy.py - Parallel clang-tidy runner ---------*- python -*--===# +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===------------------------------------------------------------------------===# +# FIXME: Integrate with clang-tidy-diff.py + +""" +Parallel clang-tidy runner +========================== + +Runs clang-tidy over all files in a compilation database. Requires clang-tidy +and clang-apply-replacements in $PATH. + +Example invocations. +- Run clang-tidy on all files in the current working directory with a default + set of checks and show warnings in the cpp files and all project headers. + run-clang-tidy.py $PWD + +- Fix all header guards. + run-clang-tidy.py -fix -checks=-*,llvm-header-guard + +- Fix all header guards included from clang-tidy and header guards + for clang-tidy headers. + run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \ + -header-filter=extra/clang-tidy + +Compilation database setup: +http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html +""" + +from __future__ import print_function + +import argparse +import glob +import json +import multiprocessing +import os +import re +import shutil +import subprocess +import sys +import tempfile +import threading +import traceback + +try: + import yaml +except ImportError: + yaml = None + +is_py2 = sys.version[0] == '2' + +if is_py2: + import Queue as queue +else: + import queue as queue + +def find_compilation_database(path): + """Adjusts the directory until a compilation database is found.""" + result = './' + while not os.path.isfile(os.path.join(result, path)): + if os.path.realpath(result) == '/': + print('Error: could not find compilation database.') + sys.exit(1) + result += '../' + return os.path.realpath(result) + + +def make_absolute(f, directory): + if os.path.isabs(f): + return f + return os.path.normpath(os.path.join(directory, f)) + + +def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path, + header_filter, extra_arg, extra_arg_before, quiet, + config): + """Gets a command line for clang-tidy.""" + start = [clang_tidy_binary] + if header_filter is not None: + start.append('-header-filter=' + header_filter) + if checks: + start.append('-checks=' + checks) + if tmpdir is not None: + start.append('-export-fixes') + # Get a temporary file. We immediately close the handle so clang-tidy can + # overwrite it. + (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir) + os.close(handle) + start.append(name) + for arg in extra_arg: + start.append('-extra-arg=%s' % arg) + for arg in extra_arg_before: + start.append('-extra-arg-before=%s' % arg) + start.append('-p=' + build_path) + if quiet: + start.append('-quiet') + if config: + start.append('-config=' + config) + start.append(f) + return start + + +def merge_replacement_files(tmpdir, mergefile): + """Merge all replacement files in a directory into a single file""" + # The fixes suggested by clang-tidy >= 4.0.0 are given under + # the top level key 'Diagnostics' in the output yaml files + mergekey="Diagnostics" + merged=[] + for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')): + content = yaml.safe_load(open(replacefile, 'r')) + if not content: + continue # Skip empty files. + merged.extend(content.get(mergekey, [])) + + if merged: + # MainSourceFile: The key is required by the definition inside + # include/clang/Tooling/ReplacementsYaml.h, but the value + # is actually never used inside clang-apply-replacements, + # so we set it to '' here. + output = { 'MainSourceFile': '', mergekey: merged } + with open(mergefile, 'w') as out: + yaml.safe_dump(output, out) + else: + # Empty the file: + open(mergefile, 'w').close() + + +def check_clang_apply_replacements_binary(args): + """Checks if invoking supplied clang-apply-replacements binary works.""" + try: + subprocess.check_call([args.clang_apply_replacements_binary, '--version']) + except: + print('Unable to run clang-apply-replacements. Is clang-apply-replacements ' + 'binary correctly specified?', file=sys.stderr) + traceback.print_exc() + sys.exit(1) + + +def apply_fixes(args, tmpdir): + """Calls clang-apply-fixes on a given directory.""" + invocation = [args.clang_apply_replacements_binary] + if args.format: + invocation.append('-format') + if args.style: + invocation.append('-style=' + args.style) + invocation.append(tmpdir) + subprocess.call(invocation) + + +def run_tidy(args, tmpdir, build_path, queue, lock, failed_files): + """Takes filenames out of queue and runs clang-tidy on them.""" + while True: + name = queue.get() + invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks, + tmpdir, build_path, args.header_filter, + args.extra_arg, args.extra_arg_before, + args.quiet, args.config) + + proc = subprocess.Popen(invocation) + proc.wait() + if proc.returncode != 0: + failed_files.append(name) + queue.task_done() + + +def main(): + parser = argparse.ArgumentParser(description='Runs clang-tidy over all files ' + 'in a compilation database. Requires ' + 'clang-tidy and clang-apply-replacements in ' + '$PATH.') + parser.add_argument('-clang-tidy-binary', metavar='PATH', + default='clang-tidy', + help='path to clang-tidy binary') + parser.add_argument('-clang-apply-replacements-binary', metavar='PATH', + default='clang-apply-replacements', + help='path to clang-apply-replacements binary') + parser.add_argument('-checks', default=None, + help='checks filter, when not specified, use clang-tidy ' + 'default') + parser.add_argument('-config', default=None, + help='Specifies a configuration in YAML/JSON format: ' + ' -config="{Checks: \'*\', ' + ' CheckOptions: [{key: x, ' + ' value: y}]}" ' + 'When the value is empty, clang-tidy will ' + 'attempt to find a file named .clang-tidy for ' + 'each source file in its parent directories.') + parser.add_argument('-header-filter', default=None, + help='regular expression matching the names of the ' + 'headers to output diagnostics from. Diagnostics from ' + 'the main file of each translation unit are always ' + 'displayed.') + if yaml: + parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes', + help='Create a yaml file to store suggested fixes in, ' + 'which can be applied with clang-apply-replacements.') + parser.add_argument('-j', type=int, default=0, + help='number of tidy instances to be run in parallel.') + parser.add_argument('files', nargs='*', default=['.*'], + help='files to be processed (regex on path)') + parser.add_argument('-fix', action='store_true', help='apply fix-its') + parser.add_argument('-format', action='store_true', help='Reformat code ' + 'after applying fixes') + parser.add_argument('-style', default='file', help='The style of reformat ' + 'code after applying fixes') + parser.add_argument('-p', dest='build_path', + help='Path used to read a compile command database.') + parser.add_argument('-extra-arg', dest='extra_arg', + action='append', default=[], + help='Additional argument to append to the compiler ' + 'command line.') + parser.add_argument('-extra-arg-before', dest='extra_arg_before', + action='append', default=[], + help='Additional argument to prepend to the compiler ' + 'command line.') + parser.add_argument('-quiet', action='store_true', + help='Run clang-tidy in quiet mode') + args = parser.parse_args() + + db_path = 'compile_commands.json' + + if args.build_path is not None: + build_path = args.build_path + else: + # Find our database + build_path = find_compilation_database(db_path) + + try: + invocation = [args.clang_tidy_binary, '-list-checks'] + invocation.append('-p=' + build_path) + if args.checks: + invocation.append('-checks=' + args.checks) + invocation.append('-') + if args.quiet: + # Even with -quiet we still want to check if we can call clang-tidy. + with open(os.devnull, 'w') as dev_null: + subprocess.check_call(invocation, stdout=dev_null) + else: + subprocess.check_call(invocation) + except: + print("Unable to run clang-tidy.", file=sys.stderr) + sys.exit(1) + + # Load the database and extract all files. + database = json.load(open(os.path.join(build_path, db_path))) + files = [make_absolute(entry['file'], entry['directory']) + for entry in database] + + max_task = args.j + if max_task == 0: + max_task = multiprocessing.cpu_count() + + tmpdir = None + if args.fix or (yaml and args.export_fixes): + check_clang_apply_replacements_binary(args) + tmpdir = tempfile.mkdtemp() + + # Build up a big regexy filter from all command line arguments. + file_name_re = re.compile('|'.join(args.files)) + + return_code = 0 + try: + # Spin up a bunch of tidy-launching threads. + task_queue = queue.Queue(max_task) + # List of files with a non-zero return code. + failed_files = [] + lock = threading.Lock() + for _ in range(max_task): + t = threading.Thread(target=run_tidy, + args=(args, tmpdir, build_path, task_queue, lock, failed_files)) + t.daemon = True + t.start() + + # Fill the queue with files. + for name in files: + if file_name_re.search(name): + task_queue.put(name) + + # Wait for all threads to be done. + task_queue.join() + if len(failed_files): + return_code = 1 + + except KeyboardInterrupt: + # This is a sad hack. Unfortunately subprocess goes + # bonkers with ctrl-c and we start forking merrily. + print('\nCtrl-C detected, goodbye.') + if tmpdir: + shutil.rmtree(tmpdir) + os.kill(0, 9) + + if yaml and args.export_fixes: + print('Writing fixes to ' + args.export_fixes + ' ...') + try: + merge_replacement_files(tmpdir, args.export_fixes) + except: + print('Error exporting fixes.\n', file=sys.stderr) + traceback.print_exc() + return_code=1 + + if args.fix: + print('Applying fixes ...') + try: + apply_fixes(args, tmpdir) + except: + print('Error applying fixes.\n', file=sys.stderr) + traceback.print_exc() + return_code=1 + + if tmpdir: + shutil.rmtree(tmpdir) + sys.exit(return_code) + +if __name__ == '__main__': + main() -- cgit v1.2.3 From 56bababcdfce097a4e08cc3d1de8d798e7999ce7 Mon Sep 17 00:00:00 2001 From: Loic Blot Date: Mon, 27 Apr 2020 08:31:37 +0200 Subject: Add MetricsBackend with prometheus counter support --- Dockerfile | 22 +++++-- README.md | 1 + builtin/settingtypes.txt | 6 ++ src/CMakeLists.txt | 26 ++++++++ src/cmake_config.h.in | 1 + src/defaultsettings.cpp | 3 + src/map.cpp | 11 +++- src/map.h | 6 +- src/server.cpp | 57 ++++++++++++---- src/server.h | 21 ++++-- src/server/mods.cpp | 1 + src/server/mods.h | 3 + src/util/CMakeLists.txt | 1 + src/util/metricsbackend.cpp | 140 ++++++++++++++++++++++++++++++++++++++++ src/util/metricsbackend.h | 140 ++++++++++++++++++++++++++++++++++++++++ util/ci/build_prometheus_cpp.sh | 13 ++++ 16 files changed, 427 insertions(+), 25 deletions(-) create mode 100644 src/util/metricsbackend.cpp create mode 100644 src/util/metricsbackend.h create mode 100755 util/ci/build_prometheus_cpp.sh (limited to 'util/ci') diff --git a/Dockerfile b/Dockerfile index 7c1107288..72343ab9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,15 +21,29 @@ WORKDIR /usr/src/minetest RUN apk add --no-cache git build-base irrlicht-dev cmake bzip2-dev libpng-dev \ jpeg-dev libxxf86vm-dev mesa-dev sqlite-dev libogg-dev \ libvorbis-dev openal-soft-dev curl-dev freetype-dev zlib-dev \ - gmp-dev jsoncpp-dev postgresql-dev && \ + gmp-dev jsoncpp-dev postgresql-dev ca-certificates && \ git clone --depth=1 -b ${MINETEST_GAME_VERSION} https://github.com/minetest/minetest_game.git ./games/minetest_game && \ - rm -fr ./games/minetest_game/.git && \ - mkdir build && \ + rm -fr ./games/minetest_game/.git + +WORKDIR /usr/src/ +RUN git clone --recursive https://github.com/jupp0r/prometheus-cpp/ && \ + mkdir prometheus-cpp/build && \ + cd prometheus-cpp/build && \ + cmake .. \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TESTING=0 && \ + make -j2 && \ + make install + +WORKDIR /usr/src/minetest +RUN mkdir build && \ cd build && \ cmake .. \ -DCMAKE_INSTALL_PREFIX=/usr/local \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SERVER=TRUE \ + -DENABLE_PROMETHEUS=TRUE \ -DBUILD_UNITTESTS=FALSE \ -DBUILD_CLIENT=FALSE && \ make -j2 && \ @@ -49,6 +63,6 @@ COPY --from=0 /usr/local/share/doc/minetest/minetest.conf.example /etc/minetest/ USER minetest:minetest -EXPOSE 30000/udp +EXPOSE 30000/udp 30000/tcp CMD ["/usr/local/bin/minetestserver", "--config", "/etc/minetest/minetest.conf"] diff --git a/README.md b/README.md index b3b2b863e..024e7b691 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,7 @@ General options and their default values: ENABLE_SPATIAL=ON - Build with LibSpatial; Speeds up AreaStores ENABLE_SOUND=ON - Build with OpenAL, libogg & libvorbis; in-game sounds ENABLE_LUAJIT=ON - Build with LuaJIT (much faster than non-JIT Lua) + ENABLE_PROMETHEUS=OFF - Build with Prometheus metrics exporter (listens on tcp/30000 by default) ENABLE_SYSTEM_GMP=ON - Use GMP from system (much faster than bundled mini-gmp) ENABLE_SYSTEM_JSONCPP=OFF - Use JsonCPP from system OPENGL_GL_PREFERENCE=LEGACY - Linux client build only; See CMake Policy CMP0072 for reference diff --git a/builtin/settingtypes.txt b/builtin/settingtypes.txt index b9228f384..165ed8c06 100644 --- a/builtin/settingtypes.txt +++ b/builtin/settingtypes.txt @@ -955,6 +955,12 @@ address (Server address) string # Note that the port field in the main menu overrides this setting. remote_port (Remote port) int 30000 1 65535 +# Prometheus listener address. +# If minetest is compiled with ENABLE_PROMETHEUS option enabled, +# enable metrics listener for Prometheus on that address. +# Metrics can be fetch on http://127.0.0.1:30000/metrics +prometheus_listener_address (Prometheus listener address) string 127.0.0.1:30000 + # Save the map received by the client on disk. enable_local_map_saving (Saving map received from server) bool false diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b416faaf3..710d9e13e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -217,6 +217,26 @@ endif(ENABLE_REDIS) find_package(SQLite3 REQUIRED) +OPTION(ENABLE_PROMETHEUS "Enable prometheus client support" FALSE) +set(USE_PROMETHEUS FALSE) + +if(ENABLE_PROMETHEUS) + find_path(PROMETHEUS_CPP_INCLUDE_DIR NAMES prometheus/counter.h) + find_library(PROMETHEUS_PULL_LIBRARY NAMES prometheus-cpp-pull) + find_library(PROMETHEUS_CORE_LIBRARY NAMES prometheus-cpp-core) + if(PROMETHEUS_CPP_INCLUDE_DIR AND PROMETHEUS_PULL_LIBRARY AND PROMETHEUS_CORE_LIBRARY) + set(PROMETHEUS_LIBRARIES ${PROMETHEUS_PULL_LIBRARY} ${PROMETHEUS_CORE_LIBRARY}) + set(USE_PROMETHEUS TRUE) + include_directories(${PROMETHEUS_CPP_INCLUDE_DIR}) + endif(PROMETHEUS_CPP_INCLUDE_DIR AND PROMETHEUS_PULL_LIBRARY AND PROMETHEUS_CORE_LIBRARY) +endif(ENABLE_PROMETHEUS) + +if(USE_PROMETHEUS) + message(STATUS "Prometheus client enabled.") +else(USE_PROMETHEUS) + message(STATUS "Prometheus client disabled.") +endif(USE_PROMETHEUS) + OPTION(ENABLE_SPATIAL "Enable SpatialIndex AreaStore backend" TRUE) set(USE_SPATIAL FALSE) @@ -597,6 +617,9 @@ if(BUILD_CLIENT) if (USE_REDIS) target_link_libraries(${PROJECT_NAME} ${REDIS_LIBRARY}) endif() + if (USE_PROMETHEUS) + target_link_libraries(${PROJECT_NAME} ${PROMETHEUS_LIBRARIES}) + endif() if (USE_SPATIAL) target_link_libraries(${PROJECT_NAME} ${SPATIAL_LIBRARY}) endif() @@ -632,6 +655,9 @@ if(BUILD_SERVER) if (USE_REDIS) target_link_libraries(${PROJECT_NAME}server ${REDIS_LIBRARY}) endif() + if (USE_PROMETHEUS) + target_link_libraries(${PROJECT_NAME}server ${PROMETHEUS_LIBRARIES}) + endif() if (USE_SPATIAL) target_link_libraries(${PROJECT_NAME}server ${SPATIAL_LIBRARY}) endif() diff --git a/src/cmake_config.h.in b/src/cmake_config.h.in index cac6335d4..cfcee4b58 100644 --- a/src/cmake_config.h.in +++ b/src/cmake_config.h.in @@ -23,6 +23,7 @@ #cmakedefine01 USE_LEVELDB #cmakedefine01 USE_LUAJIT #cmakedefine01 USE_POSTGRESQL +#cmakedefine01 USE_PROMETHEUS #cmakedefine01 USE_SPATIAL #cmakedefine01 USE_SYSTEM_GMP #cmakedefine01 USE_REDIS diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp index b6b1ce1f2..06daa3b94 100644 --- a/src/defaultsettings.cpp +++ b/src/defaultsettings.cpp @@ -334,6 +334,9 @@ void set_default_settings(Settings *settings) // Server settings->setDefault("disable_escape_sequences", "false"); settings->setDefault("strip_color_codes", "false"); +#if USE_PROMETHEUS + settings->setDefault("prometheus_listener_address", "127.0.0.1:30000"); +#endif // Network settings->setDefault("enable_ipv6", "true"); diff --git a/src/map.cpp b/src/map.cpp index 12e122124..5f1b984a4 100644 --- a/src/map.cpp +++ b/src/map.cpp @@ -144,7 +144,7 @@ bool Map::isNodeUnderground(v3s16 p) { v3s16 blockpos = getNodeBlockPos(p); MapBlock *block = getBlockNoCreateNoEx(blockpos); - return block && block->getIsUnderground(); + return block && block->getIsUnderground(); } bool Map::isValidPosition(v3s16 p) @@ -1187,7 +1187,7 @@ bool Map::isBlockOccluded(MapBlock *block, v3s16 cam_pos_nodes) ServerMap */ ServerMap::ServerMap(const std::string &savedir, IGameDef *gamedef, - EmergeManager *emerge): + EmergeManager *emerge, MetricsBackend *mb): Map(dout_server, gamedef), settings_mgr(g_settings, savedir + DIR_DELIM + "map_meta.txt"), m_emerge(emerge) @@ -1221,6 +1221,8 @@ ServerMap::ServerMap(const std::string &savedir, IGameDef *gamedef, m_savedir = savedir; m_map_saving_enabled = false; + m_save_time_counter = mb->addCounter("minetest_core_map_save_time", "Map save time (in nanoseconds)"); + try { // If directory exists, check contents and load if possible if (fs::PathExists(m_savedir)) { @@ -1777,6 +1779,8 @@ void ServerMap::save(ModifiedState save_level) return; } + u64 start_time = porting::getTimeNs(); + if(save_level == MOD_STATE_CLEAN) infostream<<"ServerMap: Saving whole map, this can take time." <increment(end_time - start_time); } void ServerMap::listAllLoadableBlocks(std::vector &dst) diff --git a/src/map.h b/src/map.h index ff6b20c4f..77ee4da9e 100644 --- a/src/map.h +++ b/src/map.h @@ -31,6 +31,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "voxel.h" #include "modifiedstate.h" #include "util/container.h" +#include "util/metricsbackend.h" #include "nodetimer.h" #include "map_settings_manager.h" #include "debug.h" @@ -45,6 +46,7 @@ class NodeMetadata; class IGameDef; class IRollbackManager; class EmergeManager; +class MetricsBackend; class ServerEnvironment; struct BlockMakeData; @@ -324,7 +326,7 @@ public: /* savedir: directory to which map data should be saved */ - ServerMap(const std::string &savedir, IGameDef *gamedef, EmergeManager *emerge); + ServerMap(const std::string &savedir, IGameDef *gamedef, EmergeManager *emerge, MetricsBackend *mb); ~ServerMap(); s32 mapType() const @@ -449,6 +451,8 @@ private: bool m_map_metadata_changed = true; MapDatabase *dbase = nullptr; MapDatabase *dbase_ro = nullptr; + + MetricCounterPtr m_save_time_counter; }; diff --git a/src/server.cpp b/src/server.cpp index af6d3e40d..a7eb52837 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -229,18 +229,46 @@ Server::Server( m_nodedef(createNodeDefManager()), m_craftdef(createCraftDefManager()), m_thread(new ServerThread(this)), - m_uptime(0), m_clients(m_con), m_admin_chat(iface), m_modchannel_mgr(new ModChannelMgr()) { - m_lag = g_settings->getFloat("dedicated_server_step"); - if (m_path_world.empty()) throw ServerError("Supplied empty world path"); if (!gamespec.isValid()) throw ServerError("Supplied invalid gamespec"); + +#if USE_PROMETHEUS + m_metrics_backend = std::unique_ptr(createPrometheusMetricsBackend()); +#else + m_metrics_backend = std::unique_ptr(new MetricsBackend()); +#endif + + m_uptime_counter = m_metrics_backend->addCounter("minetest_core_server_uptime", "Server uptime (in seconds)"); + m_player_gauge = m_metrics_backend->addGauge("minetest_core_player_number", "Number of connected players"); + + m_timeofday_gauge = m_metrics_backend->addGauge( + "minetest_core_timeofday", + "Time of day value"); + + m_lag_gauge = m_metrics_backend->addGauge( + "minetest_core_latency", + "Latency value (in seconds)"); + + m_aom_buffer_counter = m_metrics_backend->addCounter( + "minetest_core_aom_generated_count", + "Number of active object messages generated"); + + m_packet_recv_counter = m_metrics_backend->addCounter( + "minetest_core_server_packet_recv", + "Processable packets received"); + + m_packet_recv_processed_counter = m_metrics_backend->addCounter( + "minetest_core_server_packet_recv_processed", + "Valid received packets processed"); + + m_lag_gauge->set(g_settings->getFloat("dedicated_server_step")); } Server::~Server() @@ -353,7 +381,7 @@ void Server::init() MutexAutoLock envlock(m_env_mutex); // Create the Map (loads map_meta.txt, overriding configured mapgen params) - ServerMap *servermap = new ServerMap(m_path_world, this, m_emerge); + ServerMap *servermap = new ServerMap(m_path_world, this, m_emerge, m_metrics_backend.get()); // Initialize scripting infostream << "Server: Initializing Lua" << std::endl; @@ -511,9 +539,7 @@ void Server::AsyncRunStep(bool initial_step) /* Update uptime */ - { - m_uptime.set(m_uptime.get() + dtime); - } + m_uptime_counter->increment(dtime); handlePeerChanges(); @@ -527,11 +553,13 @@ void Server::AsyncRunStep(bool initial_step) */ m_time_of_day_send_timer -= dtime; - if(m_time_of_day_send_timer < 0.0) { + if (m_time_of_day_send_timer < 0.0) { m_time_of_day_send_timer = g_settings->getFloat("time_send_interval"); u16 time = m_env->getTimeOfDay(); float time_speed = g_settings->getFloat("time_speed"); SendTimeOfDay(PEER_ID_INEXISTENT, time, time_speed); + + m_timeofday_gauge->set(time); } { @@ -603,7 +631,7 @@ void Server::AsyncRunStep(bool initial_step) } m_clients.step(dtime); - m_lag += (m_lag > dtime ? -1 : 1) * dtime/100; + m_lag_gauge->increment((m_lag_gauge->get() > dtime ? -1 : 1) * dtime/100); #if USE_CURL // send masterserver announce { @@ -614,9 +642,9 @@ void Server::AsyncRunStep(bool initial_step) ServerList::AA_START, m_bind_addr.getPort(), m_clients.getPlayerNames(), - m_uptime.get(), + m_uptime_counter->get(), m_env->getGameTime(), - m_lag, + m_lag_gauge->get(), m_gamespec.id, Mapgen::getMapgenName(m_emerge->mgparams->mgtype), m_modmgr->getMods(), @@ -638,6 +666,7 @@ void Server::AsyncRunStep(bool initial_step) const RemoteClientMap &clients = m_clients.getClientList(); ScopeProfiler sp(g_profiler, "Server: update objects within range"); + m_player_gauge->set(clients.size()); for (const auto &client_it : clients) { RemoteClient *client = client_it.second; @@ -703,6 +732,8 @@ void Server::AsyncRunStep(bool initial_step) message_list->push_back(aom); } + m_aom_buffer_counter->increment(buffered_messages.size()); + m_clients.lock(); const RemoteClientMap &clients = m_clients.getClientList(); // Route data to every client @@ -943,7 +974,9 @@ void Server::Receive() } peer_id = pkt.getPeerId(); + m_packet_recv_counter->increment(); ProcessData(&pkt); + m_packet_recv_processed_counter->increment(); } catch (const con::InvalidIncomingDataException &e) { infostream << "Server::Receive(): InvalidIncomingDataException: what()=" << e.what() << std::endl; @@ -3127,7 +3160,7 @@ std::wstring Server::getStatusString() // Version os << L"version=" << narrow_to_wide(g_version_string); // Uptime - os << L", uptime=" << m_uptime.get(); + os << L", uptime=" << m_uptime_counter->get(); // Max lag estimate os << L", max_lag=" << (m_env ? m_env->getMaxLagEstimate() : 0); diff --git a/src/server.h b/src/server.h index b995aba28..71059dd30 100644 --- a/src/server.h +++ b/src/server.h @@ -33,6 +33,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "util/numeric.h" #include "util/thread.h" #include "util/basic_macros.h" +#include "util/metricsbackend.h" #include "serverenvironment.h" #include "clientiface.h" #include "chatmessage.h" @@ -203,7 +204,7 @@ public: // Connection must be locked when called std::wstring getStatusString(); - inline double getUptime() const { return m_uptime.m_value; } + inline double getUptime() const { return m_uptime_counter->get(); } // read shutdown state inline bool isShutdownRequested() const { return m_shutdown_state.is_requested; } @@ -591,9 +592,6 @@ private: float m_step_dtime = 0.0f; std::mutex m_step_dtime_mutex; - // current server step lag counter - float m_lag; - // The server mainly operates in this thread ServerThread *m_thread = nullptr; @@ -602,8 +600,6 @@ private: */ // Timer for sending time of day over network float m_time_of_day_send_timer = 0.0f; - // Uptime of server in seconds - MutexedVariable m_uptime; /* Client interface @@ -677,6 +673,19 @@ private: // ModChannel manager std::unique_ptr m_modchannel_mgr; + + // Global server metrics backend + std::unique_ptr m_metrics_backend; + + // Server metrics + MetricCounterPtr m_uptime_counter; + MetricGaugePtr m_player_gauge; + MetricGaugePtr m_timeofday_gauge; + // current server step lag + MetricGaugePtr m_lag_gauge; + MetricCounterPtr m_aom_buffer_counter; + MetricCounterPtr m_packet_recv_counter; + MetricCounterPtr m_packet_recv_processed_counter; }; /* diff --git a/src/server/mods.cpp b/src/server/mods.cpp index c8d8a28e2..6ac530739 100644 --- a/src/server/mods.cpp +++ b/src/server/mods.cpp @@ -23,6 +23,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "scripting_server.h" #include "content/subgames.h" #include "porting.h" +#include "util/metricsbackend.h" /** * Manage server mods diff --git a/src/server/mods.h b/src/server/mods.h index 2bc1aa22f..54774bd86 100644 --- a/src/server/mods.h +++ b/src/server/mods.h @@ -20,7 +20,10 @@ with this program; if not, write to the Free Software Foundation, Inc., #pragma once #include "content/mods.h" +#include +class MetricsBackend; +class MetricCounter; class ServerScripting; /** diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 199d3aeaa..cd2e468d1 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -5,6 +5,7 @@ set(UTIL_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/directiontables.cpp ${CMAKE_CURRENT_SOURCE_DIR}/enriched_string.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ieee_float.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/metricsbackend.cpp ${CMAKE_CURRENT_SOURCE_DIR}/numeric.cpp ${CMAKE_CURRENT_SOURCE_DIR}/pointedthing.cpp ${CMAKE_CURRENT_SOURCE_DIR}/quicktune.cpp diff --git a/src/util/metricsbackend.cpp b/src/util/metricsbackend.cpp new file mode 100644 index 000000000..4454557a3 --- /dev/null +++ b/src/util/metricsbackend.cpp @@ -0,0 +1,140 @@ +/* +Minetest +Copyright (C) 2013-2020 Minetest core developers team + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#include "metricsbackend.h" +#if USE_PROMETHEUS +#include +#include +#include +#include +#include "log.h" +#include "settings.h" +#endif + +MetricCounterPtr MetricsBackend::addCounter( + const std::string &name, const std::string &help_str) +{ + return std::make_shared(name, help_str); +} + +MetricGaugePtr MetricsBackend::addGauge( + const std::string &name, const std::string &help_str) +{ + return std::make_shared(name, help_str); +} + +#if USE_PROMETHEUS + +class PrometheusMetricCounter : public MetricCounter +{ +public: + PrometheusMetricCounter() = delete; + + PrometheusMetricCounter(const std::string &name, const std::string &help_str, + std::shared_ptr registry) : + MetricCounter(), + m_family(prometheus::BuildCounter() + .Name(name) + .Help(help_str) + .Register(*registry)), + m_counter(m_family.Add({})) + { + } + + virtual ~PrometheusMetricCounter() {} + + virtual void increment(double number) { m_counter.Increment(number); } + virtual double get() const { return m_counter.Value(); } + +private: + prometheus::Family &m_family; + prometheus::Counter &m_counter; +}; + +class PrometheusMetricGauge : public MetricGauge +{ +public: + PrometheusMetricGauge() = delete; + + PrometheusMetricGauge(const std::string &name, const std::string &help_str, + std::shared_ptr registry) : + MetricGauge(), + m_family(prometheus::BuildGauge() + .Name(name) + .Help(help_str) + .Register(*registry)), + m_gauge(m_family.Add({})) + { + } + + virtual ~PrometheusMetricGauge() {} + + virtual void increment(double number) { m_gauge.Increment(number); } + virtual void decrement(double number) { m_gauge.Decrement(number); } + virtual void set(double number) { m_gauge.Set(number); } + virtual double get() const { return m_gauge.Value(); } + +private: + prometheus::Family &m_family; + prometheus::Gauge &m_gauge; +}; + +class PrometheusMetricsBackend : public MetricsBackend +{ +public: + PrometheusMetricsBackend(const std::string &addr) : + MetricsBackend(), m_exposer(std::unique_ptr( + new prometheus::Exposer(addr))), + m_registry(std::make_shared()) + { + m_exposer->RegisterCollectable(m_registry); + } + + virtual ~PrometheusMetricsBackend() {} + + virtual MetricCounterPtr addCounter( + const std::string &name, const std::string &help_str); + virtual MetricGaugePtr addGauge( + const std::string &name, const std::string &help_str); + +private: + std::unique_ptr m_exposer; + std::shared_ptr m_registry; +}; + +MetricCounterPtr PrometheusMetricsBackend::addCounter( + const std::string &name, const std::string &help_str) +{ + return std::make_shared(name, help_str, m_registry); +} + +MetricGaugePtr PrometheusMetricsBackend::addGauge( + const std::string &name, const std::string &help_str) +{ + return std::make_shared(name, help_str, m_registry); +} + +MetricsBackend *createPrometheusMetricsBackend() +{ + std::string addr; + g_settings->getNoEx("prometheus_listener_address", addr); + return new PrometheusMetricsBackend(addr); +} + +#endif diff --git a/src/util/metricsbackend.h b/src/util/metricsbackend.h new file mode 100644 index 000000000..c37306392 --- /dev/null +++ b/src/util/metricsbackend.h @@ -0,0 +1,140 @@ +/* +Minetest +Copyright (C) 2013-2020 Minetest core developers team + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#pragma once +#include +#include "config.h" +#include "util/thread.h" + +class MetricCounter +{ +public: + MetricCounter() = default; + + virtual ~MetricCounter() {} + + virtual void increment(double number = 1.0) = 0; + virtual double get() const = 0; +}; + +typedef std::shared_ptr MetricCounterPtr; + +class SimpleMetricCounter : public MetricCounter +{ +public: + SimpleMetricCounter() = delete; + + virtual ~SimpleMetricCounter() {} + + SimpleMetricCounter(const std::string &name, const std::string &help_str) : + MetricCounter(), m_name(name), m_help_str(help_str), + m_counter(0.0) + { + } + + virtual void increment(double number) + { + MutexAutoLock lock(m_mutex); + m_counter += number; + } + virtual double get() const + { + MutexAutoLock lock(m_mutex); + return m_counter; + } + +private: + std::string m_name; + std::string m_help_str; + + mutable std::mutex m_mutex; + double m_counter; +}; + +class MetricGauge +{ +public: + MetricGauge() = default; + virtual ~MetricGauge() {} + + virtual void increment(double number = 1.0) = 0; + virtual void decrement(double number = 1.0) = 0; + virtual void set(double number) = 0; + virtual double get() const = 0; +}; + +typedef std::shared_ptr MetricGaugePtr; + +class SimpleMetricGauge : public MetricGauge +{ +public: + SimpleMetricGauge() = delete; + + SimpleMetricGauge(const std::string &name, const std::string &help_str) : + MetricGauge(), m_name(name), m_help_str(help_str), m_gauge(0.0) + { + } + + virtual ~SimpleMetricGauge() {} + + virtual void increment(double number) + { + MutexAutoLock lock(m_mutex); + m_gauge += number; + } + virtual void decrement(double number) + { + MutexAutoLock lock(m_mutex); + m_gauge -= number; + } + virtual void set(double number) + { + MutexAutoLock lock(m_mutex); + m_gauge = number; + } + virtual double get() const + { + MutexAutoLock lock(m_mutex); + return m_gauge; + } + +private: + std::string m_name; + std::string m_help_str; + + mutable std::mutex m_mutex; + double m_gauge; +}; + +class MetricsBackend +{ +public: + MetricsBackend() = default; + + virtual ~MetricsBackend() {} + + virtual MetricCounterPtr addCounter( + const std::string &name, const std::string &help_str); + virtual MetricGaugePtr addGauge( + const std::string &name, const std::string &help_str); +}; + +#if USE_PROMETHEUS +MetricsBackend *createPrometheusMetricsBackend(); +#endif diff --git a/util/ci/build_prometheus_cpp.sh b/util/ci/build_prometheus_cpp.sh new file mode 100755 index 000000000..edfd574cd --- /dev/null +++ b/util/ci/build_prometheus_cpp.sh @@ -0,0 +1,13 @@ +#! /bin/bash -eu + +cd /tmp +git clone --recursive https://github.com/jupp0r/prometheus-cpp +mkdir prometheus-cpp/build +cd prometheus-cpp/build +cmake .. \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_TESTING=0 +make -j2 +sudo make install + -- cgit v1.2.3 From 61d93988d8b44a9905451c4c288e02c04d41abb0 Mon Sep 17 00:00:00 2001 From: sfan5 Date: Sat, 2 May 2020 12:52:51 +0200 Subject: ci: Update Github Actions workflows (#9774) --- .github/workflows/build.yml | 45 ++++++++++++++++++++++++++++-------------- .github/workflows/cpp_lint.yml | 9 +++++---- .github/workflows/lua_lint.yml | 2 +- util/ci/build.sh | 2 +- util/ci/clang-tidy.sh | 3 +-- util/ci/common.sh | 16 --------------- 6 files changed, 38 insertions(+), 39 deletions(-) (limited to 'util/ci') diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c4d3abd1..91ed8fd43 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - - name: Install compiler + - name: Install deps run: | sudo apt-get install g++-6 gcc-6 -qyy source ./util/ci/common.sh @@ -41,7 +41,8 @@ jobs: run: | ./util/ci/build.sh env: - CMAKE_FLAGS: "-DCMAKE_C_COMPILER=gcc-6 -DCMAKE_CXX_COMPILER=g++-6" + CC: gcc-6 + CXX: g++-6 - name: Test run: | @@ -52,7 +53,7 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - - name: Install compiler + - name: Install deps run: | sudo apt-get install g++-8 gcc-8 -qyy source ./util/ci/common.sh @@ -62,7 +63,8 @@ jobs: run: | ./util/ci/build.sh env: - CMAKE_FLAGS: "-DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8" + CC: gcc-8 + CXX: g++-8 - name: Test run: | @@ -73,7 +75,7 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - - name: Install compiler + - name: Install deps run: | sudo apt-get install clang-3.9 -qyy source ./util/ci/common.sh @@ -83,7 +85,8 @@ jobs: run: | ./util/ci/build.sh env: - CMAKE_FLAGS: "-DCMAKE_C_COMPILER=clang-3.9 -DCMAKE_CXX_COMPILER=clang++-3.9" + CC: clang-3.9 + CXX: clang++-3.9 - name: Test run: | @@ -94,7 +97,7 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - - name: Install compiler + - name: Install deps run: | sudo apt-get install clang-9 valgrind -qyy source ./util/ci/common.sh @@ -106,7 +109,8 @@ jobs: run: | ./util/ci/build.sh env: - CMAKE_FLAGS: "-DCMAKE_C_COMPILER=clang-9 -DCMAKE_CXX_COMPILER=clang++-9" + CC: clang-9 + CXX: clang++-9 - name: Test run: | @@ -116,12 +120,13 @@ jobs: run: | valgrind --leak-check=full --leak-check-heuristics=all --undef-value-errors=no --error-exitcode=9 ./bin/minetest --run-unittests - + # Build with prometheus-cpp (server-only) clang_9_prometheus: + name: "clang_9 (PROMETHEUS=1)" runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - - name: Install compiler + - name: Install deps run: | sudo apt-get install clang-9 -qyy source ./util/ci/common.sh @@ -135,18 +140,21 @@ jobs: run: | ./util/ci/build.sh env: - CMAKE_FLAGS: "-DCMAKE_C_COMPILER=clang-9 -DCMAKE_CXX_COMPILER=clang++-9 -DENABLE_PROMETHEUS=1" + CC: clang-9 + CXX: clang++-9 + CMAKE_FLAGS: "-DENABLE_PROMETHEUS=1 -DBUILD_CLIENT=0" - name: Test run: | - ./bin/minetest --run-unittests + ./bin/minetestserver --run-unittests - # Some builds doesn't require freetype, ensure it compiled properly + # Build without freetype (client-only) clang_9_no_freetype: + name: "clang_9 (FREETYPE=0)" runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - - name: Install compiler + - name: Install deps run: | sudo apt-get install clang-9 -qyy source ./util/ci/common.sh @@ -156,13 +164,16 @@ jobs: run: | ./util/ci/build.sh env: - CMAKE_FLAGS: "-DCMAKE_C_COMPILER=clang-9 -DCMAKE_CXX_COMPILER=clang++-9 -DENABLE_FREETYPE=0" + CC: clang-9 + CXX: clang++-9 + CMAKE_FLAGS: "-DENABLE_FREETYPE=0 -DBUILD_SERVER=0" - name: Test run: | ./bin/minetest --run-unittests docker: + name: "Docker image" runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 @@ -171,11 +182,13 @@ jobs: docker build . win32: + name: "MinGW cross-compiler (32-bit)" runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - name: Install compiler run: | + sudo apt-get install gettext -qyy wget http://minetest.kitsunemimi.pw/mingw-w64-i686_9.2.0_ubuntu18.04.tar.xz -O mingw.tar.xz sudo tar -xaf mingw.tar.xz -C /usr @@ -187,11 +200,13 @@ jobs: NO_PACKAGE: 1 win64: + name: "MinGW cross-compiler (64-bit)" runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - name: Install compiler run: | + sudo apt-get install gettext -qyy wget http://minetest.kitsunemimi.pw/mingw-w64-x86_64_9.2.0_ubuntu18.04.tar.xz -O mingw.tar.xz sudo tar -xaf mingw.tar.xz -C /usr diff --git a/.github/workflows/cpp_lint.yml b/.github/workflows/cpp_lint.yml index 3044d5186..1f97d105a 100644 --- a/.github/workflows/cpp_lint.yml +++ b/.github/workflows/cpp_lint.yml @@ -30,19 +30,20 @@ jobs: - uses: actions/checkout@v2 - name: Install clang-format run: | - sudo apt-get install ${CLANG_FORMAT} -qyy - env: - CLANG_FORMAT: clang-format-9 + sudo apt-get install clang-format-9 -qyy + - name: Run clang-format run: | source ./util/ci/lint.sh perform_lint + env: + CLANG_FORMAT: clang-format-9 clang_tidy: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - - name: Install clang-tidy + - name: Install deps run: | sudo apt-get install clang-tidy-9 -qyy source ./util/ci/common.sh diff --git a/.github/workflows/lua_lint.yml b/.github/workflows/lua_lint.yml index 1b2c01192..738e5afff 100644 --- a/.github/workflows/lua_lint.yml +++ b/.github/workflows/lua_lint.yml @@ -13,12 +13,12 @@ on: jobs: luacheck: + name: "Builtin Luacheck and Unit Tests" runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - name: Install luarocks run: | - sudo apt-get update -qyy sudo apt-get install luarocks -qyy - name: Install luarocks tools diff --git a/util/ci/build.sh b/util/ci/build.sh index 59069b00a..ba77cd645 100755 --- a/util/ci/build.sh +++ b/util/ci/build.sh @@ -1,4 +1,4 @@ -#! /bin/bash -eu +#! /bin/bash -e mkdir cmakebuild cd cmakebuild diff --git a/util/ci/clang-tidy.sh b/util/ci/clang-tidy.sh index d048f54ee..bb4e99fef 100755 --- a/util/ci/clang-tidy.sh +++ b/util/ci/clang-tidy.sh @@ -5,8 +5,7 @@ cd cmakebuild cmake -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -DRUN_IN_PLACE=TRUE \ - -DENABLE_GETTEXT=TRUE \ - -DENABLE_SOUND=FALSE \ + -DENABLE_{GETTEXT,SOUND}=FALSE \ -DBUILD_SERVER=TRUE .. make GenerateVersion diff --git a/util/ci/common.sh b/util/ci/common.sh index 5a4f78457..a2e4beac9 100644 --- a/util/ci/common.sh +++ b/util/ci/common.sh @@ -1,21 +1,5 @@ #!/bin/bash -e -set_linux_compiler_env() { - if [[ "${COMPILER}" == "gcc-6" ]]; then - export CC=gcc-6 - export CXX=g++-6 - elif [[ "${COMPILER}" == "gcc-8" ]]; then - export CC=gcc-8 - export CXX=g++-8 - elif [[ "${COMPILER}" == "clang-3.9" ]]; then - export CC=clang-3.9 - export CXX=clang++-3.9 - elif [[ "${COMPILER}" == "clang-9" ]]; then - export CC=clang-9 - export CXX=clang++-9 - fi -} - # Linux build only install_linux_deps() { local pkgs=(libirrlicht-dev cmake libbz2-dev libpng-dev \ -- cgit v1.2.3