layers: if no settings file or call overrides make callback override loging

Default debugAction in layers is to log based on variable init. Make the
overrides work as follows:
1) app calls xglRegisterMsgCallback()
2) settings file
3) Glave (or app) calls to setlayerOption()

5 files changed, 34 insertions, 10 deletions

diff --git a/layers/draw_state.c b/layers/draw_state.c
index 233c90e6..dfa20f42 100644
--- a/layers/draw_state.c
+++ b/layers/draw_state.c
@@ -1669,8 +1669,8 @@ static void initDrawState(void)
 {
     const char *strOpt;
     // initialize DrawState options
-    g_reportingLevel = getLayerOptionEnum("DrawStateReportLevel", g_reportingLevel);
-    g_debugAction    = getLayerOptionEnum("DrawStateDebugAction", g_debugAction);
+    getLayerOptionEnum("DrawStateReportLevel", &g_reportingLevel);
+    g_actionIsDefault = getLayerOptionEnum("DrawStateDebugAction", &g_debugAction);
 
     if (g_debugAction & XGL_DBG_LAYER_ACTION_LOG_MSG)
     {
@@ -3141,6 +3141,9 @@ XGL_LAYER_EXPORT XGL_RESULT XGLAPI xglDbgRegisterMsgCallback(XGL_DBG_MSG_CALLBAC
     pNewDbgFuncNode->pUserData = pUserData;
     pNewDbgFuncNode->pNext = g_pDbgFunctionHead;
     g_pDbgFunctionHead = pNewDbgFuncNode;
+    // force callbacks if DebugAction hasn't been set already other than initial value
+    if (g_actionIsDefault)
+        g_debugAction = XGL_DBG_LAYER_ACTION_CALLBACK;
     XGL_RESULT result = nextTable.DbgRegisterMsgCallback(pfnMsgCallback, pUserData);
     return result;
 }
@@ -3163,6 +3166,13 @@ XGL_LAYER_EXPORT XGL_RESULT XGLAPI xglDbgUnregisterMsgCallback(XGL_DBG_MSG_CALLB
         pPrev = pTrav;
         pTrav = pTrav->pNext;
     }
+    if (g_pDbgFunctionHead == NULL)
+    {
+        if (g_actionIsDefault)
+            g_debugAction = XGL_DBG_LAYER_ACTION_LOG_MSG;
+        else
+            g_debugAction &= ~XGL_DBG_LAYER_ACTION_CALLBACK;
+    }
     XGL_RESULT result = nextTable.DbgUnregisterMsgCallback(pfnMsgCallback);
     return result;
 }
diff --git a/layers/layers_config.cpp b/layers/layers_config.cpp
index 14e8053d..6992f06f 100644
--- a/layers/layers_config.cpp
+++ b/layers/layers_config.cpp
@@ -81,16 +81,17 @@ const char *getLayerOption(const char *_option)
     return g_configFileObj.getOption(_option);
 }
 
-uint32_t getLayerOptionEnum(const char *_option, uint32_t optionDefault)
+bool getLayerOptionEnum(const char *_option, uint32_t *optionDefault)
 {
-    uint32_t val = 0;
+    bool res;
     const char *option = (g_configFileObj.getOption(_option));
     if (option != NULL) {
-        val = convertStringEnumVal(option);
+        *optionDefault = convertStringEnumVal(option);
+        res = false;
     } else {
-        val = optionDefault;
+        res = true;
     }
-    return val;
+    return res;
 }
 
 void setLayerOptionEnum(const char *_option, const char *_valEnum)
diff --git a/layers/layers_config.h b/layers/layers_config.h
index 722f40d3..7b608d78 100644
--- a/layers/layers_config.h
+++ b/layers/layers_config.h
@@ -23,13 +23,14 @@
  *
  **************************************************************************/
 #pragma once
+#include <stdbool.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 const char *getLayerOption(const char *_option);
-uint32_t getLayerOptionEnum(const char *_option, uint32_t optionDefault);
+bool getLayerOptionEnum(const char *_option, uint32_t *optionDefault);
 
 void setLayerOption(const char *_option, const char *_val);
 void setLayerOptionEnum(const char *_option, const char *_valEnum);
diff --git a/layers/layers_msg.h b/layers/layers_msg.h
index 933012c1..88d9fba7 100644
--- a/layers/layers_msg.h
+++ b/layers/layers_msg.h
@@ -22,10 +22,12 @@
  * DEALINGS IN THE SOFTWARE.
  */
 #include <stdio.h>
+#include <stdbool.h>
 
 static XGL_LAYER_DBG_FUNCTION_NODE *g_pDbgFunctionHead = NULL;
 static XGL_LAYER_DBG_REPORT_LEVEL g_reportingLevel = XGL_DBG_LAYER_LEVEL_INFO;
 static XGL_LAYER_DBG_ACTION g_debugAction = XGL_DBG_LAYER_ACTION_LOG_MSG;
+static bool g_actionIsDefault = true;
 static FILE *g_logFile = NULL;
 
 // Utility function to handle reporting
diff --git a/layers/mem_tracker.c b/layers/mem_tracker.c
index 45495963..8b9d4d23 100644
--- a/layers/mem_tracker.c
+++ b/layers/mem_tracker.c
@@ -770,8 +770,8 @@ static void initMemTracker(void)
 {
     const char *strOpt;
     // initialize MemTracker options
-    g_reportingLevel = getLayerOptionEnum("MemTrackerReportLevel", g_reportingLevel);
-    g_debugAction    = getLayerOptionEnum("MemTrackerDebugAction", g_debugAction);
+    getLayerOptionEnum("MemTrackerReportLevel", &g_reportingLevel);
+    g_actionIsDefault = getLayerOptionEnum("MemTrackerDebugAction", &g_debugAction);
 
     if (g_debugAction & XGL_DBG_LAYER_ACTION_LOG_MSG)
     {
@@ -1966,6 +1966,9 @@ XGL_LAYER_EXPORT XGL_RESULT XGLAPI xglDbgRegisterMsgCallback(XGL_DBG_MSG_CALLBAC
     pNewDbgFuncNode->pUserData = pUserData;
     pNewDbgFuncNode->pNext = g_pDbgFunctionHead;
     g_pDbgFunctionHead = pNewDbgFuncNode;
+    // force callbacks if DebugAction hasn't been set already other than initial value
+    if (g_actionIsDefault)
+        g_debugAction = XGL_DBG_LAYER_ACTION_CALLBACK;
     XGL_RESULT result = nextTable.DbgRegisterMsgCallback(pfnMsgCallback, pUserData);
     return result;
 }
@@ -1985,6 +1988,13 @@ XGL_LAYER_EXPORT XGL_RESULT XGLAPI xglDbgUnregisterMsgCallback(XGL_DBG_MSG_CALLB
         pPrev = pTrav;
         pTrav = pTrav->pNext;
     }
+    if (g_pDbgFunctionHead == NULL)
+    {
+        if (g_actionIsDefault)
+            g_debugAction = XGL_DBG_LAYER_ACTION_LOG_MSG;
+        else
+            g_debugAction &= ~XGL_DBG_LAYER_ACTION_CALLBACK;
+    }
     XGL_RESULT result = nextTable.DbgUnregisterMsgCallback(pfnMsgCallback);
     return result;
 }