layers: if no settings file or call overrides make callback override loging

Default debugAction in layers is to log based on variable init. Make the
overrides work as follows:
1) app calls xglRegisterMsgCallback()
2) settings file
3) Glave (or app) calls to setlayerOption()

6 files changed, 56 insertions, 12 deletions

diff --git a/layers/draw_state.c b/layers/draw_state.c
index 233c90e6..dfa20f42 100644
--- a/layers/draw_state.c
+++ b/layers/draw_state.c
@@ -1669,8 +1669,8 @@ static void initDrawState(void)
 {
     const char *strOpt;
     // initialize DrawState options
-    g_reportingLevel = getLayerOptionEnum("DrawStateReportLevel", g_reportingLevel);
-    g_debugAction    = getLayerOptionEnum("DrawStateDebugAction", g_debugAction);
+    getLayerOptionEnum("DrawStateReportLevel", &g_reportingLevel);
+    g_actionIsDefault = getLayerOptionEnum("DrawStateDebugAction", &g_debugAction);
 
     if (g_debugAction & XGL_DBG_LAYER_ACTION_LOG_MSG)
     {
@@ -3141,6 +3141,9 @@ XGL_LAYER_EXPORT XGL_RESULT XGLAPI xglDbgRegisterMsgCallback(XGL_DBG_MSG_CALLBAC
     pNewDbgFuncNode->pUserData = pUserData;
     pNewDbgFuncNode->pNext = g_pDbgFunctionHead;
     g_pDbgFunctionHead = pNewDbgFuncNode;
+    // force callbacks if DebugAction hasn't been set already other than initial value
+    if (g_actionIsDefault)
+        g_debugAction = XGL_DBG_LAYER_ACTION_CALLBACK;
     XGL_RESULT result = nextTable.DbgRegisterMsgCallback(pfnMsgCallback, pUserData);
     return result;
 }
@@ -3163,6 +3166,13 @@ XGL_LAYER_EXPORT XGL_RESULT XGLAPI xglDbgUnregisterMsgCallback(XGL_DBG_MSG_CALLB
         pPrev = pTrav;
         pTrav = pTrav->pNext;
     }
+    if (g_pDbgFunctionHead == NULL)
+    {
+        if (g_actionIsDefault)
+            g_debugAction = XGL_DBG_LAYER_ACTION_LOG_MSG;
+        else
+            g_debugAction &= ~XGL_DBG_LAYER_ACTION_CALLBACK;
+    }
     XGL_RESULT result = nextTable.DbgUnregisterMsgCallback(pfnMsgCallback);
     return result;
 }
diff --git a/layers/layers_config.cpp b/layers/layers_config.cpp
index 14e8053d..6992f06f 100644
--- a/layers/layers_config.cpp
+++ b/layers/layers_config.cpp
@@ -81,16 +81,17 @@ const char *getLayerOption(const char *_option)
     return g_configFileObj.getOption(_option);
 }
 
-uint32_t getLayerOptionEnum(const char *_option, uint32_t optionDefault)
+bool getLayerOptionEnum(const char *_option, uint32_t *optionDefault)
 {
-    uint32_t val = 0;
+    bool res;
     const char *option = (g_configFileObj.getOption(_option));
     if (option != NULL) {
-        val = convertStringEnumVal(option);
+        *optionDefault = convertStringEnumVal(option);
+        res = false;
     } else {
-        val = optionDefault;
+        res = true;
     }
-    return val;
+    return res;
 }
 
 void setLayerOptionEnum(const char *_option, const char *_valEnum)
diff --git a/layers/layers_config.h b/layers/layers_config.h
index 722f40d3..7b608d78 100644
--- a/layers/layers_config.h
+++ b/layers/layers_config.h
@@ -23,13 +23,14 @@
  *
  **************************************************************************/
 #pragma once
+#include <stdbool.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 const char *getLayerOption(const char *_option);
-uint32_t getLayerOptionEnum(const char *_option, uint32_t optionDefault);
+bool getLayerOptionEnum(const char *_option, uint32_t *optionDefault);
 
 void setLayerOption(const char *_option, const char *_val);
 void setLayerOptionEnum(const char *_option, const char *_valEnum);
diff --git a/layers/layers_msg.h b/layers/layers_msg.h
index 933012c1..88d9fba7 100644
--- a/layers/layers_msg.h
+++ b/layers/layers_msg.h
@@ -22,10 +22,12 @@
  * DEALINGS IN THE SOFTWARE.
  */
 #include <stdio.h>
+#include <stdbool.h>
 
 static XGL_LAYER_DBG_FUNCTION_NODE *g_pDbgFunctionHead = NULL;
 static XGL_LAYER_DBG_REPORT_LEVEL g_reportingLevel = XGL_DBG_LAYER_LEVEL_INFO;
 static XGL_LAYER_DBG_ACTION g_debugAction = XGL_DBG_LAYER_ACTION_LOG_MSG;
+static bool g_actionIsDefault = true;
 static FILE *g_logFile = NULL;
 
 // Utility function to handle reporting
diff --git a/layers/mem_tracker.c b/layers/mem_tracker.c
index 45495963..8b9d4d23 100644
--- a/layers/mem_tracker.c
+++ b/layers/mem_tracker.c
@@ -770,8 +770,8 @@ static void initMemTracker(void)
 {
     const char *strOpt;
     // initialize MemTracker options
-    g_reportingLevel = getLayerOptionEnum("MemTrackerReportLevel", g_reportingLevel);
-    g_debugAction    = getLayerOptionEnum("MemTrackerDebugAction", g_debugAction);
+    getLayerOptionEnum("MemTrackerReportLevel", &g_reportingLevel);
+    g_actionIsDefault = getLayerOptionEnum("MemTrackerDebugAction", &g_debugAction);
 
     if (g_debugAction & XGL_DBG_LAYER_ACTION_LOG_MSG)
     {
@@ -1966,6 +1966,9 @@ XGL_LAYER_EXPORT XGL_RESULT XGLAPI xglDbgRegisterMsgCallback(XGL_DBG_MSG_CALLBAC
     pNewDbgFuncNode->pUserData = pUserData;
     pNewDbgFuncNode->pNext = g_pDbgFunctionHead;
     g_pDbgFunctionHead = pNewDbgFuncNode;
+    // force callbacks if DebugAction hasn't been set already other than initial value
+    if (g_actionIsDefault)
+        g_debugAction = XGL_DBG_LAYER_ACTION_CALLBACK;
     XGL_RESULT result = nextTable.DbgRegisterMsgCallback(pfnMsgCallback, pUserData);
     return result;
 }
@@ -1985,6 +1988,13 @@ XGL_LAYER_EXPORT XGL_RESULT XGLAPI xglDbgUnregisterMsgCallback(XGL_DBG_MSG_CALLB
         pPrev = pTrav;
         pTrav = pTrav->pNext;
     }
+    if (g_pDbgFunctionHead == NULL)
+    {
+        if (g_actionIsDefault)
+            g_debugAction = XGL_DBG_LAYER_ACTION_LOG_MSG;
+        else
+            g_debugAction &= ~XGL_DBG_LAYER_ACTION_CALLBACK;
+    }
     XGL_RESULT result = nextTable.DbgUnregisterMsgCallback(pfnMsgCallback);
     return result;
 }
diff --git a/xgl-layer-generate.py b/xgl-layer-generate.py
index 6cb2bef2..eef53ad2 100755
--- a/xgl-layer-generate.py
+++ b/xgl-layer-generate.py
@@ -148,6 +148,9 @@ class Subcommand(object):
         r_body.append('    pNewDbgFuncNode->pUserData = pUserData;')
         r_body.append('    pNewDbgFuncNode->pNext = g_pDbgFunctionHead;')
         r_body.append('    g_pDbgFunctionHead = pNewDbgFuncNode;')
+        r_body.append('    // force callbacks if DebugAction hasn\'t been set already other than initial value')
+        r_body.append('    if (g_actionIsDefault)')
+        r_body.append('        g_debugAction = XGL_DBG_LAYER_ACTION_CALLBACK;')
         r_body.append('    XGL_RESULT result = nextTable.DbgRegisterMsgCallback(pfnMsgCallback, pUserData);')
         r_body.append('    return result;')
         r_body.append('}')
@@ -170,6 +173,13 @@ class Subcommand(object):
         ur_body.append('        pPrev = pTrav;')
         ur_body.append('        pTrav = pTrav->pNext;')
         ur_body.append('    }')
+        ur_body.append('    if (g_pDbgFunctionHead == NULL)')
+        ur_body.append('    {')
+        ur_body.append('        if (g_actionIsDefault)')
+        ur_body.append('            g_debugAction = XGL_DBG_LAYER_ACTION_LOG_MSG;')
+        ur_body.append('        else')
+        ur_body.append('            g_debugAction &= ~XGL_DBG_LAYER_ACTION_CALLBACK;')
+        ur_body.append('    }')
         ur_body.append('    XGL_RESULT result = nextTable.DbgUnregisterMsgCallback(pfnMsgCallback);')
         ur_body.append('    return result;')
         ur_body.append('}')
@@ -580,6 +590,9 @@ class Subcommand(object):
                         using_line += '    pNewDbgFuncNode->pUserData = pUserData;\n'
                         using_line += '    pNewDbgFuncNode->pNext = g_pDbgFunctionHead;\n'
                         using_line += '    g_pDbgFunctionHead = pNewDbgFuncNode;\n'
+                        using_line += '    // force callbacks if DebugAction hasn\'t been set already other than initial value\n'
+                        using_line += '    if (g_actionIsDefault)\n'
+                        using_line += '        g_debugAction = XGL_DBG_LAYER_ACTION_CALLBACK;\n'
                     elif 'DbgUnregisterMsgCallback' in proto.name:
                         using_line =  '    XGL_LAYER_DBG_FUNCTION_NODE *pTrav = g_pDbgFunctionHead;\n'
                         using_line += '    XGL_LAYER_DBG_FUNCTION_NODE *pPrev = pTrav;\n'
@@ -594,6 +607,13 @@ class Subcommand(object):
                         using_line += '        pPrev = pTrav;\n'
                         using_line += '        pTrav = pTrav->pNext;\n'
                         using_line += '    }\n'
+                        using_line += '    if (g_pDbgFunctionHead == NULL)\n'
+                        using_line += '    {\n'
+                        using_line += '        if (g_actionIsDefault)\n'
+                        using_line += '            g_debugAction = XGL_DBG_LAYER_ACTION_LOG_MSG;\n'
+                        using_line += '        else\n'
+                        using_line += '            g_debugAction &= ~XGL_DBG_LAYER_ACTION_CALLBACK;\n'
+                        using_line += '    }\n'
                     # Special cases for API funcs that don't use an object as first arg
                     elif True in [no_use_proto in proto.name for no_use_proto in ['GlobalOption', 'CreateInstance', 'QueueSubmit', 'QueueSetGlobalMemReferences', 'QueueWaitIdle', 'CreateDevice', 'SignalQueueSemaphore', 'WaitQueueSemaphore', 'WsiX11QueuePresent']]:
                         using_line = ''
@@ -888,8 +908,8 @@ class Subcommand(object):
         if init_opts:
             func_body.append('    const char *strOpt;')
             func_body.append('    // initialize %s options' % name)
-            func_body.append('    g_reportingLevel = getLayerOptionEnum("%sReportLevel", g_reportingLevel);' % name)
-            func_body.append('    g_debugAction    = getLayerOptionEnum("%sDebugAction", g_debugAction);' % name)
+            func_body.append('    getLayerOptionEnum("%sReportLevel", &g_reportingLevel);' % name)
+            func_body.append('    g_actionIsDefault = getLayerOptionEnum("%sDebugAction", &g_debugAction);' % name)
             func_body.append('')
             func_body.append('    if (g_debugAction & XGL_DBG_LAYER_ACTION_LOG_MSG)')
             func_body.append('    {')