demos: Fix vulkaninfo on win32 to use existing console

The vulkaninfo utility is compiled as /SUBSYSTEM:APPLICATION but
attempts to allocate its own console and print to it. Not only
does this appear to be unreliable (sometimes no output is printed),
it is unnecessary if the utility is compiled as /SUBSYSTEM:CONSOLE.

This removes unnecessary console manipulation code. By using
/SUBSYSTEM:CONSOLE, the ordinary main() entrypoint function can be
used.

The console is still enlarged if it was opened exclusively for the
process by detecting whether it was so opened. Likewise, the process
still sleeps forever if the console is exclusive to the process.
The process behaves like a normal command line process when executed
from the command line.

Fixes lunarg issue 350.

2 files changed, 45 insertions, 59 deletions

diff --git a/demos/CMakeLists.txt b/demos/CMakeLists.txt
index ebc406b7..e8424d65 100644
--- a/demos/CMakeLists.txt
+++ b/demos/CMakeLists.txt
@@ -87,11 +87,7 @@ if(WIN32)
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_CRT_SECURE_NO_WARNINGS -D_USE_MATH_DEFINES")
 endif()
 
-if(UNIX)
-    add_executable(vulkaninfo vulkaninfo.c)
-else()
-    add_executable(vulkaninfo WIN32 vulkaninfo.c)
-endif()
+add_executable(vulkaninfo vulkaninfo.c)
 target_link_libraries(vulkaninfo ${LIBRARIES})
 
 if(UNIX)
diff --git a/demos/vulkaninfo.c b/demos/vulkaninfo.c
index 8549b3a5..dcba5d48 100644
--- a/demos/vulkaninfo.c
+++ b/demos/vulkaninfo.c
@@ -48,11 +48,17 @@
 
 #define snprintf _snprintf
 
-bool consoleCreated = false;
+// Returns nonzero if the console is used only for this process. Will return
+// zero if another process (such as cmd.exe) is also attached.
+static int ConsoleIsExclusive(void) {
+    DWORD pids[2];
+    DWORD num_pids = GetConsoleProcessList(pids, ARRAYSIZE(pids));
+    return num_pids <= 1;
+}
 
 #define WAIT_FOR_CONSOLE_DESTROY                                               \
     do {                                                                       \
-        if (consoleCreated)                                                    \
+        if (ConsoleIsExclusive())                                                    \
             Sleep(INFINITE);                                                   \
     } while (0)
 #else
@@ -1133,6 +1139,32 @@ static void app_gpu_dump(const struct app_gpu *gpu) {
     app_dev_dump(&gpu->dev);
 }
 
+#ifdef _WIN32
+// Enlarges the console window to have a large scrollback size.
+static void ConsoleEnlarge() {
+    HANDLE consoleHandle = GetStdHandle(STD_OUTPUT_HANDLE);
+
+    // make the console window bigger
+    CONSOLE_SCREEN_BUFFER_INFO csbi;
+    COORD bufferSize;
+    if (GetConsoleScreenBufferInfo(consoleHandle, &csbi))
+    {
+        bufferSize.X = csbi.dwSize.X + 30;
+        bufferSize.Y = 20000;
+        SetConsoleScreenBufferSize(consoleHandle, bufferSize);
+    }
+
+    SMALL_RECT r;
+    r.Left = r.Top = 0;
+    r.Right = csbi.dwSize.X - 1 + 30;
+    r.Bottom = 50;
+    SetConsoleWindowInfo(consoleHandle, true, &r);
+
+    // change the console window title
+    SetConsoleTitle(TEXT(APP_SHORT_NAME));
+}
+#endif
+
 int main(int argc, char **argv) {
     unsigned int major, minor, patch;
     struct app_gpu gpus[MAX_GPUS];
@@ -1141,6 +1173,11 @@ int main(int argc, char **argv) {
     VkResult err;
     struct app_instance inst;
 
+#ifdef _WIN32
+    if (ConsoleIsExclusive())
+        ConsoleEnlarge();
+#endif
+
     major = VK_API_VERSION >> 22;
     minor = (VK_API_VERSION >> 12) & 0x3ff;
     patch = VK_API_VERSION & 0xfff;
@@ -1200,58 +1237,11 @@ int main(int argc, char **argv) {
 
     app_destroy_instance(&inst);
 
-    return 0;
-}
-
-#ifdef _WIN32
-
-// Create a console window with a large scrollback size to which to send stdout.
-// Returns true if console window was successfully created, false otherwise.
-bool SetStdOutToNewConsole() {
-    // don't do anything if we already have a console
-    if (GetStdHandle(STD_OUTPUT_HANDLE))
-        return false;
-
-    // allocate a console for this app
-    if (!AllocConsole())
-        return false;
-
-    // redirect unbuffered STDOUT to the console
-    HANDLE consoleHandle = GetStdHandle(STD_OUTPUT_HANDLE);
-    int fileDescriptor = _open_osfhandle((intptr_t)consoleHandle, _O_TEXT);
-    FILE *fp = _fdopen(fileDescriptor, "w");
-    *stdout = *fp;
-    setvbuf(stdout, NULL, _IONBF, 0);
-
-    // make the console window bigger
-    CONSOLE_SCREEN_BUFFER_INFO csbi;
-    COORD bufferSize;
-    if (GetConsoleScreenBufferInfo(consoleHandle, &csbi))
-    {
-        bufferSize.X = csbi.dwSize.X + 30;
-        bufferSize.Y = 20000;
-        SetConsoleScreenBufferSize(consoleHandle, bufferSize);
-    }
-
-    SMALL_RECT r;
-    r.Left = r.Top = 0;
-    r.Right = csbi.dwSize.X - 1 + 30;
-    r.Bottom = 50;
-    SetConsoleWindowInfo(consoleHandle, true, &r);
-
-    // change the console window title
-    SetConsoleTitle(TEXT(APP_SHORT_NAME));
-
-    return true;
-}
-
-int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, PSTR pCmdLine,
-                   int nCmdShow) {
-    char *argv = pCmdLine;
-    consoleCreated = SetStdOutToNewConsole();
-    main(1, &argv);
     fflush(stdout);
-    if (consoleCreated)
+#ifdef _WIN32
+    if (ConsoleIsExclusive())
         Sleep(INFINITE);
-}
 #endif
+
+    return 0;
+}