add BURSTDOG_USE_TOTAL mode

1 files changed, 139 insertions, 35 deletions

diff --git a/burstdog.c b/burstdog.c
index f64b8a7..da8dedc 100644
--- a/burstdog.c
+++ b/burstdog.c
@@ -17,6 +17,8 @@
 #define BURSTDOG_BURST       98  // cpu time percentage considered a burst
 #define BURSTDOG_BURST_END   90  // cpu time percentage considered the end of a burst
 #define BURSTDOG_SAMPLES      5  // how many samples a burst needs to persist for to be logged
+#define BURSTDOG_USE_TOTAL    1  // 0: consider processes individually 1: consider total cpu usage and log top processes
+#define BURSTDOG_CULPRITS     5  // for BURSTDOG_USE_TOTAL: how many processes are logged on burst
 
 #if BURSTDOG_SAMPLES < 2
 #error must consider at least 2 samples
@@ -30,6 +32,7 @@ struct process
 {
     unsigned int pid;
     int fd;
+    char name[32];
     uint64_t time[BURSTDOG_SAMPLES];
 };
 
@@ -43,7 +46,7 @@ static unsigned int pids[MAX_PROCESS];
 static struct process_tab process_tabs[2];
 static char dirbuffer[DIRBUFFER_SIZE];
 static char statbuffer[BUFSIZ];
-static char linebuffer[BUFSIZ];
+static char msgbuffer[BUFSIZ];
 
 struct linux_dirent64 {
     ino64_t        d_ino;    /* 64-bit inode number */
@@ -53,6 +56,20 @@ struct linux_dirent64 {
     char           d_name[]; /* Filename (null-terminated) */
 };
 
+enum cpu_stat_kind
+{
+    CS_USER,
+    CS_NICE,
+    CS_SYSTEM,
+    CS_IDLE,
+    CS_IOWAIT,
+    CS_IRQ,
+    CS_SOFTIRQ,
+    CS_STEAL,
+    CS_GUEST,
+    CS_GUEST_NICE,
+};
+
 static int compare_int(const void *pa, const void *pb)
 {
     int a = *(const int *) pa;
@@ -62,7 +79,7 @@ static int compare_int(const void *pa, const void *pb)
 
 static bool is_sep(char c)
 {
-    return c == ' ' || c == '\0';
+    return c == ' ' || c == '\n' || c == '\0';
 }
 
 static char *nth_word(unsigned int n, char *ptr, size_t size)
@@ -82,9 +99,21 @@ static char *nth_word(unsigned int n, char *ptr, size_t size)
     return &ptr[start];
 }
 
+static bool write_all(int fd, char *buf, size_t n)
+{
+    while (n) {
+        int written = write(fd, buf, n);
+        if (written == -1)
+            return false;
+        buf += written;
+        n -= written;
+    }
+    return true;
+}
+
 int main(int argc, char **argv)
 {
-    long clock_tick = sysconf(_SC_CLK_TCK);
+    long nproc = sysconf(_SC_NPROCESSORS_ONLN);
 
     char *logfile = argc > 1 ? argv[1] : "burstdog.log";
     int logfd = open(logfile, O_WRONLY | O_CREAT | O_APPEND | O_DIRECT, 0644);
@@ -99,21 +128,51 @@ int main(int argc, char **argv)
         return EXIT_FAILURE;
     }
 
-    uint64_t times[BURSTDOG_SAMPLES];
+    int sysstatfd = openat(dirfd, "stat", O_RDONLY);
+    if (sysstatfd == -1) {
+        perror("open /proc");
+        return EXIT_FAILURE;
+    }
+
     size_t num_samples = 0;
-    unsigned int bursting_pid = 0;
+    struct {
+        uint64_t busy;
+        uint64_t total;
+    } cpu_stats[BURSTDOG_SAMPLES] = {};
+    unsigned int bursting = 0;
     struct process_tab *procs = &process_tabs[0], *oldprocs = &process_tabs[1];
     for (;;) {
-        size_t move_samples = num_samples;
-        if (num_samples < BURSTDOG_SAMPLES)
-            num_samples++;
-        else
-            move_samples--;
-        if (num_samples)
-            memmove(&times[1], &times[0], sizeof(unsigned int) * move_samples);
-        struct timespec ts;
-        clock_gettime(CLOCK_BOOTTIME, &ts);
-        times[0] = ts.tv_sec * clock_tick + ts.tv_nsec * clock_tick / 1000000000;
+        time_t time_v = time(NULL);
+        struct tm time_s;
+        localtime_r(&time_v, &time_s);
+        char timebuf[128];
+        strftime(timebuf, 128, "%c", &time_s);
+
+        ssize_t n_read = pread(sysstatfd, statbuffer, BUFSIZ-1, 0);
+        if (n_read == -1) {
+            perror("pread /proc/stat");
+            return EXIT_FAILURE;
+        }
+        statbuffer[n_read] = '\0';
+
+        uint64_t cpu[10];
+        for (size_t i = 0; i < 10; i++) {
+            char *str = nth_word(2+i, statbuffer, n_read+1);
+            if (!str) {
+                fprintf(stderr, "failed to parse /proc/stat\n");
+                return EXIT_FAILURE;
+            }
+            cpu[i] = atoll(str);
+        }
+
+        uint64_t idle = cpu[CS_IDLE] + cpu[CS_IOWAIT];
+        uint64_t nonidle = cpu[CS_USER] + cpu[CS_NICE] + cpu[CS_SYSTEM] + cpu[CS_IRQ] + cpu[CS_SOFTIRQ] + cpu[CS_STEAL];
+
+        memmove(&cpu_stats[1], &cpu_stats[0], sizeof(*cpu_stats) * (BURSTDOG_SAMPLES-1));
+        cpu_stats[0].busy = nonidle;
+        cpu_stats[0].total = idle+nonidle;
+
+        if (num_samples < BURSTDOG_SAMPLES) num_samples++;
 
         if (lseek(dirfd, 0, SEEK_SET) != 0) {
             perror("seek /proc");
@@ -185,41 +244,86 @@ int main(int argc, char **argv)
             proc->pid = pids[i];
             proc->fd = statfd;
             proc->time[0] = atoll(utime) + atoll(stime);
+            strncpy(proc->name, name, 32);
 
             if (oldproc) {
-                memcpy(&proc->time[1], &oldproc->time[0], (BURSTDOG_SAMPLES-1) * sizeof(int));
+                memcpy(&proc->time[1], &oldproc->time[0], sizeof(*proc->time) * (BURSTDOG_SAMPLES-1));
             } else {
                 for (size_t j = 1; j < BURSTDOG_SAMPLES; j++)
                     proc->time[j] = proc->time[0];
             }
 
+#if !BURSTDOG_USE_TOTAL
             if (num_samples == BURSTDOG_SAMPLES) {
-                uint64_t total = times[0] - times[BURSTDOG_SAMPLES-1];
-                uint64_t subset = proc->time[0] - proc->time[BURSTDOG_SAMPLES-1];
-                uint64_t share = subset * 100 / total;
-                if (bursting_pid == proc->pid && share < BURSTDOG_BURST_END) {
-                    bursting_pid = 0;
-                } else if (bursting_pid != proc->pid && share >= BURSTDOG_BURST) {
-                    time_t time_v = time(NULL);
-                    struct tm time_s;
-                    localtime_r(&time_v, &time_s);
-                    char timebuf[128];
-                    strftime(timebuf, 128, "%c", &time_s);
-
-                    int n_line = snprintf(linebuffer, BUFSIZ,
-                        "%s: %5d %s is bursting: %"PRIu64" of %"PRIu64" ticks (%"PRIu64"%%)\n",
-                        timebuf, proc->pid, name, subset, total, share);
-
-                    int n_written = write(logfd, linebuffer, n_line);
-                    (void) n_written;
-                    bursting_pid = proc->pid;
+                uint64_t total = cpu_stats[0].total - cpu_stats[BURSTDOG_SAMPLES-1].total;
+                uint64_t proc_busy = proc->time[0] - proc->time[BURSTDOG_SAMPLES-1];
+                uint64_t proc_share = nproc * proc_busy * 100 / total;
+                if (bursting == proc->pid && proc_share < BURSTDOG_BURST_END) {
+                    bursting = 0;
+                } else if (bursting != proc->pid && proc_share >= BURSTDOG_BURST) {
+                    int n_msg = snprintf(msgbuffer, BUFSIZ,
+                        "%s: %5d %s is bursting: %"PRIu64" of %"PRIu64" ticks (%"PRIu64"%% of %ld cores)\n",
+                        timebuf, proc->pid, proc->name, proc_busy, total, proc_share, nproc);
+                    write_all(logfd, msgbuffer, n_msg);
+                    bursting = proc->pid;
                 }
             }
+#endif
         }
 
         while (oldproc_idx < oldprocs->num)
             close(oldprocs->arr[oldproc_idx++].fd);
 
+#if BURSTDOG_USE_TOTAL
+        if (num_samples == BURSTDOG_SAMPLES) {
+            uint64_t total = cpu_stats[0].total - cpu_stats[BURSTDOG_SAMPLES-1].total;
+            uint64_t busy = cpu_stats[0].busy - cpu_stats[BURSTDOG_SAMPLES-1].busy;
+            uint64_t share = nproc * busy * 100 / total;
+            uint64_t level = 2*share/BURSTDOG_BURST;
+            bool do_log = false;
+
+            if (bursting && share < BURSTDOG_BURST_END) {
+                bursting = 0;
+            } else if (share >= BURSTDOG_BURST) {
+                if (level > bursting)
+                    do_log = true;
+                bursting = level;
+            }
+
+            if (do_log) {
+                struct process *culprits[BURSTDOG_CULPRITS] = {};
+                memset(culprits, 0, sizeof(culprits));
+                for (size_t i = 0; i < procs->num; i++) {
+                    struct process *proc = &procs->arr[i];
+                    uint64_t proc_busy = proc->time[0] - proc->time[BURSTDOG_SAMPLES-1];
+                    for (size_t j = 0; j < BURSTDOG_CULPRITS; j++) {
+                        if (!culprits[j] || (culprits[j]->time[0] - culprits[j]->time[BURSTDOG_SAMPLES-1]) < proc_busy) {
+                            memmove(&culprits[j+1], &culprits[j], sizeof(*culprits) * (BURSTDOG_CULPRITS-j-1));
+                            culprits[j] = proc;
+                            break;
+                        }
+                    }
+                }
+
+                int n_msg = 0;
+                n_msg += snprintf(msgbuffer+n_msg, BUFSIZ > n_msg ? BUFSIZ-n_msg : 0,
+                    "%s: cpu is bursting: %"PRIu64" of %"PRIu64" ticks (%"PRIu64"%% of %ld cores)\n",
+                    timebuf, busy, total, share, nproc);
+                for (size_t i = 0; i < BURSTDOG_CULPRITS; i++) {
+                    struct process *proc = culprits[i];
+                    if (!proc) break;
+                    uint64_t proc_busy = proc->time[0] - proc->time[BURSTDOG_SAMPLES-1];
+                    uint64_t proc_share = nproc * proc_busy * 100 / total;
+                    n_msg += snprintf(msgbuffer+n_msg, BUFSIZ > n_msg ? BUFSIZ-n_msg : 0,
+                        "%s: top %zu: %5d %s got: %"PRIu64" of %"PRIu64" ticks (%"PRIu64"%% of %ld cores)\n",
+                        timebuf, i+1, proc->pid, proc->name, proc_busy, total, proc_share, nproc);
+                }
+                n_msg += snprintf(msgbuffer+n_msg, BUFSIZ > n_msg ? BUFSIZ-n_msg : 0, "\n");
+                write_all(logfd, msgbuffer, n_msg);
+            }
+        }
+#endif
+
         struct process_tab *tmp = oldprocs;
         oldprocs = procs;
         procs = tmp;