tool: trace process termination by default

`sched_process_exit` tracepoint is called when thread terminates.
So exitsnoop shows line per each thread termination if the process
is multi-thread process. This is not useful when people wants to
know why process terminates, not thread.

So this changes exitsnoop default behavior which traces process termination
instead of thread termination. And add `--per-thread` option which behaves
as original exitsnoop implementation.

man/man8/exitsnoop.8

@@ -2,7 +2,7 @@
 .SH NAME
 exitsnoop \- Trace all process termination (exit, fatal signal). Uses Linux eBPF/bcc.
 .SH SYNOPSIS
-.B exitsnoop [\-h] [\-t] [\-\-utc] [\-x] [\-p PID] [\-\-label LABEL]
+.B exitsnoop [\-h] [\-t] [\-\-utc] [\-x] [\-p PID] [\-\-label LABEL] [\-\-per\-thread]
 .SH DESCRIPTION
 exitsnoop traces process termination, showing the command name and reason for
 termination, either an exit or a fatal signal.
@@ -35,6 +35,9 @@ Trace this process ID only (filtered in-kernel).
 .TP
 \-\-label LABEL
 Label each line with LABEL (default 'exit') in first column (2nd if timestamp is present).
+.TP
+\-\-per\-thread
+Trace per thread termination
 .SH EXAMPLES
 .TP
 Trace all process termination
@@ -56,6 +59,10 @@ Trace PID 181 only:
 Label each output line with 'EXIT':
 #
 .B exitsnoop \-\-label EXIT
+.TP
+Trace per thread termination
+#
+.B exitsnoop \-\-per\-thread
 .SH FIELDS
 .TP
 TIME-TZ

tools/exitsnoop.py

@@ -27,6 +27,7 @@ _examples = """examples:
     exitsnoop --utc          # include timestamps (UTC)
     exitsnoop -p 181         # only trace PID 181
     exitsnoop --label=exit   # label each output line with 'exit'
+    exitsnoop --per-thread   # trace per thread termination
 """
 """
   Exit status (from <include/sysexits.h>):
@@ -62,6 +63,7 @@ def _getParser():
     a("-p", "--pid",                            help="trace this PID only")
     a("--label",                                help="label each line")
     a("-x", "--failed",    action="store_true", help="trace only fails, exclude exit(0)")
+    a("--per-thread",      action="store_true", help="trace per thread termination")
     # print the embedded C program and exit, for debugging
     a("--ebpf",            action="store_true", help=argparse.SUPPRESS)
     # RHEL 7.6 keeps task->start_time as struct timespec, convert to u64 nanoseconds
@@ -140,11 +142,20 @@ def _embedded_c(args):
     extern int bpf_static_assert[(condition) ? 1 : -1]
     #endif
     """
+
+    if Global.args.pid:
+        if Global.args.per_thread:
+            filter_pid = "task->tgid != %s" % Global.args.pid
+        else:
+            filter_pid = "!(task->tgid == %s && task->pid == task->tgid)" % Global.args.pid
+    else:
+        filter_pid = '0' if Global.args.per_thread else 'task->pid != task->tgid'
+
     code_substitutions = [
         ('EBPF_COMMENT', '' if not Global.args.ebpf else _ebpf_comment()),
         ("BPF_STATIC_ASSERT_DEF", bpf_static_assert_def),
         ("CTYPES_SIZEOF_DATA", str(ct.sizeof(Data))),
-        ('FILTER_PID', '0' if not Global.args.pid else "task->tgid != %s" % Global.args.pid),
+        ('FILTER_PID', filter_pid),
         ('FILTER_EXIT_CODE', '0' if not Global.args.failed else 'task->exit_code == 0'),
         ('PROCESS_START_TIME_NS', 'task->start_time' if not Global.args.timespec else
              '(task->start_time.tv_sec * 1000000000L) + task->start_time.tv_nsec'),

tools/exitsnoop_example.txt

@@ -67,7 +67,7 @@ TIME-UTC     LABEL PCOMM            PID    PPID   TID    AGE(s)  EXIT_CODE
 USAGE message:
 
 # ./exitsnoop.py -h
-usage: exitsnoop.py [-h] [-t] [--utc] [-p PID] [--label LABEL] [-x]
+usage: exitsnoop.py [-h] [-t] [--utc] [-p PID] [--label LABEL] [-x] [--per-thread]
 
 Trace all process termination (exit, fatal signal)
 
@@ -78,6 +78,7 @@ optional arguments:
   -p PID, --pid PID  trace this PID only
   --label LABEL      label each line
   -x, --failed       trace only fails, exclude exit(0)
+  --per-thread       trace per thread termination
 
 examples:
     exitsnoop                # trace all process termination
@@ -86,6 +87,7 @@ examples:
     exitsnoop --utc          # include timestamps (UTC)
     exitsnoop -p 181         # only trace PID 181
     exitsnoop --label=exit   # label each output line with 'exit'
+    exitsnoop --per-thread   # trace per thread termination
 
 Exit status: