Reference guide
options
help
tokunori@tokunori-desktop:~$ bpftrace --help
USAGE:
bpftrace [options] filename
bpftrace [options] - <stdin input>
bpftrace [options] -e 'program'
OPTIONS:
-B MODE output buffering mode ('full', 'none')
-f FORMAT output format ('text', 'json')
-o file redirect bpftrace output to file
-e 'program' execute this program
-h, --help show this help message
-I DIR add the directory to the include search path
--include FILE add an #include file before preprocessing
-l [search] list probes
-p PID enable USDT probes on PID
-c 'CMD' run CMD and enable USDT probes on resulting process
--usdt-file-activation
activate usdt semaphores based on file path
--unsafe allow unsafe builtin functions
-q keep messages quiet
--info Print information about kernel BPF support
-k emit a warning when a bpf helper returns an error (except read functions)
-kk check all bpf helper functions
-V, --version bpftrace version
--no-warnings disable all warning messages
TROUBLESHOOTING OPTIONS:
-v verbose messages
-d (dry run) debug info
-dd (dry run) verbose debug info
--emit-elf FILE (dry run) generate ELF file with bpf programs and write to FILE
--emit-llvm FILE write LLVM IR to FILE.original.ll and FILE.optimized.ll
ENVIRONMENT:
BPFTRACE_STRLEN [default: 64] bytes on BPF stack per str()
BPFTRACE_NO_CPP_DEMANGLE [default: 0] disable C++ symbol demangling
BPFTRACE_MAP_KEYS_MAX [default: 4096] max keys in a map
BPFTRACE_CAT_BYTES_MAX [default: 10k] maximum bytes read by cat builtin
BPFTRACE_MAX_PROBES [default: 512] max number of probes
BPFTRACE_MAX_BPF_PROGS [default: 512] max number of generated BPF programs
BPFTRACE_LOG_SIZE [default: 1000000] log size in bytes
BPFTRACE_PERF_RB_PAGES [default: 64] pages per CPU to allocate for ring buffer
BPFTRACE_NO_USER_SYMBOLS [default: 0] disable user symbol resolution
BPFTRACE_CACHE_USER_SYMBOLS [default: auto] enable user symbol cache
BPFTRACE_VMLINUX [default: none] vmlinux path used for kernel symbol resolution
BPFTRACE_BTF [default: none] BTF file
EXAMPLES:
bpftrace -l '*sleep*'
list probes containing "sleep"
bpftrace -e 'kprobe:do_nanosleep { printf("PID %d sleeping...\n", pid); }'
trace processes calling sleep
bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); }'
count syscalls by process name
program
nsecs
root@tokunori-desktop:/home/tokunori# bpftrace -e 'kprobe:nvme_submit_user_cmd { @start = nsecs; } tracepoint:nvme:nvme_complete_rq /@start != 0/ { printf("latency: %u usec\n", (nsecs - @start) / 1e3); @start = 0; }'
Attaching 2 probes...
latency: 115 usec
latency: 88 usec
latency: 9190 usec
^C
@start: 0
root@tokunori-desktop:/home/tokunori# bpftrace -e 'kprobe:nvme_submit_user_cmd { @start = nsecs; } tracepoint:nvme:nvme_complete_rq /@start != 0/ { printf("latency: %u usec\n", (nsecs - @start) / 1e3); exit(); }'
Attaching 2 probes...
latency: 62 usec
@start: 4610675857935
tokunori@tokunori-desktop:~$ sudo bpftrace -e 'kprobe:nvme_submit_user_cmd { printf("nvme_submit_user_cmd: %d usecs\n", nsecs / 1e3); } kprobe:nvme_pci_complete_rq { printf("nvme_pci_complete_rq: %d usecs\n", nsecs / 1e3); }'
Attaching 2 probes...
nvme_pci_complete_rq: 288380020 usecs
nvme_submit_user_cmd: 289193121 usecs
nvme_pci_complete_rq: 290822995 usecs
nvme_pci_complete_rq: 290823400 usecs
...
nvme_pci_complete_rq: 308935959 usecs
nvme_submit_user_cmd: 310082640 usecs
nvme_pci_complete_rq: 310296300 usecs
nvme_pci_complete_rq: 312702398 usecs
count()
2004 sudo bpftrace -e 'kprobe:nvme_submit_user_cmd { @[str(arg1)] = count(); }'
2005 sudo bpftrace -e 'kprobe:nvme_dev_release { @[str(arg1)] = count(); }'
2006 sudo bpftrace -e 'kprobe:nvme_pci_complete_rq { @[str(arg1)] = count(); }'
2009 sudo bpftrace -e 'kprobe:nvme_complete_rq { @[str(arg1)] = count(); }'
2010 sudo bpftrace -e 'kprobe:nvme_complete_async_event { @[str(arg1)] = count(); }'
2011 sudo bpftrace -e 'kprobe:nvme_complete_batch_req { @[str(arg1)] = count(); }'
2012 sudo bpftrace -e 'kprobe:nvme_dev_release { @[str(arg1)] = count(); }'
2013 sudo bpftrace -e 'kprobe:nvme_dev_ioctl { @[str(arg1)] = count(); }'
2014 sudo bpftrace -e 'kprobe:nvme_ctrl_ioctl { @[str(arg1)] = count(); }'
2015 sudo bpftrace -e 'kprobe:nvme_ioctl { @[str(arg1)] = count(); }'
2016 sudo bpftrace -e 'kprobe:nvme_irq { @[str(arg1)] = count(); }'
tools
biolatency.bt
tokunori@tokunori-desktop:~$ sudo bpftrace/tools/biolatency.bt
bpftrace/tools/biolatency.bt:21:2-14: ERROR: Can not access field 'sector' on type '(ctx) struct _tracepoint_block_block_bio_queue *'. Try dereferencing it first, or using '->'
@start[args.sector] = nsecs;
~~~~~~~~~~~~
bpftrace/tools/biolatency.bt:26:1-14: ERROR: Can not access field 'sector' on type '(ctx) struct _tracepoint_block_block_bio_complete *'. Try dereferencing it first, or using '->'
/@start[args.sector]/
~~~~~~~~~~~~~
bpftrace/tools/biolatency.bt:28:25-37: ERROR: Can not access field 'sector' on type '(ctx) struct _tracepoint_block_block_bio_complete *'. Try dereferencing it first, or using '->'
@usecs = hist((nsecs - @start[args.sector]) / 1000);
~~~~~~~~~~~~
bpftrace/tools/biolatency.bt:29:2-21: ERROR: Can not access field 'sector' on type '(ctx) struct _tracepoint_block_block_bio_complete *'. Try dereferencing it first, or using '->'
delete(@start[args.sector]);
~~~~~~~~~~~~~~~~~~~
biolatency-kp.bt
tokunori@tokunori-desktop:~$ sudo bpftrace/tools/biolatency-kp.bt
bpftrace/tools/biolatency-kp.bt:17-19: WARNING: blk_account_io_start is not traceable (either non-existing, inlined, or marked as "notrace"); attaching to it will likely fail
bpftrace/tools/biolatency-kp.bt:19-20: WARNING: __blk_account_io_start is not traceable (either non-existing, inlined, or marked as "notrace"); attaching to it will likely fail
bpftrace/tools/biolatency-kp.bt:23-25: WARNING: blk_account_io_done is not traceable (either non-existing, inlined, or marked as "notrace"); attaching to it will likely fail
bpftrace/tools/biolatency-kp.bt:25-26: WARNING: __blk_account_io_done is not traceable (either non-existing, inlined, or marked as "notrace"); attaching to it will likely fail
Attaching 6 probes...
cannot attach kprobe, probe entry may not exist
WARNING: could not attach probe kprobe:__blk_account_io_done, skipping.
cannot attach kprobe, Invalid argument
WARNING: could not attach probe kprobe:blk_account_io_done, skipping.
cannot attach kprobe, probe entry may not exist
WARNING: could not attach probe kprobe:__blk_account_io_start, skipping.
cannot attach kprobe, probe entry may not exist
WARNING: could not attach probe kprobe:blk_account_io_start, skipping.
Tracing block device I/O... Hit Ctrl-C to end.
^C
biosnoop.bt
tokunori@tokunori-desktop:~$ sudo bpftrace/tools/biosnoop.bt
bpftrace/tools/biosnoop.bt:21-23: WARNING: blk_account_io_start is not traceable (either non-existing, inlined, or marked as "notrace"); attaching to it will likely fail
bpftrace/tools/biosnoop.bt:23-24: WARNING: __blk_account_io_start is not traceable (either non-existing, inlined, or marked as "notrace"); attaching to it will likely fail
bpftrace/tools/biosnoop.bt:30-32: WARNING: blk_account_io_done is not traceable (either non-existing, inlined, or marked as "notrace"); attaching to it will likely fail
bpftrace/tools/biosnoop.bt:32-33: WARNING: __blk_account_io_done is not traceable (either non-existing, inlined, or marked as "notrace"); attaching to it will likely fail
Attaching 6 probes...
cannot attach kprobe, probe entry may not exist
WARNING: could not attach probe kprobe:__blk_account_io_done, skipping.
cannot attach kprobe, Invalid argument
WARNING: could not attach probe kprobe:blk_account_io_done, skipping.
cannot attach kprobe, probe entry may not exist
WARNING: could not attach probe kprobe:__blk_account_io_start, skipping.
cannot attach kprobe, probe entry may not exist
WARNING: could not attach probe kprobe:blk_account_io_start, skipping.
TIME(ms) DISK COMM PID LAT(ms)
^C
nvme-cli
dsm
tokunori@tokunori-desktop:~/nvme-cli-2$ sudo .build/nvme dsm /dev/nvme0n1 -b 256 -L
ioctl: 4e40 (ID) latency: 3 us
IO Command opcode: 09 (Dataset Management) latency: 79 us
NVMe DSM: success
root@tokunori-desktop:/home/tokunori/nvme-cli-2# .build/nvme dsm /dev/nvme0n1 -b 256 -L
ioctl: 4e40 (ID) latency: 3 us
IO Command opcode: 09 (Dataset Management) latency: 87 us
NVMe DSM: success
bpftrace
root@tokunori-desktop:/home/tokunori# bpftrace -e 'kprobe:nvme_submit_user_cmd { @start = nsecs; } tracepoint:nvme:nvme_complete_rq /@start != 0/ { printf("latency: %u usec\n", (nsecs - @start) / 1e3); exit(); }'
Attaching 2 probes...
latency: 62 usec
@start: 4610675857935
dsm (4 ranges)
root@tokunori-desktop:/home/tokunori/nvme-cli-2# .build/nvme dsm /dev/nvme0n1 -b 0x100000,0x100000,0x100000,0x100000 -s 0x0,0x100000,0x200000,0x300000 -L -d
ioctl: 4e40 (ID) latency: 3 us
IO Command opcode: 09 (Dataset Management) latency: 3480 us
NVMe DSM: success
bpftrace
root@tokunori-desktop:/home/tokunori# bpftrace -e 'kprobe:nvme_submit_user_cmd { @start = nsecs; } tracepoint:nvme:nvme_complete_rq /@start != 0/ { printf("latency: %u usec\n", (nsecs - @start) / 1e3); @start = 0; }'
Attaching 2 probes...
latency: 3456 usec
^C
@start: 0
format
root@tokunori-desktop:/home/tokunori/nvme-cli-2# .build/nvme format /dev/nvme0n1 -L
Admin Command opcode: 06 (Identify) latency: 175 us
ioctl: 4e40 (ID) latency: 1 us
Admin Command opcode: 06 (Identify) latency: 102 us
You are about to format nvme0n1, namespace 0x1.
WARNING: Format may irrevocably delete this device's data.
You have 10 seconds to press Ctrl-C to cancel this operation.
Use the force [--force] option to suppress this warning.
Sending format operation ...
Admin Command opcode: 80 (Format NVM) latency: 330815 us
Success formatting namespace:1
bpftrace
root@tokunori-desktop:/home/tokunori# bpftrace -e 'kprobe:nvme_submit_user_cmd { @start = nsecs; } tracepoint:nvme:nvme_complete_rq /@start != 0/ { printf("latency: %u usec\n", (nsecs - @start) / 1e3); exit(); }'
Attaching 2 probes...
latency: 135 usec
@start: 5694705011354
format (secure erase: user data erase)
root@tokunori-desktop:/home/tokunori/nvme-cli-2# .build/nvme format /dev/nvme0n1 -s 1 -L
Admin Command opcode: 06 (Identify) latency: 138 us
ioctl: 4e40 (ID) latency: 1 us
Admin Command opcode: 06 (Identify) latency: 98 us
You are about to format nvme0n1, namespace 0x1.
WARNING: Format may irrevocably delete this device's data.
You have 10 seconds to press Ctrl-C to cancel this operation.
Use the force [--force] option to suppress this warning.
Sending format operation ...
Admin Command opcode: 80 (Format NVM) latency: 9883314 us
Success formatting namespace:1
bpftrace
root@tokunori-desktop:/home/tokunori# bpftrace -e 'kprobe:nvme_submit_user_cmd { @start = nsecs; } tracepoint:nvme:nvme_complete_rq /@start != 0/ { printf("latency: %u usec\n", (nsecs - @start) / 1e3); @start = 0; }'
Attaching 2 probes...
latency: 115 usec
latency: 88 usec
latency: 9190 usec
^C
@start: 0