1# 保存成文件
2trace-cmd record
3trace-cmd report

1# 从tracefs读取
2trace-cmd start
3trace-cmd stop
4trace-cmd show
5
6trace-cmd clear
7trace-cmd restart
8trace-cmd show

 1# 内存分配
 2-n kmem_cache_alloc -n __kmalloc -n kmalloc_trace -n kfree -n kmalloc_slab -n __kmalloc_node_track_caller -n vmalloc -n vfree
 3
 4# 调度
 5-n __cond_resched -n __wake_up
 6
 7# 中断
 8-n irq_enter_rcu -n irq_exit_rcu
 9
10# 锁
11-n '*_*lock' -n *_lock_irqsave -n *_unlock_irqrestore -n down_write -n up_write
12
13-n mutex_lock -n mutex_unlock \
14-n _raw_spin_lock -n _raw_spin_unlock \
15-n _raw_spin_lock_irqsave -n _raw_spin_unlock_irqrestore \
16-n __rcu_read_lock -n __rcu_read_unlock \

1PID=$(pidof sh)
2trace-cmd start -p function_graph --max-graph-depth 5 -P $PID
3trace-cmd stop
4trace-cmd show
5trace-cmd clear

1PID=$(pidof sh)
2trace-cmd start -p function -l vfs_open -l filp_close -P $PID
3trace-cmd stop
4trace-cmd show
5trace-cmd clear

1trace-cmd record --temp /tmp/ -p function_graph --max-graph-depth 4 -g "do_exec*" -g load_elf_binary -e "task:*" -n __check_object_size -n __cond_resched -n '*_*lock' sh -c ls
2trace-cmd report
3trace-cmd hist      # 输出内容较多，可以重定向到文件

1export LD_LIBRARY_PATH=$(pwd)
2
3trace-cmd record --temp /tmp/ -p function_graph -e sched:* sh -c "echo hello; sleep 3"
4trace-cmd report | grep -E 'sched_waking:.*comm=rcu_preempt|sched_switch:.*==.*rcu_preempt'
5# 受实际结果影响，以下命令可能计算错误
6trace-cmd report | grep -E 'sched_waking:.*comm=rcu_preempt|sched_switch:.*==.*rcu_preempt' | grep -o -E '[0-9]*\.[0-9]*: sched_waking|[0-9]*\.[0-9]*: sched_switch' | sed -e 'N;s/\n/ : /'

1# 可以加入-T来查看调用栈
2# 注意-f要只作用于前一个-e指定的event
3trace-cmd record -e kmalloc -f 'bytes_alloc <= 256' -e kfree ls
4trace-cmd record -e kmalloc -f 'bytes_alloc <= 256' -e kmem_cache_alloc* -f 'bytes_alloc <= 256' -e kfree ls

 1trace-cmd version 3.2.0 (not-a-git-repo)
 2
 3usage:
 4  trace-cmd [COMMAND] ...
 5
 6  commands:
 7     record - record a trace into a trace.dat file
 8     set - set a ftrace configuration parameter
 9     start - start tracing without recording into a file
10     extract - extract a trace from the kernel
11     stop - stop the kernel from recording trace data
12     restart - restart the kernel trace data recording
13     show - show the contents of the kernel tracing buffer
14     reset - disable all kernel tracing and clear the trace buffers
15     clear - clear the trace buffers
16     report - read out the trace stored in a trace.dat file
17     stream - Start tracing and read the output directly
18     profile - Start profiling and read the output directly
19     hist - show a histogram of the trace.dat information
20     stat - show the status of the running tracing (ftrace) system
21     split - parse a trace.dat file into smaller file(s)
22     options - list the plugin options available for trace-cmd report
23     listen - listen on a network socket for trace clients
24     agent - listen on a vsocket for trace clients
25     setup-guest - create FIFOs for tracing guest VMs
26     list - list the available events, plugins or options
27     restore - restore a crashed record
28     snapshot - take snapshot of running trace
29     stack - output, enable or disable kernel stack tracing
30     check-events - parse trace event formats
31     dump - read out the meta data from a trace file
32     attach - Attach a host and guest trace.dat file
33     convert - convert trace file to different version

 1usage:
 2 trace-cmd list [-e [regex]][-t][-o][-f [regex]]
 3          -e list available events
 4            -F show event format
 5            --full show the print fmt with -F
 6            -R show event triggers
 7            -l show event filters
 8          -t list available tracers
 9          -o list available options
10          -f [regex] list available functions to filter on
11          -P list loaded plugin files (by path)
12          -O list plugin options
13          -B list defined buffer instances
14          -C list the defined clocks (and active one)
15          -c list the supported trace file compression algorithms

 1usage:
 2 trace-cmd record [-v][-e event [-f filter]][-p plugin][-F][-d][-D][-o file] \
 3           [-q][-s usecs][-O option ][-l func][-g func][-n func] \
 4           [-P pid][-N host:port][-t][-r prio][-b size][-B buf] \
 5           [--proxy vsock][command ...]
 6           [-m max][-C clock]
 7          -e run command with event enabled
 8          -f filter for previous -e event
 9          -R trigger for previous -e event
10          -p run command with plugin enabled
11          -F filter only on the given process
12          -P trace the given pid like -F for the command
13          -c also trace the children of -F (or -P if kernel supports it)
14          -C set the trace clock
15          -T do a stacktrace on all events
16          -l filter function name
17          -g set graph function
18          -n do not trace function
19          -m max size per CPU in kilobytes
20          -M set CPU mask to trace
21          -v will negate all -e (disable those events) and -B (delete those instances) after it
22          -d disable function tracer when running
23          -D Full disable of function tracing (for all users)
24          -o data output file [default trace.dat]
25          -O option to enable (or disable)
26          -r real time priority to run the capture threads
27          -s sleep interval between recording (in usecs) [default: 1000]
28          -S used with --profile, to enable only events in command line
29          -N host:port to connect to (see listen)
30          -V cid:port to connect to via vsocket (see listen)
31          -t used with -N, forces use of tcp in live trace
32          -b change kernel buffersize (in kilobytes per CPU)
33          -B create sub buffer and following events will be enabled here
34          -k do not reset the buffers after tracing.
35          -i do not fail if an event is not found
36          -q print no output to the screen
37          -G when profiling, set soft and hard irqs as global
38          --quiet print no output to the screen
39          --temp specify a directory to store the temp files used to create trace.dat
40          --module filter module name
41          --by-comm used with --profile, merge events for related comms
42          --profile enable tracing options needed for report --profile
43          --func-stack perform a stack trace for function tracer
44             (use with caution)
45          --max-graph-depth limit function_graph depth
46          --cmdlines-size change kernel saved_cmdlines_size
47          --no-filter include trace-cmd threads in the trace
48          --proc-map save the traced processes address map into the trace.dat file
49          --user execute the specified [command ...] as given user
50          --tsc2nsec Convert the current clock to nanoseconds, using tsc multiplier and shift from the Linux
51               kernel's perf interface
52          --tsync-interval set the loop interval, in ms, for timestamps synchronization with guests:
53               If a negative number is specified, timestamps synchronization is disabled
54               If 0 is specified, no loop is performed - timestamps offset is calculated only twice,
55                                                         at the beginnig and at the end of the trace
56          --poll don't block while reading from the trace buffer
57          --name used with -A to give the agent a specific name
58          --file-version set the desired trace file version
59          --compression compress the trace output file, one of these strings can be passed:
60                            any  - auto select the best available compression algorithm
61                            none - do not compress the trace file
62                            name - the name of the desired compression algorithms
63                        available algorithms can be listed with trace-cmd list -c
64          --proxy vsocket to reach the agent. Acts the same as -A (for an agent)
65              but will send the proxy connection to the agent.

1usage:
2 trace-cmd start [-e event][-p plugin][-d][-O option ][-P pid]
3          Uses same options as record.
4          It only enables the tracing and exits
5
6        --fork: If a command is specified, then return right after it forks
7        --verbose 'level' Set the desired log level