How to trace a command or particular system call in the command on Linux ?
In this post, we will check how to trace a command in Linux or a specific system call used by that command.
1. Tracing a command say “ls -l /etc/hosts”
[root@nglinux ~]# strace ls -l /etc/hosts
execve("/bin/ls", ["ls", "-l", "/etc/hosts"], [/* 28 vars */]) = 0
brk(0) = 0x857e000
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77b5000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=66865, ...}) = 0
mmap2(NULL, 66865, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb77a4000
close(3) = 0
open("/lib/libselinux.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0P\264\276\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=126540, ...}) = 0
mmap2(0xbe7000, 125988, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xbe7000
mmap2(0xc04000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1d000) = 0xc04000
close(3) = 0
open("/lib/librt.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\310\247\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=42356, ...}) = 0
mmap2(0xa7b000, 33336, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xa7b000
mmap2(0xa82000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0xa82000
close(3) = 0
open("/lib/libcap.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\\\353F4\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=14328, ...}) = 0
mmap2(0x46eb5000, 15664, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x46eb5000
mmap2(0x46eb8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x46eb8000
close(3) = 0
open("/lib/libacl.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\226\245\0044\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=31988, ...}) = 0
mmap2(0x4a58000, 33092, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x4a58000
mmap2(0x4a5f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x4a5f000
close(3) = 0
open("/lib/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\3\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200N\215\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1912920, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77a3000
mmap2(0x8be000, 1665484, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x8be000
mmap2(0xa4f000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x191000) = 0xa4f000
mmap2(0xa52000, 10700, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xa52000
close(3) = 0
open("/lib/libdl.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`J\247\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=20284, ...}) = 0
mmap2(0xa74000, 16500, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xa74000
mmap2(0xa77000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0xa77000
close(3) = 0
open("/lib/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\276\245\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=133980, ...}) = 0
mmap2(0xa57000, 107044, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xa57000
mmap2(0xa6e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0xa6e000
mmap2(0xa70000, 4644, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xa70000
close(3) = 0
open("/lib/libattr.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0P>\314\0044\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=19456, ...}) = 0
mmap2(0x4cc3000, 20660, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x4cc3000
mmap2(0x4cc7000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x4cc7000
close(3) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77a2000
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77a1000
set_thread_area({entry_number:-1 -> 6, base_addr:0xb77a1720, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
access("/etc/sysconfig/32bit_ssse3_memcpy_via_32bit_ssse3_memmove", F_OK) = -1 ENOENT (No such file or directory)
access("/etc/sysconfig/32bit_ssse3_memcpy_via_32bit_ssse3_memmove", F_OK) = -1 ENOENT (No such file or directory)
access("/etc/sysconfig/32bit_ssse3_memcpy_via_32bit_ssse3_memmove", F_OK) = -1 ENOENT (No such file or directory)
access("/etc/sysconfig/32bit_ssse3_memcpy_via_32bit_ssse3_memmove", F_OK) = -1 ENOENT (No such file or directory)
access("/etc/sysconfig/32bit_ssse3_memcpy_via_32bit_ssse3_memmove", F_OK) = -1 ENOENT (No such file or directory)
mprotect(0xc04000, 4096, PROT_READ) = 0
mprotect(0xa82000, 4096, PROT_READ) = 0
mprotect(0x4a5f000, 4096, PROT_READ) = 0
mprotect(0xa4f000, 8192, PROT_READ) = 0
mprotect(0xa77000, 4096, PROT_READ) = 0
mprotect(0x8b6000, 4096, PROT_READ) = 0
mprotect(0xa6e000, 4096, PROT_READ) = 0
mprotect(0x4cc7000, 4096, PROT_READ) = 0
munmap(0xb77a4000, 66865) = 0
set_tid_address(0xb77a1788) = 21765
set_robust_list(0xb77a1790, 12) = 0
futex(0xbfd53d64, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0xbfd53d64, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 1, NULL, bfd53d74) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0xa5b7d0, [], SA_SIGINFO}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0xa5bcd0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=10240*1024, rlim_max=RLIM_INFINITY}) = 0
uname({sys="Linux", node="ngelinux-new", ...}) = 0
statfs64("/selinux", 84, {f_type=0xf97cff8c, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4128}) = 0
statfs64("/selinux", 84, {f_type=0xf97cff8c, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4128}) = 0
stat64("/selinux", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
brk(0) = 0x857e000
brk(0x859f000) = 0x859f000
open("/usr/lib/locale/locale-archive", O_RDONLY|O_LARGEFILE) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=99174416, ...}) = 0
mmap2(NULL, 2097152, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb75a1000
close(3) = 0
ioctl(1, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TIOCGWINSZ, {ws_row=31, ws_col=96, ws_xpixel=0, ws_ypixel=0}) = 0
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=2512, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77b4000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2512
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb77b4000, 4096) = 0
open("/usr/share/locale/en_US.UTF-8/LC_TIME/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_TIME/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_TIME/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_TIME/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_TIME/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_TIME/coreutils.mo", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=435, ...}) = 0
mmap2(NULL, 435, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb77b4000
close(3) = 0
open("/usr/lib/gconv/gconv-modules.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=26058, ...}) = 0
mmap2(NULL, 26058, PROT_READ, MAP_SHARED, 3, 0) = 0xb77ad000
close(3) = 0
futex(0xa51f10, FUTEX_WAKE_PRIVATE, 2147483647) = 0
lstat64("/etc/hosts", {st_mode=S_IFREG|0644, st_size=241, ...}) = 0
lgetxattr("/etc/hosts", "security.selinux", "system_u:object_r:net_conf_t:s0", 255) = 32
open("/selinux/mls", O_RDONLY|O_LARGEFILE) = 3
read(3, "1", 19) = 1
close(3) = 0
futex(0xc05c08, FUTEX_WAKE_PRIVATE, 2147483647) = 0
socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC, 0) = 3
connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/setrans/.setrans-unix"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
lstat64("/etc/hosts", {st_mode=S_IFREG|0644, st_size=241, ...}) = 0
lgetxattr("/etc/hosts", "system.posix_acl_access", 0x0, 0) = -1 EOPNOTSUPP (Operation not supported)
socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
open("/etc/nsswitch.conf", O_RDONLY|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=1688, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77ac000
read(3, "#\n# /etc/nsswitch.conf\n#\n# An ex"..., 4096) = 1688
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb77ac000, 4096) = 0
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=66865, ...}) = 0
mmap2(NULL, 66865, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7590000
close(3) = 0
open("/lib/libnss_files.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\240\32\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=59172, ...}) = 0
mmap2(NULL, 53964, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x1ec000
mmap2(0x1f8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x1f8000
close(3) = 0
mprotect(0x1f8000, 4096, PROT_READ) = 0
munmap(0xb7590000, 66865) = 0
open("/etc/passwd", O_RDONLY|O_CLOEXEC) = 3
fcntl64(3, F_GETFD) = 0x1 (flags FD_CLOEXEC)
fstat64(3, {st_mode=S_IFREG|0644, st_size=1920, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77ac000
read(3, "root:x:0:0:root:/root:/bin/bash\n"..., 4096) = 1920
close(3) = 0
munmap(0xb77ac000, 4096) = 0
socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
open("/etc/group", O_RDONLY|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=923, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77ac000
read(3, "root:x:0:nglinux\nbin:x:1:bin,dae"..., 4096) = 923
close(3) = 0
munmap(0xb77ac000, 4096) = 0
fstat64(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 1), ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77ac000
open("/etc/localtime", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=2819, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=2819, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77ab000
read(3, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\0"..., 4096) = 2819
_llseek(3, -24, [2795], SEEK_CUR) = 0
read(3, "\nPST8PDT,M3.2.0,M11.1.0\n", 4096) = 24
close(3) = 0
munmap(0xb77ab000, 4096) = 0
clock_gettime(CLOCK_REALTIME, {1518155893, 243672407}) = 0
write(1, "-rw-r--r--. 1 root root 241 Jan "..., 52-rw-r--r--. 1 root root 241 Jan 8 01:12 /etc/hosts
) = 52
close(1) = 0
munmap(0xb77ac000, 4096) = 0
close(2) = 0
exit_group(0) = ?
+++ exited with 0 +++
[root@nglinux ~]#
2. Tracing a particular system call i.e. fstat
ls command uses fstat or fstat64 syscall to get inode information.
So lets try to trace this particular system call fstat64.
[root@nglinux ~]# strace -e fstat64 ls -l /etc/hosts
fstat64(3, {st_mode=S_IFREG|0644, st_size=66865, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0755, st_size=126540, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0755, st_size=42356, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0755, st_size=14328, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0755, st_size=31988, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0755, st_size=1912920, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0755, st_size=20284, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0755, st_size=133980, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0755, st_size=19456, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=99174416, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=2512, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=435, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=26058, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=1688, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=66865, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0755, st_size=59172, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=1920, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=923, ...}) = 0
fstat64(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 1), ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=2819, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=2819, ...}) = 0
-rw-r--r--. 1 root root 241 Jan 8 01:12 /etc/hosts
+++ exited with 0 +++
[root@nglinux ~]#