Solved: Identify which filesystem causing df command to hang on linux ?

By Published Linux/Unix

Today we will look how to troubleshoot a hanging df command.

And how to identify which filesystem makes it hang.

1. Check which filesystem gets hanged.

[root@ngelinux001 ~]# df -h
^C
[root@ngelinux001 ~]# ^C


[root@ngelinux001 ~]# strace df -h
execve("/bin/df", ["df", "-h"], [/* 43 vars */]) = 0
brk(0)                                  = 0x1812000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc                                                         35db75000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/products/oracle/11.2.0.3_Client/lib/tls/x86_64/libc.so.6", O_RDONLY) = -1                                                          ENOENT (No such file or directory)
stat("/products/oracle/11.2.0.3_Client/lib/tls/x86_64", 0x7ffcf5d1abb0) = -1 ENO                                                         ENT (No such file or directory)
open("/products/oracle/11.2.0.3_Client/lib/tls/libc.so.6", O_RDONLY) = -1 ENOENT                                                          (No such file or directory)
stat("/products/oracle/11.2.0.3_Client/lib/tls", 0x7ffcf5d1abb0) = -1 ENOENT (No                                                          such file or directory)
open("/products/oracle/11.2.0.3_Client/lib/x86_64/libc.so.6", O_RDONLY) = -1 ENO                                                         ENT (No such file or directory)
stat("/products/oracle/11.2.0.3_Client/lib/x86_64", 0x7ffcf5d1abb0) = -1 ENOENT                                                          (No such file or directory)
open("/products/oracle/11.2.0.3_Client/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No                                                          such file or directory)
stat("/products/oracle/11.2.0.3_Client/lib", 0x7ffcf5d1abb0) = -1 ENOENT (No suc                                                         h file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=50140, ...}) = 0
mmap(NULL, 50140, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fc35db68000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY)      = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\356\1@=\0\0\0"..., 83                                                         2) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1930416, ...}) = 0
mmap(0x3d40000000, 3750184, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0                                                         ) = 0x3d40000000
mprotect(0x3d4018b000, 2093056, PROT_NONE) = 0
mmap(0x3d4038a000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWR                                                         ITE, 3, 0x18a000) = 0x3d4038a000
mmap(0x3d40390000, 14632, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYM                                                         OUS, -1, 0) = 0x3d40390000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc                                                         35db67000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc                                                         35db66000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc                                                         35db65000
arch_prctl(ARCH_SET_FS, 0x7fc35db66700) = 0
mprotect(0x3d4038a000, 16384, PROT_READ) = 0
mprotect(0x3d3fe20000, 4096, PROT_READ) = 0
munmap(0x7fc35db68000, 50140)           = 0
open("/usr/lib/locale/locale-archive", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=99174448, ...}) = 0
mmap(NULL, 99174448, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fc357cd0000
close(3)                                = 0
brk(0)                                  = 0x1812000
brk(0x1833000)                          = 0x1833000
open("/etc/mtab", O_RDONLY|O_CLOEXEC)   = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=35143, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc                                                         35db74000
read(3, "/dev/mapper/rootvg-rootlv / ext3"..., 4096) = 4096
read(3, "s rw,nolock,soft,addr=10.56.62.4"..., 4096) = 4096
read(3, "nfs rw,nolock,soft,addr=10.56.50"..., 4096) = 4096
read(3, "9 0 0\nssvud934:/mtpuat1/Fusion /"..., 4096) = 4096
read(3, " /DCCI/DCCEXL2/OUBI nfs rw,noloc"..., 4096) = 4096
read(3, " /workspace/MTPDEV1/MWM nfs ro,b"..., 4096) = 4096
read(3, ".56.8.147 0 0\nngelt411:/sofnft1/"..., 4096) = 4096
read(3, "fs rw,nolock,soft,addr=10.56.11."..., 4096) = 4096
read(3, "6.49.253 0 0\nclsut042-t:/dccexl2"..., 4096) = 2375
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x7fc35db74000, 4096)            = 0
statfs("/", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=6515275, f_bfree=                                                         5469124, f_bavail=5138242, f_files=1654784, f_ffree=1632653, f_fsid={-811497880,                                                          700899867}, f_namelen=255, f_frsize=4096}) = 0
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2512, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc                                                         35db74000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2512
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x7fc35db74000, 4096)            = 0
open("/usr/share/locale/en_GB/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (                                                         No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=435, ...}) = 0
mmap(NULL, 435, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fc35db74000
close(3)                                = 0
statfs("/sys", {f_type="SYSFS_MAGIC", f_bsize=4096, f_blocks=0, f_bfree=0, f_bav                                                         ail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096}) = 0
statfs("/dev/pts", {f_type="DEVPTS_SUPER_MAGIC", f_bsize=4096, f_blocks=0, f_bfr                                                         ee=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4                                                         096}) = 0
statfs("/dev/shm", {f_type=0x1021994, f_bsize=4096, f_blocks=1007643, f_bfree=10                                                         07643, f_bavail=1007643, f_files=1007643, f_ffree=1007642, f_fsid={0, 0}, f_name                                                         len=255, f_frsize=4096}) = 0
statfs("/boot", {f_type="EXT2_SUPER_MAGIC", f_bsize=1024, f_blocks=99150, f_bfre                                                         e=22683, f_bavail=17563, f_files=25688, f_ffree=25647, f_fsid={-1664345186, 3717                                                         33116}, f_namelen=255, f_frsize=1024}) = 0
statfs("/home", {f_type="EXT2_SUPER_MAGIC", f_bsize=1024, f_blocks=126931, f_bfr                                                         ee=119001, f_bavail=119001, f_files=32768, f_ffree=30971, f_fsid={1806820415, 16                                                         47329693}, f_namelen=255, f_frsize=1024}) = 0
statfs("/home/metron", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=258022                                                         , f_bfree=249291, f_bavail=249291, f_files=65536, f_ffree=65433, f_fsid={-141191                                                         8556, -1300942330}, f_namelen=255, f_frsize=4096}) = 0

statfs("/NHHDA/samba_FTP", {f_type="NFS_SUPER_MAGIC", f_bsize=1048576, f_blocks=                                                         102400, f_bfree=19050, f_bavail=17904, f_files=4977040, f_ffree=4876561, f_fsid=                                                         {0, 0}, f_namelen=255, f_frsize=1048576}) = 0
statfs("/TWS/DEV/NGELUSR/ngelt132", {f_type="NFS_SUPER_MAGIC", f_bsize=65536, f_b                                                         locks=487424, f_bfree=188033, f_bavail=188033, f_files=3019615, f_ffree=2959450,                                                          f_fsid={0, 0}, f_namelen=255, f_frsize=65536}) = 0
statfs("/NGEL3/PREPROD/FILELOAD", ^C^C^C


^C^C^C


^C


Process 6763 detached
 
[root@ngelinux001 ~]#

The filesystem name in the end where our strace command hanged is the one which is causing issue on the server. So lets umount and mount it to resolve the issue.

2. Now umount the filesystem.

If it is not umounted forcefully, use lazy option to umount its entry from the system mount file.

[root@ngelinux001 ~]# mount | grep -i NGEL3
10.56.34.98:/app/releases/fileload on /NGEL3/PREPROD/FILELOAD type nfs (rw,addr=10.56.34.98)

[root@ngelinux001 ~]# umount /NGEL3/PREPROD/FILELOAD
umount.nfs: /NGEL3/PREPROD/FILELOAD: device is busy
umount.nfs: /NGEL3/PREPROD/FILELOAD: device is busy

t@ngelinux001 ~]# umount -f /NGEL3/PREPROD/FILELOAD
umount2: Device or resource busy
umount.nfs: /NGEL3/PREPROD/FILELOAD: device is busy
umount2: Device or resource busy
umount.nfs: /NGEL3/PREPROD/FILELOAD: device is busy

[root@ngelinux001 ~]# ls -l /NGEL3/PREPROD/FILELOAD
total 0
[root@ngelinux001 ~]# umount -l /NGEL3/PREPROD/FILELOAD
umount: /NGEL3/PREPROD/FILELOAD: not mounted

 

3. Now mount it back.

[root@ngelinux001 ~]# mount /NGEL3/PREPROD/FILELOAD
mount: can't find /NGEL3/PREPROD/FILELOAD in /etc/fstab or /etc/mtab
[root@ngelinux001 ~]#
0 0 votes
Article Rating

Tags:

Subscribe
Notify of
guest

1 Comment
Newest
Oldest Most Voted
Inline Feedbacks
View all comments
Bhargav Garikipati
Bhargav Garikipati
3 years ago

it is showing as can’t find under fstab or mnttab because previously /NGEL3/PREPROD/FILELOAD might had added as temporary mount and there is no entry specified under /etc/fstab.
So as soon as the server got reboot, The server will not look to remount this mount while booting up.

1
Reply