This happened while an mdadm array was syncing, all access from writing a new blank file to opening a small .txt file was very slow:

[222117.312078] kjournald starting.  Commit interval 5 seconds
[222117.685060] EXT3-fs (md0): using internal journal
[222117.685096] EXT3-fs (md0): mounted filesystem with ordered data mode
[222122.376847] kjournald starting.  Commit interval 5 seconds
[222122.602825] EXT3-fs (md2): using internal journal
[222122.602866] EXT3-fs (md2): mounted filesystem with ordered data mode
[223081.568032] INFO: task kjournald:352 blocked for more than 120 seconds.
[223081.568069] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[223081.568121] kjournald       D ef1e41c0     0   352      2 0x00000000
[223081.568157]  eee0dac0 00000046 ef1e41c0 ef1e41c0 0000cab8 00000000 e971ee70 22650000
[223081.568219]  e8c11c9f ccb941c0 eee0dc74 ccb941c0 eee0dac0 ccb941c0 ccb941c0 00000000
[223081.568280]  eee0daf4 eee0daf4 ef1e4204 ff48eeaf c102e1e0 e971ee70 00000012 ff48eeaf
[223081.568341] Call Trace:
[223081.568375]  [<c102e1e0>] ? dequeue_task_fair+0x149/0x1d6
[223081.568411]  [<c11939a6>] ? journal_commit_transaction+0x12c/0xd02
[223081.568446]  [<c1605d15>] ? __schedule+0x5cb/0x5f2
[223081.568478]  [<c1047ef5>] ? wake_up_bit+0x56/0x56
[223081.568510]  [<c103bfe1>] ? lock_timer_base+0x19/0x34
[223081.568541]  [<c103c48c>] ? try_to_del_timer_sync+0x5c/0x63
[223081.568574]  [<c1197306>] ? kjournald+0xb5/0x1cc
[223081.568604]  [<c1047ef5>] ? wake_up_bit+0x56/0x56
[223081.568633]  [<c1197251>] ? commit_timeout+0x5/0x5
[223081.568663]  [<c1047b9f>] ? kthread+0x63/0x68
[223081.568692]  [<c1047b3c>] ? kthread_worker_fn+0x114/0x114
[223081.568725]  [<c160cbbe>] ? kernel_thread_helper+0x6/0xd
[223081.568756] INFO: task vi:530 blocked for more than 120 seconds.
[223081.568787] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[223081.568838] vi              D 00000002     0   530  32563 0x00000004
[223081.568873]  cc72dee0 00000086 00000003 00000002 c1098a00 00000000 00204033 22636000
[223081.572601]  00000002 ccb941c0 cc72e094 ccb941c0 cc72dee0 ccb941c0 ccb941c0 eec86180
[223081.572662]  c10284ce 00000002 00000001 00000005 cc263dec 00000000 00000001 eec03870
[223081.572723] Call Trace:
[223081.572751]  [<c1098a00>] ? __alloc_pages_nodemask+0x121/0x572
[223081.572785]  [<c10284ce>] ? find_busiest_group+0x244/0x7e5
[223081.572818]  [<c104e4e8>] ? timekeeping_get_ns+0x10/0x48
[223081.572850]  [<c104f849>] ? ktime_get_ts+0x76/0x7d
[223081.572880]  [<c1605fff>] ? io_schedule+0x6c/0xa4
[223081.572911]  [<c10e40f0>] ? sleep_on_buffer+0x5/0x8
[223081.572941]  [<c1606255>] ? __wait_on_bit_lock+0x2f/0x68
[223081.572973]  [<c10e40eb>] ? unmap_underlying_metadata+0x41/0x41
[223081.573005]  [<c10e40eb>] ? unmap_underlying_metadata+0x41/0x41
[223081.573037]  [<c1606329>] ? out_of_line_wait_on_bit_lock+0x9b/0xa3
[223081.573071]  [<c1047f1e>] ? autoremove_wake_function+0x29/0x29
[223081.573103]  [<c10e4210>] ? __lock_buffer+0x1c/0x1e
[223081.573133]  [<c1192e25>] ? do_get_write_access+0x4f/0x305
[223081.573166]  [<c11928e1>] ? start_this_handle+0x27a/0x2d6
[223081.573197]  [<c1196d5a>] ? journal_add_journal_head+0x85/0xec
[223081.573230]  [<c11930f3>] ? journal_get_write_access+0x18/0x26
[223081.573263]  [<c1154fde>] ? __ext3_journal_get_write_access+0x16/0x36
[223081.573298]  [<c114ae9a>] ? ext3_orphan_add+0x78/0x13e
[223081.573328]  [<c1192a4e>] ? journal_start+0x80/0xa7
[223081.573359]  [<c1147da7>] ? ext3_setattr+0x180/0x2ad
[223081.573391]  [<c10d627e>] ? notify_change+0x182/0x252
[223081.573424]  [<c10c40d6>] ? do_truncate+0x61/0x77
[223081.573455]  [<c10ce12b>] ? do_last+0x4c4/0x52f
[223081.573485]  [<c10ce620>] ? path_openat+0x9f/0x285
[223081.573514]  [<c10ce8aa>] ? do_filp_open+0x21/0x5d
[223081.573545]  [<c10c10a8>] ? kmem_cache_alloc+0x21/0xd9
[223081.573576]  [<c10cc7ed>] ? getname_flags+0x24/0xd5
[223081.573606]  [<c10c3779>] ? do_sys_open+0x58/0xd1
[223081.573636]  [<c10c3836>] ? sys_open+0x1e/0x23
[223081.573666]  [<c160711c>] ? syscall_call+0x7/0x7


[224161.572060] INFO: task kjournald:349 blocked for more than 120 seconds.
[224161.572099] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[224161.572151] kjournald       D ef1e41c0     0   349      2 0x00000000
[224161.572187]  eee0f7a0 00000046 ef1e41c0 ef1e41c0 0000cbae 00000000 e07bdd83 22650000
[224161.572248]  dfcdf1fd ccb941c0 eee0f954 ccb941c0 eee0f7a0 ccb941c0 ccb941c0 00000000
[224161.572310]  eee0f7d4 eee0f7d4 ef1e4204 ff52147a c102e1e0 e07bdd83 0000001d ff52147a
[224161.572371] Call Trace:
[224161.572405]  [<c102e1e0>] ? dequeue_task_fair+0x149/0x1d6
[224161.572441]  [<c11939a6>] ? journal_commit_transaction+0x12c/0xd02
[224161.572477]  [<c1605d15>] ? __schedule+0x5cb/0x5f2
[224161.572509]  [<c1047ef5>] ? wake_up_bit+0x56/0x56
[224161.572541]  [<c103bfe1>] ? lock_timer_base+0x19/0x34
[224161.572573]  [<c103c48c>] ? try_to_del_timer_sync+0x5c/0x63
[224161.572605]  [<c1197306>] ? kjournald+0xb5/0x1cc
[224161.572635]  [<c1047ef5>] ? wake_up_bit+0x56/0x56
[224161.572664]  [<c1197251>] ? commit_timeout+0x5/0x5
[224161.572694]  [<c1047b9f>] ? kthread+0x63/0x68
[224161.572723]  [<c1047b3c>] ? kthread_worker_fn+0x114/0x114
[224161.572757]  [<c160cbbe>] ? kernel_thread_helper+0x6/0xd
[224161.572789] INFO: task vi:791 blocked for more than 120 seconds.
[224161.572821] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[224161.572872] vi              D 00000010     0   791    552 0x00000000
[224161.572907]  cc47f380 00000086 00000040 00000010 c1820380 00000000 00000000 22629000
[224161.572968]  00000002 ccb941c0 cc47f534 ccb941c0 cc47f380 ccb941c0 ccb941c0 ef1b8c40
[224161.573029]  eec03870 eec44a40 c14747c5 eec03860 eec86100 c10284ce 00000002 00000001
[224161.573090] Call Trace:
[224161.573117]  [<c14747c5>] ? cpumask_next_and+0x20/0x2b
[224161.573150]  [<c10284ce>] ? find_busiest_group+0x244/0x7e5
[224161.573183]  [<c104e4e8>] ? timekeeping_get_ns+0x10/0x48
[224161.573214]  [<c104f849>] ? ktime_get_ts+0x76/0x7d
[224161.573244]  [<c1605fff>] ? io_schedule+0x6c/0xa4
[224161.573275]  [<c10e40f0>] ? sleep_on_buffer+0x5/0x8
[224161.573305]  [<c1606255>] ? __wait_on_bit_lock+0x2f/0x68
[224161.573337]  [<c10e40eb>] ? unmap_underlying_metadata+0x41/0x41
[224161.573370]  [<c10e40eb>] ? unmap_underlying_metadata+0x41/0x41
[224161.573402]  [<c1606329>] ? out_of_line_wait_on_bit_lock+0x9b/0xa3
[224161.573435]  [<c1047f1e>] ? autoremove_wake_function+0x29/0x29
[224161.573468]  [<c10e4210>] ? __lock_buffer+0x1c/0x1e
[224161.573498]  [<c1192e25>] ? do_get_write_access+0x4f/0x305
[224161.573530]  [<c1196d5a>] ? journal_add_journal_head+0x85/0xec
[224161.573562]  [<c11930f3>] ? journal_get_write_access+0x18/0x26
[224161.573596]  [<c1154fde>] ? __ext3_journal_get_write_access+0x16/0x36
[224161.573630]  [<c1145c46>] ? ext3_reserve_inode_write+0x2d/0x5a
[224161.573663]  [<c114aeb2>] ? ext3_orphan_add+0x90/0x13e
[224161.573693]  [<c1147da7>] ? ext3_setattr+0x180/0x2ad
[224161.573726]  [<c10d627e>] ? notify_change+0x182/0x252
[224161.573758]  [<c10c40d6>] ? do_truncate+0x61/0x77
[224161.573790]  [<c10ce12b>] ? do_last+0x4c4/0x52f
[224161.573819]  [<c10ce7b1>] ? path_openat+0x230/0x285
[224161.573849]  [<c10ce8aa>] ? do_filp_open+0x21/0x5d
[224161.573880]  [<c10cc7ed>] ? getname_flags+0x24/0xd5
[224161.573910]  [<c10c3779>] ? do_sys_open+0x58/0xd1
[224161.573940]  [<c10c3836>] ? sys_open+0x1e/0x23
[224161.573970]  [<c160711c>] ? syscall_call+0x7/0x7
[224161.573999] INFO: task grub:817 blocked for more than 120 seconds.
[224161.574031] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[224161.574082] grub            D 00000002     0   817  32563 0x00000004
[224161.574117]  cc47eb40 00000082 c109a37f 00000002 0000000e 00000000 c9eabec4 22643000
[224161.574178]  00000000 ccb941c0 cc47ecf4 ccb941c0 cc47eb40 ccb941c0 ccb941c0 c9eabf1c
[224161.574239]  c1093d65 0000000e 00000001 c9eabf38 00000000 c9eabf34 00000000 c8270270
[224161.574300] Call Trace:
[224161.574325]  [<c109a37f>] ? write_cache_pages+0x10d/0x2be
[224161.574358]  [<c1093d65>] ? find_get_pages_tag+0x37/0xd1
[224161.574390]  [<c1025f08>] ? __wake_up+0x2c/0x3b
[224161.574419]  [<c1048067>] ? prepare_to_wait+0x14/0x52
[224161.574450]  [<c11971b3>] ? log_wait_commit+0xa9/0xf1
[224161.574480]  [<c1047ef5>] ? wake_up_bit+0x56/0x56
[224161.574510]  [<c1153556>] ? ext3_sync_fs+0x5d/0x67
[224161.574541]  [<c10f9d6f>] ? dquot_quota_sync+0x193/0x235
[224161.574572]  [<c10f9bdc>] ? dquot_file_open+0x25/0x25
[224161.574603]  [<c10e1fc4>] ? __sync_filesystem+0x69/0x69
[224161.574634]  [<c10e1f8b>] ? __sync_filesystem+0x30/0x69
[224161.574664]  [<c10c6a1f>] ? iterate_supers+0x4e/0x8d
[224161.574695]  [<c10e2005>] ? sys_sync+0x36/0x49
[224161.574724]  [<c160711c>] ? syscall_call+0x7/0x7

 

blocked, iowait, timeout, linux, diskthis, mdadm, array, syncing, txt, kjournald, interval, ext, fs, md, mounted, filesystem, mode, info, task, quot, echo, proc, sys, kernel, hung_task_timeout_secs, disables, ef, eee, dac, ee, ccb, dc, daf, ff, eeaf, dequeue_task_fair, journal_commit_transaction, xd, __schedule, cb, wake_up_bit, bfe, lock_timer_base, try_to_del_timer_sync, xb, cc, commit_timeout, kthread, kthread_worker_fn, cbbe, kernel_thread_helper, vi, dee, eec, ce, dec, __alloc_pages_nodemask, find_busiest_group, timekeeping_get_ns, ktime_get_ts, fff, io_schedule, xa, sleep_on_buffer, __wait_on_bit_lock, eb, unmap_underlying_metadata, out_of_line_wait_on_bit_lock, autoremove_wake_function, __lock_buffer, do_get_write_access, start_this_handle, journal_add_journal_head, xec, journal_get_write_access, fde, __ext, _journal_get_write_access, ae, _orphan_add, journal_start, _setattr, notify_change, do_truncate, do_last, path_openat, aa, do_filp_open, kmem_cache_alloc, getname_flags, do_sys_open, sys_open, syscall_call, cbae, bdd, dfcdf, fd, cpumask_next_and, _reserve_inode_write, aeb, grub, eabec, ecf, eabf, write_cache_pages, find_get_pages_tag, __wake_up, prepare_to_wait, log_wait_commit, xf, _sync_fs, dquot_quota_sync, bdc, dquot_file_open, fc, __sync_filesystem, iterate_supers, sys_sync,

Latest Articles