Fixed
Created: Dec 19, 2016
Updated: Dec 3, 2018
Resolved Date: Jan 23, 2017
Found In Version: 6.0.0.11
Fix Version: 6.0.0.33
Severity: Standard
Applicable for: Wind River Linux 6
Component/s: Kernel
When customer was trying to reboot their board (with command 'reboot -f'), RCU stalls occured as below.
INFO: rcu_preempt self-detected stall on CPU { 0} (t=40531929 jiffies g=125002 c=125001 q=1490)
sending NMI to all CPUs:
NMI backtrace for cpu 3
CPU: 3 PID: 0 Comm: swapper/3 Not tainted 3.10.38-ltsi-WR6.0.0.11_standard #1
Hardware name: To be filled by O.E.M. To be filled by O.E.M./Tiger Hill, BIOS 4.6.5 04/24/2013
task: ffff880179b58000 ti: ffff880179b54000 task.ti: ffff880179b54000
RIP: 0010:[<ffffffff813cf91e>] [<ffffffff813cf91e>] do_raw_spin_lock+0x8e/0x140
RSP: 0018:ffff88017fd83d28 EFLAGS: 00000206
RAX: 000000000000aafa RBX: ffffffffa0263468 RCX: 000000000000aafd
RDX: 000000000000aafd RSI: 0000000000000100 RDI: ffffffff81eeea20
RBP: ffff88017fd83d40 R08: 0000000000000002 R09: 0000000000000001
R10: 61206f7420494d4e R11: 0000000000000503 R12: 000000006f4248d8
R13: 000000006f4248d8 R14: 0000000000000000 R15: ffff880142dc0910
FS: 0000000000000000(0000) GS:ffff88017fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007ffe2c8ee000 CR3: 0000000001e0c000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffffffffa0263468 ffffffffa0263420 0000000000000001 ffff88017fd83d60
ffffffff818db5e8 ffffffffa024eabb ffffffffa0263468 ffff88017fd83d80
ffffffffa024eabb ffff880142dc0800 ffff8801598ae400 ffff88017fd83dc0
Call Trace:
<IRQ>
[<ffffffff818db5e8>] _raw_spin_lock_bh+0x48/0x50
[<ffffffffa024eabb>] ? tipc_bearer_blocked+0x1b/0x30 [tipc]
[<ffffffffa024eabb>] tipc_bearer_blocked+0x1b/0x30 [tipc]
[<ffffffffa0252971>] tipc_link_send_proto_msg+0x421/0x510 [tipc]
[<ffffffffa0252f90>] ? link_start+0x40/0x40 [tipc]
[<ffffffffa0252ee6>] link_state_event+0x486/0x4f0 [tipc]
[<ffffffffa0252ffd>] link_timeout+0x6d/0x160 [tipc]
[<ffffffff8104cd0f>] call_timer_fn+0x7f/0x1f0
[<ffffffff8104cc95>] ? call_timer_fn+0x5/0x1f0
[<ffffffffa0252f90>] ? link_start+0x40/0x40 [tipc]
[<ffffffff8104d174>] run_timer_softirq+0x1b4/0x2c0
[<ffffffff81045098>] __do_softirq+0xf8/0x2b0
[<ffffffff818e4d0c>] call_softirq+0x1c/0x30
[<ffffffff81003e75>] do_softirq+0x65/0xa0
[<ffffffff810453c6>] irq_exit+0x96/0xc0
[<ffffffff818e531b>] smp_apic_timer_interrupt+0x6b/0x9b
[<ffffffff818e45ef>] apic_timer_interrupt+0x6f/0x80
<EOI>
[<ffffffff81076305>] ? sched_clock_cpu+0xb5/0x100
[<ffffffff81718ed4>] ? cpuidle_enter_state+0x54/0xc0
[<ffffffff81718ed0>] ? cpuidle_enter_state+0x50/0xc0
[<ffffffff81719015>] cpuidle_idle_call+0xd5/0x290
[<ffffffff8100abce>] arch_cpu_idle+0xe/0x30
[<ffffffff8108de1e>] cpu_startup_entry+0x15e/0x2b0
[<ffffffff818c9aae>] start_secondary+0x1b2/0x1b5
Code: 48 70 c9 81 48 89 df e8 6b 4f 50 00 e8 2c 7b c5 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 89 d1 74 0a f3 90 0f b7 03 <66> 39 c1 75 f6 eb 13 0f 1f 00 8d 8a 00 00 01 00 89 d0 f0 0f b1
NMI backtrace for cpu 2
CPU: 2 PID: 0 Comm: swapper/2 Not tainted 3.10.38-ltsi-WR6.0.0.11_standard #1
Hardware name: To be filled by O.E.M. To be filled by O.E.M./Tiger Hill, BIOS 4.6.5 04/24/2013
task: ffff880179b45e80 ti: ffff880179b52000 task.ti: ffff880179b52000
RIP: 0010:[<ffffffff813cf91b>] [<ffffffff813cf91b>] do_raw_spin_lock+0x8b/0x140
RSP: 0018:ffff88017fd03d28 EFLAGS: 00000202
RAX: 000000000000aafa RBX: ffffffffa0263468 RCX: 000000000000aafc
RDX: 000000000000aafc RSI: 0000000000000000 RDI: 0000000000000001
RBP: ffff88017fd03d40 R08: 0000000000000002 R09: 0000000000000001
R10: ffffffff820803cd R11: 0000000000000503 R12: 000000006f4248d8
R13: 000000006f4248d8 R14: 0000000000000000 R15: ffff8801602bd110
FS: 0000000000000000(0000) GS:ffff88017fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00000037362de040 CR3: 0000000001e0c000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffffffffa0263468 ffffffffa0263420 0000000000000001 ffff88017fd03d60
ffffffff818db5e8 ffffffffa024eabb ffffffffa0263468 ffff88017fd03d80
ffffffffa024eabb ffff8801602bd000 ffff880159873000 ffff88017fd03dc0
Call Trace:
<IRQ>
[<ffffffff818db5e8>] _raw_spin_lock_bh+0x48/0x50
[<ffffffffa024eabb>] ? tipc_bearer_blocked+0x1b/0x30 [tipc]
[<ffffffffa024eabb>] tipc_bearer_blocked+0x1b/0x30 [tipc]
[<ffffffffa0252971>] tipc_link_send_proto_msg+0x421/0x510 [tipc]
[<ffffffffa0252f90>] ? link_start+0x40/0x40 [tipc]
[<ffffffffa0252ee6>] link_state_event+0x486/0x4f0 [tipc]
[<ffffffffa0252ffd>] link_timeout+0x6d/0x160 [tipc]
[<ffffffff8104cd0f>] call_timer_fn+0x7f/0x1f0
[<ffffffff8104cc95>] ? call_timer_fn+0x5/0x1f0
[<ffffffffa0252f90>] ? link_start+0x40/0x40 [tipc]
[<ffffffff8104d174>] run_timer_softirq+0x1b4/0x2c0
[<ffffffff81045098>] __do_softirq+0xf8/0x2b0
[<ffffffff818e4d0c>] call_softirq+0x1c/0x30
[<ffffffff81003e75>] do_softirq+0x65/0xa0
[<ffffffff810453c6>] irq_exit+0x96/0xc0
[<ffffffff818e531b>] smp_apic_timer_interrupt+0x6b/0x9b
[<ffffffff818e45ef>] apic_timer_interrupt+0x6f/0x80
<EOI>
[<ffffffff81076305>] ? sched_clock_cpu+0xb5/0x100
[<ffffffff81718ed4>] ? cpuidle_enter_state+0x54/0xc0
[<ffffffff81718ed0>] ? cpuidle_enter_state+0x50/0xc0
[<ffffffff81719015>] cpuidle_idle_call+0xd5/0x290
[<ffffffff8100abce>] arch_cpu_idle+0xe/0x30
[<ffffffff8108de1e>] cpu_startup_entry+0x15e/0x2b0
[<ffffffff818c9aae>] start_secondary+0x1b2/0x1b5
Code: 48 c7 c6 48 70 c9 81 48 89 df e8 6b 4f 50 00 e8 2c 7b c5 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 89 d1 74 0a f3 90 <0f> b7 03 66 39 c1 75 f6 eb 13 0f 1f 00 8d 8a 00 00 01 00 89 d0
NMI backtrace for cpu 0
CPU: 0 PID: 4 Comm: kworker/0:0 Not tainted 3.10.38-ltsi-WR6.0.0.11_standard #1
Hardware name: To be filled by O.E.M. To be filled by O.E.M./Tiger Hill, BIOS 4.6.5 04/24/2013
Workqueue: events linkwatch_event
task: ffff880179b0de80 ti: ffff880179b2a000 task.ti: ffff880179b2a000
RIP: 0010:[<ffffffff813ca250>] [<ffffffff813ca250>] __bitmap_empty+0x70/0x80
RSP: 0018:ffff88017fc03e00 EFLAGS: 00000006
RAX: ffff880179b0de80 RBX: 0000000000002710 RCX: 0000000000000004
RDX: 0000000000010603 RSI: 0000000000000100 RDI: ffffffff81eeea20
RBP: ffff88017fc03e00 R08: 0000000000000002 R09: 0000000000000001
R10: 61206f7420494d4e R11: 0000000000000000 R12: ffffffff81e4da00
R13: ffffffff81e4da00 R14: ffff88017fc0d940 R15: 00000000000005d2
FS: 0000000000000000(0000) GS:ffff88017fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 0000000000000138 CR3: 0000000141c5f000 CR4: 00000000000007f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88017fc03e18 ffffffff810274b0 ffffffff81eeed80 ffff88017fc03e80
ffffffff810d1d75 ffffffff81067f70 ffff88017fc13540 ffff880179b0de80
0000000000000000 ffff880179b2a000 0000000000000000 ffff880179b0de80
Call Trace:
<IRQ>
[<ffffffff810274b0>] arch_trigger_all_cpu_backtrace+0x80/0x90
[<ffffffff810d1d75>] rcu_check_callbacks+0x305/0x780
[<ffffffff81067f70>] ? hrtimer_run_queues+0x20/0x180
[<ffffffff8104df27>] update_process_times+0x47/0x80
[<ffffffff81097f46>] tick_sched_handle.isra.15+0x36/0x50
[<ffffffff81097fa1>] tick_sched_timer+0x41/0x60
[<ffffffff810670d4>] __run_hrtimer+0x84/0x280
[<ffffffff81097f60>] ? tick_sched_handle.isra.15+0x50/0x50
[<ffffffff81067c4f>] hrtimer_interrupt+0xff/0x240
[<ffffffff818e5316>] smp_apic_timer_interrupt+0x66/0x9b
[<ffffffff818e45ef>] apic_timer_interrupt+0x6f/0x80
<EOI>
[<ffffffff818dc4e0>] ? retint_restore_args+0xe/0xe
[<ffffffff813cf91b>] ? do_raw_spin_lock+0x8b/0x140
[<ffffffff813cf904>] ? do_raw_spin_lock+0x74/0x140
[<ffffffff818db5e8>] _raw_spin_lock_bh+0x48/0x50
[<ffffffffa024efba>] ? tipc_block_bearer+0x9a/0x110 [tipc]
[<ffffffffa024efba>] tipc_block_bearer+0x9a/0x110 [tipc]
[<ffffffffa025e893>] recv_notification+0xb3/0xe0 [tipc]
[<ffffffff818df90c>] notifier_call_chain+0x4c/0x70
[<ffffffff810696d6>] raw_notifier_call_chain+0x16/0x20
[<ffffffff81763edd>] call_netdevice_notifiers+0x2d/0x60
[<ffffffff817671d3>] netdev_state_change+0x23/0x40
[<ffffffff8177aef8>] linkwatch_do_dev+0x38/0x50
[<ffffffff8177b10f>] __linkwatch_run_queue+0xef/0x180
[<ffffffff8177b1c5>] linkwatch_event+0x25/0x30
[<ffffffff8105c356>] process_one_work+0x1d6/0x530
[<ffffffff8105c2e3>] ? process_one_work+0x163/0x530
[<ffffffff8105d311>] worker_thread+0x121/0x3a0
[<ffffffff8105d1f0>] ? manage_workers.isra.23+0x2b0/0x2b0
[<ffffffff810637cb>] kthread+0xdb/0xe0
[<ffffffff810636f0>] ? kthread_create_on_node+0x130/0x130
[<ffffffff818e381c>] ret_from_fork+0x7c/0xb0
[<ffffffff810636f0>] ? kthread_create_on_node+0x130/0x130
Code: fa 1f c1 ea 1a 8d 0c 16 83 e1 3f 29 d1 ba 01 00 00 00 48 d3 e2 48 83 ea 01 48 85 14 c7 75 0c b8 01 00 00 00 5d c3 0f 1f 44 00 00 <31> c0 5d c3 31 c0 eb c2 0f 1f 84 00 00 00 00 00 8d 4e 3f 85 f6
NMI backtrace for cpu 1
CPU: 1 PID: 0 Comm: swapper/1 Not tainted 3.10.38-ltsi-WR6.0.0.11_standard #1
Hardware name: To be filled by O.E.M. To be filled by O.E.M./Tiger Hill, BIOS 4.6.5 04/24/2013
task: ffff880179b43f00 ti: ffff880179b50000 task.ti: ffff880179b50000
RIP: 0010:[<ffffffff813cf91b>] [<ffffffff813cf91b>] do_raw_spin_lock+0x8b/0x140
RSP: 0018:ffff88017fc83d28 EFLAGS: 00000202
RAX: 000000000000aafa RBX: ffffffffa0263468 RCX: 000000000000aafb
RDX: 000000000000aafb RSI: 0000000000000100 RDI: ffffffff81eeea20
RBP: ffff88017fc83d40 R08: 0000000000000002 R09: 0000000000000001
R10: ffffffff82075518 R11: 0000000000000503 R12: 000000006f4248d8
R13: 000000006f4248d8 R14: 0000000000000000 R15: ffff880175725110
FS: 0000000000000000(0000) GS:ffff88017fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007ffe2c8ee000 CR3: 0000000001e0c000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffffffffa0263468 ffffffffa0263420 0000000000000001 ffff88017fc83d60
ffffffff818db5e8 ffffffffa024eabb ffffffffa0263468 ffff88017fc83d80
ffffffffa024eabb ffff880175725000 ffff8801601f4600 ffff88017fc83dc0
Call Trace:
<IRQ>
[<ffffffff818db5e8>] _raw_spin_lock_bh+0x48/0x50
[<ffffffffa024eabb>] ? tipc_bearer_blocked+0x1b/0x30 [tipc]
[<ffffffffa024eabb>] tipc_bearer_blocked+0x1b/0x30 [tipc]
[<ffffffffa0252971>] tipc_link_send_proto_msg+0x421/0x510 [tipc]
[<ffffffffa0252f90>] ? link_start+0x40/0x40 [tipc]
[<ffffffffa0252ee6>] link_state_event+0x486/0x4f0 [tipc]
[<ffffffffa0252ffd>] link_timeout+0x6d/0x160 [tipc]
[<ffffffff8104cd0f>] call_timer_fn+0x7f/0x1f0
[<ffffffff8104cc95>] ? call_timer_fn+0x5/0x1f0
[<ffffffffa0252f90>] ? link_start+0x40/0x40 [tipc]
[<ffffffff8104d174>] run_timer_softirq+0x1b4/0x2c0
[<ffffffff81045098>] __do_softirq+0xf8/0x2b0
[<ffffffff818e4d0c>] call_softirq+0x1c/0x30
[<ffffffff81003e75>] do_softirq+0x65/0xa0
[<ffffffff810453c6>] irq_exit+0x96/0xc0
[<ffffffff818e531b>] smp_apic_timer_interrupt+0x6b/0x9b
[<ffffffff818e45ef>] apic_timer_interrupt+0x6f/0x80
<EOI>
[<ffffffff81076305>] ? sched_clock_cpu+0xb5/0x100
[<ffffffff81718ed4>] ? cpuidle_enter_state+0x54/0xc0
[<ffffffff81718ed0>] ? cpuidle_enter_state+0x50/0xc0
[<ffffffff81719015>] cpuidle_idle_call+0xd5/0x290
[<ffffffff8100abce>] arch_cpu_idle+0xe/0x30
[<ffffffff8108de1e>] cpu_startup_entry+0x15e/0x2b0
[<ffffffff818c9aae>] start_secondary+0x1b2/0x1b5
Code: 48 c7 c6 48 70 c9 81 48 89 df e8 6b 4f 50 00 e8 2c 7b c5 ff b8 00 00 01 00 f0 0f c1 03 89 c2 c1 ea 10 66 39 c2 89 d1 74 0a f3 90 <0f> b7 03 66 39 c1 75 f6 eb 13 0f 1f 00 8d 8a 00 00 01 00 89 d0
