Fixed
Created: Jul 25, 2016
Updated: Dec 3, 2018
Resolved Date: Jul 25, 2016
Found In Version: 6.0.0.30
Fix Version: 6.0.0.31
Severity: Severe
Applicable for: Wind River Linux 6
Component/s: BSP, Kernel
Using the SabreSD NXP reference board, one of the system tests causes a hard lock up within 4 to 8 hours.
This system is using the stock Wind River preempt-rt kernel, no customer's modifications. The issue has also been seen with a standard (non-RT) WRL5 kernel.
This issue has NOT been reproduced on stock NXP reference BSPs on the SabreSD board, which is why customer opened a case against Wind River Linux 5.
NXP kernels tested by customer: v3.0.35, 3.14.52 (TBC), and v4.1.15.
Customer worked with NXP and ran a test with the NXP kernel that was supplied to WindRiver and they could not reproduce the issue. They also could not reproduce the issue with WRL6 and WRL8.
The "smaller" test case that they can reproduce on the SabreSD board has 2 process with 3 threads each accessing some shared memory protected by a mutex. The test flips between using a process specific shared memory and system wide shared memory on every other run through the test loop.
Basics of the test loop:
Grab mutex;
Increment access count;
Ensure access count == 1;
Decrement access count;
Ensure access count == 0;
Release mutex;
This test will fail generally in 4 to 8 hours, always less than a day.
The board OEM managed to read out the DAP register in fail conditions.
Please find attached the content of the register. They have read out DBGPCSR registers via Lauterbach and see following values of DBGPCSR (core0-3) printed if they run Lauterbach script. From kernel's System.map, corresponding PC of all cores maps to the following functions:
PC System.map source file
DBGPCSR CPU0 0x805A7C20 805a7c18 T _mutex_lock kernel/rt.c:89
DBGPCSR CPU1 0x805A79A8 805a7940 T rt_spin_trylock_bh kernel/rtmutex.c:884
DBGPCSR CPU2 0x805A7A7C 805a7a74 T __rt_spin_lock kernel/rtmutex.c:833
DBGPCSR CPU3 0x805A7A7C 805a7a74 T __rt_spin_lock kernel/rtmutex.c:833
DBGPCSR CPU0 0x0FFFF0280 <======= ISR ???
DBGPCSR CPU1 0x805A79A8 805a7940 T rt_spin_trylock_bh kernel/rtmutex.c:884
DBGPCSR CPU2 0x805A7A7C 805a7a74 T __rt_spin_lock kernel/rtmutex.c:833
DBGPCSR CPU3 0x805A7A7C 805a7a74 T __rt_spin_lock kernel/rtmutex.c:833
DBGPCSR CPU0 0x805A7C20 805a7c18 T _mutex_lock kernel/rt.c:89
DBGPCSR CPU1 0x805A79B8 805a7940 T rt_spin_trylock_bh kernel/rtmutex.c:884
DBGPCSR CPU2 0x805A7A7C 805a7a74 T __rt_spin_lock kernel/rtmutex.c:833
DBGPCSR CPU3 0x805A7A7C 805a7a74 T __rt_spin_lock kernel/rtmutex.c:833
Reverse code tracing needs to be done to see how the kernel end up in this lock.
Patch from the board OEM to save/restore extra CP15 registers across supend/resume cycles
diff --git a/arch/arm/mach-mx6/mx6_suspend.S b/arch/arm/mach-mx6/mx6_suspend.S
index ca90014..cd4dd68 100644
--- a/arch/arm/mach-mx6/mx6_suspend.S
+++ b/arch/arm/mach-mx6/mx6_suspend.S
@@ -1056,6 +1056,11 @@ ddr_io_save_done:
mov r6, lr @ Store lr
stmfd r0!, {r4-r6}
+ /* c15 registers */
+ mrc p15, 0, r4, c15, c0, 1 @ Diagnostic register
+ mrc p15, 0, r5, c15, c0, 0 @ Power register
+ stmfd r0!, {r4-r5}
+
/* c1 and c2 registers */
mrc p15, 0, r4, c1, c0, 2 @ CPACR
mrc p15, 0, r5, c2, c0, 0 @ TTBR0
@@ -1524,6 +1529,11 @@ poll_dvfs_clear_2:
msr spsr_cxsf, r5 @ Restore spsr
mov lr, r6 @ Restore lr
+ /* c15 registers */
+ ldmea r0!, {r4-r5}
+ mcr p15, 0, r4, c15, c0, 1 @ Diagnostic register
+ mcr p15, 0, r5, c15, c0, 0 @ Power register
+
/* c1 and c2 registers */
ldmea r0!, {r4-r7}
mcr p15, 0, r4, c1, c0, 2 @ CPACR
Configure line from platform build:
--enable-board=fsl-imx6 --enable-build=production --enable-kernel=preempt-rt --enable-reconfig --enable-rootfs=glibc_std_sato --with-template=feature/gpu-viv-bin-mx6q-x11 --with-rcpl-version=0031
#################################
# How to run bspd on SabreSD
#################################
# Partition SD card
# delete any existing partitions
sudo fdisk /dev/sdx
d
#create new partition
n
p
1
#start 8MB into SD card
16384
# use default end secctor
<enter>
w # save changes
# create file system on partition
sudo mkfs.ext4 /dev/sdx1
#untar IAVS root partition
sudo mount /dev/sdx1 /mnt/sdcard
cd /mnt/sdcard
sudo tar -xjf <path to sabresd _prj>/export/images/wrlinux-image-glibc-std-sato-fsl-imx6.tar.bz2 --numeric-owner
# add module tree for SabreSD kernel
sudo tar -xzf <path to sabresd _prj>/export/images/modules-<version>.tgz --numeric-owner
# overwrite IAVS /etc/fstab with SabreSD project file
cd /mnt/sdcard/etc
sudo cp <path to sabresd _prj>/export/dist/etc/fstab .
# edit etc/inittab to have console instead of tty_console on agetty line
# copy Board Services tarball to target file system
sudo cp BoardServices-armv7hl-2.7.454-3-PL2.0.324-3.tar.bz2 /mnt/sdcard/root
sudo cp Tools-BS2.7.454-10003-PL2.0.324-3.tar.bz2 /mnt/sdcard/root
sudo cp logging.sh /mnt/sdcard/root
chmod +x /mnt/sdcard/root/logging.sh
# copy simulated MIBs to target file system
scp MIB_IAVS.bin /mnt/sdcard/root
scp MIB_7_Inch_Monitor_DB3_RE338107.bin /mnt/sdcard/root
# write out u-boot
cd <path to Freescale uboot packages>/L3.0.101_4.1.1_141016_images_MX6/
sudo dd if=u-boot-mx6q-sabresd.bin of=/dev/sdx bs=512 seek=2 skip=2 conv=fsync
# write out kernel
cd <path to sabresd _prj>/export/images/
sudo dd if=uImage-fsl-imx6.bin of=/dev/sdx bs=512 seek=2048 conv=fsync
sync
sudo eject /dev/sdx
# To boot from flash, stop boot in uboot and update bootcmd_mmc to use mmc 2 and size of 0x5000
setenv bootcmd_mmc run bootargs_base bootargs_mmc\; mmc dev 2\; mmc read \$\{loadaddr\} 0x800 0x5000\; bootm
# set board to use mmcblk1p1 as the rootfs
setenv bootargs_mmc setenv bootargs \$\{bootargs\} root=/dev/mmcblk1p1 rootwait
# add support for HDMI monitor
setenv bootargs_base setenv bootargs console=ttymxc0,115200 video=mxcfb0:dev=hdmi,1920x1080M@60,if=RGB24
# update uboot to boot from flash by default
setenv bootcmd run bootcmd_mmc
saveenv
# after first boot, setup static IP address
vim /etc/network/interfaces
iface eth0 inet static
address 128.224.xx.xx
netmask 255.255.254.0
network 128.224.xx.0
# install BoardServices
cd /
tar -xjf /root/BoardServices-armv7hl-2.7.454-3-PL2.0.324-3.tar.bz2
mkdir -p /root/artifacts/tool
cd /root/artifacts/tool
tar -xjf /root/Tools-BS2.7.454-10003-PL2.0.324-3.tar.bz2
#reboot for run once scripts to run
poweroff
# update hostname
echo SabreSD > /etc/hostname
# update link for xorg.conf
cd /etc/X11
ln -sf xorg_800x600.conf xorg.conf
#on target
cd /etc/MIBData/
cp /root/MIB_* .
ln -sf MIB_IAVS.bin SimulatorMIB.bin
ln -sf MIB_7_Inch_Monitor_DB3_RE338107.bin SimulatorMIBMonitor0.bin
# Set simulator flags
vim /etc/profile
export BSP_SIMULATOR=1
export DEERE_VBS_SERVER_IP_ADDR=<windows ip address>
# reboot, bspd should start on boot now
# to verify
ps aux |grep bspd
# ensure a process is listed
#################################
# How to run customer's test case on SabreSD
#################################
# log into SabreSD target
# go to tools directory
cd /root/artifacts/tool
# run test
./IPCClientTest -m=3 -d=1209600 &
# run logging script to show that the system is running
/root/logging.sh
