Acknowledged
Created: Jul 15, 2025
Updated: Oct 28, 2025
Found In Version: 10.25.33.1
Severity: Standard
Applicable for: Wind River Linux LTS 25
Component/s: BSP
When running stress case with rt kernel, there is below error:
root@nvidia-orin-agx:~# /opt/wr-test/main_test.py -k rt_pi_stress
... ...
nvgpu: 17000000.gpu nvgpu_timeout_expired_msg_cpu:94 [ERR] Timeout detected @ gv11b_fifo_is_preempt_pending+0x2c8/0x3d8 [nvgpu]
nvgpu: 17000000.gpu gv11b_fifo_preempt_poll_eng:307 [ERR] preempt timeout eng: 0 ctx_stat: 3 tsgid: 0
nvgpu: 17000000.gpu nvgpu_fifo_preempt_tsg:117 [ERR] preempt timed out for tsgid: 0, ctxsw timeout will trigger recovery if needed
nvgpu: 17000000.gpu nvgpu_channel_suspend_all_serviceable_ch:1953 [ERR] failed to preempt channel/TSG
nvgpu: 17000000.gpu nvgpu_pmu_rpc_execute:669 [WRN] PMU is not ready to process RPC
nvgpu: 17000000.gpu ga10b_pmu_pg_post_init:218 [ERR] Failed to execute RPC status=0x16
nvgpu: 17000000.gpu ga10b_pmu_pg_init_send:257 [ERR] Failed to execute PG_POST_INIT RPC
nvgpu: 17000000.gpu nvgpu_timeout_expired_msg_cpu:94 [ERR] Timeout detected @ nvgpu_pmu_wait_fw_ack_status+0xc8/0x160 [nvgpu]
nvgpu: 17000000.gpu nvgpu_pmu_rpc_execute:749 [ERR] PMU wait timeout expired.
nvgpu: 17000000.gpu ga10b_pmu_elpg_statistics:396 [ERR] Failed to execute RPC status=0xffffff92
nvgpu: 17000000.gpu pmu_response_handle:152 [ERR] msg for an unknown sequence 0
nvgpu: 17000000.gpu gk20a_pmu_handle_interrupts:530 [ERR] nvgpu_pmu_process_message failed err=-22
nvgpu: 17000000.gpu nvgpu_timeout_expired_msg_cpu:94 [ERR] Timeout detected @ gm20b_gr_falcon_ctx_wait_ucode+0xfc/0x420 [nvgpu]
nvgpu: 17000000.gpu nvgpu_timeout_expired_msg_cpu:94 [ERR] Timeout detected @ nvgpu_pmu_wait_fw_ack_status+0xc8/0x160 [nvgpu]
nvgpu: 17000000.gpu nvgpu_pmu_rpc_execute:749 [ERR] PMU wait timeout expired.
nvgpu: 17000000.gpu ga10b_pmu_elpg_statistics:396 [ERR] Failed to execute RPC status=0xffffff92
nvgpu: 17000000.gpu pmu_response_handle:152 [ERR] msg for an unknown sequence 0
nvgpu: 17000000.gpu gk20a_pmu_handle_interrupts:530 [ERR] nvgpu_pmu_process_message failed err=-22
nvgpu: 17000000.gpu nvgpu_timeout_expired_msg_cpu:94 [ERR] Timeout detected @ nvgpu_pmu_wait_fw_ack_status+0xc8/0x160 [nvgpu]
nvgpu: 17000000.gpu nvgpu_pmu_rpc_execute:749 [ERR] PMU wait timeout expired.
nvgpu: 17000000.gpu ga10b_pmu_pg_disallow:145 [ERR] Failed to execute RPC status=0xffffff92
nvgpu: 17000000.gpu nvgpu_pmu_disable_elpg:450 [ERR] PMU_PG_ELPG_CMD_DISALLOW cmd post failed
nvgpu: 17000000.gpu nvgpu_pmu_pg_destroy:1058 [ERR] failed to set disable elpg
nvgpu: 17000000.gpu ga10b_intr_log_pending_intrs:306 [ERR] Pending TOP[0]: 0x00000000, LEAF[4]: 0x01000000
nvgpu: 17000000.gpu nvgpu_timeout_expired_msg_cpu:94 [ERR] Timeout detected @ gv11b_fifo_is_preempt_pending+0x2c8/0x3d8 [nvgpu]
nvgpu: 17000000.gpu gv11b_fifo_preempt_poll_eng:307 [ERR] preempt timeout eng: 0 ctx_stat: 3 tsgid: 0
nvgpu: 17000000.gpu nvgpu_fifo_preempt_tsg:117 [ERR] preempt timed out for tsgid: 0, ctxsw timeout will trigger recovery if needed
nvgpu: 17000000.gpu nvgpu_channel_suspend_all_serviceable_ch:1953 [ERR] failed to preempt channel/TSG
nvgpu: 17000000.gpu nvgpu_timeout_expired_msg_cpu:94 [ERR] Timeout detected @ nvgpu_pmu_wait_fw_ack_status+0xc8/0x160 [nvgpu]
nvgpu: 17000000.gpu nvgpu_pmu_rpc_execute:749 [ERR] PMU wait timeout expired.
nvgpu: 17000000.gpu ga10b_pmu_elpg_statistics:396 [ERR] Failed to execute RPC status=0xffffff92
nvgpu: 17000000.gpu ga10b_intr_log_pending_intrs:306 [ERR] Pending TOP[0]: 0x00000000, LEAF[4]: 0x01000000
nvgpu: 17000000.gpu nvgpu_timeout_expired_msg_cpu:94 [ERR] Timeout detected @ gv11b_fifo_is_preempt_pending+0x2c8/0x3d8 [nvgpu]
nvgpu: 17000000.gpu gv11b_fifo_preempt_poll_eng:307 [ERR] preempt timeout eng: 0 ctx_stat: 3 tsgid: 0
nvgpu: 17000000.gpu nvgpu_fifo_preempt_tsg:117 [ERR] preempt timed out for tsgid: 0, ctxsw timeout will trigger recovery if needed
Workaround:
When running stress case with rt kernel，please enhance the performance and power of board with below command
/usr/sbin/nvpmodel -m 0
INFO: HTTP shared image: http://pek-lpgtest7408.wrs.com/buildarea1/SharedImage/LTS25/BSP/nvidia-orin-agx_preempt-rt_glibc-std-sato_GIT/20250715_144851
/lpg-build/cdc/fast_prod/WRL10_25/WRL10_25_GIT/wrlinux-10/setup.sh --machines nvidia-orin-agx --distro wrlinux-graphics --templates feature/docker --dl-layers=1 --accept-eula=yes
source environment-setup-x86_64-wrlinuxsdk-linux
source oe-init-build-env build
echo 'PREFERRED_PROVIDER_virtual/kernel = "linux-yocto-rt"' >> conf/local.conf
echo 'BB_NUMBER_THREADS ?= "24"' >> conf/local.conf
echo 'PARALLEL_MAKE ?= "-j 24"' >> conf/local.conf
export https_proxy=147.11.252.42:9090
export http_proxy=147.11.252.42:9090
bitbake-layers add-layer /net/pek-lpgtest16.wrs.com/buildarea1/wgao/wassp-linux/testcases/wrlinux/wr-testing/bts-dev
#####To Clone a whole new testcases layer#####
git clone --depth 1 -b master git://lxgit.wrs.com/wrlinux-testing/testcases.git ../layers/testcases/wrlinux/
bitbake-layers add-layer ../layers/testcases/wrlinux/wr-testing/bts-dev
#####End#####
echo '
require templates/feature/bts/template.conf
IMAGE_FSTYPES += " tar.bz2"
bitbake-layers add-layer /net/pek-lpgtest7408/buildarea1/SharedImage/wrlinux-testing-dl
echo 'KERNEL_FEATURES:append = " features/intel-e1xxxx/intel-e1xxxx.scc"
LICENSE_FLAGS_ACCEPTED += "commercial_faad2 commercial_x264 commercial"
CUSTOMER_RECIPES:networking-layer += 'mosquitto'
CUSTOMER_RECIPES:meta-python += 'python3-kconfiglib python3-smbus2'
CUSTOMER_RECIPES:openembedded-layer += 'libglvnd faad2 libwebsockets uthash'
CUSTOMER_RECIPES:tegra += 'tegra-flashtools tegra-bootfiles tegra-minimal-initramfs tegra-espimage edk2-firmware-tegra tegra-tools nvidia-kernel-oot-dtb nvidia-kernel-oot'
CUSTOMER_RECIPES:tegra += 'edk2-firmware-tegra-minimal tegra-initrd-flash-initramfs tos-optee cuda-samples argus-samples l4t-graphics-demos nvgstapps cudnn-samples nvidia-docker'
CUSTOMER_RECIPES:tegra += 'optee-nvsamples optee-test optee-client optee-os optee-os-tadevkit standalone-mm-optee-tegra'
CUSTOMER_RECIPES:tegra += 'nativesdk-packagegroup-cuda-sdk-host tegra-firmware tegra-minimal-init tegra-flash-init'
CUSTOMER_RECIPES:tegra += 'libgcc-for-nvcc gcc-for-nvcc-runtime gcc-for-nvcc-cross-aarch64 gcc-for-nvcc gcc-for-nvcc-cross-canadian-aarch64 gcc-source-for-nvcc-10.3.0'
CUSTOMER_RECIPES:tegra += 'tegra-nvfancontrol tegra-helper-scripts tegra-flashvars tegra-binaries tegra-eks-image tegra-storage-layout'
CUSTOMER_RECIPES:tegra += 'arm-trusted-firmware tar-l4t-workaround-native cudnn libgstnvcustomhelper cupva'
CUSTOMER_RECIPES:tegra += 'cuda-profiler-api cuda-cudart cuda-compatibility-workarounds cuda-crt cuda-compiler cuda-libraries'
CUSTOMER_RECIPES:tegra += 'tegra-mmapi tegra-cmake-overrides tegra-libraries-multimedia-utils tegra-drm-headers tegra-libraries-nvdsseimeta tegra-libraries-multimedia'
CUSTOMER_RECIPES:tegra += 'tegra-wifi tegra-nvstartup tegra-bluetooth tegra-nvsciipc tegra-nvphs tegra-nvs-service tegra-configs tegra-redundant-boot tegra-redundant-boot-base tegra-libraries-cuda'
CUSTOMER_RECIPES:tegra += 'gstreamer1.0-plugins-nvvideosinks gstreamer1.0-plugins-nvvidconv gstreamer1.0-plugins-nvdrmvideosink gstreamer1.0-plugins-nvarguscamerasrc'
CUSTOMER_RECIPES:tegra += 'gstreamer1.0-plugins-nvcompositor gstreamer1.0-plugins-nveglgles gstreamer1.0-plugins-nvipcpipeline gstreamer1.0-plugins-nvjpeg gstreamer1.0-plugins-nvtee'
CUSTOMER_RECIPES:tegra += 'gstreamer1.0-plugins-nvunixfd gstreamer1.0-plugins-nvv4l2camerasrc gstreamer1.0-plugins-nvvideo4linux2 gstreamer1.0-plugins-tegra gstreamer1.0-plugins-tegra-binaryonly'
CUSTOMER_RECIPES:tegra += 'tensorrt-trtexec tensorrt-samples tegra-mmapi-samples libcublas nv-tegra-release tegra-nvpmodel nvidia-container-toolkit tegra-libraries-multimedia-v4l'
CUSTOMER_RECIPES:tegra += 'libusbgx-tegra-initrd-flash tegra-flash-reboot tegra-storage-layout-base tegra-nvpower tegra-nvpmodel-base tegra-eks-image-base tegra-configs-alsa'
CUSTOMER_RECIPES:tegra += 'cuda-cccl cuda-cuxxfilt cuda-driver cuda-nvcc-headers cuda-nvprune cuda-nvvm cuda-cuobjdump cuda-nvcc cuda-nvml cuda-nvrtc cuda-target-environment'
CUSTOMER_RECIPES:tegra += 'libnvjitlink libnvvpi3 libcusolver libcurand libcudla libcufft libcusparse libcufile libnpp libnvidia-container libnvfatbin'
CUSTOMER_RECIPES:tegra += 'l4t-nvidia-glheaders l4t-graphics-demos l4t-launcher-rootfs-ab-config l4t-launcher-extlinux'
CUSTOMER_RECIPES:tegra += 'tegra-libraries-glescore tegra-argus-daemon tegra-eeprom-tool tegra-libraries-camera tegra-libraries-core tegra-libraries-dla-compiler tegra-libraries-eglcore'
CUSTOMER_RECIPES:tegra += 'tegra-libraries-gbm-backend tegra-libraries-glxcore tegra-libraries-multimedia-ds tegra-libraries-nvml tegra-libraries-nvsci tegra-libraries-pva'
CUSTOMER_RECIPES:tegra += 'egl-wayland nvidia-drm-loadconf egl-gbm tegra-nvs-base tegra-nvsciipc-base tegra-nvphs-base tegra-cuda-utils tegra-container-passthrough'
CUSTOMER_RECIPES:tegra += 'xserver-xorg-video-nvidia nativesdk-cuda-environment tegra-nv-boot-control-config setup-nv-boot-control tensorrt-plugins tensorrt-core eglexternalplatform'
CUSTOMER_RECIPES:tegra-community += 'gstreamer-tests tegra-mmapi-tests vpi3-tests tensorrt-tests deepstream-tests python3-jetson-stats nvidia-docker-tests'
CUSTOMER_RECIPES:tegra-community += 'vpi3-samples deepstream-7.1-pyds deepstream-7.1 python3-cuda openssl111 yaml-cpp-070 python3-pyclibrary'
BUILD_NVIDIA_PROPRIETARY_PACKAGE = "1"
BB_NO_NETWORK = '0'' >> conf/local.conf
bitbake wrlinux-image-std-sato
