Steps on taking stack trace for a core file when appliance restarted due to memory and versa-service.log is filled with lcore messages before service restart
1.
[admin@Branch: ~] $ cd /var/tmp/versa-cores
[admin@Branch: versa-cores] $ ls -ltr
total 702636
-rw-rw-rw- 1 root root 719491911 May 18 12:11 core.versa-vsmd.3765.Branch.1652893605.gz
2.
[admin@Branch: versa-cores] $ sudo gunzip core.versa-vsmd.3765.Branch.1652893605.gz
[sudo] password for admin:
[admin@Branch: versa-cores] $ ls -ltr
total 14995392
-rw-rw-rw- 1 root root 15355277312 May 18 12:11 core.versa-vsmd.3765.Branch.1652893605
[admin@Branch: versa-cores] $ sudo gunzip core.versa-vsmd.3765.Branch.1652893605.gz
[sudo] password for admin:
[admin@Branch: versa-cores] $ ls -ltr
total 14995392
-rw-rw-rw- 1 root root 15355277312 May 18 12:11 core.versa-vsmd.3765.Branch.1652893605
3.
[admin@Branch: versa-cores] $ sudo gdb /opt/versa/bin/versa-vsmd core.versa-vsmd.3765.Branch.1652893605
GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.3) 7.7.1
Copyright (C) 2014 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /opt/versa/bin/versa-vsmd...done.
[New LWP 3891]
[New LWP 3776]
[New LWP 3778]
[New LWP 3782]
[New LWP 3781]
[New LWP 3779]
[New LWP 3780]
[New LWP 3775]
[New LWP 3883]
[New LWP 3885]
[New LWP 3888]
[New LWP 3889]
[New LWP 3765]
[New LWP 3777]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/opt/versa/bin/versa-vsmd -N -H 0 -f 1'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007f075f02f1fb in raise () from /lib/x86_64-linux-gnu/libpthread.so.0
(gdb) bt
#0 0x00007f075f02f1fb in raise () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 <signal handler called>
#2 0x00000000016fcf70 in vs_thrm_dead_lcore (data=0x0, id_core=5) at ../usr/lib/libvsthrm/vs_thrm.c:3366
#3 0x0000000001b5a2cd in rte_keepalive_dispatch_pings (ptr_timer=ptr_timer@entry=0x0, ptr_data=0x1e8bd5cc0) at ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c:82
#4 0x00000000016fcf2e in vs_thrm_watchdog_main_loop (arg=<optimized out>) at ../usr/lib/libvsthrm/vs_thrm.c:3451
#5 0x00007f075f027184 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007f075b80e03d in clone () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) gdb_vsm_dump_dead_lcore_info
[Switching to thread 1 (Thread 0x7f06cf571700 (LWP 3891))]
#0 0x00007f075f02f1fb in raise () from /lib/x86_64-linux-gnu/libpthread.so.0
#0 0x00007f075f02f1fb in raise () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 <signal handler called>
#2 0x00000000016fcf70 in vs_thrm_dead_lcore (data=0x0, id_core=5) at ../usr/lib/libvsthrm/vs_thrm.c:3366
#3 0x0000000001b5a2cd in rte_keepalive_dispatch_pings (ptr_timer=ptr_timer@entry=0x0, ptr_data=0x1e8bd5cc0) at ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c:82
#4 0x00000000016fcf2e in vs_thrm_watchdog_main_loop (arg=<optimized out>) at ../usr/lib/libvsthrm/vs_thrm.c:3451
#5 0x00007f075f027184 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007f075b80e03d in clone () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x0000000001b5a2cd in rte_keepalive_dispatch_pings (ptr_timer=ptr_timer@entry=0x0, ptr_data=0x1e8bd5cc0) at ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c:82
82 ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c: No such file or directory.
rte lcore keepalive context
---------------------------
$1 = {
live_data = {{
core_state = 0x0
}, {
core_state = 0x1
}, {
core_state = 0x1
}, {
core_state = 0x1
}, {
core_state = 0x1
}, {
core_state = 0x3
}, {
core_state = 0x1
}, {
core_state = 0x1
}, {
core_state = 0x0
} <repeats 56 times>},
last_alive = {0x0, 0x7c1d87bb105d72, 0x7c1d87bb105ff8, 0x7c1d87bb10601a, 0x7c1d87bb106038, 0x7c1d493c82db0c, 0x7c1d86b4cc4a58, 0x7c1d86b4cc4ac0, 0x0 <repeats 56 times>},
active_cores = {0x0, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0 <repeats 56 times>},
callback = 0x16fcf50,
callback_data = 0x0,
tsc_initial = 0x525c145936,
tsc_mhz = 0x2191a8,
relay_callback = 0x1702910,
relay_callback_data = 0x1e8bd5cc0
}
Id Target Id Frame
14 Thread 0x7f0757f7d700 (LWP 3777) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
13 Thread 0x7f0759fd2580 (LWP 3765) vs_sdwan_fp_vl_entry_delete (list_head=<optimized out>, fp_vl=0x7f06c3e10400) at ../usr/module/pbf/vs_sdwan_fp.c:1779
12 Thread 0x7f07483ff700 (LWP 3889) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
11 Thread 0x7f074a40c700 (LWP 3888) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
---Type <return> to continue, or q <return> to quit---
10 Thread 0x7f07501ff700 (LWP 3885) 0x00007f075b80e6d3 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
9 Thread 0x7f0753577700 (LWP 3883) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
8 Thread 0x7f0758f7f700 (LWP 3775) 0x00007f075b80e6d3 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
7 Thread 0x7f075677a700 (LWP 3780) 0x00007f075b874690 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
6 Thread 0x7f0756f7b700 (LWP 3779) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
5 Thread 0x7f0755f79700 (LWP 3781) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
4 Thread 0x7f0755778700 (LWP 3782) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
3 Thread 0x7f075777c700 (LWP 3778) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
2 Thread 0x7f075877e700 (LWP 3776) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
* 1 Thread 0x7f06cf571700 (LWP 3891) 0x00007f075f02f1fb in raise () from /lib/x86_64-linux-gnu/libpthread.so.0
Finding the lcore being stuck...
--------------------------------
[Switching to thread 1 (Thread 0x7f06cf571700 (LWP 3891))]
#3 0x0000000001b5a2cd in rte_keepalive_dispatch_pings (ptr_timer=ptr_timer@entry=0x0, ptr_data=0x1e8bd5cc0) at ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c:82
82 in ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c
[Switching to thread 2 (Thread 0x7f075877e700 (LWP 3776))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 3 (Thread 0x7f075777c700 (LWP 3778))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 4 (Thread 0x7f0755778700 (LWP 3782))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 5 (Thread 0x7f0755f79700 (LWP 3781))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 6 (Thread 0x7f0756f7b700 (LWP 3779))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 7 (Thread 0x7f075677a700 (LWP 3780))]
#0 0x00007f075b874690 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
The below lcore 5 is stuck
---------------------------
Thread 7 backtrace, lcore 5
------------------------------
#0 0x00007f075b874690 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x0000000000776a1e in vsm_p2mp_slam_endpt_cmp (key1=<optimized out>, key2=<optimized out>) at ../usr/sbin/vsm/vsm_p2mp.c:6444
#2 0x000000000321c43a in do_htable_lookup_unsafe (search_key=0x7f0756367130, bucket=0x7f07317f7d30, ht=0x7f07317f7800) at ../usr/lib/libutils/vs_htable.c:195
#3 vs_htable_get (ht=0x7f07317f7800, search_key=search_key@entry=0x7f0756367130) at ../usr/lib/libutils/vs_htable.c:228
#4 0x0000000000781671 in vsm_p2mp_slam_endpt_lookup (rmt_ip=<optimized out>, lan_fib_id=<optimized out>) at ../usr/sbin/vsm/vsm_p2mp.c:6359
#5 vsm_p2mp_slam_state (slam_type=<optimized out>, lan_fib_id=<optimized out>, peer_ip=<optimized out>, is_slam_up=0x7f0756367240) at ../usr/sbin/vsm/vsm_p2mp.c:6459
#6 0x000000000086bdcc in vs_pbf_sdwan_data_driven_path_update (pbf_sess_ext=pbf_sess_ext@entry=0x7f06c5bbf010, sdwan_flow_rx=<optimized out>,
sdwan_flow_tx=sdwan_flow_tx@entry=0x7f06c5bbf038, pol_sess=pol_sess@entry=0x7f06c5cde390, m=m@entry=0x1f911e0c0, dctxt=dctxt@entry=0x7f0756367610, ingress_ifp=<optimized out>,
ingress_ifp@entry=0x7f06d1c89800, egress_ifp=<optimized out>, egress_ifp@entry=0x7f06d1b10000, first_pkt=<optimized out>, first_pkt@entry=false, fp=<optimized out>)
at ../usr/module/pbf/vs_pbf_nfp_hdlr.c:4143
#7 0x00000000031d9dda in vs_pbf_sdwan_path_sel (pbf_sess_ext=pbf_sess_ext@entry=0x7f06c5bbf010, sdwan_flow_rx=sdwan_flow_rx@entry=0x7f06c5bbf038,
sdwan_flow_tx=sdwan_flow_tx@entry=0x7f06c5bbf038, pol_sess=0x7f06c5cde390, m=0x1f911e0c0, dctxt=<optimized out>, ingress_ifp=<optimized out>, egress_ifp=<optimized out>,
first_pkt=<optimized out>) at ../usr/module/pbf/vs_pbf_nfp_hdlr.c:4322
---Type <return> to continue, or q <return> to quit---
#8 0x0000000000874243 in vs_pbf_apply_sdwan_policy_results (first_pkt=<optimized out>, egress_ifp=<optimized out>, ingress_ifp=<optimized out>, dctxt=<optimized out>, m=<optimized out>,
pol_sess=<optimized out>, sdwan_flow_tx=<optimized out>, sdwan_flow_rx=<optimized out>, pbf_sess_ext=<optimized out>) at ../usr/module/pbf/vs_pbf_nfp_hdlr.c:4509
#9 vs_sdwan_pkt_to_branch_hndlr (app_ctx=app_ctx@entry=0x7f06c5ddd470, m=m@entry=0x1f911e0c0, dctxt=dctxt@entry=0x7f0756367610, ingress_ifp=<optimized out>, egress_ifp=0x7f06d1b10000)
at ../usr/module/pbf/vs_pbf_nfp_hdlr.c:5149
#10 0x00000000031db778 in vs_pbf_nfp_pkt_handler (app_ctx=0x7f06c5ddd470, m=0x1f911e0c0, ingress_flag=<optimized out>) at ../usr/module/pbf/vs_pbf_nfp_hdlr.c:6866
#11 0x00000000031f855d in vs_nfp_pkt_input_chain_v4 (mctx=mctx@entry=0x7f0756717800) at ../usr/lib/libvsf/vs_nfp_pkt_svc.c:2987
#12 0x00000000031e7be6 in vs_ipvx_input_post_hook (mctx=0x7f0756717800, nstage=0x7f0756367840 "\0027V\a\177") at ../usr/lib/libvsf/vs_nfp.c:2401
#13 0x00000000031ee5d3 in vs_nfp_enter_post_vmlh (shdl=shdl@entry=139667063417605, m=m@entry=0x1f911e0c0) at ../usr/lib/libvsf/vs_nfp.c:4818
#14 0x00000000031ef71f in vs_nfp_enter (shdl=139667063417605, m=0x1f911e0c0) at ../usr/lib/libvsf/vs_nfp.c:5099
#15 0x00000000031d417a in process_local_pkts (nmbs=2859853318, mbq=<optimized out>) at ../usr/sbin/vsm/vsm_thrm_pl.c:3793
#16 0x00000000031d9031 in vsm_pl_proc_io_pkts (works=<optimized out>, npkts=<optimized out>) at ../usr/sbin/vsm/vsm_thrm_pl.c:4659
#17 0x000000000071ed26 in vsm_process_thrm_work (works=works@entry=0x7f0756367c00, nworks=nworks@entry=1, work_type=work_type@entry=VS_THRM_WORK_TYPE_PKT)
at ../usr/sbin/vsm/vsm_thrm.c:3800
#18 0x0000000003219eda in read_n_dispatch_spsc_ring_pkts (pset=pset@entry=0x7f07539e54d8, works=0x7f0756367c00, dispatch_fn=0x71ec60 <vsm_process_thrm_work>, quota=<optimized out>,
quota@entry=64) at ../usr/lib/libvsthrm/vs_worker_threads.c:136
#19 0x000000000321a422 in process_io_rx_workqs (quota=64, tinfo=0x7f07539e5400, ctx=<optimized out>) at ../usr/lib/libvsthrm/vs_worker_threads.c:190
#20 vs_worker_thread_routine (arg=0x7f07539e5400) at ../usr/lib/libvsthrm/vs_worker_threads.c:648
#21 0x00000000016fd280 in vs_generic_thread_routine (arg=0x7f07539e5400) at ../usr/lib/libvsthrm/vs_thrm.c:256
#22 0x00000000016ff593 in vs_thrm_start_routine (handle=<optimized out>, tgid=<optimized out>, tgid@entry=VS_THREAD_GROUP_MAX, shared_cpu=shared_cpu@entry=false)
at ../usr/lib/libvsthrm/vs_thrm.c:1774
#23 0x000000000072d517 in vsm_thrm_start (arg=<optimized out>) at ../usr/sbin/vsm/vsm_thrm.c:5858
#24 0x0000000001b63f44 in eal_thread_loop (arg=<optimized out>) at ../usr/lib/DPDK/lib/librte_eal/linuxapp/eal/eal_thread.c:182
#25 0x00007f075f027184 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#26 0x00007f075b80e03d in clone () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 8 (Thread 0x7f0758f7f700 (LWP 3775))]
#0 0x00007f075b80e6d3 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 9 (Thread 0x7f0753577700 (LWP 3883))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 10 (Thread 0x7f07501ff700 (LWP 3885))]
#0 0x00007f075b80e6d3 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 11 (Thread 0x7f074a40c700 (LWP 3888))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 12 (Thread 0x7f07483ff700 (LWP 3889))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[admin@Branch: versa-cores] $ sudo gdb /opt/versa/bin/versa-vsmd core.versa-vsmd.3765.Branch.1652893605
GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.3) 7.7.1
Copyright (C) 2014 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /opt/versa/bin/versa-vsmd...done.
[New LWP 3891]
[New LWP 3776]
[New LWP 3778]
[New LWP 3782]
[New LWP 3781]
[New LWP 3779]
[New LWP 3780]
[New LWP 3775]
[New LWP 3883]
[New LWP 3885]
[New LWP 3888]
[New LWP 3889]
[New LWP 3765]
[New LWP 3777]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/opt/versa/bin/versa-vsmd -N -H 0 -f 1'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007f075f02f1fb in raise () from /lib/x86_64-linux-gnu/libpthread.so.0
(gdb) bt
#0 0x00007f075f02f1fb in raise () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 <signal handler called>
#2 0x00000000016fcf70 in vs_thrm_dead_lcore (data=0x0, id_core=5) at ../usr/lib/libvsthrm/vs_thrm.c:3366
#3 0x0000000001b5a2cd in rte_keepalive_dispatch_pings (ptr_timer=ptr_timer@entry=0x0, ptr_data=0x1e8bd5cc0) at ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c:82
#4 0x00000000016fcf2e in vs_thrm_watchdog_main_loop (arg=<optimized out>) at ../usr/lib/libvsthrm/vs_thrm.c:3451
#5 0x00007f075f027184 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007f075b80e03d in clone () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) gdb_vsm_dump_dead_lcore_info
[Switching to thread 1 (Thread 0x7f06cf571700 (LWP 3891))]
#0 0x00007f075f02f1fb in raise () from /lib/x86_64-linux-gnu/libpthread.so.0
#0 0x00007f075f02f1fb in raise () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 <signal handler called>
#2 0x00000000016fcf70 in vs_thrm_dead_lcore (data=0x0, id_core=5) at ../usr/lib/libvsthrm/vs_thrm.c:3366
#3 0x0000000001b5a2cd in rte_keepalive_dispatch_pings (ptr_timer=ptr_timer@entry=0x0, ptr_data=0x1e8bd5cc0) at ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c:82
#4 0x00000000016fcf2e in vs_thrm_watchdog_main_loop (arg=<optimized out>) at ../usr/lib/libvsthrm/vs_thrm.c:3451
#5 0x00007f075f027184 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#6 0x00007f075b80e03d in clone () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x0000000001b5a2cd in rte_keepalive_dispatch_pings (ptr_timer=ptr_timer@entry=0x0, ptr_data=0x1e8bd5cc0) at ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c:82
82 ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c: No such file or directory.
rte lcore keepalive context
---------------------------
$1 = {
live_data = {{
core_state = 0x0
}, {
core_state = 0x1
}, {
core_state = 0x1
}, {
core_state = 0x1
}, {
core_state = 0x1
}, {
core_state = 0x3
}, {
core_state = 0x1
}, {
core_state = 0x1
}, {
core_state = 0x0
} <repeats 56 times>},
last_alive = {0x0, 0x7c1d87bb105d72, 0x7c1d87bb105ff8, 0x7c1d87bb10601a, 0x7c1d87bb106038, 0x7c1d493c82db0c, 0x7c1d86b4cc4a58, 0x7c1d86b4cc4ac0, 0x0 <repeats 56 times>},
active_cores = {0x0, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0 <repeats 56 times>},
callback = 0x16fcf50,
callback_data = 0x0,
tsc_initial = 0x525c145936,
tsc_mhz = 0x2191a8,
relay_callback = 0x1702910,
relay_callback_data = 0x1e8bd5cc0
}
Id Target Id Frame
14 Thread 0x7f0757f7d700 (LWP 3777) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
13 Thread 0x7f0759fd2580 (LWP 3765) vs_sdwan_fp_vl_entry_delete (list_head=<optimized out>, fp_vl=0x7f06c3e10400) at ../usr/module/pbf/vs_sdwan_fp.c:1779
12 Thread 0x7f07483ff700 (LWP 3889) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
11 Thread 0x7f074a40c700 (LWP 3888) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
---Type <return> to continue, or q <return> to quit---
10 Thread 0x7f07501ff700 (LWP 3885) 0x00007f075b80e6d3 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
9 Thread 0x7f0753577700 (LWP 3883) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
8 Thread 0x7f0758f7f700 (LWP 3775) 0x00007f075b80e6d3 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
7 Thread 0x7f075677a700 (LWP 3780) 0x00007f075b874690 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
6 Thread 0x7f0756f7b700 (LWP 3779) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
5 Thread 0x7f0755f79700 (LWP 3781) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
4 Thread 0x7f0755778700 (LWP 3782) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
3 Thread 0x7f075777c700 (LWP 3778) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
2 Thread 0x7f075877e700 (LWP 3776) 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
* 1 Thread 0x7f06cf571700 (LWP 3891) 0x00007f075f02f1fb in raise () from /lib/x86_64-linux-gnu/libpthread.so.0
Finding the lcore being stuck...
--------------------------------
[Switching to thread 1 (Thread 0x7f06cf571700 (LWP 3891))]
#3 0x0000000001b5a2cd in rte_keepalive_dispatch_pings (ptr_timer=ptr_timer@entry=0x0, ptr_data=0x1e8bd5cc0) at ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c:82
82 in ../usr/lib/DPDK/lib/librte_eal/common/rte_keepalive.c
[Switching to thread 2 (Thread 0x7f075877e700 (LWP 3776))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 3 (Thread 0x7f075777c700 (LWP 3778))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 4 (Thread 0x7f0755778700 (LWP 3782))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 5 (Thread 0x7f0755f79700 (LWP 3781))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 6 (Thread 0x7f0756f7b700 (LWP 3779))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 7 (Thread 0x7f075677a700 (LWP 3780))]
#0 0x00007f075b874690 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
The below lcore 5 is stuck
---------------------------
Thread 7 backtrace, lcore 5
------------------------------
#0 0x00007f075b874690 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x0000000000776a1e in vsm_p2mp_slam_endpt_cmp (key1=<optimized out>, key2=<optimized out>) at ../usr/sbin/vsm/vsm_p2mp.c:6444
#2 0x000000000321c43a in do_htable_lookup_unsafe (search_key=0x7f0756367130, bucket=0x7f07317f7d30, ht=0x7f07317f7800) at ../usr/lib/libutils/vs_htable.c:195
#3 vs_htable_get (ht=0x7f07317f7800, search_key=search_key@entry=0x7f0756367130) at ../usr/lib/libutils/vs_htable.c:228
#4 0x0000000000781671 in vsm_p2mp_slam_endpt_lookup (rmt_ip=<optimized out>, lan_fib_id=<optimized out>) at ../usr/sbin/vsm/vsm_p2mp.c:6359
#5 vsm_p2mp_slam_state (slam_type=<optimized out>, lan_fib_id=<optimized out>, peer_ip=<optimized out>, is_slam_up=0x7f0756367240) at ../usr/sbin/vsm/vsm_p2mp.c:6459
#6 0x000000000086bdcc in vs_pbf_sdwan_data_driven_path_update (pbf_sess_ext=pbf_sess_ext@entry=0x7f06c5bbf010, sdwan_flow_rx=<optimized out>,
sdwan_flow_tx=sdwan_flow_tx@entry=0x7f06c5bbf038, pol_sess=pol_sess@entry=0x7f06c5cde390, m=m@entry=0x1f911e0c0, dctxt=dctxt@entry=0x7f0756367610, ingress_ifp=<optimized out>,
ingress_ifp@entry=0x7f06d1c89800, egress_ifp=<optimized out>, egress_ifp@entry=0x7f06d1b10000, first_pkt=<optimized out>, first_pkt@entry=false, fp=<optimized out>)
at ../usr/module/pbf/vs_pbf_nfp_hdlr.c:4143
#7 0x00000000031d9dda in vs_pbf_sdwan_path_sel (pbf_sess_ext=pbf_sess_ext@entry=0x7f06c5bbf010, sdwan_flow_rx=sdwan_flow_rx@entry=0x7f06c5bbf038,
sdwan_flow_tx=sdwan_flow_tx@entry=0x7f06c5bbf038, pol_sess=0x7f06c5cde390, m=0x1f911e0c0, dctxt=<optimized out>, ingress_ifp=<optimized out>, egress_ifp=<optimized out>,
first_pkt=<optimized out>) at ../usr/module/pbf/vs_pbf_nfp_hdlr.c:4322
---Type <return> to continue, or q <return> to quit---
#8 0x0000000000874243 in vs_pbf_apply_sdwan_policy_results (first_pkt=<optimized out>, egress_ifp=<optimized out>, ingress_ifp=<optimized out>, dctxt=<optimized out>, m=<optimized out>,
pol_sess=<optimized out>, sdwan_flow_tx=<optimized out>, sdwan_flow_rx=<optimized out>, pbf_sess_ext=<optimized out>) at ../usr/module/pbf/vs_pbf_nfp_hdlr.c:4509
#9 vs_sdwan_pkt_to_branch_hndlr (app_ctx=app_ctx@entry=0x7f06c5ddd470, m=m@entry=0x1f911e0c0, dctxt=dctxt@entry=0x7f0756367610, ingress_ifp=<optimized out>, egress_ifp=0x7f06d1b10000)
at ../usr/module/pbf/vs_pbf_nfp_hdlr.c:5149
#10 0x00000000031db778 in vs_pbf_nfp_pkt_handler (app_ctx=0x7f06c5ddd470, m=0x1f911e0c0, ingress_flag=<optimized out>) at ../usr/module/pbf/vs_pbf_nfp_hdlr.c:6866
#11 0x00000000031f855d in vs_nfp_pkt_input_chain_v4 (mctx=mctx@entry=0x7f0756717800) at ../usr/lib/libvsf/vs_nfp_pkt_svc.c:2987
#12 0x00000000031e7be6 in vs_ipvx_input_post_hook (mctx=0x7f0756717800, nstage=0x7f0756367840 "\0027V\a\177") at ../usr/lib/libvsf/vs_nfp.c:2401
#13 0x00000000031ee5d3 in vs_nfp_enter_post_vmlh (shdl=shdl@entry=139667063417605, m=m@entry=0x1f911e0c0) at ../usr/lib/libvsf/vs_nfp.c:4818
#14 0x00000000031ef71f in vs_nfp_enter (shdl=139667063417605, m=0x1f911e0c0) at ../usr/lib/libvsf/vs_nfp.c:5099
#15 0x00000000031d417a in process_local_pkts (nmbs=2859853318, mbq=<optimized out>) at ../usr/sbin/vsm/vsm_thrm_pl.c:3793
#16 0x00000000031d9031 in vsm_pl_proc_io_pkts (works=<optimized out>, npkts=<optimized out>) at ../usr/sbin/vsm/vsm_thrm_pl.c:4659
#17 0x000000000071ed26 in vsm_process_thrm_work (works=works@entry=0x7f0756367c00, nworks=nworks@entry=1, work_type=work_type@entry=VS_THRM_WORK_TYPE_PKT)
at ../usr/sbin/vsm/vsm_thrm.c:3800
#18 0x0000000003219eda in read_n_dispatch_spsc_ring_pkts (pset=pset@entry=0x7f07539e54d8, works=0x7f0756367c00, dispatch_fn=0x71ec60 <vsm_process_thrm_work>, quota=<optimized out>,
quota@entry=64) at ../usr/lib/libvsthrm/vs_worker_threads.c:136
#19 0x000000000321a422 in process_io_rx_workqs (quota=64, tinfo=0x7f07539e5400, ctx=<optimized out>) at ../usr/lib/libvsthrm/vs_worker_threads.c:190
#20 vs_worker_thread_routine (arg=0x7f07539e5400) at ../usr/lib/libvsthrm/vs_worker_threads.c:648
#21 0x00000000016fd280 in vs_generic_thread_routine (arg=0x7f07539e5400) at ../usr/lib/libvsthrm/vs_thrm.c:256
#22 0x00000000016ff593 in vs_thrm_start_routine (handle=<optimized out>, tgid=<optimized out>, tgid@entry=VS_THREAD_GROUP_MAX, shared_cpu=shared_cpu@entry=false)
at ../usr/lib/libvsthrm/vs_thrm.c:1774
#23 0x000000000072d517 in vsm_thrm_start (arg=<optimized out>) at ../usr/sbin/vsm/vsm_thrm.c:5858
#24 0x0000000001b63f44 in eal_thread_loop (arg=<optimized out>) at ../usr/lib/DPDK/lib/librte_eal/linuxapp/eal/eal_thread.c:182
#25 0x00007f075f027184 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#26 0x00007f075b80e03d in clone () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 8 (Thread 0x7f0758f7f700 (LWP 3775))]
#0 0x00007f075b80e6d3 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 9 (Thread 0x7f0753577700 (LWP 3883))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 10 (Thread 0x7f07501ff700 (LWP 3885))]
#0 0x00007f075b80e6d3 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 11 (Thread 0x7f074a40c700 (LWP 3888))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6
[Switching to thread 12 (Thread 0x7f07483ff700 (LWP 3889))]
#0 0x00007f075b7d4d4d in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6