|
Description
|
panic[cpu0]/thread=3002a2c63e0:
assertion failed: cvp->cv_waiters <= 1024, file: ../../common/os/condvar.c, line
: 169
000002a102e68fc0 genunix:assfail+74 (12e17b8, 12e15e0, a9, 1889400, 12f9c00, 0)
%l0-3: 0000000000001aa0 0000000001919330 0000000001919000 00000000000001aa
%l4-7: 000001830421bdaa 0000000000000000 00000000018df000 0000000000000000
000002a102e69070 genunix:cv_block+170 (1, 3002a2c63e0, dead, 191add8, 12e15e0, 1
91add0)
%l0-3: 0000000000001aa0 0000000001919330 0000000001919000 00000000000001aa
%l4-7: 000001830421bdaa 0000018004198a95 000000000180c000 000000000000deae
000002a102e69120 genunix:cv_wait+44 (60010662a54, 60010d7b3a0, 1982870, 12e1400,
0, 0)
%l0-3: 000000007bbae0a0 0000000000000000 0000000000000000 0000060010a8d8c0
%l4-7: 000000007bbae0a0 0000000000000223 0000000000000003 0000000000000001
000002a102e691d0 neti:wait_for_nini_inprogress+78 (600106629f8, 1982870, deadbee
f, 124, 60010d7b3a0, 7bbae000)
%l0-3: 000000007bbae0a0 0000000000000000 0000000000000000 0000060010a8d8c0
%l4-7: 000000007bbae0a0 0000000000000223 0000060010662a54 0000000000000001
000002a102e69280 neti:neti_stack_apply_shutdown+ac (1982870, 60010d7b3a0, 60012b
24010, 600106629f8, 8, 7bbae000)
%l0-3: 000000007bbae0a0 0000000000000000 0000000000000000 0000060010a8d8c0
%l4-7: 0000000000000044 0000000000000004 0000000000000124 0000000000000134
000002a102e69330 neti:neti_apply_all_instances+48 (60010d7b3a0, 7bbad860, 7bbae0
00, 600106629f8, 1982870, 60010d7b3d0)
%l0-3: 0000000000000001 0000000000000000 0000000000000000 0000060010a8d8c0
%l4-7: 0000000000000044 0000000000000004 0000000000000124 0000000000000134
000002a102e693e0 neti:neti_stack_shutdown+c0 (1982800, 60010d7b3a0, 7bbad800, 7b
bae0a0, 8, 0)
%l0-3: 0000000000000001 0000000000000000 0000000000000000 0000060010a8d8c0
%l4-7: 0000000000000044 0000000000000004 0000000000000124 0000000000000134
000002a102e69490 genunix:netstack_apply_shutdown+108 (18dee50, 60011ec3d80, 18de
ca0, 60011ec3e00, 2, 60011ec3e40)
%l0-3: 0000060011ec3e08 0000000000000002 00000000012f6f28 0000060011ec3e00
%l4-7: 000000007bbacef4 0000060010d7b3a0 000000000000099d 0000000000000010
000002a102e69540 genunix:apply_all_modules_reverse+24 (18dee50, 11ca6e0, 0, 0, 6
0011ec3d80, 2)
%l0-3: 00000000018dec60 00000000018dec50 00000000012f6f28 0000060011ec3e00
%l4-7: 0000000000000010 00000000018dee50 0000000000000014 0000000000000002
000002a102e695f0 genunix:netstack_zone_shutdown+154 (60011ec3e40, 60011ec3d80, 6
0011ec3d80, 11ca400, 11ca080, 12f6c00)
%l0-3: 00000000018dec60 00000000018dec50 00000000012f6f28 0000060011ec3e00
%l4-7: 0000000000000010 00000000018dee50 0000000000000014 0000000000000002
000002a102e696a0 genunix:zsd_apply_shutdown+174 (0, 1, 60010961c40, 60011ec3d80,
1, 60010961c58)
%l0-3: 0000060010961d10 0000060010961c58 0000060011e83580 0000000000000004
%l4-7: 0000000000000000 0000000000000010 0000000001315a20 0000000000000002
000002a102e69750 genunix:zsd_apply_all_keys+30 (12c777c, 60010961c40, 60010961cf
8, 60010961d10, 60011e83580, 60010961c58)
%l0-3: 0000060010961d10 0000060010961c58 0000000000000000 000000007b76336c
%l4-7: 0000000000000000 0000000000000010 0000000000000000 0000000001915000
000002a102e69800 genunix:zone_zsd_callbacks+140 (12c7964, 12c7800, 60010961c40,
12c7400, 0, 1315a20)
%l0-3: 0000060010961d10 0000060010961c58 0000000000000000 000000007b76336c
%l4-7: 0000000000000000 0000000000000010 0000000000000000 0000000001915000
000002a102e698b0 genunix:zone_shutdown+1a4 (99d, 1e5d4, 187ec00, 4, ff3e1c2c, 19
13518)
%l0-3: 0000060010961c40 000002a102e69b90 0000060010961c58 0000000000000002
%l4-7: 00000000018c5b38 0000000000000000 0000000000000000 0000000001915000
000002a102e69960 genunix:zone+1c0 (5, 99d, 5, 1f8, ff3e1c2c, 12ce000)
%l0-3: 000000000100a314 000002a102e69b90 000006001285c418 0000000000000016
%l4-7: 00000000018c5b38 00000000feb7baa0 000003002a2c63e0 00000000012ce348
syncing file systems...
19
2
done
dumping to /dev/dsk/c0t0d0s1, offset 1888157696, content: kernel
> 60012b24010::whatis
60012b24010 is 60012b24010+0, bufctl 3058079adc8 freed from kmem_alloc_96
> 3058079adc8::bufctl -v
ADDR BUFADDR TIMESTAMP THREAD
CACHE LASTLOG CONTENTS
3058079adc8 60012b24010 1d9b3c621658 2a10010fca0
300000f8020 300059cee80 30011e3db18
kmem_cache_free+0x84
net_instance_unregister+0x1c4
recall_fini+0x20
recall_expire+0x24
callout_execute+0xb0
taskq_thread+0x1e4
> 600106629f8::whatis
600106629f8 is 600106629f8+0, bufctl 3001b86f0b0 freed from kmem_alloc_96
> 3001b86f0b0::bufctl -v
ADDR BUFADDR TIMESTAMP THREAD
CACHE LASTLOG CONTENTS
3001b86f0b0 600106629f8 1d9b3c6200b8 2a10010fca0
300000f8020 300059ced80 30011e3da58
kmem_cache_free+0x84
net_instance_unregister+0x1a0
recall_fini+0x20
recall_expire+0x24
callout_execute+0xb0
taskq_thread+0x1e4
> 600106629f8::print -t net_instance_int_t
{
struct nini_next = {
struct net_instance_int_s *le_next = 0xdeadbeefdeadbeef
struct net_instance_int_s **le_prev = 0xdeadbeefdeadbeef
}
> 60010d7b3a0::whatis
60010d7b3a0 is 60010d7b3a0+0, bufctl 30024cdd208 allocated from kmem_alloc_80
> 30024cdd208::bufctl -v
ADDR BUFADDR TIMESTAMP THREAD
CACHE LASTLOG CONTENTS
30024cdd208 60010d7b3a0 1d9a67ceebbc 3009bf4d180
300000f2020 30006b61900 3000d82c3d8
kmem_cache_alloc+0x90
kmem_zalloc+0x2c
neti_stack_init+0xc
netstack_apply_create+0xf8
apply_all_modules+0x28
netstack_zone_create+0x210
zsd_apply_create+0x148
zsd_apply_all_keys+0x30
zsched+0x5c0
> 60010a8d8c0::whatis
60010a8d8c0 is 60010a8d8c0+0, bufctl 3002618e748 allocated from kmem_alloc_40
> 3002618e748::bufctl -v
ADDR BUFADDR TIMESTAMP THREAD
CACHE LASTLOG CONTENTS
3002618e748 60010a8d8c0 206607ea94 2a102581ca0
300000da020 30004cf1900 30010822510
kmem_cache_alloc+0x90
kmem_zalloc+0x2c
net_instance_alloc+0x18
ipf_attach+0xcc
devi_attach+0xc0
attach_node+0x164
i_ndi_config_node+0x150
i_ddi_attachchild+0x74
devi_attach_node+0x13c
config_immediate_children+0x104
devi_config_common+0xb0
mt_config_thread+0x60
> 60010a8d8c0::print -t net_instance_t
{
int nin_version = 0x1
char *nin_name = 0x7aa2e158 "ipf"
int (*)() nin_create = ipf_stack_create
int (*)() nin_destroy = ipf_stack_destroy
int (*)() nin_shutdown = 0
}
> 60010662a54::whatis
60010662a54 is 600106629f8+5c, bufctl 3001b86f0b0 freed from kmem_alloc_96
> 3001b86f0b0::bufctl -v
ADDR BUFADDR TIMESTAMP THREAD
CACHE LASTLOG CONTENTS
3001b86f0b0 600106629f8 1d9b3c6200b8 2a10010fca0
300000f8020 300059ced80 30011e3da58
kmem_cache_free+0x84
net_instance_unregister+0x1a0
recall_fini+0x20
recall_expire+0x24
callout_execute+0xb0
taskq_thread+0x1e4
The problem here turned out to be that the net_instance_int_t was being free'd while waiting for the INPROGRESS flags clear in wait_for_nini_inprogress. The problem is that there is (or was) nothing to indicate that any threads were actually inside this function.
What happens is that we get into wait_for_nini_inprogress() while, for example, neti_apply_stack_shutdown() is being called. If the thread inside wait_for_nini_inprogress doesn't get executed before neti_apply_stack_destroy() is called, and there is no reason why it should, then it is possible for wait_for_nini_inprogress() to wakeup after its net_instance_int_t has been free'd.
To fix this we can use nini_ref in the following way: before calling wait_for_nini_inprogress, bump it up by one in each of the neti_apply_* functions. Then when exiting these functions, we drop nini_ref by one. The complimentary change to this is inside net_instance_int_free(), where we need to not free the structure if nini_ref is > 0 (for clones) but set a condemned flag. When exiting the neti_apply_* functions, a check is made on the condemned flag and net_instance_int_free() called if set.
The algorithm in use is similar to both that used for netstack and that used with zones. So why doesn't this problem occur there? Most notably, when zsd_apply_all_keys calls wait_for_zsd_inprogress(), it passes a NULL lock pointer, meaning that the locks are never dropped. Thus there isn't any chance for the structure holding the flags to be grabbed by another thread and free'd.
|
