Kernel Memory
From pressy's brainbackup
From time to time I see some solaris server using a lot of memory "just" for the kernel. You could take a deeper look to see where the kernel memory is used:
root@server:~# mdb -k
Loading modules: [ unix genunix specfs dtrace zfs scsi_vhci sd mpt_sas mac px ldc ip hook neti ds arp usba kssl stmf stmf_sbd random sockfs md niumx idm
cpc crypto fcip fctl fcp mdesc vldc smbsrv nfs zvmm ufs logindmux ptm ii nsctl sppp nsmb rdc sdbc sv lofs ipc ]
>
> ::kmastat
cache buf buf buf memory alloc alloc
name size in use total in use succeed fail
------------------------- ------ ------ ------ ---------- --------- -----
kmem_magazine_1 16 13787 39546 638976B 2212036 0
kmem_magazine_3 32 13293 149017 4825088B 20882637 0
kmem_magazine_7 64 63818 687708 44711936B 74367950 0
kmem_magazine_15 128 81728 154728 20119552B 40689829 0
kmem_magazine_31 256 0 0 0B 0 0
kmem_magazine_47 384 0 0 0B 0 0
kmem_magazine_63 512 0 0 0B 0 0
kmem_magazine_95 768 0 0 0B 0 0
kmem_magazine_127 1024 0 0 0B 0 0
kmem_magazine_143 1152 0 0 0B 0 0
kmem_magazine_179 1440 0 0 0B 0 0
kmem_magazine_255 2048 0 0 0B 0 0
kmem_magazine_361 2896 0 0 0B 0 0
kmem_magazine_492 3944 0 0 0B 0 0
kmem_slab_cache 72 21964 41888 3063808B 1905292 0
kmem_bufctl_cache 24 60474 84162 2039808B 1966131 0
kmem_bufctl_audit_cache 128 0 0 0B 0 0
kmem_va_8192 8192 6617 15040 123207680B 92929 0
kmem_va_16384 16384 12 32 524288B 22 0
kmem_va_24576 24576 11 50 1310720B 147 0
kmem_va_32768 32768 6 64 2097152B 1152 0
kmem_va_40960 40960 9 192 8388608B 4001 0
kmem_va_49152 49152 9 200 10485760B 7542 0
kmem_va_57344 57344 1 48 3145728B 1143 0
kmem_va_65536 65536 18 144 9437184B 4846 0
kmem_alloc_8 8 225785 229390 1851392B 1460654149 0
kmem_alloc_16 16 70050 74022 1196032B 1802895553 0
kmem_alloc_24 24 176486 244712 5931008B 4193514113 0
kmem_alloc_32 32 40268 45034 1458176B 3716814702 0
kmem_alloc_40 40 31285 47705 1925120B 1528348140 0
kmem_alloc_48 48 67277 128102 6209536B 1790577690 0
kmem_alloc_56 56 91174 141955 8019968B 858181106 0
kmem_alloc_64 64 151639 323946 21061632B 1933863414 0
kmem_alloc_80 80 187667 261994 21250048B 596556789 0
kmem_alloc_96 96 9275 29064 2834432B 1763944514 0
kmem_alloc_112 112 8553 16056 1826816B 242740674 0
kmem_alloc_128 128 108755 111636 14516224B 241660346 0
kmem_alloc_160 160 4200 18000 2949120B 378366571 0
kmem_alloc_192 192 60314 130536 25460736B 516119668 0
kmem_alloc_224 224 1682 2232 507904B 498796577 0
kmem_alloc_256 256 2237 51305 13557760B 1199808091 0
kmem_alloc_320 320 4686 18475 6053888B 4005106805 0
kmem_alloc_384 384 664 2289 892928B 1743958062 0
kmem_alloc_448 448 1487 1890 860160B 156543851 0
kmem_alloc_512 512 6052 6645 3629056B 283536972 0
kmem_alloc_640 640 48054 63216 43155456B 49035869 0
kmem_alloc_768 768 181 510 417792B 533346028 0
kmem_alloc_896 896 106 423 385024B 2201063 0
kmem_alloc_1152 1152 3164 3563 4169728B 758869183 0
kmem_alloc_1344 1344 449 768 1048576B 134708639 0
kmem_alloc_1600 1600 130 490 802816B 12434972 0
kmem_alloc_2048 2048 862 1080 2211840B 49268582 0
kmem_alloc_2688 2688 147 471 1286144B 228164685 0
kmem_alloc_4096 4096 2311 2738 11214848B 599289299 0
kmem_alloc_8192 8192 6843 7355 60252160B 823767889 0
kmem_alloc_12288 12288 24 286 3514368B 94265288 0
kmem_alloc_16384 16384 448 4864 79691776B 372203661 0
kmem_alloc_24576 24576 247 358 8798208B 20182582 0
kmem_alloc_32768 32768 625 829 27164672B 67372957 0
kmem_alloc_40960 40960 292 497 20357120B 10594602 0
kmem_alloc_49152 49152 264 424 20840448B 18226119 0
kmem_alloc_57344 57344 57 140 8028160B 7752775 0
kmem_alloc_65536 65536 83 146 9568256B 68801457 0
kmem_alloc_73728 73728 254 318 23445504B 1224400 0
kmem_alloc_81920 81920 12 75 6144000B 855948 0
kmem_alloc_90112 90112 16 65 5857280B 772093 0
kmem_alloc_98304 98304 72 130 12779520B 673798 0
kmem_alloc_106496 106496 5 60 6389760B 649065 0
kmem_alloc_114688 114688 9 63 7225344B 796342 0
kmem_alloc_122880 122880 3 59 7249920B 388702 0
kmem_alloc_131072 131072 20 352 46137344B 46679183 0
kmem_alloc_262144 262144 25 89 23330816B 22305 0
kmem_alloc_524288 524288 10 91 47710208B 292417 0
kmem_alloc_1048576 1048576 14 83 87031808B 151526 0
streams_mblk 64 11628 17892 1163264B 4222106588 0
streams_dblk_64 192 1880 3948 770048B 1560260262 0
streams_dblk_128 256 6 744 196608B 2094673871 0
streams_dblk_192 320 2 500 163840B 407759363 0
streams_dblk_256 384 206 1911 745472B 384014686 0
streams_dblk_320 448 0 360 163840B 916550415 0
streams_dblk_512 640 0 312 212992B 122220096 0
streams_dblk_1024 1152 0 322 376832B 60312175 0
streams_dblk_1536 1664 2 468 851968B 4237545508 0
streams_dblk_1920 2048 0 180 368640B 2582208 0
streams_dblk_2560 2688 0 276 753664B 9535544 0
streams_dblk_4096 4224 0 81 368640B 10214770 0
streams_dblk_8192 128 0 693 90112B 319193334 0
streams_dblk_12288 12416 0 98 1261568B 17628988 0
streams_dblk_16384 128 0 315 40960B 2089403 0
streams_dblk_20480 20608 0 56 1179648B 9157299 0
streams_dblk_24576 128 0 315 40960B 23840 0
streams_dblk_28672 28800 0 28 819200B 37109 0
streams_dblk_32768 128 0 567 73728B 72012 0
streams_dblk_36864 36992 0 56 2097152B 1752035 0
streams_dblk_40960 128 0 378 49152B 89538 0
streams_dblk_45056 45184 0 21 958464B 122283 0
streams_dblk_49152 128 0 378 49152B 72414 0
streams_dblk_53248 53376 0 49 2637824B 36207829 0
streams_dblk_57344 128 0 63 8192B 80 0
streams_dblk_61440 61568 0 21 1302528B 30352 0
streams_dblk_65536 128 0 63 8192B 136 0
streams_dblk_69632 69760 0 0 0B 0 0
streams_dblk_73728 128 0 0 0B 0 0
streams_dblk_esb 128 2048 2709 352256B 310293725 0
streams_dblk_mdc 128 0 0 0B 0 0
streams_fthdr 408 0 0 0B 0 0
streams_ftblk 376 0 0 0B 0 0
multidata 248 0 0 0B 0 0
multidata_pdslab 7112 0 0 0B 0 0
multidata_pattbl 32 0 0 0B 0 0
log_cons_cache 48 29 1352 65536B 943715 0
taskq_ent_cache 56 17219 36685 2072576B 105416892 0
taskq_cache 280 307 348 98304B 417 0
id32_cache 32 7 759 24576B 61202888 0
One_wallet_cache 68 3257 4480 327680B 93172593 0
Pac_nopredictor_pool 505536 1 7 3538944B 1 0
Mo_cache 288 0 0 0B 0 0
Monode_prealloc_cache 104 0 0 0B 0 0
Mo_policy_cache 72 0 0 0B 0 0
Mo_resident_cache 72 0 0 0B 0 0
Mo_capture_cache 376 0 0 0B 0 0
Mo_caphead_cache 64 0 0 0B 0 0
Mw_later_cache 128 0 0 0B 0 0
Mw_cache 128 0 0 0B 0 0
fakemw 88 0 460 40960B 306059268 0
mvec_tracking 8 0 2030 16384B 1104075934 0
mvec_tag 48 0 0 0B 0 0
Memseg_cache 64 0 0 0B 0 0
bp_map_8192 8192 0 0 0B 0 0
bp_map_16384 16384 0 80 1310720B 1823 0
bp_map_24576 24576 0 80 2097152B 2443 0
bp_map_32768 32768 0 80 2621440B 3048 0
bp_map_40960 40960 0 0 0B 0 0
bp_map_49152 49152 0 0 0B 0 0
bp_map_57344 57344 0 0 0B 0 0
bp_map_65536 65536 0 0 0B 0 0
mod_hash_entries 24 1635 3042 73728B 11631688 0
ipp_mod 304 0 0 0B 0 0
ipp_action 368 0 0 0B 0 0
ipp_packet 64 0 0 0B 0 0
mmuctxdom_cache 696 8 11 8192B 8 0
sfmmuid_cache 1176 370 636 868352B 132755208 0
sfmmu_tsbinfo_cache 64 384 1638 106496B 277806748 0
sfmmu_tsb8k_cache 8192 0 0 0B 0 0
sfmmu_tsb_cache 8192 104 218 1785856B 130478770 0
sfmmu8_cache 320 359954 380900 124813312B 230714082 0
sfmmu1_cache 96 1160768 1277472 124583936B 975785584 0
pa_hment_cache 64 384 1764 114688B 152854640 0
ism_blk_cache 336 0 0 0B 0 0
ism_ment_cache 32 0 0 0B 0 0
srd_cache 2192 120 407 909312B 36959896 0
region_cache 144 186 840 122880B 38085849 0
scd_cache 2192 0 0 0B 0 0
seg_cache 112 27575 33264 3784704B 2966412882 0
seg_pcache 104 0 624 65536B 52128 0
vfs_cache 240 100 561 139264B 24676 0
vn_cache 216 99646 140492 37126144B 346774691 0
shadow_cache 72 0 0 0B 0 0
vsk_anchor_cache 40 39 203 8192B 138 0
nep_cache 384 5 294 114688B 1429 0
dev_info_node_cache 760 283 310 253952B 830 0
ndi_fm_entry_cache 32 5392 6831 221184B 170484658 0
kcf_sreq_cache 56 0 126 8192B 1542463 0
kcf_areq_cache 296 0 25 8192B 25 0
kcf_context_cache 112 0 0 0B 0 0
object_handle 80 2798905 3152109 255664128B 931834012 0
object_debug_handle 216 0 0 0B 0 0
object_event 40 0 0 0B 0 0
segkmem_ppa_262144 262144 0 12 3145728B 16 0
segkp_8192 8192 375 512 4194304B 73688442 0
segkp_16384 16384 0 0 0B 0 0
segkp_24576 24576 0 0 0B 0 0
segkp_32768 32768 0 0 0B 0 0
segkp_40960 40960 3323 3363 146931712B 507168 0
umem_np_8192 8192 0 128 1048576B 555389 0
umem_np_16384 16384 0 80 1310720B 24633 0
umem_np_24576 24576 0 0 0B 0 0
umem_np_32768 32768 0 104 3407872B 484312 0
umem_np_40960 40960 0 90 3932160B 459456 0
umem_np_49152 49152 0 0 0B 0 0
umem_np_57344 57344 0 0 0B 0 0
umem_np_65536 65536 0 68 4456448B 24633 0
thread_cache 1040 915 1545 1687552B 107833858 0
wbuf32_cache 512 741 900 491520B 82343977 0
wbuf64_cache 1024 775 1141 1335296B 1121736 0
lwp_cache 1048 1516 1785 1949696B 5360544 0
turnstile_cache 64 3245 4788 311296B 87013637 0
rw_reentrd_cache 136 3245 4130 573440B 93172717 0
tslabel_cache 48 2 169 8192B 2 0
cred_cache 184 1137 3520 655360B 111662850 0
proc_ac_cache 64 301 1512 98304B 81421934 0
rctl_cache 48 6392 8957 434176B 1288617008 0
rctl_val_cache 64 13614 18018 1171456B 2940692401 0
task_cache 160 160 800 131072B 1083893 0
kmem_defrag_cache 224 2 36 8192B 2 0
kmem_move_cache 56 0 18705 1056768B 6862115 0
i_dmahdl 2648 0 0 0B 0 0
timeout_request 128 0 0 0B 0 0
cyclic_id_cache 80 263 303 24576B 329 0
callout_cachebabecafe 80 3239 3276 425984B 3239 0
callout_lcachebabecafe 48 10163 10206 663552B 10163 0
bounds_predictor 505536 3 7 3538944B 3 0
dnlc_space_cache 24 0 0 0B 0 0
file_cache 72 5236 7392 540672B 2852162077 0
stream_head_cache 376 518 798 311296B 122978665 0
queue_cache 664 1176 1536 1048576B 128578727 0
syncq_cache 168 81 432 73728B 20512 0
qband_cache 64 2 126 8192B 2 0
linkinfo_cache 48 50 507 24576B 5064 0
ciputctrl_cache 1024 0 0 0B 0 0
serializer_cache 64 55 1008 65536B 163607 0
as_cache 352 371 667 237568B 132754981 0
marker_cache 128 0 378 49152B 715850 0
anon_cache 48 164399 212771 10313728B 2491485009 0
anonmap_cache 120 15546 19363 2367488B 3655840909 0
segvn_cache 224 27575 32724 7446528B 2756160318 0
segvn_szc_cache1 64 0 882 57344B 772856386 0
segvn_szc_cache2 512 0 0 0B 0 0
segvn_szc_cache3 4096 0 88 360448B 35897056 0
segvn_szc_cache4 32768 0 0 0B 0 0
segvn_szc_cache5 262144 0 0 0B 0 0
segvn_szc_cache6 2097152 0 0 0B 0 0
flk_edges 48 0 169 8192B 409 0
fdb_cache 104 0 0 0B 0 0
timer_cache 176 4 46 8192B 31 0
vmu_bound_cache 56 6650 7540 425984B 15900 0
vmu_object_cache 88 2305 2484 221184B 3234 0
physio_buf_cache 248 0 416 106496B 74677 0
process_cache 4168 383 468 2129920B 92796816 0
numaio_obj_cache 328 112 192 65536B 425 0
numaio_grp_cache 144 9 56 8192B 27 0
mac_impl_cache 13488 6 9 122880B 8 0
mac_ring_cache 480 8 30 16384B 12 0
mac_block_cache 152 0 0 0B 0 0
mac_descriptor_cache 64 0 0 0B 0 0
mac_packet_pool_cache 1184 0 0 0B 0 0
mac_magazine_cache 552 0 0 0B 0 0
flow_tab_cache_0 184 5 42 8192B 7 0
flow_entry_cache_0 22440 12 20 450560B 26 0
mac_bcast_grp_cache 80 7 101 8192B 19 0
mac_client_impl_cache 2064 6 11 24576B 8 0
mac_promisc_impl_cache 120 0 0 0B 0 0
ip_minor_arena_sa_1 1 46 384 384B 1721432 0
ip_minor_arena_la_1 1 65 1088 1088B 959164 0
ip_conn_cache 744 3 60 49152B 1193 0
tcp_conn_cache 2120 314 693 1548288B 737336 0
udp_conn_cache 1256 69 324 442368B 2592397 0
rawip_conn_cache 1096 0 140 163840B 17159 0
rts_conn_cache 816 8 27 24576B 30 0
ire_cache 352 153 168 65536B 202 0
ncec_cache 200 110 279 73728B 6158 0
nce_cache 112 117 378 49152B 6454 0
rt_entry 152 133 168 32768B 180 0
radix_mask 32 8 253 8192B 15 0
radix_node 120 5 67 8192B 5 0
ipsec_actions 88 0 0 0B 0 0
ipsec_selectors 80 0 0 0B 0 0
ipsec_policy 80 0 0 0B 0 0
tcp_timercache 88 1239 2024 180224B 726650 0
tcp_notsack_blk_cache 24 1 1690 40960B 2083719 0
squeue_cache 168 100 126 24576B 100 0
sctp_conn_cache 2608 0 0 0B 0 0
sctp_faddr_cache 472 0 0 0B 0 0
sctp_set_cache 24 0 0 0B 0 0
sctp_ftsn_set_cache 16 0 0 0B 0 0
dce_cache 152 179 265 40960B 239 0
ire_gw_secattr_cache 24 0 0 0B 0 0
ldc_memhdl_cache 48 0 0 0B 0 0
ldc_memseg_cache 64 0 0 0B 0 0
fnode_cache 176 9 84 16384B 121213 0
pipe_cache 320 66 350 114688B 57668150 0
snode_cache 152 858 1590 245760B 208309593 0
clnt_clts_endpnt_cache 88 0 0 0B 0 0
bpmap_cache 200 0 0 0B 0 0
zio_cache 912 26 76848 78692352B 2230982187 0
zio_link_cache 48 0 82472 3997696B 3836987868 0
sa_cache 56 87949 132675 7495680B 197787650 0
dnode_t 696 954000 954371 710746112B 104337805 0
dmu_buf_impl_t 216 1769940 2097530 464404480B 206659141 0
arc_elink_t 32 967948 1389982 45006848B 3414386709 0
arc_buf_t 168 992393 1401744 239230976B 559040333 0
arc_ref_t 72 1769981 2097760 153436160B 54845411 0
arc_ghost_t 64 314633 317646 20652032B 208588997 0
arc_meta 184 956283 1262932 235134976B 57748154 0
arc_data 184 915092 924264 172081152B 109836972 0
arc_data_512 16 743953 756951 12230656B 122178970 0
arc_meta_512 16 843315 1134159 18325504B 82077912 0
arc_data_1024 16 3656 12168 196608B 1870065 0
arc_meta_1024 16 380 2535 40960B 291201 0
arc_data_1536 16 2409 8112 131072B 1191893 0
arc_meta_1536 16 72 1521 24576B 43830 0
arc_data_2048 16 1994 8112 131072B 1125267 0
arc_meta_2048 16 307 2028 32768B 76657 0
arc_data_3072 16 2210 11154 180224B 3490145 0
arc_meta_3072 16 50 1521 24576B 20353 0
arc_data_4096 16 1369 15210 245760B 5126190 0
arc_meta_4096 16 5662 7098 114688B 1601065 0
arc_data_6144 16 1864 10647 172032B 2382865 0
arc_meta_6144 16 20 1521 24576B 40232 0
arc_data_8192 16 998 7605 122880B 1823731 0
arc_meta_8192 16 15 1014 16384B 3434 0
arc_data_12288 16 958 7605 122880B 1551591 0
arc_meta_12288 16 32 1014 16384B 8738 0
arc_data_16384 16 513 5577 90112B 1210128 0
arc_meta_16384 16 90237 305214 4931584B 15250828 0
arc_data_24576 16 458 20787 335872B 10999937 0
arc_meta_24576 16 19 1014 16384B 5337 0
arc_data_32768 16 428 25857 417792B 7619052 0
arc_meta_32768 16 10 1014 16384B 2976 0
arc_data_40960 16 163 22308 360448B 15904627 0
arc_meta_40960 16 13 1521 24576B 22600 0
arc_data_49152 16 227 19266 311296B 7580554 0
arc_meta_49152 16 1 507 8192B 205 0
arc_data_57344 16 2956 13182 212992B 3697718 0
arc_meta_57344 16 0 507 8192B 175 0
arc_data_65536 16 1638 3042 49152B 967486 0
arc_meta_65536 16 0 507 8192B 359 0
arc_data_73728 16 94 2028 32768B 569502 0
arc_meta_73728 16 4 507 8192B 356 0
arc_data_81920 16 448 1521 24576B 528592 0
arc_meta_81920 16 3 507 8192B 542 0
arc_data_90112 16 1492 2535 40960B 675906 0
arc_meta_90112 16 1 507 8192B 204 0
arc_data_98304 16 68 1521 24576B 428205 0
arc_meta_98304 16 1 507 8192B 151 0
arc_data_106496 16 47 1521 24576B 381247 0
arc_meta_106496 16 1 507 8192B 248 0
arc_data_114688 16 32 1521 24576B 387053 0
arc_meta_114688 16 1 1014 16384B 5162 0
arc_data_122880 16 23 1521 24576B 396002 0
arc_meta_122880 16 1 507 8192B 263 0
arc_data_131072 16 126094 142467 2301952B 3965382018 0
arc_meta_131072 16 29 1521 24576B 316793 0
arc_data_139264 16 0 0 0B 0 0
arc_meta_139264 16 0 0 0B 0 0
arc_data_262144 16 0 0 0B 0 0
arc_meta_262144 16 0 0 0B 0 0
arc_data_524288 16 0 0 0B 0 0
arc_meta_524288 16 0 0 0B 0 0
arc_data_1048576 16 0 0 0B 0 0
arc_meta_1048576 16 0 0 0B 0 0
l2arc_seg_t 96 0 0 0B 0 0
l2arc_buf_t 80 0 0 0B 0 0
zfetch_trigger_t 80 189 2525 204800B 61215486 0
space_seg_cache 64 35260 482076 31342592B 2637153477 0
dsl_share_t 328 1 24 8192B 1 0
dsl_share_state_t 48 0 0 0B 0 0
zil_lwb_cache 208 6 624 131072B 736315 0
zil_train_cache 64 2 1386 90112B 2295082 0
zil_car_cache 56 2 1885 106496B 3394521 0
zil_ian_cache 80 32 1818 147456B 63267405 0
vdev_disk_cache 256 0 341 90112B 882654057 0
zfs_znode_cache 328 87949 91200 31129600B 243237064 0
dls_link_cache 344 6 23 8192B 9 0
dls_devnet_cache 368 6 22 8192B 8 0
px0_px0_0_cache1 8192 12 32 262144B 112535534 0
px0_px0_0_cache2 16384 3 8 131072B 9249 0
px0_px0_0_cache8 65536 2 2 131072B 2 0
dv_node_cache 184 664 748 139264B 1211 0
px0_mpt_sas0_2_cache1 8192 2 32 262144B 296721559 0
px0_mpt_sas0_2_cache2 16384 0 16 262144B 99973488 0
pkt_cache_mpt_sas_0 720 0 110 81920B 290898182 0
px0_mpt_sas1_3_cache1 8192 4 48 393216B 737561670 0
px0_mpt_sas1_3_cache2 16384 0 32 524288B 281684410 0
pkt_cache_mpt_sas_1 720 2 132 98304B 763554977 0
sdev_node_cache 248 905 992 253952B 51623 0
audit_proc 48 385 1521 73728B 80880005 0
drv_secobj_cache 296 0 0 0B 0 0
dld_str_cache 320 11 150 49152B 147472 0
exacct_object_cache 40 0 0 0B 0 0
rw_numa_cache 128 19911 20853 2711552B 2551080 0
kssl_cache 1624 0 0 0B 0 0
stmf_task_event_cache 64 0 0 0B 0 0
stmf_task_cache 3296 0 0 0B 0 0
stmf_ref_node_cache 16 2 507 8192B 6 0
sbd_task_cache 1304 0 0 0B 0 0
namefs_inodes_1 1 47 1152 1152B 74469 0
port_cache 80 13 101 8192B 51 0
socket_cache 792 321 720 589824B 1700687 0
socktpi_cache 1096 0 7 8192B 6 0
socktpi_unix_cache 1096 32 350 409600B 170699 0
sock_sod_cache 656 0 0 0B 0 0
tl_cache 448 78 396 180224B 171292 0
keysock_1 1 0 64 64B 1 0
spdsock_1 1 0 64 64B 5 0
rds_alloc_cache 88 0 0 0B 0 0
dtrace_state_cache 262144 0 14 3670016B 29 0
idm_buf_cache 256 0 0 0B 0 0
idm_task_cache 1408 0 0 0B 0 0
idm_tx_pdu_cache 400 0 0 0B 0 0
idm_rx_pdu_cache 596 0 0 0B 0 0
softmac_cache 568 5 14 8192B 7 0
softmac_upper_cache 232 0 0 0B 0 0
fctl_cache 112 0 0 0B 0 0
vldc_cookie_buf_cache 262144 0 65 17039360B 33963567 0
authkern_cache 72 0 784 57344B 205205946 0
authnone_cache 72 0 0 0B 0 0
authloopback_cache 72 0 0 0B 0 0
authdes_cache_handle 80 0 0 0B 0 0
rnode_cache 680 982 990 737280B 61171 0
nfs_access_cache 56 182 3190 180224B 145036 0
client_handle_cache 32 17 506 16384B 1850 0
rnode4_cache 1032 0 0 0B 0 0
svnode_cache 40 0 0 0B 0 0
nfs4_access_cache 56 0 0 0B 0 0
client_handle4_cache 32 0 0 0B 0 0
nfs4_ace4vals_cache 48 0 0 0B 0 0
nfs4_ace4_list_cache 264 0 0 0B 0 0
NFS_idmap_cache 56 0 0 0B 0 0
lm_xprt_10003c62cf40 32 0 0 0B 0 0
lm_vnode_10003c62cf40 184 0 0 0B 0 0
lm_sysid_10003c62cf40 160 0 50 8192B 1 0
lm_client_10003c62cf40 128 0 63 8192B 1 0
lm_async_10003c62cf40 32 0 0 0B 0 0
lm_sleep_10003c62cf40 96 0 0 0B 0 0
lm_config_10003c62cf40 80 2 101 8192B 2 0
uvfs_uvnode_cache 392 0 0 0B 0 0
uvfs_task_sync_cache 16 0 0 0B 0 0
uvfs_task_rootvp_cache 16 0 0 0B 0 0
uvfsvfs_cache 280 0 0 0B 0 0
ufs_inode_cache 368 0 0 0B 0 0
directio_buf_cache 272 0 0 0B 0 0
lufs_save 24 0 0 0B 0 0
lufs_bufs 256 0 0 0B 0 0
lufs_mapentry_cache 112 0 0 0B 0 0
pty_map 64 54 756 49152B 10729 0
sppptun_map 440 0 0 0B 0 0
Hex0x100030bf3428_minor_1 1 0 0 0B 0 0
Hex0x100030bf3430_minor_1 1 0 0 0B 0 0
px0_igb3_4_cache1 8192 1793 1840 15073280B 169264446 0
px0_igb3_4_cache2 16384 1 24 393216B 194088248 0
px0_igb0_5_cache1 8192 1793 2000 16384000B 3804956026 0
px0_igb0_5_cache2 16384 1 24 393216B 653588623 0
iscsit_status_pdu_cache 400 0 0 0B 0 0
stp_2_0_987 1712 0 0 0B 0 0
stp_m2_0_987 56 0 0 0B 0 0
audit_buffer 152 0 212 32768B 690 0
lnode_cache 32 12 1265 40960B 247872400 0
flow_tab_cache_1 184 1 42 8192B 1 0
flow_entry_cache_1 22440 1 4 90112B 1 0
lm_xprt_10003c62fa40 32 0 0 0B 0 0
lm_vnode_10003c62fa40 184 0 0 0B 0 0
lm_sysid_10003c62fa40 160 0 0 0B 0 0
lm_client_10003c62fa40 128 0 0 0B 0 0
lm_async_10003c62fa40 32 0 0 0B 0 0
lm_sleep_10003c62fa40 96 0 0 0B 0 0
lm_config_10003c62fa40 80 1 101 8192B 1 0
vnic_cache 1544 1 5 8192B 1 0
crypto_session_cache 104 0 0 0B 0 0
sdp_generic_table 32 0 0 0B 0 0
sdp_advt_cache 80 0 0 0B 0 0
sdp_advt_table 24 0 0 0B 0 0
sdp_conn_cache 1944 0 0 0B 0 0
Hex0x10004c7d1428_minor_1 1 0 0 0B 0 0
Hex0x10004c7d1430_minor_1 1 0 0 0B 0 0
stp_3_1_3841 1712 0 0 0B 0 0
stp_m3_1_3841 56 0 0 0B 0 0
stp_2_1_3866 1712 0 0 0B 0 0
stp_m2_1_3866 56 0 0 0B 0 0
fcsm_job_cache 104 0 0 0B 0 0
aggr_port_cache 1032 0 0 0B 0 0
aggr_grp_cache 1008 0 0 0B 0 0
iptun_cache 288 0 0 0B 0 0
smb_shr_notify_cache 72 0 0 0B 0 0
smb_share_cache 168 1 48 8192B 1 0
smb_vfs_cache 48 0 0 0B 0 0
smb_mc_cache 96 0 0 0B 0 0
smb_uio_cache 752 0 0 0B 0 0
smb_node_cache 752 0 0 0B 0 0
smb_txreq 66592 0 0 0B 0 0
vxlan_grp_cache 200 0 0 0B 0 0
vxlan_cache 904 0 0 0B 0 0
zvsdir_zvnode_cache 96 0 0 0B 0 0
px0_igb1_6_cache1 8192 0 32 262144B 3584 0
px0_igb1_6_cache2 16384 0 8 131072B 1 0
px0_igb2_7_cache1 8192 0 32 262144B 3584 0
px0_igb2_7_cache2 16384 0 8 131072B 1 0
------------------------- ------ ------ ------ ---------- --------- -----
Total [hat_memload] 124813312B 230714082 0
Total [kmem_msb] 76464128B 148885992 0
Total [kmem_va] 158597120B 111782 0
Total [kmem_default] 3346472960B 2897004240 0
Total [bp_map] 6029312B 7314 0
Total [kmem_tsb_default] 1785856B 130478770 0
Total [hat_memload1] 124583936B 975785584 0
Total [segkmem_ppa] 3145728B 16 0
Total [umem_np] 14155776B 1548423 0
Total [id32] 24576B 61202888 0
Total [segkp] 151126016B 74195610 0
Total [ip_minor_arena_sa] 384B 1721432 0
Total [ip_minor_arena_la] 1088B 959164 0
Total [px0_px0_0_vmem_top] 524288B 112544785 0
Total [px0_mpt_sas0_2_vmem_top] 524288B 396695047 0
Total [px0_mpt_sas1_3_vmem_top] 917504B 1019246080 0
Total [namefs_inodes] 1152B 74469 0
Total [keysock] 64B 1 0
Total [spdsock] 64B 5 0
Total [px0_igb3_4_vmem_top] 15466496B 363352694 0
Total [px0_igb0_5_vmem_top] 16777216B 163577353 0
Total [px0_igb1_6_vmem_top] 393216B 3585 0
Total [px0_igb2_7_vmem_top] 393216B 3585 0
------------------------- ------ ------ ------ ---------- --------- -----
vmem memory memory memory alloc alloc
name in use total import succeed fail
------------------------- ---------- ----------- ---------- --------- -----
heap 4418913714176B 17592186044416B 0B 445761 0
vmem_metadata 14204928B 14417920B 14417920B 1625 0
vmem_seg 42156032B 42156032B 42156032B 5141 0
vmem_hash 5945344B 5955584B 5955584B 378 0
vmem_vmem 443992B 513592B 475136B 171 0
heap_alloc 61760B 65536B 65536B 198 0
hat_memload 124813312B 124813312B 124813312B 17940 0
kstat 1317944B 1351680B 1286144B 1091062 0
kmem_metadata 61620224B 97779712B 97779712B 115733 0
kmem_msb 76464128B 76464128B 76464128B 130218 0
kmem_audit 0B 0B 0B 0 0
kmem_cache 2236176B 3907584B 3907584B 552 0
kmem_hash 530944B 540672B 540672B 1038 0
kmem_log 4204640B 4210688B 4210688B 6 0
kmem_firewall_va 0B 0B 0B 0 0
kmem_firewall 0B 0B 0B 0 0
mod_sysfile 8B 8192B 8192B 1 0
kmem_oversize 20114661376B 20122435584B 20122435584B 333208 0
kmem_va 210501632B 210501632B 210501632B 43084 0
kmem_default 3346472960B 3925639168B 3925639168B 5790429 0
little_endian 1162048B 1228800B 1228800B 104879936 0
big_endian 36849813B 58744832B 58744832B 60100613 0
bp_map 6029312B 6029312B 6029312B 6402 0
ksyms 3267768B 3317760B 3317760B 418 0
ctf 278772B 327680B 327680B 420 0
kmem_bigtsb 0B 0B 0B 0 0
kmem_bigtsb_default 0B 0B 0B 0 0
kmem_tsb 12582912B 12582912B 12582912B 610 0
kmem_tsb_default 10174464B 12582912B 12582912B 28176218 579
hat_memload1 124583936B 124583936B 124583936B 15214 0
KOM firewall 0B 0B 0B 0 0
segkmem_ppa 3145728B 3145728B 3145728B 3 0
umem_np 14155776B 14155776B 14155776B 24604 0
contig_mem_arena 30666816B 222298112B 0B 133475242 0
contig_mem_arena_le 0B 0B 0B 0 0
defdump_arena 1147854848B 1147854848B 1147854848B 6 0
defdump_metadata_arena 1147854848B 1147854848B 1147854848B 6 0
lppool 154394624B 154394624B 0B 31840 138056
heap32 10772480B 134217728B 0B 126 0
id32 24576B 24576B 24576B 3 0
module_data 8638543B 8814592B 8290304B 596 0
promplat 0B 0B 0B 79 0
trapstat 0B 0B 0B 0 0
heaptext 33562624B 134217728B 0B 17 0
module_text 33551680B 37863424B 0B 421 0
logminor_space 61B 262137B 0B 942755 0
taskq_id_arena 146B 2147483647B 0B 240 0
heap_lp 4026531840B 4397241204736B 0B 16 0
kmem_lp 4026531840B 4026531840B 4026531840B 7466 4150
segkp 151420928B 2147483648B 0B 4905 0
rctl_ids 44B 32767B 0B 44 0
zoneid_space 1B 9998B 0B 1 0
taskid_space 160B 999999B 0B 1048209 0
pool_ids 0B 999998B 0B 0 0
contracts 159B 2147483646B 0B 853997 0
regspec 9175040B 5368709120B 0B 31 0
mac_minor_ids 116B 130070B 0B 792 0
ip_minor_arena_sa 384B 262140B 0B 6 0
ip_minor_arena_la 1088B 4294705152B 0B 17 0
px0_px0_0_vmem_top 655360B 1878917120B 0B 2135 8
px0_px0_0_vmem_16 0B 0B 0B 2125 0
px0_px0_0_vmem_c 0B 268435456B 0B 0 9
px0_mpt_sas0_2_vmem_top 1441792B 939393024B 0B 41273125 8
px0_mpt_sas0_2_vmem_16 262144B 262144B 262144B 35835132 0
px0_mpt_sas0_2_vmem_c 0B 134217728B 0B 0 9
px0_mpt_sas1_3_vmem_top 1835008B 939393024B 0B 110855090 8
px0_mpt_sas1_3_vmem_16 262144B 262144B 262144B 94440293 0
px0_mpt_sas1_3_vmem_c 0B 134217728B 0B 0 9
lib_va_32 7954432B 2031599616B 0B 20 0
lib_va_64 283328512B 2251793356234752B 0B 211 0
namefs_inodes 1152B 65536B 0B 18 0
tl_minor_space 78B 262138B 0B 163787 0
keysock 64B 4294967295B 0B 1 0
spdsock 64B 4294967295B 0B 1 0
dtrace 104429B 4294967295B 0B 1219732 0
dtrace_minor 0B 4294967293B 0B 28 0
syseventd_channel 15B 101B 0B 24474 0
syseventd_channel 1B 2B 0B 1 0
idm_taskid_space 0B 65536B 0B 0 0
module_text_holesrc_2 0B 4194304B 0B 0 0
ktext_hole_2 1862280B 4194304B 0B 156 0
module_text_holesrc_0 0B 4194304B 0B 0 0
ktext_hole_0 2155704B 4194304B 0B 20 0
ibcm_local_sid 0B 4294967295B 0B 0 0
ibcm_ip_sid 0B 65535B 0B 0 0
lmsysid_space 1B 16383B 0B 3 0
module_text_holesrc_3 0B 4194304B 0B 0 0
ktext_hole_3 1924624B 4194304B 0B 154 0
module_text_holesrc_4 0B 4194304B 0B 0 0
ktext_hole_4 2109924B 4194304B 0B 72 0
module_text_holesrc_1 0B 4194304B 0B 0 0
ktext_hole_1 236992B 4194304B 0B 32 0
logdmux_minor 34B 256B 0B 5028 0
ptms_minor 54B 128B 0B 10703 3
sppptun_minor 0B 16B 0B 0 0
syseventconfd_door 1B 101B 0B 1 0
syseventconfd_door 1B 2B 0B 1 0
devfsadm_event_channel 1B 101B 0B 1 0
devfsadm_event_channel 1B 2B 0B 1 0
Hex0x100030bf3428_minor 0B 4294967294B 0B 0 0
Hex0x100030bf3430_minor 0B 4294967294B 0B 0 0
px0_igb3_4_vmem_top 15597568B 939393024B 0B 12335257 8
px0_igb3_4_vmem_16 0B 0B 0B 12334856 0
px0_igb3_4_vmem_c 0B 134217728B 0B 0 9
px0_igb0_5_vmem_top 16908288B 939393024B 0B 362737 8
px0_igb0_5_vmem_16 0B 0B 0B 362449 0
px0_igb0_5_vmem_c 0B 134217728B 0B 0 9
iscsit_tsih_pool 0B 65535B 0B 0 0
ipnet_minor_space 6B 262141B 0B 6 0
crypto 0B 16B 0B 136281 0
lofi_id 0B 16383B 0B 0 0
ds_minors 0B 262140B 0B 0 0
Hex0x10004c7d1428_minor 0B 4294967294B 0B 0 0
Hex0x10004c7d1430_minor 0B 4294967294B 0B 0 0
semids 90B 128B 0B 90 0
mdesc_minor 0B 256B 0B 120953 0
aggr_portids 0B 65534B 0B 0 0
aggr_key_ids 0B 64535B 0B 0 0
zvmm_minor_space 0B 262142B 0B 0 0
px0_igb1_6_vmem_top 524288B 939393024B 0B 115 8
px0_igb1_6_vmem_16 0B 0B 0B 0 0
px0_igb1_6_vmem_c 0B 134217728B 0B 0 9
px0_igb2_7_vmem_top 524288B 939393024B 0B 115 8
px0_igb2_7_vmem_16 0B 0B 0B 0 0
px0_igb2_7_vmem_c 0B 134217728B 0B 0 9
msqids 0B 128B 0B 0 0
shmids 0B 128B 0B 0 0
------------------------- ---------- ----------- ---------- --------- -----
>
>
> ::kmastat !awk '!/Total/ {print $4 " " $1}' | sort -n | tail
76464128B kmem_msb
97779712B kmem_metadata
124583936B hat_memload1
124813312B hat_memload
210501632B kmem_va
1147854848B defdump_arena
1147854848B defdump_metadata_arena
3925639168B kmem_default
4026531840B kmem_lp
20122435584B kmem_oversize
>
>
> ::vmem ! grep kmem_oversize
000003000008a000 kmem_oversize 20114661376 20122435584 333212 0
> 000003000008a000::print vmem_t vm_kstat.vk_free.value.l
vm_kstat.vk_free.value.l = 0x50304
> 000003000008a000::print vmem_t vm_kstat.vk_alloc.value.l
vm_kstat.vk_alloc.value.l = 0x5159d
In newer solaris releases we see the ZFS buffers as own lines in mdb... that's part of the kernel memory:
> ::memstat Page Summary Pages Bytes %Tot ----------------- ---------------- ---------------- ---- Kernel 3129843 23.8G 76% Defdump prealloc 140119 1.0G 3% ZFS Metadata 292857 2.2G 7% ZFS File Data 2251900 17.1G 55% Anon 158898 1.2G 4% Exec and libs 6226 48.6M 0% Page cache 9878 77.1M 0% failed to read 'mrp_svc'; module not present Free (cachelist) 52 416k 0% Free (freelist) 121576 949.8M 3% Total 4128768 31.5G >
What is happening there? Well... IO IO IO :-)
root@server:~# dtrace -n 'fbt::vmem_alloc:entry { @[args[0]->vm_name] = sum(arg1); }'
dtrace: description 'fbt::vmem_alloc:entry ' matched 1 probe
^C
ip_minor_arena_la 1
namefs_inodes 1
ip_minor_arena_sa 2
logminor_space 2
little_endian 119168
segkp 188416
big_endian 1397837
heap 3145728
kmem_oversize 3145728
px0_mpt_sas0_2_vmem_16 5767168
px0_mpt_sas0_2_vmem_top 5767168
px0_mpt_sas1_3_vmem_16 15990784
px0_mpt_sas1_3_vmem_top 15990784
root@server:~#
root@server:~# dtrace -n 'fbt::vmem_alloc:entry /args[0]->vm_name == "px0_mpt_sas1_3_vmem_top"/ { @[stack()] = count(); }'
dtrace: description 'fbt::vmem_alloc:entry ' matched 1 probe
^C
genunix`vmem_xalloc+0x670
genunix`vmem_alloc+0x21c
px`px_dvma_pool_default_dvma_alloc+0x180
px`px_atu_dvma_alloc+0x94
px`px_dvma_map+0x54
px`px_dma_bindhdl+0xbc
genunix`ddi_dma_buf_bind_handle+0x54
scsi`scsi_cache_bind+0x24
scsi`scsi_cache_init_pkt+0x2d4
scsi`scsi_init_pkt+0x4c
scsi_vhci`vhci_bind_transport+0x9ac
scsi_vhci`vhci_scsi_start+0x350
sd`sd_start_cmds+0x3a4
sd`sd_core_iostart+0x228
sd`sd_mapblockaddr_iostart+0x210
sd`xbuf_iostart+0x20c
zfs`vdev_disk_strategy+0x30
zfs`vdev_disk_io_start+0x26c
zfs`zio_execute+0xf4
zfs`vdev_queue_io_done+0xb4
9
genunix`vmem_xalloc+0x670
genunix`vmem_alloc+0x21c
px`px_dvma_pool_default_dvma_alloc+0x180
px`px_atu_dvma_alloc+0x94
px`px_dvma_map+0x54
px`px_dma_bindhdl+0xbc
genunix`ddi_dma_buf_bind_handle+0x54
scsi`scsi_cache_bind+0x24
scsi`scsi_cache_init_pkt+0x2d4
scsi`scsi_init_pkt+0x4c
scsi_vhci`vhci_bind_transport+0x9ac
scsi_vhci`vhci_scsi_start+0x350
sd`sd_start_cmds+0x3a4
sd`sd_core_iostart+0x228
sd`sd_mapblockaddr_iostart+0x210
sd`xbuf_iostart+0x20c
zfs`vdev_disk_strategy+0x30
zfs`vdev_disk_io_start+0x26c
zfs`zio_execute+0xf4
zfs`vdev_raidz_io_start+0x26c
16
genunix`vmem_xalloc+0x670
genunix`vmem_alloc+0x21c
px`px_dvma_pool_default_dvma_alloc+0x180
px`px_atu_dvma_alloc+0x94
px`px_dvma_map+0x54
px`px_dma_bindhdl+0xbc
genunix`ddi_dma_buf_bind_handle+0x54
scsi`scsi_cache_bind+0x24
scsi`scsi_cache_init_pkt+0x2d4
scsi`scsi_init_pkt+0x4c
scsi_vhci`vhci_bind_transport+0x9ac
scsi_vhci`vhci_scsi_start+0x350
sd`sd_start_cmds+0x3a4
sd`sd_core_iostart+0x228
sd`sd_mapblockaddr_iostart+0x210
sd`xbuf_iostart+0x20c
zfs`vdev_disk_strategy+0x30
zfs`vdev_disk_io_start+0x26c
zfs`zio_execute+0xf4
zfs`vdev_queue_io_done+0x94
110
root@server:~# dtrace -n 'fbt::vmem_alloc:entry /args[0]->vm_name == "kmem_oversize"/ { @[stack()] = count(); }'
dtrace: description 'fbt::vmem_alloc:entry ' matched 1 probe
^C
genunix`kmem_alloc+0x160
genunix`kmem_zalloc+0x120
autofs`auto_calldaemon+0x1d0
autofs`auto_null_request+0x24
autofs`unmount_tree+0x60
autofs`unmount_zone_tree+0xc
unix`thread_start+0x4
1
root@server:~#