NetBSD Problem Report #54818

From tsutsui@ceres.dti.ne.jp  Wed Jan  1 05:56:31 2020
Return-Path: <tsutsui@ceres.dti.ne.jp>
Received: from mail.netbsd.org (mail.netbsd.org [199.233.217.200])
	(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
	(Client CN "mail.NetBSD.org", Issuer "mail.NetBSD.org CA" (not verified))
	by mollari.NetBSD.org (Postfix) with ESMTPS id 3FE477A174
	for <gnats-bugs@gnats.NetBSD.org>; Wed,  1 Jan 2020 05:56:31 +0000 (UTC)
Message-Id: <202001010556.0015uOHO018475@ceres.dti.ne.jp>
Date: Wed, 1 Jan 2020 14:56:24 +0900 (JST)
From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
Reply-To: tsutsui@ceres.dti.ne.jp
To: gnats-bugs@NetBSD.org
Subject: 9.0_RC1 pagedaemon spins
X-Send-Pr-Version: 3.95

>Number:         54818
>Category:       kern
>Synopsis:       9.0_RC1 pagedaemon spins
>Confidential:   no
>Severity:       critical
>Priority:       low
>Responsible:    ad
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Wed Jan 01 06:00:00 +0000 2020
>Last-Modified:  Wed May 20 20:15:01 +0000 2020
>Originator:     Izumi Tsutsui
>Release:        NetBSD 9.0_RC1
>Organization:
>Environment:
System: NetBSD mirage 9.0_RC1 NetBSD 9.0_RC1 (GENERIC) #10: Sun Dec 29 03:07:32 JST 2019 tsutsui@mirage:/s/netbsd-9/src/sys/arch/i386/compile/GENERIC i386
Architecture: i386
Machine: i386
>Description:
On some unknown condition, NetBSD/i386 9.0_RC1 GENERIC kernel
on my i386 PC spins in pagedaemon and consumes ~100% CPU:

---
load averages:  0.41,  0.59,  0.59;               up 1+08:57:10        14:34:52
418 threads: 18 idle, 1 runnable, 381 sleeping, 2 stopped, 12 zombie, 4 on CPU
CPU states:  0.0% user,  0.0% nice, 26.0% system,  0.4% interrupt, 73.5% idle
Memory: 1765M Act, 865M Inact, 32M Wired, 200M Exec, 1547M File, 20M Free
Swap: 8972M Total, 130M Used, 8842M Free

  PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
    0    98 root     126 CPU/3    231:35 99.02% 99.02% pgdaemon  [system]
 :

---
load averages:  0.54,  0.61,  0.60;               up 1+08:57:26        14:35:08
98 processes: 94 sleeping, 2 stopped, 2 on CPU
CPU states:  2.4% user,  0.0% nice, 30.4% system,  2.0% interrupt, 65.1% idle
Memory: 1753M Act, 863M Inact, 32M Wired, 200M Exec, 1535M File, 33M Free
Swap: 8972M Total, 130M Used, 8842M Free

  PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
    0 root     126    0     0K   23M CPU/0    246:37 99.37% 99.37% [system]
 :

---
 UID   PID %CPU LNAME           WCHAN  PPID   CPU  LID NLWP PRI NI     VSZ    RSS WCHAN    STAT TTY       LTIME COMMAND
   0     0 99.0 (zombie)            0     0     0  112   97 221  0       0  25256 -        ZK-  ?       0:00.00 [system]
   0     0 99.0 nfsio               0     0     0  111   97  96  0       0  25256 nfsiod   DK-  ?       0:00.00 [system]
   0     0 99.0 nfsio               0     0     0  110   97  96  0       0  25256 nfsiod   DK-  ?       0:00.00 [system]
   0     0 99.0 nfsio               0     0     0  109   97  96  0       0  25256 nfsiod   DK-  ?       0:00.00 [system]
   0     0 99.0 nfsio               0     0     0  108   97  96  0       0  25256 nfsiod   DK-  ?       0:00.00 [system]
   0     0 99.0 physiod             0     0     0  107   97 123  0       0  25256 physiod  DK-  ?       0:00.00 [system]
   0     0 99.0 radeonfb            0     0     0  106   97  43  0       0  25256 radeonfb DK-  ?       0:00.00 [system]
   0     0 99.0 radeon-crtc         0     0     0  104   97  43  0       0  25256 radeon-c DK-  ?       0:00.00 [system]
   0     0 99.0 radeon-crtc         0     0     0  103   97  43  0       0  25256 radeon-c DK-  ?       0:00.00 [system]
   0     0 99.0 ttm_swap            0     0     0  102   97  43  0       0  25256 ttm_swap DK-  ?       0:00.00 [system]
   0     0 99.0 pooldrain           0     0     0  101   97 125  0       0  25256 xclocv   DK-  ?       0:04.20 [system]
   0     0 99.0 aiodoned            0     0     0  100   97 125  0       0  25256 aiodoned DK-  ?       0:09.50 [system]
   0     0 99.0 ioflush             0     0     0   99   97 124  0       0  25256 syncer   DK-  ?       3:26.00 [system]
   0     0 99.0 pgdaemon            0     0     0   98   97 126  0       0  25256 -        OK-  ?     246:48.38 [system]
   0     0 99.0 raidio0             0     0     0   95   97 221  0       0  25256 raidiow  DK-  ?       0:29.36 [system]
   0     0 99.0 raid0               0     0     0   94   97 221  0       0  25256 rfnodeq  DK-  ?       0:10.94 [system]
   0     0 99.0 atapibus0           0     0     0   93   97  96  0       0  25256 sccomp   DK-  ?       0:00.00 [system]
   0     0 99.0 usb8                0     0     0   88   97  96  0       0  25256 usbevt   DK-  ?       0:00.00 [system]
   0     0 99.0 usb6                0     0     0   87   97  96  0       0  25256 usbevt   DK-  ?       0:00.00 [system]
   0     0 99.0 usb3                0     0     0   86   97  96  0       0  25256 usbevt   DK-  ?       0:00.00 [system]
   0     0 99.0 usb4                0     0     0   85   97  96  0       0  25256 usbevt   DK-  ?       0:00.00 [system]
   0     0 99.0 usb1                0     0     0   84   97  96  0       0  25256 usbevt   DK-  ?       0:00.01 [system]
   0     0 99.0 usb5                0     0     0   83   97  96  0       0  25256 usbevt   DK-  ?       0:00.01 [system]
   0     0 99.0 usb0                0     0     0   82   97  96  0       0  25256 usbevt   DK-  ?       0:00.01 [system]
   0     0 99.0 usb2                0     0     0   81   97  96  0       0  25256 usbevt   DK-  ?       0:00.01 [system]
   0     0 99.0 usb7                0     0     0   80   97  96  0       0  25256 usbevt   DK-  ?       0:00.00 [system]
   0     0 99.0 amdtemp0            0     0     0   79   97  43  0       0  25256 amdtemp0 DK-  ?       0:00.02 [system]
   0     0 99.0 npfgc-0             0     0     0   78   97  96  0       0  25256 npfgccv  DK-  ?       0:02.31 [system]
   0     0 99.0 rt_free             0     0     0   77   97 222  0       0  25256 rt_free  DK-  ?       0:00.00 [system]
   0     0 99.0 unpgc               0     0     0   76   97  96  0       0  25256 unpgc    DK-  ?       0:00.00 [system]
   0     0 99.0 key_timehandler     0     0     0   75   97 222  0       0  25256 key_time DK-  ?       0:00.44 [system]
   0     0 99.0 icmp6_wqinput/3     0     0     0   74   97 222  0       0  25256 icmp6_wq DK-  ?       0:00.00 [system]
   0     0 99.0 icmp6_wqinput/2     0     0     0   73   97 222  0       0  25256 icmp6_wq DK-  ?       0:00.00 [system]
   0     0 99.0 icmp6_wqinput/1     0     0     0   72   97 222  0       0  25256 icmp6_wq DK-  ?       0:00.00 [system]
   0     0 99.0 icmp6_wqinput/0     0     0     0   71   97 222  0       0  25256 icmp6_wq DK-  ?       0:00.21 [system]
   0     0 99.0 nd6_timer           0     0     0   70   97 222  0       0  25256 nd6_time DK-  ?       0:08.22 [system]
   0     0 99.0 carp6_wqinput/3     0     0     0   69   97 222  0       0  25256 carp6_wq DK-  ?       0:00.00 [system]
   0     0 99.0 carp6_wqinput/2     0     0     0   68   97 222  0       0  25256 carp6_wq DK-  ?       0:00.00 [system]
   0     0 99.0 carp6_wqinput/1     0     0     0   67   97 222  0       0  25256 carp6_wq DK-  ?       0:00.00 [system]
   0     0 99.0 carp6_wqinput/0     0     0     0   66   97 222  0       0  25256 carp6_wq DK-  ?       0:00.00 [system]
   0     0 99.0 carp_wqinput/3      0     0     0   65   97 222  0       0  25256 carp_wqi DK-  ?       0:00.00 [system]
   0     0 99.0 carp_wqinput/2      0     0     0   64   97 222  0       0  25256 carp_wqi DK-  ?       0:00.00 [system]
   0     0 99.0 carp_wqinput/1      0     0     0   63   97 222  0       0  25256 carp_wqi DK-  ?       0:00.00 [system]
   0     0 99.0 carp_wqinput/0      0     0     0   62   97 222  0       0  25256 carp_wqi DK-  ?       0:00.00 [system]
   0     0 99.0 icmp_wqinput/3      0     0     0   61   97 222  0       0  25256 icmp_wqi DK-  ?       0:00.00 [system]
   0     0 99.0 icmp_wqinput/2      0     0     0   60   97 222  0       0  25256 icmp_wqi DK-  ?       0:00.00 [system]
   0     0 99.0 icmp_wqinput/1      0     0     0   59   97 222  0       0  25256 icmp_wqi DK-  ?       0:00.00 [system]
   0     0 99.0 icmp_wqinput/0      0     0     0   58   97 222  0       0  25256 icmp_wqi DK-  ?       0:00.00 [system]
   0     0 99.0 rt_timer            0     0     0   57   97 222  0       0  25256 rt_timer DK-  ?       0:00.27 [system]
   0     0 99.0 vmem_rehash         0     0     0   56   97 125  0       0  25256 vmem_reh DK-  ?       0:00.05 [system]
   0     0 99.0 atabus5             0     0     0   47   97  96  0       0  25256 atath    DK-  ?       0:00.00 [system]
   0     0 99.0 atabus4             0     0     0   46   97  96  0       0  25256 atath    DK-  ?       0:00.00 [system]
   0     0 99.0 iic0                0     0     0   45   97  96  0       0  25256 iicintr  DK-  ?       1:36.97 [system]
   0     0 99.0 usbtask-dr          0     0     0   44   97  96  0       0  25256 usbtsk   DK-  ?       0:00.00 [system]
   0     0 99.0 usbtask-hc          0     0     0   43   97  96  0       0  25256 usbtsk   DK-  ?       0:00.00 [system]
   0     0 99.0 atabus3             0     0     0   42   97  96  0       0  25256 atath    DK-  ?       0:00.00 [system]
   0     0 99.0 atabus2             0     0     0   41   97  96  0       0  25256 atath    DK-  ?       0:00.00 [system]
   0     0 99.0 atabus1             0     0     0   40   97  96  0       0  25256 atath    DK-  ?       0:00.00 [system]
   0     0 99.0 atabus0             0     0     0   39   97  96  0       0  25256 atath    DK-  ?       0:00.00 [system]
   0     0 99.0 apm0                0     0     0   38   97  96  0       0  25256 apmev    DK-  ?       0:05.33 [system]
   0     0 99.0 xcall/3             0     0     0   37   97 127  0       0  25256 xcall    DK-  ?       0:00.87 [system]
   0     0 99.0 softser/3           0     0     0   36   97 223  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 softclk/3           0     0     0   35   97 220  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 softbio/3           0     0     0   34   97 221  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 softnet/3           0     0     0   33   97 222  0       0  25256 -        RK-  ?       0:00.01 [system]
   0     0 99.0 idle/3              0     0     0   32   97   0  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 xcall/2             0     0     0   31   97 127  0       0  25256 xcall    DK-  ?       0:00.80 [system]
   0     0 99.0 softser/2           0     0     0   30   97 223  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 softclk/2           0     0     0   29   97 220  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 softbio/2           0     0     0   28   97 221  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 softnet/2           0     0     0   27   97 222  0       0  25256 -        RK-  ?       0:00.01 [system]
   0     0 99.0 idle/2              0     0     0   26   97   0  0       0  25256 -        OK-  ?       0:00.00 [system]
   0     0 99.0 xcall/1             0     0     0   25   97 127  0       0  25256 xcall    DK-  ?       0:00.92 [system]
   0     0 99.0 softser/1           0     0     0   24   97 223  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 softclk/1           0     0     0   23   97 220  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 softbio/1           0     0     0   22   97 221  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 softnet/1           0     0     0   21   97 222  0       0  25256 -        RK-  ?       0:00.01 [system]
   0     0 99.0 idle/1              0     0     0   20   97   0  0       0  25256 -        OK-  ?       0:00.00 [system]
   0     0 99.0 lnxpwrwq            0     0     0   19   97  43  0       0  25256 lnxpwrwq DK-  ?       0:01.51 [system]
   0     0 99.0 lnxlngwq            0     0     0   18   97  43  0       0  25256 lnxlngwq DK-  ?       0:00.00 [system]
   0     0 99.0 lnxsyswq            0     0     0   17   97  43  0       0  25256 lnxsyswq DK-  ?       8:16.45 [system]
   0     0 99.0 lnxrcugc            0     0     0   16   97  43  0       0  25256 lnxrcugc DK-  ?       0:01.90 [system]
   0     0 99.0 sysmon              0     0     0   15   97  96  0       0  25256 smtaskq  DK-  ?       0:00.00 [system]
   0     0 99.0 pmfsuspend          0     0     0   14   97  43  0       0  25256 pmfsuspe DK-  ?       0:00.00 [system]
   0     0 99.0 pmfevent            0     0     0   13   97  43  0       0  25256 pmfevent DK-  ?       0:00.00 [system]
   0     0 99.0 sopendfree          0     0     0   12   97  96  0       0  25256 sopendfr DK-  ?       0:04.72 [system]
   0     0 99.0 nfssilly            0     0     0   11   97  43  0       0  25256 nfssilly DK-  ?       0:00.00 [system]
   0     0 99.0 cachegc             0     0     0   10   97 125  0       0  25256 cachegc  DK-  ?       0:00.53 [system]
   0     0 99.0 vdrain              0     0     0    9   97 125  0       0  25256 vdrain   DK-  ?       0:00.70 [system]
   0     0 99.0 modunload           0     0     0    8   97 125  0       0  25256 mod_unld DK-  ?       0:00.00 [system]
   0     0 99.0 xcall/0             0     0     0    7   97 127  0       0  25256 -        RK-  ?       0:01.03 [system]
   0     0 99.0 softser/0           0     0     0    6   97 223  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 softclk/0           0     0     0    5   97 220  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 softbio/0           0     0     0    4   97 221  0       0  25256 -        RK-  ?       0:00.01 [system]
   0     0 99.0 softnet/0           0     0     0    3   97 222  0       0  25256 -        RK-  ?       0:00.21 [system]
   0     0 99.0 idle/0              0     0     0    2   97   0  0       0  25256 -        RK-  ?       0:00.00 [system]
   0     0 99.0 swapper             0     0     0    1   97 125  0       0  25256 uvm      DK   ?       0:11.99 [system]
   0     1  0.0 -                   0     0     0    1    1  85  0    5960   1240 wait     I    ?       0:00.01 init 
 :

---

dmesg is here:
 https://dmesgd.nycbug.org/index.cgi?do=view&id=5276

Note root is on raid0 (wd0 and wd1, 2TB x2),
swap is on wd2 (SSD 3.0Gb/s). 

I also saw the similar spins on NetBSD/i386 GENERIC.

>How-To-Repeat:
NetBSD/i386 kernel with several load?

Maybe sound (HTML5 video on Firefox) and serial interrupts
(serial console teriminal via kermit) related?

>Fix:
No idea even what's going on.

---
Izumi Tsutsui

>Release-Note:

>Audit-Trail:
From: Andrew Doran <ad@netbsd.org>
To: gnats-bugs@netbsd.org
Cc: kern-bug-people@netbsd.org, gnats-admin@netbsd.org,
	netbsd-bugs@netbsd.org
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Wed, 1 Jan 2020 13:35:27 +0000

 If it happens again could you please collect the output of "vmstat -s", wait
 a few seconds, and then capture "vmstat -s" again?  That could be useful.

 Hmm, I wonder if backporting a change from -current to use marker pages when
 scanning the inactive queue in uvmpdpol_selectvictim() could help here.

 Andrew

From: Andrew Doran <ad@netbsd.org>
To: gnats-bugs@netbsd.org
Cc: kern-bug-people@netbsd.org, gnats-admin@netbsd.org,
	netbsd-bugs@netbsd.org
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Wed, 1 Jan 2020 16:07:56 +0000

 Could you please try the following?

 	http://www.netbsd.org/~ad/2020/pdaemon-9.diff

 Andrew

From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
To: ad@netbsd.org
Cc: gnats-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Thu, 2 Jan 2020 06:42:39 +0900

 > If it happens again could you please collect the output of "vmstat -s", wait
 > a few seconds, and then capture "vmstat -s" again?  That could be useful.

 Here is output of "vmstat -s 1" 10 times:
 ---
      4096 bytes per page
         8 page colors
    832700 pages managed
    117235 pages free
    342925 pages active
    183082 pages inactive
         0 pages paging
     65844 pages wired
     97392 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    118567 kernel pool pages
    361159 anonymous pages
    180316 cached file pages
     50376 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    139630 swap pages in use
    159606 swap allocations
 458503384 total faults taken
 458961173 traps
   9552277 device interrupts
  96460867 CPU context switches
  15626258 software interrupts
 930547937 system calls
    155893 pagein requests
     17376 pageout requests
         0 pages swapped in
    276978 pages swapped out
    887490 forks total
    520251 forks blocked parent
    520251 forks shared address space with parent
  57121049 pagealloc zero wanted and avail
  99809329 pagealloc zero wanted and not avail
      6630 aborts of idle page zeroing
 370962565 pagealloc desired color avail
  21949517 pagealloc desired color not avail
 316537267 pagealloc local cpu avail
  76374815 pagealloc local cpu not avail
       133 faults with no memory
         0 faults with no anons
        31 faults had to wait on pages
         0 faults found released page
    305643 faults relock (302934 ok)
  25034954 anon page faults
    155001 anon retry faults
  31940079 amap copy faults
  19191744 neighbour anon page faults
 389315776 neighbour object page faults
  98755999 locked pager get faults
    149720 unlocked pager get faults
  16939718 anon faults
   8095274 anon copy on write faults
  76579891 object faults
  22173678 promote copy faults
 150460952 promote zero fill faults
      1085 times daemon wokeup
     71617 revolutions of the clock hand
   4994032 pages freed by daemon
  13125360 pages scanned by daemon
    269619 anonymous pages scanned by daemon
   4724413 object pages scanned by daemon
   2215383 pages reactivated
       167 pages found busy by daemon
    259601 total pending pageouts
  14625650 pages deactivated
 523823913 total name lookups
 494694075 good hits
  27128967 negative hits
    279115 bad hits
     21338 false hits
   1293870 miss
    406548 too long
    295970 pass2 hits
    446837 2passes
           cache hits (94% pos + 5% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
    118948 pages free
    341224 pages active
    183069 pages inactive
         0 pages paging
     65844 pages wired
     97392 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    118569 kernel pool pages
    359445 anonymous pages
    180316 cached file pages
     50376 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    139630 swap pages in use
    159606 swap allocations
 458503461 total faults taken
 458961250 traps
   9552377 device interrupts
  96461807 CPU context switches
  15626634 software interrupts
 930553597 system calls
    155893 pagein requests
     17376 pageout requests
         0 pages swapped in
    276978 pages swapped out
    887490 forks total
    520251 forks blocked parent
    520251 forks shared address space with parent
  57121049 pagealloc zero wanted and avail
  99809329 pagealloc zero wanted and not avail
      6630 aborts of idle page zeroing
 370962612 pagealloc desired color avail
  21949517 pagealloc desired color not avail
 316537314 pagealloc local cpu avail
  76374815 pagealloc local cpu not avail
       133 faults with no memory
         0 faults with no anons
        31 faults had to wait on pages
         0 faults found released page
    305643 faults relock (302934 ok)
  25034992 anon page faults
    155001 anon retry faults
  31940083 amap copy faults
  19191744 neighbour anon page faults
 389315776 neighbour object page faults
  98755999 locked pager get faults
    149720 unlocked pager get faults
  16939756 anon faults
   8095274 anon copy on write faults
  76579891 object faults
  22173678 promote copy faults
 150460952 promote zero fill faults
      1085 times daemon wokeup
     71617 revolutions of the clock hand
   4994032 pages freed by daemon
  13125360 pages scanned by daemon
    269619 anonymous pages scanned by daemon
   4724413 object pages scanned by daemon
   2215383 pages reactivated
       167 pages found busy by daemon
    259601 total pending pageouts
  14625650 pages deactivated
 523823920 total name lookups
 494694080 good hits
  27128969 negative hits
    279115 bad hits
     21338 false hits
   1293870 miss
    406548 too long
    295970 pass2 hits
    446837 2passes
           cache hits (94% pos + 5% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
    118674 pages free
    341431 pages active
    183042 pages inactive
         0 pages paging
     65932 pages wired
     97201 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    118572 kernel pool pages
    359713 anonymous pages
    180316 cached file pages
     50376 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    139630 swap pages in use
    159606 swap allocations
 458519831 total faults taken
 458977623 traps
   9552530 device interrupts
  96465487 CPU context switches
  15627043 software interrupts
 930580429 system calls
    155893 pagein requests
     17376 pageout requests
         0 pages swapped in
    276978 pages swapped out
    887490 forks total
    520251 forks blocked parent
    520251 forks shared address space with parent
  57121240 pagealloc zero wanted and avail
  99809329 pagealloc zero wanted and not avail
      6630 aborts of idle page zeroing
 370979690 pagealloc desired color avail
  21949517 pagealloc desired color not avail
 316553121 pagealloc local cpu avail
  76376086 pagealloc local cpu not avail
       133 faults with no memory
         0 faults with no anons
        31 faults had to wait on pages
         0 faults found released page
    305643 faults relock (302934 ok)
  25035040 anon page faults
    155001 anon retry faults
  31940098 amap copy faults
  19191744 neighbour anon page faults
 389315776 neighbour object page faults
  98756001 locked pager get faults
    149720 unlocked pager get faults
  16939804 anon faults
   8095274 anon copy on write faults
  76579893 object faults
  22173678 promote copy faults
 150461140 promote zero fill faults
      1085 times daemon wokeup
     71617 revolutions of the clock hand
   4994032 pages freed by daemon
  13125360 pages scanned by daemon
    269619 anonymous pages scanned by daemon
   4724413 object pages scanned by daemon
   2215383 pages reactivated
       167 pages found busy by daemon
    259601 total pending pageouts
  14625650 pages deactivated
 523824179 total name lookups
 494694307 good hits
  27128984 negative hits
    279115 bad hits
     21338 false hits
   1293870 miss
    406565 too long
    295970 pass2 hits
    446837 2passes
           cache hits (94% pos + 5% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
    118635 pages free
    341498 pages active
    183020 pages inactive
         0 pages paging
     65916 pages wired
     97141 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    118576 kernel pool pages
    359742 anonymous pages
    180316 cached file pages
     50376 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    139630 swap pages in use
    159606 swap allocations
 458534252 total faults taken
 458992045 traps
   9552732 device interrupts
  96471307 CPU context switches
  15627559 software interrupts
 930617318 system calls
    155893 pagein requests
     17376 pageout requests
         0 pages swapped in
    276978 pages swapped out
    887490 forks total
    520251 forks blocked parent
    520251 forks shared address space with parent
  57121300 pagealloc zero wanted and avail
  99809329 pagealloc zero wanted and not avail
      6630 aborts of idle page zeroing
 370994701 pagealloc desired color avail
  21949517 pagealloc desired color not avail
 316568114 pagealloc local cpu avail
  76376104 pagealloc local cpu not avail
       133 faults with no memory
         0 faults with no anons
        31 faults had to wait on pages
         0 faults found released page
    305643 faults relock (302934 ok)
  25035240 anon page faults
    155001 anon retry faults
  31940112 amap copy faults
  19191744 neighbour anon page faults
 389315776 neighbour object page faults
  98756001 locked pager get faults
    149720 unlocked pager get faults
  16940004 anon faults
   8095274 anon copy on write faults
  76579893 object faults
  22173678 promote copy faults
 150461200 promote zero fill faults
      1085 times daemon wokeup
     71617 revolutions of the clock hand
   4994032 pages freed by daemon
  13125360 pages scanned by daemon
    269619 anonymous pages scanned by daemon
   4724413 object pages scanned by daemon
   2215383 pages reactivated
       167 pages found busy by daemon
    259601 total pending pageouts
  14625650 pages deactivated
 523824315 total name lookups
 494694427 good hits
  27128987 negative hits
    279115 bad hits
     21338 false hits
   1293870 miss
    406578 too long
    295970 pass2 hits
    446837 2passes
           cache hits (94% pos + 5% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
    118715 pages free
    341514 pages active
    183006 pages inactive
         0 pages paging
     65828 pages wired
     97138 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    118559 kernel pool pages
    359656 anonymous pages
    180316 cached file pages
     50376 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    139630 swap pages in use
    159606 swap allocations
 458539603 total faults taken
 458997396 traps
   9552915 device interrupts
  96472810 CPU context switches
  15628005 software interrupts
 930625317 system calls
    155893 pagein requests
     17376 pageout requests
         0 pages swapped in
    276978 pages swapped out
    887490 forks total
    520251 forks blocked parent
    520251 forks shared address space with parent
  57121303 pagealloc zero wanted and avail
  99809329 pagealloc zero wanted and not avail
      6630 aborts of idle page zeroing
 371000164 pagealloc desired color avail
  21949517 pagealloc desired color not avail
 316573576 pagealloc local cpu avail
  76376105 pagealloc local cpu not avail
       133 faults with no memory
         0 faults with no anons
        31 faults had to wait on pages
         0 faults found released page
    305643 faults relock (302934 ok)
  25035272 anon page faults
    155001 anon retry faults
  31940117 amap copy faults
  19191744 neighbour anon page faults
 389315776 neighbour object page faults
  98756001 locked pager get faults
    149720 unlocked pager get faults
  16940036 anon faults
   8095274 anon copy on write faults
  76579893 object faults
  22173678 promote copy faults
 150461202 promote zero fill faults
      1085 times daemon wokeup
     71617 revolutions of the clock hand
   4994032 pages freed by daemon
  13125360 pages scanned by daemon
    269619 anonymous pages scanned by daemon
   4724413 object pages scanned by daemon
   2215383 pages reactivated
       167 pages found busy by daemon
    259601 total pending pageouts
  14625650 pages deactivated
 523824315 total name lookups
 494694427 good hits
  27128987 negative hits
    279115 bad hits
     21338 false hits
   1293870 miss
    406578 too long
    295970 pass2 hits
    446837 2passes
           cache hits (94% pos + 5% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
    118712 pages free
    341528 pages active
    182995 pages inactive
         0 pages paging
     65828 pages wired
     97131 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    118556 kernel pool pages
    359659 anonymous pages
    180316 cached file pages
     50376 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    139630 swap pages in use
    159606 swap allocations
 458543032 total faults taken
 459000826 traps
   9553020 device interrupts
  96473904 CPU context switches
  15628384 software interrupts
 930631548 system calls
    155893 pagein requests
     17376 pageout requests
         0 pages swapped in
    276978 pages swapped out
    887490 forks total
    520251 forks blocked parent
    520251 forks shared address space with parent
  57121310 pagealloc zero wanted and avail
  99809329 pagealloc zero wanted and not avail
      6630 aborts of idle page zeroing
 371003665 pagealloc desired color avail
  21949517 pagealloc desired color not avail
 316577073 pagealloc local cpu avail
  76376109 pagealloc local cpu not avail
       133 faults with no memory
         0 faults with no anons
        31 faults had to wait on pages
         0 faults found released page
    305643 faults relock (302934 ok)
  25035300 anon page faults
    155001 anon retry faults
  31940129 amap copy faults
  19191744 neighbour anon page faults
 389315776 neighbour object page faults
  98756001 locked pager get faults
    149720 unlocked pager get faults
  16940064 anon faults
   8095274 anon copy on write faults
  76579893 object faults
  22173678 promote copy faults
 150461205 promote zero fill faults
      1085 times daemon wokeup
     71617 revolutions of the clock hand
   4994032 pages freed by daemon
  13125360 pages scanned by daemon
    269619 anonymous pages scanned by daemon
   4724413 object pages scanned by daemon
   2215383 pages reactivated
       167 pages found busy by daemon
    259601 total pending pageouts
  14625650 pages deactivated
 523824321 total name lookups
 494694432 good hits
  27128987 negative hits
    279115 bad hits
     21338 false hits
   1293870 miss
    406579 too long
    295970 pass2 hits
    446837 2passes
           cache hits (94% pos + 5% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
    118732 pages free
    341522 pages active
    182981 pages inactive
         0 pages paging
     65828 pages wired
     97127 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    118572 kernel pool pages
    359639 anonymous pages
    180316 cached file pages
     50376 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    139630 swap pages in use
    159606 swap allocations
 458546595 total faults taken
 459004389 traps
   9553143 device interrupts
  96474875 CPU context switches
  15628773 software interrupts
 930636950 system calls
    155893 pagein requests
     17376 pageout requests
         0 pages swapped in
    276978 pages swapped out
    887490 forks total
    520251 forks blocked parent
    520251 forks shared address space with parent
  57121314 pagealloc zero wanted and avail
  99809329 pagealloc zero wanted and not avail
      6630 aborts of idle page zeroing
 371007257 pagealloc desired color avail
  21949517 pagealloc desired color not avail
 316580661 pagealloc local cpu avail
  76376113 pagealloc local cpu not avail
       133 faults with no memory
         0 faults with no anons
        31 faults had to wait on pages
         0 faults found released page
    305643 faults relock (302934 ok)
  25035312 anon page faults
    155001 anon retry faults
  31940132 amap copy faults
  19191744 neighbour anon page faults
 389315776 neighbour object page faults
  98756001 locked pager get faults
    149720 unlocked pager get faults
  16940076 anon faults
   8095274 anon copy on write faults
  76579893 object faults
  22173678 promote copy faults
 150461205 promote zero fill faults
      1085 times daemon wokeup
     71617 revolutions of the clock hand
   4994032 pages freed by daemon
  13125360 pages scanned by daemon
    269619 anonymous pages scanned by daemon
   4724413 object pages scanned by daemon
   2215383 pages reactivated
       167 pages found busy by daemon
    259601 total pending pageouts
  14625650 pages deactivated
 523824321 total name lookups
 494694432 good hits
  27128987 negative hits
    279115 bad hits
     21338 false hits
   1293870 miss
    406579 too long
    295970 pass2 hits
    446837 2passes
           cache hits (94% pos + 5% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
    118676 pages free
    341557 pages active
    182949 pages inactive
         0 pages paging
     65884 pages wired
     97116 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    118547 kernel pool pages
    359698 anonymous pages
    180316 cached file pages
     50376 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    139630 swap pages in use
    159606 swap allocations
 458551558 total faults taken
 459009353 traps
   9553301 device interrupts
  96476981 CPU context switches
  15629214 software interrupts
 930649540 system calls
    155893 pagein requests
     17376 pageout requests
         0 pages swapped in
    276978 pages swapped out
    887490 forks total
    520251 forks blocked parent
    520251 forks shared address space with parent
  57121325 pagealloc zero wanted and avail
  99809329 pagealloc zero wanted and not avail
      6630 aborts of idle page zeroing
 371012340 pagealloc desired color avail
  21949517 pagealloc desired color not avail
 316585739 pagealloc local cpu avail
  76376118 pagealloc local cpu not avail
       133 faults with no memory
         0 faults with no anons
        31 faults had to wait on pages
         0 faults found released page
    305643 faults relock (302934 ok)
  25035356 anon page faults
    155001 anon retry faults
  31940155 amap copy faults
  19191744 neighbour anon page faults
 389315776 neighbour object page faults
  98756001 locked pager get faults
    149720 unlocked pager get faults
  16940120 anon faults
   8095274 anon copy on write faults
  76579893 object faults
  22173678 promote copy faults
 150461212 promote zero fill faults
      1085 times daemon wokeup
     71617 revolutions of the clock hand
   4994032 pages freed by daemon
  13125360 pages scanned by daemon
    269619 anonymous pages scanned by daemon
   4724413 object pages scanned by daemon
   2215383 pages reactivated
       167 pages found busy by daemon
    259601 total pending pageouts
  14625650 pages deactivated
 523824436 total name lookups
 494694535 good hits
  27128989 negative hits
    279115 bad hits
     21338 false hits
   1293870 miss
    406589 too long
    295970 pass2 hits
    446837 2passes
           cache hits (94% pos + 5% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
    118701 pages free
    341601 pages active
    182932 pages inactive
         0 pages paging
     65828 pages wired
     97074 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    118536 kernel pool pages
    359669 anonymous pages
    180316 cached file pages
     50376 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    139630 swap pages in use
    159606 swap allocations
 458565635 total faults taken
 459023432 traps
   9553487 device interrupts
  96481705 CPU context switches
  15629681 software interrupts
 930679821 system calls
    155893 pagein requests
     17376 pageout requests
         0 pages swapped in
    276978 pages swapped out
    887490 forks total
    520251 forks blocked parent
    520251 forks shared address space with parent
  57121367 pagealloc zero wanted and avail
  99809329 pagealloc zero wanted and not avail
      6630 aborts of idle page zeroing
 371026749 pagealloc desired color avail
  21949517 pagealloc desired color not avail
 316600142 pagealloc local cpu avail
  76376124 pagealloc local cpu not avail
       133 faults with no memory
         0 faults with no anons
        31 faults had to wait on pages
         0 faults found released page
    305643 faults relock (302934 ok)
  25035392 anon page faults
    155001 anon retry faults
  31940161 amap copy faults
  19191744 neighbour anon page faults
 389315776 neighbour object page faults
  98756001 locked pager get faults
    149720 unlocked pager get faults
  16940156 anon faults
   8095274 anon copy on write faults
  76579893 object faults
  22173678 promote copy faults
 150461250 promote zero fill faults
      1085 times daemon wokeup
     71617 revolutions of the clock hand
   4994032 pages freed by daemon
  13125360 pages scanned by daemon
    269619 anonymous pages scanned by daemon
   4724413 object pages scanned by daemon
   2215383 pages reactivated
       167 pages found busy by daemon
    259601 total pending pageouts
  14625650 pages deactivated
 523824436 total name lookups
 494694535 good hits
  27128989 negative hits
    279115 bad hits
     21338 false hits
   1293870 miss
    406589 too long
    295970 pass2 hits
    446837 2passes
           cache hits (94% pos + 5% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
    118702 pages free
    341620 pages active
    182917 pages inactive
         0 pages paging
     65828 pages wired
     97064 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    118509 kernel pool pages
    359673 anonymous pages
    180316 cached file pages
     50376 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    139630 swap pages in use
    159606 swap allocations
 458571475 total faults taken
 459029274 traps
   9553585 device interrupts
  96482704 CPU context switches
  15630064 software interrupts
 930685464 system calls
    155893 pagein requests
     17376 pageout requests
         0 pages swapped in
    276978 pages swapped out
    887490 forks total
    520251 forks blocked parent
    520251 forks shared address space with parent
  57121377 pagealloc zero wanted and avail
  99809329 pagealloc zero wanted and not avail
      6630 aborts of idle page zeroing
 371032625 pagealloc desired color avail
  21949517 pagealloc desired color not avail
 316604666 pagealloc local cpu avail
  76377476 pagealloc local cpu not avail
       133 faults with no memory
         0 faults with no anons
        31 faults had to wait on pages
         0 faults found released page
    305643 faults relock (302934 ok)
  25035443 anon page faults
    155001 anon retry faults
  31940173 amap copy faults
  19191744 neighbour anon page faults
 389315776 neighbour object page faults
  98756001 locked pager get faults
    149720 unlocked pager get faults
  16940207 anon faults
   8095274 anon copy on write faults
  76579893 object faults
  22173678 promote copy faults
 150461254 promote zero fill faults
      1085 times daemon wokeup
     71617 revolutions of the clock hand
   4994032 pages freed by daemon
  13125360 pages scanned by daemon
    269619 anonymous pages scanned by daemon
   4724413 object pages scanned by daemon
   2215383 pages reactivated
       167 pages found busy by daemon
    259601 total pending pageouts
  14625650 pages deactivated
 523824436 total name lookups
 494694535 good hits
  27128989 negative hits
    279115 bad hits
     21338 false hits
   1293870 miss
    406589 too long
    295970 pass2 hits
    446837 2passes
           cache hits (94% pos + 5% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 ---

 More data is here:
  https://gist.github.com/tsutsui/d5d671fbd9361db222f44be968463d8c

 > Hmm, I wonder if backporting a change from -current to use marker pages when
 > scanning the inactive queue in uvmpdpol_selectvictim() could help here.

 Note I forgot to note in the first mail, it happened also on NetBSD 8.1.

 I'll try your suggested patch later. (not so easy to reproduce)

 ---
 Izumi Tsutsui

From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
To: ad@netbsd.org
Cc: gnats-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Thu, 2 Jan 2020 19:25:00 +0900

 >  Could you please try the following?
 >  
 >  	http://www.netbsd.org/~ad/2020/pdaemon-9.diff

 I guess uvm/uvm_pdpolicy_clockpro.c part (empty uvmpdpol_scanfini())
 isn't necessary, right?

 Unfortunately the spin still occurs with the above diff on load:
 (Firefox with HTML5 video + build.sh -j4 release + serial terminal output)

 ---
 load averages:  5.17,  4.40,  3.02;               up 0+03:00:37        18:08:10
 449 threads: 20 idle, 3 runnable, 410 sleeping, 1 stopped, 11 zombie, 4 on CPU
 CPU states: 38.1% user,  0.0% nice, 37.4% system,  1.9% interrupt, 22.5% idle
 Memory: 1540M Act, 752M Inact, 314M Wired, 211M Exec, 756M File, 16M Free
 Swap: 8972M Total, 179M Used, 8793M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/2     17:28 98.88% 98.88% pgdaemon  [system]
  7082    20 tsutsui   25 RUN/3      8:54 46.73% 46.73% Composito firefox
  7082   226 tsutsui   29 parked/1   0:16 20.63% 20.51% -         firefox
  1086     1 tsutsui   79 select/3  37:03 13.48% 13.48% -         X
  7082   227 tsutsui   28 RUN/0      0:14 12.43% 12.35% -         firefox
  7082     1 tsutsui   80 select/0   4:10 10.99% 10.99% -         firefox
  7082   228 tsutsui   31 CPU/0      0:15  9.38%  9.33% -         firefox
  1106     1 tsutsui   84 select/1   6:50  3.56%  3.56% -         ruby26
  2485     1 tsutsui   83 uvn_fp/3   0:00 13.23%  2.93% -         mnews

 ---
 load averages:  4.62,  4.56,  3.29;               up 0+03:02:58        18:10:31
 449 threads: 20 idle, 1 runnable, 412 sleeping, 1 stopped, 11 zombie, 4 on CPU
 CPU states: 34.9% user,  0.0% nice, 34.0% system,  0.1% interrupt, 30.9% idle
 Memory: 1516M Act, 746M Inact, 314M Wired, 198M Exec, 678M File, 49M Free
 Swap: 8972M Total, 186M Used, 8786M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/1     19:48 98.58% 98.58% pgdaemon  [system]
  7082    20 tsutsui   26 CPU/2     10:09 52.44% 52.44% Composito firefox
  7082   233 tsutsui   29 parked/3   0:19 20.57% 20.51% -         firefox
  7082   235 tsutsui   27 parked/3   0:18 20.42% 20.36% -         firefox
  1086     1 tsutsui   79 select/2  37:28 19.29% 19.29% -         X
  7082   234 tsutsui   30 parked/3   0:17 13.47% 13.43% -         firefox
  7082     1 tsutsui   81 select/2   4:28 12.11% 12.11% -         firefox

 ---
 load averages:  4.21,  4.68,  3.72;               up 0+03:07:29        18:15:02
 467 threads: 20 idle, 6 runnable, 423 sleeping, 1 stopped, 13 zombie, 4 on CPU
 CPU states: 51.4% user,  0.0% nice, 40.4% system,  0.1% interrupt,  8.0% idle
 Memory: 1424M Act, 750M Inact, 316M Wired, 215M Exec, 599M File, 135M Free
 Swap: 8972M Total, 186M Used, 8786M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/3     24:20 98.93% 98.93% pgdaemon  [system]
  7082    20 tsutsui   27 RUN/0     12:21 53.42% 53.42% Composito firefox
  1086     1 tsutsui   33 CPU/0     38:16 18.75% 18.75% -         X
  7082     1 tsutsui   82 select/1   5:10 15.82% 15.82% -         firefox
  1106     1 tsutsui   85 select/1   7:07  3.17%  3.17% -         ruby26
 13282     1 tsutsui   28 RUN/3      0:00  9.00%  0.44% -         cc1
  1677     1 tsutsui   27 RUN/2      0:00  7.00%  0.34% -         cc1
 24773     1 tsutsui   28 CPU/2      0:00  5.00%  0.24% -         cc1

 ---

 [ build.sh -j4 release is suspended here]

 ---
 load averages:  4.23,  4.65,  3.74;               up 0+03:07:59        18:15:32
 463 threads: 19 idle, 5 runnable, 391 sleeping, 32 stopped, 12 zombie, 4 on CPU
 CPU states: 26.4% user,  0.0% nice, 29.6% system,  1.0% interrupt, 42.9% idle
 Memory: 1450M Act, 751M Inact, 318M Wired, 213M Exec, 609M File, 107M Free
 Swap: 8972M Total, 186M Used, 8786M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/3     24:50 98.88% 98.88% pgdaemon  [system]
  7082    20 tsutsui   77 select/0  12:34 44.73% 44.73% Composito firefox
  7082     1 tsutsui   36 parked/2   5:15 12.65% 12.65% -         firefox
  1086     1 tsutsui   30 RUN/0     38:20 12.45% 12.45% -         X
  7082   265 tsutsui   35 parked/3   0:03 14.26% 11.08% -         firefox
  7082   263 tsutsui   34 parked/1   0:04 10.74%  8.35% -         firefox
  7082   264 tsutsui   34 CPU/1      0:03  7.79%  6.05% -         firefox
  1118     2 tsutsui   85 select/0   0:20  5.47%  5.47% -         pulseaudio
     0    99 root     124 syncer/3   0:29  3.56%  3.56% ioflush   [system]

 ---
 load averages:  3.26,  4.29,  3.67;               up 0+03:08:59        18:16:32
 465 threads: 20 idle, 1 runnable, 396 sleeping, 32 stopped, 12 zombie, 4 on CPU
 CPU states: 25.4% user,  0.0% nice, 28.5% system,  0.0% interrupt, 45.9% idle
 Memory: 1485M Act, 749M Inact, 316M Wired, 213M Exec, 609M File, 77M Free
 Swap: 8972M Total, 186M Used, 8786M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/1     25:50 99.02% 99.02% pgdaemon  [system]
  7082    20 tsutsui   26 CPU/2     13:08 47.66% 47.66% Composito firefox
  7082   263 tsutsui   31 parked/0   0:14 15.36% 15.19% -         firefox
  7082   264 tsutsui   36 parked/2   0:12 15.06% 14.89% -         firefox
  7082     1 tsutsui   36 CPU/3      5:24 14.45% 14.45% -         firefox
  1086     1 tsutsui   83 select/2  38:31 13.04% 13.04% -         X
  7082   265 tsutsui   34 parked/0   0:12  6.02%  5.96% -         firefox
  1106     1 tsutsui   85 select/0   7:11  1.90%  1.90% -         ruby26

 ---
 load averages:  2.92,  4.02,  3.61;               up 0+03:10:00        18:17:33
 464 threads: 20 idle, 2 runnable, 394 sleeping, 32 stopped, 12 zombie, 4 on CPU
 CPU states: 15.3% user,  0.0% nice, 27.8% system,  0.0% interrupt, 56.8% idle
 Memory: 1448M Act, 759M Inact, 315M Wired, 213M Exec, 609M File, 103M Free
 Swap: 8972M Total, 186M Used, 8786M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/0     26:50 99.02% 99.02% pgdaemon  [system]
  7082    20 tsutsui   29 RUN/3     13:44 52.29% 52.29% Composito firefox
  7082     1 tsutsui   36 CPU/2      5:33 15.87% 15.87% -         firefox
  1086     1 tsutsui   82 select/3  38:42 13.28% 13.28% -         X
  7082   273 tsutsui   38 parked/2   0:02  8.71%  5.66% -         firefox
  7082   274 tsutsui   40 CPU/1      0:02  5.55%  3.61% -         firefox
  7082   177 tsutsui   43 parked/3   0:10  1.56%  1.56% MediaPDec firefox
  7082   170 tsutsui   43 parked/3   0:09  0.73%  0.73% MediaPDec firefox
  7082   163 tsutsui   43 parked/2   0:08  0.24%  0.24% MediaDeco firefox
  1118     2 tsutsui   85 select/2   0:21  0.10%  0.10% -         pulseaudio
  1106     1 tsutsui   85 select/1   7:12  0.00%  0.00% -         ruby26

 ---
 load averages:  2.54,  3.69,  3.51;               up 0+03:11:00        18:18:33
 467 threads: 18 idle, 4 runnable, 398 sleeping, 32 stopped, 11 zombie, 4 on CPU
 CPU states: 34.8% user,  0.0% nice, 30.8% system,  0.0% interrupt, 34.3% idle
 Memory: 1495M Act, 750M Inact, 316M Wired, 213M Exec, 609M File, 65M Free
 Swap: 8972M Total, 186M Used, 8786M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/3     27:50 99.02% 99.02% pgdaemon  [system]
  7082    20 tsutsui   76 select/1  14:23 63.96% 63.96% Composito firefox
  1086     1 tsutsui   32 RUN/0     38:54 21.97% 21.97% -         X
  7082   275 tsutsui   34 parked/0   0:11 20.67% 20.31% -         firefox
  7082   274 tsutsui   36 parked/1   0:11 14.61% 14.36% -         firefox
  7082     1 tsutsui   37 parked/2   5:42 13.48% 13.48% -         firefox
  7082   273 tsutsui   34 RUN/0      0:12 11.48% 11.28% -         firefox
  7082   179 tsutsui   43 parked/1   0:03  0.05%  0.05% MediaPlay firefox
  7082   164 tsutsui   43 parked/1   0:03  0.05%  0.05% MediaPlay firefox
  1106     1 tsutsui   85 select/0   7:13  0.00%  0.00% -         ruby26

 ---
 load averages:  2.49,  3.45,  3.43;               up 0+03:12:00        18:19:33
 464 threads: 20 idle, 2 runnable, 396 sleeping, 32 stopped, 11 zombie, 3 on CPU
 CPU states: 27.3% user,  0.0% nice, 30.5% system,  0.0% interrupt, 42.1% idle
 Memory: 1435M Act, 751M Inact, 316M Wired, 213M Exec, 553M File, 123M Free
 Swap: 8972M Total, 190M Used, 8782M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/1     28:50 98.97% 98.97% pgdaemon  [system]
  7082    20 tsutsui   76 select/2  14:57 56.69% 56.69% Composito firefox
  7082     1 tsutsui   82 select/3   5:51 17.33% 17.33% -         firefox
  1086     1 tsutsui   34 CPU/0     39:05 16.36% 16.36% -         X
  7082   284 tsutsui   39 parked/0   0:01 10.82%  5.18% -         firefox
  7082   283 tsutsui   38 parked/3   0:01  8.78%  4.20% -         firefox
  1106     1 tsutsui   85 select/0   7:13  1.46%  1.46% -         ruby26
  6548     1 tsutsui   85 select/2   0:02  1.17%  1.17% -         medit

 ---
 load averages:  2.54,  3.30,  3.37;               up 0+03:13:00        18:20:33
 462 threads: 19 idle, 2 runnable, 393 sleeping, 32 stopped, 12 zombie, 4 on CPU
 CPU states: 32.6% user,  0.0% nice, 31.7% system,  0.1% interrupt, 35.5% idle
 Memory: 1494M Act, 750M Inact, 316M Wired, 213M Exec, 553M File, 66M Free
 Swap: 8972M Total, 190M Used, 8782M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/2     29:50 99.02% 99.02% pgdaemon  [system]
  7082    20 tsutsui   76 select/1  15:34 57.91% 57.91% Composito firefox
  1086     1 tsutsui   32 CPU/3     39:16 17.43% 17.43% -         X
  7082   285 tsutsui   37 parked/1   0:10 17.44% 16.99% -         firefox
  7082   283 tsutsui   38 RUN/2      0:10 13.98% 13.62% -         firefox
  7082     1 tsutsui   33 parked/3   6:01 13.33% 13.33% -         firefox
  7082   284 tsutsui   41 parked/0   0:09 12.03% 11.72% -         firefox
  1118     2 tsutsui   85 select/1   0:23  0.78%  0.78% -         pulseaudio
   488     1 tsutsui   85 select/3   0:09  0.24%  0.24% -         ibus-ui-gtk3
  1086     3 tsutsui   85 select/1   0:09  0.05%  0.05% -         X
  1106     1 tsutsui   85 select/1   7:14  0.00%  0.00% -         ruby26

 ---
 load averages:  2.67,  3.20,  3.33;               up 0+03:14:00        18:21:33
 460 threads: 18 idle, 3 runnable, 390 sleeping, 32 stopped, 13 zombie, 4 on CPU
 CPU states: 27.2% user,  0.0% nice, 31.2% system,  0.0% interrupt, 41.4% idle
 Memory: 1411M Act, 750M Inact, 316M Wired, 213M Exec, 553M File, 149M Free
 Swap: 8972M Total, 190M Used, 8782M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/2     30:50 98.93% 98.93% pgdaemon  [system]
  7082    20 tsutsui   27 RUN/3     16:11 60.21% 60.21% Composito firefox
  1086     1 tsutsui   80 select/0  39:29 16.99% 16.99% -         X
  7082     1 tsutsui   82 select/1   6:12 16.46% 16.46% -         firefox
  7082   178 tsutsui   85 select/3   0:01  1.32%  1.32% -         firefox
  7082   170 tsutsui   43 parked/0   0:10  0.34%  0.34% MediaPDec firefox
  1106     1 tsutsui   85 select/3   7:15  0.00%  0.00% -         ruby26
     0    17 root      43 lnxsys/1   1:31  0.00%  0.00% lnxsyswq  [system]
     0    99 root     124 syncer/3   0:30  0.00%  0.00% ioflush   [system]
  1118     2 tsutsui   85 select/3   0:24  0.00%  0.00% -         pulseaudio

 ---

 [ firefox + HTML5 video stopped here ]

 ---
 load averages:  1.36,  2.72,  3.14;               up 0+03:15:00        18:22:33
 455 threads: 18 idle, 1 runnable, 387 sleeping, 32 stopped, 13 zombie, 4 on CPU
 CPU states:  0.3% user,  0.0% nice, 25.3% system,  0.0% interrupt, 74.3% idle
 Memory: 1340M Act, 669M Inact, 310M Wired, 213M Exec, 554M File, 307M Free
 Swap: 8972M Total, 190M Used, 8782M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/1     31:51 99.02% 99.02% pgdaemon  [system]
  1106     1 tsutsui   85 select/0   7:16  2.64%  2.64% -         ruby26
  7082    20 tsutsui   43 parked/3  16:12  2.54%  2.54% Composito firefox
  1086     1 tsutsui   85 select/0  39:31  1.46%  1.46% -         X
  7082     1 tsutsui   85 select/3   6:15  1.46%  1.46% -         firefox
   918     1 tsutsui   85 ttyraw/2   0:02  0.44%  0.44% -         kermit
     0    17 root      43 lnxsys/0   1:31  0.00%  0.00% lnxsyswq  [system]
     0    99 root     124 syncer/0   0:30  0.00%  0.00% ioflush   [system]
  1118     2 tsutsui   85 select/0   0:24  0.00%  0.00% -         pulseaudio
   160     1 tsutsui   85 select/3   0:18  0.00%  0.00% -         kterm

 ---
 load averages:  0.69,  2.27,  2.94;               up 0+03:16:00        18:23:33
 451 threads: 18 idle, 3 runnable, 382 sleeping, 32 stopped, 12 zombie, 4 on CPU
 CPU states:  0.8% user,  0.0% nice, 25.4% system,  0.0% interrupt, 73.8% idle
 Memory: 1342M Act, 664M Inact, 310M Wired, 213M Exec, 556M File, 309M Free
 Swap: 8972M Total, 190M Used, 8782M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    98 root     126 CPU/1     32:51 99.02% 99.02% pgdaemon  [system]
  1106     1 tsutsui   85 select/2   7:18  4.44%  4.44% -         ruby26
  1086     1 tsutsui   85 select/0  39:31  0.00%  0.00% -         X
  7082    20 tsutsui   43 parked/3  16:12  0.00%  0.00% Composito firefox
  7082     1 tsutsui   85 select/0   6:16  0.00%  0.00% -         firefox
     0    17 root      43 lnxsys/0   1:31  0.00%  0.00% lnxsyswq  [system]
     0    99 root     124 syncer/0   0:30  0.00%  0.00% ioflush   [system]
  1118     2 tsutsui   85 select/0   0:24  0.00%  0.00% -         pulseaudio
   160     1 tsutsui   85 select/3   0:18  0.00%  0.00% -         kterm
   867     4 tsutsui   85 select/0   0:17  0.00%  0.00% -         ibus-daemon
   867     1 tsutsui   85 select/0   0:14  0.00%  0.00% -         ibus-daemon
  1086     3 tsutsui   85 select/0   0:10  0.00%  0.00% -         X

 ---

  :

 ---
 load averages:  0.91,  0.76,  0.74;               up 0+04:07:30        19:15:03
 124 processes: 90 sleeping, 32 stopped, 2 on CPU
 CPU states:  0.0% user,  0.0% nice, 25.0% system,  0.0% interrupt, 74.9% idle
 Memory: 1437M Act, 771M Inact, 308M Wired, 213M Exec, 722M File, 116M Free
 Swap: 8972M Total, 187M Used, 8785M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/1     87:28 99.61% 99.61% [system]
  1106 tsutsui   43    0  1079M  378M parked/0  12:25  5.18%  5.18% ruby26
  1086 tsutsui   85    0   197M  102M select/0  41:19  4.25%  4.25% X

 ---

 vmstat -s 60 output during CPU=~100%:
 ---
      4096 bytes per page
         8 page colors
    832700 pages managed
     60342 pages free
    357560 pages active
    174830 pages inactive
         0 pages paging
     79437 pages wired
     48677 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    155348 kernel pool pages
    397762 anonymous pages
    159684 cached file pages
     54381 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
     48090 swap pages in use
     38097 swap allocations
 571167736 total faults taken
 572025051 traps
   6151654 device interrupts
  76097176 CPU context switches
   7620552 software interrupts
 784213321 system calls
     37949 pagein requests
      7809 pageout requests
         0 pages swapped in
    123557 pages swapped out
   1153923 forks total
    833401 forks blocked parent
    833401 forks shared address space with parent
  34779238 pagealloc zero wanted and avail
 174312295 pagealloc zero wanted and not avail
      5505 aborts of idle page zeroing
 440945942 pagealloc desired color avail
  41317461 pagealloc desired color not avail
 371972703 pagealloc local cpu avail
 110290700 pagealloc local cpu not avail
        17 faults with no memory
         0 faults with no anons
        15 faults had to wait on pages
         0 faults found released page
    233969 faults relock (232471 ok)
  20588058 anon page faults
     37521 anon retry faults
  42871646 amap copy faults
  15367225 neighbour anon page faults
 506973389 neighbour object page faults
 127416617 locked pager get faults
    196005 unlocked pager get faults
  13246783 anon faults
   7341632 anon copy on write faults
 102744734 object faults
  24669311 promote copy faults
 201209401 promote zero fill faults
       793 times daemon wokeup
    114784 revolutions of the clock hand
   4038337 pages freed by daemon
  10901027 pages scanned by daemon
    122868 anonymous pages scanned by daemon
   3915470 object pages scanned by daemon
   2279625 pages reactivated
      3280 pages found busy by daemon
    115747 total pending pageouts
  11899207 pages deactivated
 1173949525 total name lookups
 1110996690 good hits
  56424088 negative hits
   1018792 bad hits
    599001 false hits
   4635176 miss
    275778 too long
    531266 pass2 hits
   1006000 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
      7441 pages free
    391703 pages active
    193576 pages inactive
         0 pages paging
     79401 pages wired
      2499 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    155258 kernel pool pages
    398418 anonymous pages
    211804 cached file pages
     54458 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
     48088 swap pages in use
     38106 swap allocations
 571481759 total faults taken
 572339088 traps
   6190161 device interrupts
  76356058 CPU context switches
   7671880 software interrupts
 784505780 system calls
     37958 pagein requests
      7809 pageout requests
         0 pages swapped in
    123557 pages swapped out
   1153924 forks total
    833402 forks blocked parent
    833402 forks shared address space with parent
  34785858 pagealloc zero wanted and avail
 174314899 pagealloc zero wanted and not avail
      5608 aborts of idle page zeroing
 441315623 pagealloc desired color avail
  41354429 pagealloc desired color not avail
 372091719 pagealloc local cpu avail
 110578333 pagealloc local cpu not avail
        17 faults with no memory
         0 faults with no anons
        15 faults had to wait on pages
         0 faults found released page
    234000 faults relock (232502 ok)
  20589617 anon page faults
     37530 anon retry faults
  42872364 amap copy faults
  15367225 neighbour anon page faults
 506973825 neighbour object page faults
 127416751 locked pager get faults
    196027 unlocked pager get faults
  13248342 anon faults
   7341632 anon copy on write faults
 102744851 object faults
  24669328 promote copy faults
 201218590 promote zero fill faults
       793 times daemon wokeup
    118887 revolutions of the clock hand
   4291215 pages freed by daemon
  11607144 pages scanned by daemon
    122868 anonymous pages scanned by daemon
   4168348 object pages scanned by daemon
   2412568 pages reactivated
      3303 pages found busy by daemon
    115747 total pending pageouts
  12624344 pages deactivated
 1173953475 total name lookups
 1111000536 good hits
  56424113 negative hits
   1018811 bad hits
    599001 false hits
   4635229 miss
    275785 too long
    531266 pass2 hits
   1006005 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
     13293 pages free
    384629 pages active
    194766 pages inactive
         0 pages paging
     79397 pages wired
     11813 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    155293 kernel pool pages
    395048 anonymous pages
    208907 cached file pages
     54837 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
     48072 swap pages in use
     38129 swap allocations
 571546372 total faults taken
 572403707 traps
   6195009 device interrupts
  76397706 CPU context switches
   7687158 software interrupts
 784707192 system calls
     37981 pagein requests
      7809 pageout requests
         0 pages swapped in
    123557 pages swapped out
   1153926 forks total
    833404 forks blocked parent
    833404 forks shared address space with parent
  34790304 pagealloc zero wanted and avail
 174315985 pagealloc zero wanted and not avail
      5613 aborts of idle page zeroing
 441363218 pagealloc desired color avail
  41356991 pagealloc desired color not avail
 372119451 pagealloc local cpu avail
 110600758 pagealloc local cpu not avail
        17 faults with no memory
         0 faults with no anons
        15 faults had to wait on pages
         0 faults found released page
    234085 faults relock (232586 ok)
  20590298 anon page faults
     37552 anon retry faults
  42872809 amap copy faults
  15367227 neighbour anon page faults
 506974947 neighbour object page faults
 127417227 locked pager get faults
    196089 unlocked pager get faults
  13249022 anon faults
   7341632 anon copy on write faults
 102745137 object faults
  24669518 promote copy faults
 201224091 promote zero fill faults
       793 times daemon wokeup
    118888 revolutions of the clock hand
   4295483 pages freed by daemon
  11611504 pages scanned by daemon
    122868 anonymous pages scanned by daemon
   4172616 object pages scanned by daemon
   2412581 pages reactivated
      3303 pages found busy by daemon
    115747 total pending pageouts
  12628162 pages deactivated
 1173956237 total name lookups
 1111003060 good hits
  56424173 negative hits
   1018820 bad hits
    599001 false hits
   4635300 miss
    275883 too long
    531269 pass2 hits
   1006022 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
      7332 pages free
    392330 pages active
    193080 pages inactive
         0 pages paging
     79309 pages wired
      5862 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    155375 kernel pool pages
    396569 anonymous pages
    213313 cached file pages
     54837 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
     48070 swap pages in use
     38303 swap allocations
 571609612 total faults taken
 572466955 traps
   6200615 device interrupts
  76457315 CPU context switches
   7703597 software interrupts
 785023135 system calls
     38152 pagein requests
      7809 pageout requests
         0 pages swapped in
    123557 pages swapped out
   1153926 forks total
    833404 forks blocked parent
    833404 forks shared address space with parent
  34796650 pagealloc zero wanted and avail
 174319457 pagealloc zero wanted and not avail
      5619 aborts of idle page zeroing
 441434855 pagealloc desired color avail
  41357844 pagealloc desired color not avail
 372163237 pagealloc local cpu avail
 110629462 pagealloc local cpu not avail
        17 faults with no memory
         0 faults with no anons
        15 faults had to wait on pages
         0 faults found released page
    234291 faults relock (232792 ok)
  20591101 anon page faults
     37723 anon retry faults
  42873789 amap copy faults
  15367227 neighbour anon page faults
 506975130 neighbour object page faults
 127417316 locked pager get faults
    196124 unlocked pager get faults
  13249825 anon faults
   7341632 anon copy on write faults
 102745226 object faults
  24669518 promote copy faults
 201233875 promote zero fill faults
       793 times daemon wokeup
    118894 revolutions of the clock hand
   4299298 pages freed by daemon
  11674014 pages scanned by daemon
    122868 anonymous pages scanned by daemon
   4176431 object pages scanned by daemon
   2440817 pages reactivated
      3303 pages found busy by daemon
    115747 total pending pageouts
  12690889 pages deactivated
 1173959731 total name lookups
 1111006292 good hits
  56424233 negative hits
   1018821 bad hits
    599001 false hits
   4635302 miss
    276082 too long
    531269 pass2 hits
   1006026 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
      6928 pages free
    392318 pages active
    193483 pages inactive
         0 pages paging
     79309 pages wired
      5501 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    155303 kernel pool pages
    396630 anonymous pages
    213643 cached file pages
     54837 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
     48064 swap pages in use
     38323 swap allocations
 571659315 total faults taken
 572516639 traps
   6206759 device interrupts
  76524523 CPU context switches
   7719472 software interrupts
 785362761 system calls
     38172 pagein requests
      7809 pageout requests
         0 pages swapped in
    123557 pages swapped out
   1153926 forks total
    833404 forks blocked parent
    833404 forks shared address space with parent
  34798919 pagealloc zero wanted and avail
 174320897 pagealloc zero wanted and not avail
      5634 aborts of idle page zeroing
 441485663 pagealloc desired color avail
  41358775 pagealloc desired color not avail
 372199678 pagealloc local cpu avail
 110644760 pagealloc local cpu not avail
        17 faults with no memory
         0 faults with no anons
        15 faults had to wait on pages
         0 faults found released page
    234328 faults relock (232828 ok)
  20591171 anon page faults
     37742 anon retry faults
  42874205 amap copy faults
  15367228 neighbour anon page faults
 506975191 neighbour object page faults
 127417387 locked pager get faults
    196141 unlocked pager get faults
  13249894 anon faults
   7341632 anon copy on write faults
 102745297 object faults
  24669518 promote copy faults
 201237532 promote zero fill faults
       793 times daemon wokeup
    118894 revolutions of the clock hand
   4299298 pages freed by daemon
  11674014 pages scanned by daemon
    122868 anonymous pages scanned by daemon
   4176431 object pages scanned by daemon
   2440817 pages reactivated
      3303 pages found busy by daemon
    115747 total pending pageouts
  12690889 pages deactivated
 1173964655 total name lookups
 1111010750 good hits
  56424273 negative hits
   1018822 bad hits
    599001 false hits
   4635309 miss
    276500 too long
    531269 pass2 hits
   1006027 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

      4096 bytes per page
         8 page colors
    832700 pages managed
     11139 pages free
    387981 pages active
    193564 pages inactive
         0 pages paging
     79345 pages wired
      9438 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    155380 kernel pool pages
    401648 anonymous pages
    204405 cached file pages
     54837 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
     48063 swap pages in use
     38334 swap allocations
 571795544 total faults taken
 572652856 traps
   6217762 device interrupts
  76615644 CPU context switches
   7742758 software interrupts
 785837015 system calls
     38182 pagein requests
      7809 pageout requests
         0 pages swapped in
    123557 pages swapped out
   1153928 forks total
    833405 forks blocked parent
    833405 forks shared address space with parent
  34809731 pagealloc zero wanted and avail
 174330339 pagealloc zero wanted and not avail
      5635 aborts of idle page zeroing
 441623057 pagealloc desired color avail
  41363612 pagealloc desired color not avail
 372299528 pagealloc local cpu avail
 110687141 pagealloc local cpu not avail
        17 faults with no memory
         0 faults with no anons
        15 faults had to wait on pages
         0 faults found released page
    234428 faults relock (232926 ok)
  20592568 anon page faults
     37752 anon retry faults
  42875771 amap copy faults
  15367288 neighbour anon page faults
 506976395 neighbour object page faults
 127417869 locked pager get faults
    196231 unlocked pager get faults
  13251234 anon faults
   7341689 anon copy on write faults
 102745681 object faults
  24669614 promote copy faults
 201257608 promote zero fill faults
       793 times daemon wokeup
    118897 revolutions of the clock hand
   4309683 pages freed by daemon
  11775017 pages scanned by daemon
    122868 anonymous pages scanned by daemon
   4186816 object pages scanned by daemon
   2463970 pages reactivated
      3303 pages found busy by daemon
    115747 total pending pageouts
  12792264 pages deactivated
 1173968522 total name lookups
 1111014253 good hits
  56424377 negative hits
   1018827 bad hits
    599001 false hits
   4635343 miss
    276721 too long
    531269 pass2 hits
   1006029 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%


 ---
 Izumi Tsutsui

From: Andrew Doran <ad@netbsd.org>
To: gnats-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp
Cc: martin@netbsd.org
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Wed, 15 Jan 2020 13:19:48 +0000

 On Thu, Jan 02, 2020 at 10:30:01AM +0000, Izumi Tsutsui wrote:

 > The following reply was made to PR kern/54818; it has been noted by GNATS.
 > 
 > From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
 > To: ad@netbsd.org
 > Cc: gnats-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp
 > Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
 > Date: Thu, 2 Jan 2020 19:25:00 +0900
 > 
 >  >  Could you please try the following?
 >  >  
 >  >  	http://www.netbsd.org/~ad/2020/pdaemon-9.diff
 >  
 >  I guess uvm/uvm_pdpolicy_clockpro.c part (empty uvmpdpol_scanfini())
 >  isn't necessary, right?

 It's needed to make it compile, but it does not work on 9.0 or -current.

 >  Unfortunately the spin still occurs with the above diff on load:
 >  (Firefox with HTML5 video + build.sh -j4 release + serial terminal output)

 Thank you for the vmstat output, that was very helpful.  I think it trying
 to balance the queues but your other processes are competing with it, and
 because of uvm_pageqlock nobody can make progress.  This is not a problem on
 -current any more because uvm_pageqlock is gone..  I have updated the patch
 to try and help this situation.  I have not compiled it because I don't have
 a 9.0 system at the moment.  Could you please try it again?

 	http://www.netbsd.org/~ad/2020/uvm-9.diff 

 Thanks,
 Andrew

From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
To: ad@netbsd.org
Cc: gnats-bugs@netbsd.org, martin@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Wed, 15 Jan 2020 23:32:29 +0900

 ad@ wrote:

 > >  I guess uvm/uvm_pdpolicy_clockpro.c part (empty uvmpdpol_scanfini())
 > >  isn't necessary, right?
 > 
 > It's needed to make it compile, but it does not work on 9.0 or -current.

 Ah, I see for options PDPOLICY_CLOCKPRO case.

 > Thank you for the vmstat output, that was very helpful.  I think it trying
 > to balance the queues but your other processes are competing with it, and
 > because of uvm_pageqlock nobody can make progress.  This is not a problem on
 > -current any more because uvm_pageqlock is gone..  I have updated the patch
 > to try and help this situation.  I have not compiled it because I don't have
 > a 9.0 system at the moment.  Could you please try it again?
 > 
 > 	http://www.netbsd.org/~ad/2020/uvm-9.diff 

 Thanks, but unfortunately it fails because this change includes
 "TAILQ_ENTRY(vm_page) pdqueue" not available in netbsd-9:

 ---
 #   compile  GENERIC/uvm_pdpolicy_clock.o
 /s/netbsd-9/src/obj.i386/tooldir.NetBSD-9.0_RC1-i386/bin/i486--netbsdelf-gcc -msoft-float -mno-mmx -mno-sse -mno-avx -mindirect-branch=thunk -mindirect-branch-register -ffreestanding -fno-zero-initialized-in-bss -fno-delete-null-pointer-checks -O2 -fno-omit-frame-pointer -fstack-protector -Wstack-protector --param ssp-buffer-size=1 -fno-strict-aliasing -fno-common -std=gnu99 -Werror -Wall -Wno-main -Wno-format-zero-length -Wpointer-arith -Wmissing-prototypes -Wstrict-prototypes -Wold-style-definition -Wswitch -Wshadow -Wcast-qual -Wwrite-strings -Wno-unreachable-code -Wno-pointer-sign -Wno-attributes -Wextra -Wno-unused-parameter -Wold-style-definition -Wno-sign-compare --sysroot=/s/netbsd-9/src/obj.i386/destdir.i386 -Di386 -I. -I../../../../external/bsd/acpica/dist -I../../../../external/bsd/libnv/dist -I../../../../../common/lib/libx86emu -I../../../../../common/lib/libc/misc -I../../../../../common/include -I../../../../arch -I../../../.. -nostdinc -DCOMPAT_UTILS -DCOMPAT!
  _44 -D_KERNEL -D_KERNEL_OPT -std=gnu99 -I../../../../lib/libkern/../../../common/lib/libc/quad -I../../../../lib/libkern/../../../common/lib/libc/string -I../../../../lib/libkern/../../../common/lib/libc/arch/i386/string -D_FORTIFY_SOURCE=2 -I../../../../external/isc/atheros_hal/dist -I../../../../external/isc/atheros_hal/ic -I../../../../external/bsd/common/include -I../../../../external/bsd/common/include -I../../../../external/bsd/drm2/include -I../../../../external/bsd/drm2/include -I../../../../external/bsd/drm2/include/drm -I../../../../external/bsd/common/include -I../../../../external/bsd/drm2/dist/include -I../../../../external/bsd/drm2/dist/include/drm -I../../../../external/bsd/drm2/dist/uapi -I../../../../external/bsd/drm2/dist -D__KERNEL__ -DCONFIG_BACKLIGHT_CLASS_DEVICE=0 -DCONFIG_BACKLIGHT_CLASS_DEVICE_MODULE=0 -DCONFIG_DRM_FBDEV_EMULATION=1 -DCONFIG_FB=0 -I../../../../../common/include -I../../../../external/bsd/libnv/dist -I../../../../external/bsd/drm2/i91!
  5drm -I../../../../external/bsd/drm2/dist/drm/i915 -DCONFIG_DR!
  M_I915_FBDEV=1 -DCONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT=0 -DCONFIG_DRM_FBDEV_EMULATION=1 -I../../../../external/bsd/drm2/include/radeon -I../../../../external/bsd/drm2/radeon -I../../../../external/bsd/drm2/dist/drm/amd/include -I../../../../external/bsd/drm2/dist/drm/radeon -I../../../../external/bsd/drm2/dist/drm/nouveau -I../../../../external/bsd/drm2/dist/drm/nouveau/include -I../../../../external/bsd/drm2/dist/drm/nouveau/include/nvkm -I../../../../external/bsd/drm2/dist/drm/nouveau/nvkm -I../../../../external/bsd/drm2/nouveau -DCONFIG_NOUVEAU_DEBUG=5 -DCONFIG_NOUVEAU_DEBUG_DEFAULT=3 -I../../../../external/bsd/acpica/dist/include -c ../../../../uvm/uvm_pdpolicy_clock.c -o uvm_pdpolicy_clock.o
 In file included from ../../../../sys/timevar.h:64:0,
                  from ../../../../sys/time.h:307,
                  from ../../../../sys/param.h:145,
                  from ../../../../uvm/uvm_pdpolicy_clock.c:74:
 ../../../../uvm/uvm_pdpolicy_clock.c: In function 'uvmpdpol_balancequeue':
 ../../../../sys/queue.h:503:12: error: 'struct vm_page' has no member named 'pdqueue'
   if (((elm)->field.tqe_next = (head)->tqh_first) != TAILQ_END(head))\
             ^
 ../../../../uvm/uvm_pdpolicy_clock.c:270:2: note: in expansion of macro 'TAILQ_INSERT_HEAD'
   TAILQ_INSERT_HEAD(&pdpol_state.s_activeq, &marker, pdqueue);
   ^~~~~~~~~~~~~~~~~
 ../../../../sys/queue.h:504:20: error: 'struct vm_page' has no member named 'pdqueue'
    (head)->tqh_first->field.tqe_prev =   \
                     ^
 ../../../../uvm/uvm_pdpolicy_clock.c:270:2: note: in expansion of macro 'TAILQ_INSERT_HEAD'
   TAILQ_INSERT_HEAD(&pdpol_state.s_activeq, &marker, pdqueue);
   ^~~~~~~~~~~~~~~~~
  :
 ---

 Is it possible to keep struct vm_page for netbsd-9 branch?

 Thanks,
 ---
 Izumi Tsutsui

From: Andrew Doran <ad@netbsd.org>
To: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
Cc: gnats-bugs@netbsd.org, martin@netbsd.org
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Wed, 15 Jan 2020 14:41:17 +0000

 On Wed, Jan 15, 2020 at 11:32:29PM +0900, Izumi Tsutsui wrote:

 > ad@ wrote:
 > 
 > > >  I guess uvm/uvm_pdpolicy_clockpro.c part (empty uvmpdpol_scanfini())
 > > >  isn't necessary, right?
 > > 
 > > It's needed to make it compile, but it does not work on 9.0 or -current.
 > 
 > Ah, I see for options PDPOLICY_CLOCKPRO case.
 > 
 > > Thank you for the vmstat output, that was very helpful.  I think it trying
 > > to balance the queues but your other processes are competing with it, and
 > > because of uvm_pageqlock nobody can make progress.  This is not a problem on
 > > -current any more because uvm_pageqlock is gone..  I have updated the patch
 > > to try and help this situation.  I have not compiled it because I don't have
 > > a 9.0 system at the moment.  Could you please try it again?
 > > 
 > > 	http://www.netbsd.org/~ad/2020/uvm-9.diff 
 > 
 > Thanks, but unfortunately it fails because this change includes
 > "TAILQ_ENTRY(vm_page) pdqueue" not available in netbsd-9:

 Oops.  Updated patch is here:

 	http://www.netbsd.org/~ad/2020/uvm-9.diff

 Thank you,
 Andrew

From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
To: ad@netbsd.org
Cc: gnats-bugs@netbsd.org, martin@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Thu, 16 Jan 2020 01:04:34 +0900

 > Oops.  Updated patch is here:
 > 
 > 	http://www.netbsd.org/~ad/2020/uvm-9.diff

 This version successfully builds. I'll check how it goes.

 Thanks,

 ---
 Izumi Tsutsui

From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
To: ad@netbsd.org
Cc: gnats-bugs@netbsd.org, martin@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Sat, 18 Jan 2020 01:45:58 +0900

 > > Oops.  Updated patch is here:
 > > 
 > > 	http://www.netbsd.org/~ad/2020/uvm-9.diff
 > 
 > This version successfully builds. I'll check how it goes.

 The pgdaemon spin still occurs with this patch.

 Here is a log of "date; top -1n; vmstat -s" per 30 sec when the spin started:

 ---
 Fri Jan 17 03:56:23 JST 2020
 load averages:  11.2,  10.9,  10.3;               up 0+19:30:09        03:56:24
 164 processes: 6 runnable, 154 sleeping, 4 on CPU
 CPU0 states: 85.1% user,  0.0% nice, 14.9% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 85.0% user,  0.0% nice, 15.0% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 91.1% user,  0.0% nice,  8.9% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 93.1% user,  0.0% nice,  6.9% system,  0.0% interrupt,  0.0% idle
 Memory: 1364M Act, 776M Inact, 309M Wired, 320M Exec, 575M File, 229M Free
 Swap: 8972M Total, 1391M Used, 7581M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
 11758 tsutsui   25    0   373M  131M RUN/0      7:40 54.26% 54.25% npviewer.bin
   892 tsutsui   25    0  1419M  660M RUN/0     92:22 48.73% 48.73% firefox
  1669 tsutsui   43    0   764M  312M parked/2  14:22 18.07% 18.07% firefox68
   881 tsutsui   25    0   241M   54M RUN/0     18:36 12.06% 12.06% X
 15660 tsutsui   85    0   155M   23M kqueue/0   0:06  2.64%  2.64% plugin-container
     0 root     124    0     0K   31M syncer/3   8:42  0.29%  0.29% [system]
 26793 root      25    0    32M   22M RUN/3      0:00  5.00%  0.24% cc1
  1087 tsutsui   85    0   962M  226M select/0  37:45  0.00%  0.00% ruby26
  1427 tsutsui   85    0    27M   11M select/0   0:18  0.00%  0.00% jwm
   876 tsutsui   85    0    18M 6384K select/0   0:18  0.00%  0.00% mlterm

      4096 bytes per page
         8 page colors
    832700 pages managed
     58591 pages free
    349365 pages active
    198589 pages inactive
         0 pages paging
     79120 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    141907 kernel pool pages
    398150 anonymous pages
    147061 cached file pages
     81863 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    356116 swap pages in use
    360514 swap allocations
 650180712 total faults taken
 651191960 traps
   5213640 device interrupts
  99101950 CPU context switches
  17911491 software interrupts
 1330414319 system calls
    356562 pagein requests
     43430 pageout requests
         0 pages swapped in
    693735 pages swapped out
   1775952 forks total
   1258088 forks blocked parent
   1258088 forks shared address space with parent
  31070706 pagealloc zero wanted and avail
 323645945 pagealloc zero wanted and not avail
      1391 aborts of idle page zeroing
 438957480 pagealloc desired color avail
  45020534 pagealloc desired color not avail
 366246088 pagealloc local cpu avail
 117731926 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       814 faults had to wait on pages
         0 faults found released page
    550806 faults relock (547733 ok)
  34293491 anon page faults
    355106 anon retry faults
  68456388 amap copy faults
  24831987 neighbour anon page faults
 791848936 neighbour object page faults
 200003437 locked pager get faults
    193430 unlocked pager get faults
  22570168 anon faults
  11722977 anon copy on write faults
 160212736 object faults
  39785779 promote copy faults
 342588366 promote zero fill faults
       736 times daemon wokeup
       735 revolutions of the clock hand
   3189940 pages freed by daemon
   9278147 pages scanned by daemon
    686571 anonymous pages scanned by daemon
   2503369 object pages scanned by daemon
   2508064 pages reactivated
      1509 pages found busy by daemon
    650305 total pending pageouts
  10297406 pages deactivated
 1797003112 total name lookups
 1703355097 good hits
  87397367 negative hits
   1053584 bad hits
    676254 false hits
   4216444 miss
    304366 too long
    640842 pass2 hits
   1114346 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 03:56:54 JST 2020
 load averages:  10.8,  10.8,  10.3;               up 0+19:30:40        03:56:55
 164 processes: 6 runnable, 154 sleeping, 4 on CPU
 CPU0 states: 80.2% user,  0.0% nice, 17.8% system,  2.0% interrupt,  0.0% idle
 CPU1 states: 89.0% user,  0.0% nice, 11.0% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 95.0% user,  0.0% nice,  5.0% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 98.0% user,  0.0% nice,  2.0% system,  0.0% interrupt,  0.0% idle
 Memory: 1530M Act, 775M Inact, 309M Wired, 320M Exec, 509M File, 63M Free
 Swap: 8972M Total, 1454M Used, 7518M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
   892 tsutsui   25    0  1412M  526M RUN/2     92:42 87.35% 70.61% firefox
 11758 tsutsui   25    0   374M  131M RUN/2      7:56 50.73% 50.73% npviewer.bin
 11203 root      25    0   164M  153M RUN/0      0:08 39.11% 25.44% cc1
 25294 root      25    0   130M  121M CPU/2      0:04 45.75% 18.02% cc1
  1669 tsutsui   43    0   771M  277M parked/2  14:24 15.04% 15.04% firefox68
  9417 root      25    0   118M  108M RUN/2      0:03 33.23% 13.09% cc1
   881 tsutsui   25    0   241M   52M RUN/0     18:39  9.67%  9.67% X
     0 root     221    0     0K   30M rfnode/1   8:43  7.76%  7.76% [system]
 22267 root      25    0    61M   50M CPU/3      0:01 28.79%  5.22% cc1
 28938 tsutsui   79    0    34M   21M select/0   0:15  3.96%  3.96% xv

      4096 bytes per page
         8 page colors
    832700 pages managed
     16211 pages free
    391650 pages active
    198489 pages inactive
         0 pages paging
     79048 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    142114 kernel pool pages
    456945 anonymous pages
    130379 cached file pages
     81863 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    372351 swap pages in use
    360978 swap allocations
 651573200 total faults taken
 652586299 traps
   5223350 device interrupts
  99373429 CPU context switches
  17930911 software interrupts
 1332623461 system calls
    357026 pagein requests
     45770 pageout requests
         0 pages swapped in
    731092 pages swapped out
   1777287 forks total
   1259267 forks blocked parent
   1259267 forks shared address space with parent
  31074257 pagealloc zero wanted and avail
 324692215 pagealloc zero wanted and not avail
      1392 aborts of idle page zeroing
 440136590 pagealloc desired color avail
  45077629 pagealloc desired color not avail
 367041731 pagealloc local cpu avail
 118172488 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       817 faults had to wait on pages
         0 faults found released page
    551489 faults relock (548411 ok)
  34304729 anon page faults
    355568 anon retry faults
  68536430 amap copy faults
  24838097 neighbour anon page faults
 792660155 neighbour object page faults
 200187362 locked pager get faults
    193646 unlocked pager get faults
  22578008 anon faults
  11726371 anon copy on write faults
 160377029 object faults
  39805410 promote copy faults
 343628985 promote zero fill faults
       749 times daemon wokeup
       748 revolutions of the clock hand
   3250066 pages freed by daemon
   9379936 pages scanned by daemon
    723764 anonymous pages scanned by daemon
   2526302 object pages scanned by daemon
   2542833 pages reactivated
      1509 pages found busy by daemon
    685322 total pending pageouts
  10398934 pages deactivated
 1800556371 total name lookups
 1706746533 good hits
  87554884 negative hits
   1054683 bad hits
    676464 false hits
   4219233 miss
    304574 too long
    641491 pass2 hits
   1115151 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 03:57:25 JST 2020
 load averages:  10.2,  10.7,  10.2;               up 0+19:31:11        03:57:26
 152 processes: 2 runnable, 144 sleeping, 2 zombie, 4 on CPU
 CPU0 states: 61.4% user,  0.0% nice, 35.6% system,  1.0% interrupt,  2.0% idle
 CPU1 states: 42.0% user,  0.0% nice, 50.0% system,  0.0% interrupt,  8.0% idle
 CPU2 states: 73.3% user,  0.0% nice, 23.8% system,  0.0% interrupt,  3.0% idle
 CPU3 states: 51.0% user,  0.0% nice, 47.0% system,  0.0% interrupt,  2.0% idle
 Memory: 1313M Act, 774M Inact, 309M Wired, 295M Exec, 577M File, 281M Free
 Swap: 8972M Total, 1462M Used, 7510M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
 11758 tsutsui   84    0   370M  129M select/0   8:16 52.69% 52.69% npviewer.bin
   892 tsutsui   26    0  1611M  698M CPU/1     93:01 54.50% 52.39% firefox
   881 tsutsui   26    0   241M   52M RUN/1     18:45 14.84% 14.84% X
  1669 tsutsui   43    0   765M  268M parked/2  14:26  3.08%  3.08% firefox68
     0 root     221    0     0K   30M rfnode/3   8:43  0.98%  0.98% [system]
 20852 root      26    0    31M   20M CPU/2      0:00 13.00%  0.63% cc1
 28938 tsutsui   79    0    34M   21M select/0   0:17  0.49%  0.49% xv
  3799 root      85    0  7192K 1428K select/0   0:00  0.44%  0.44% nbmake
  9897 root      77    0  9780K 5156K pipe_w/1   0:00  1.88%  0.34% nbmake
  7718 root      26    0    26M   14M RUN/0      0:00  3.00%  0.15% cc1

      4096 bytes per page
         8 page colors
    832700 pages managed
     71775 pages free
    336283 pages active
    198136 pages inactive
         0 pages paging
     79048 pages wired
      6582 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    142426 kernel pool pages
    390190 anonymous pages
    147628 cached file pages
     75649 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    374204 swap pages in use
    362125 swap allocations
 653351619 total faults taken
 654367698 traps
   5236058 device interrupts
  99699492 CPU context switches
  17950439 software interrupts
 1335711321 system calls
    358173 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1782773 forks total
   1263943 forks blocked parent
   1263943 forks shared address space with parent
  31106956 pagealloc zero wanted and avail
 325738629 pagealloc zero wanted and not avail
      1392 aborts of idle page zeroing
 441504269 pagealloc desired color avail
  45099801 pagealloc desired color not avail
 368080948 pagealloc local cpu avail
 118523122 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       817 faults had to wait on pages
         0 faults found released page
    553234 faults relock (550147 ok)
  34353420 anon page faults
    356706 anon retry faults
  68725217 amap copy faults
  24869687 neighbour anon page faults
 794667492 neighbour object page faults
 200692437 locked pager get faults
    194244 unlocked pager get faults
  22609034 anon faults
  11744027 anon copy on write faults
 160783950 object faults
  39903551 promote copy faults
 344673336 promote zero fill faults
       751 times daemon wokeup
       750 revolutions of the clock hand
   3259111 pages freed by daemon
   9391960 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2526303 object pages scanned by daemon
   2545805 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10410891 pages deactivated
 1804911824 total name lookups
 1710888843 good hits
  87743171 negative hits
   1059789 bad hits
    677512 false hits
   4234946 miss
    307563 too long
    643890 pass2 hits
   1120211 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 03:57:56 JST 2020
 load averages:  10.2,  10.6,  10.2;               up 0+19:31:42        03:57:57
 163 processes: 7 runnable, 152 sleeping, 4 on CPU
 CPU0 states: 77.2% user,  0.0% nice, 21.8% system,  1.0% interrupt,  0.0% idle
 CPU1 states: 88.0% user,  0.0% nice, 12.0% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 86.1% user,  0.0% nice, 13.9% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 99.0% user,  0.0% nice,  1.0% system,  0.0% interrupt,  0.0% idle
 Memory: 1225M Act, 774M Inact, 309M Wired, 320M Exec, 572M File, 369M Free
 Swap: 8972M Total, 1461M Used, 7511M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
   892 tsutsui   26    0  1411M  520M RUN/3     93:22 52.63% 52.20% firefox
 11758 tsutsui   26    0   371M  129M RUN/1      8:33 47.51% 45.26% npviewer.bin
 28938 tsutsui   80    0    34M   21M select/0   0:19 18.41% 18.41% xv
   881 tsutsui   26    0   243M   54M RUN/0     18:50 12.70% 12.70% X
     0 root     126    0     0K   29M pgdaem/0   8:49  9.62%  9.62% [system]
  3133 root      26    0    55M   46M CPU/3      0:01 34.71%  6.30% cc1
 21221 root      26    0    34M   26M RUN/1      0:00 34.00%  1.66% cc1
  1669 tsutsui   43    0   773M  273M parked/2  14:28  1.42%  1.42% firefox68
 26996 root      26    0    28M   20M RUN/0      0:00 20.00%  0.98% cc1
  1087 tsutsui   85    0   986M  246M select/0  37:47  0.54%  0.54% ruby26

      4096 bytes per page
         8 page colors
    832700 pages managed
     94441 pages free
    313712 pages active
    198071 pages inactive
         0 pages paging
     79120 pages wired
      2783 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    142171 kernel pool pages
    362632 anonymous pages
    146408 cached file pages
     81863 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    374091 swap pages in use
    363158 swap allocations
 655031862 total faults taken
 656050164 traps
   5246032 device interrupts
  99988849 CPU context switches
  17968286 software interrupts
 1338760500 system calls
    359202 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1784994 forks total
   1266074 forks blocked parent
   1266074 forks shared address space with parent
  31128356 pagealloc zero wanted and avail
 326893463 pagealloc zero wanted and not avail
      1393 aborts of idle page zeroing
 442904338 pagealloc desired color avail
  45104226 pagealloc desired color not avail
 369333750 pagealloc local cpu avail
 118674814 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       817 faults had to wait on pages
         0 faults found released page
    554527 faults relock (551428 ok)
  34363124 anon page faults
    357723 anon retry faults
  68846819 amap copy faults
  24873111 neighbour anon page faults
 796114364 neighbour object page faults
 201023832 locked pager get faults
    194508 unlocked pager get faults
  22616801 anon faults
  11745953 anon copy on write faults
 161077554 object faults
  39941343 promote copy faults
 345834955 promote zero fill faults
       753 times daemon wokeup
       750 revolutions of the clock hand
   3259111 pages freed by daemon
   9391960 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2526303 object pages scanned by daemon
   2545805 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10410891 pages deactivated
 1812088016 total name lookups
 1717717722 good hits
  88079805 negative hits
   1062291 bad hits
    678847 false hits
   4241419 miss
    307932 too long
    645204 pass2 hits
   1122155 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 03:58:28 JST 2020
 load averages:  11.0,  10.8,  10.3;               up 0+19:32:14        03:58:29
 160 processes: 6 runnable, 150 sleeping, 4 on CPU
 CPU0 states: 45.0% user,  0.0% nice, 55.0% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 80.0% user,  0.0% nice, 20.0% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 40.6% user,  0.0% nice, 54.5% system,  0.0% interrupt,  5.0% idle
 CPU3 states: 90.0% user,  0.0% nice, 10.0% system,  0.0% interrupt,  0.0% idle
 Memory: 1275M Act, 755M Inact, 309M Wired, 320M Exec, 588M File, 339M Free
 Swap: 8972M Total, 1449M Used, 7523M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
   892 tsutsui   30    0  1405M  533M RUN/0     93:41 45.75% 45.75% firefox
 11758 tsutsui   25    0   371M  129M CPU/3      8:51 44.81% 44.34% npviewer.bin
     0 root     126    0     0K   30M CPU/0      9:00 44.24% 44.24% [system]
   881 tsutsui   26    0   243M   54M RUN/0     18:54 11.04% 11.04% X
  1669 tsutsui   84    0   773M  274M select/2  14:30 10.25% 10.25% firefox68
  2402 root      25    0    57M   47M RUN/0      0:01 43.32%  7.86% cc1
 28938 tsutsui   82    0    34M   21M select/2   0:20  3.76%  3.76% xv
 26803 root      25    0    52M   41M RUN/2      0:00 23.46%  3.27% cc1
 15660 tsutsui   85    0   155M   23M kqueue/0   0:07  0.78%  0.78% plugin-container
  9924 root      85    0  7356K 1408K select/3   0:03  0.44%  0.44% nbmake

      4096 bytes per page
         8 page colors
    832700 pages managed
     87491 pages free
    325765 pages active
    193234 pages inactive
         0 pages paging
     79116 pages wired
       474 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    141936 kernel pool pages
    365705 anonymous pages
    153651 cached file pages
     78760 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    371071 swap pages in use
    377745 swap allocations
 656412181 total faults taken
 657432928 traps
   5267621 device interrupts
 100266160 CPU context switches
  17998881 software interrupts
 1341384041 system calls
    373789 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1786789 forks total
   1267724 forks blocked parent
   1267724 forks shared address space with parent
  31146161 pagealloc zero wanted and avail
 327829160 pagealloc zero wanted and not avail
      1393 aborts of idle page zeroing
 444052695 pagealloc desired color avail
  45119233 pagealloc desired color not avail
 370317599 pagealloc local cpu avail
 118854328 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       898 faults had to wait on pages
         0 faults found released page
    569495 faults relock (566250 ok)
  34390313 anon page faults
    372247 anon retry faults
  68944465 amap copy faults
  24878847 neighbour anon page faults
 797244493 neighbour object page faults
 201282317 locked pager get faults
    194808 unlocked pager get faults
  22640642 anon faults
  11749157 anon copy on write faults
 161306037 object faults
  39971342 promote copy faults
 346776486 promote zero fill faults
       755 times daemon wokeup
       750 revolutions of the clock hand
   3259111 pages freed by daemon
   9391960 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2526303 object pages scanned by daemon
   2545805 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10410891 pages deactivated
 1817821136 total name lookups
 1723181875 good hits
  88342432 negative hits
   1063696 bad hits
    679521 false hits
   4245491 miss
    308121 too long
    646138 pass2 hits
   1123392 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 03:58:59 JST 2020
 load averages:  11.0,  10.8,  10.3;               up 0+19:32:45        03:59:00
 164 processes: 8 runnable, 152 sleeping, 4 on CPU
 CPU0 states: 36.6% user,  0.0% nice, 62.4% system,  1.0% interrupt,  0.0% idle
 CPU1 states: 32.7% user,  0.0% nice, 67.3% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 92.0% user,  0.0% nice,  8.0% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 87.1% user,  0.0% nice, 12.9% system,  0.0% interrupt,  0.0% idle
 Memory: 1272M Act, 755M Inact, 309M Wired, 320M Exec, 613M File, 341M Free
 Swap: 8972M Total, 1449M Used, 7523M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
 11758 tsutsui   26    0   371M  129M CPU/2      9:10 55.45% 55.32% npviewer.bin
   892 tsutsui   27    0  1402M  538M CPU/0     94:03 43.46% 43.46% firefox
     0 root     126    0     0K   29M CPU/1      9:06 36.38% 36.38% [system]
   881 tsutsui   26    0   244M   55M RUN/0     18:59 11.77% 11.77% X
  3432 root      25    0    37M   28M RUN/3      0:00 11.91%  2.64% cc1
  1669 tsutsui   43    0   775M  278M RUN/1     14:32  2.39%  2.39% firefox68
 28938 tsutsui   79    0    34M   21M select/0   0:22  0.49%  0.49% xv
  1087 tsutsui   85    0   986M  246M select/0  37:47  0.00%  0.00% ruby26
  1427 tsutsui   85    0    27M   11M select/0   0:18  0.00%  0.00% jwm
   876 tsutsui   85    0    18M 6384K select/3   0:18  0.00%  0.00% mlterm

      4096 bytes per page
         8 page colors
    832700 pages managed
     87384 pages free
    325566 pages active
    193198 pages inactive
         0 pages paging
     79075 pages wired
       754 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    142213 kernel pool pages
    359088 anonymous pages
    156888 cached file pages
     81863 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    371052 swap pages in use
    377875 swap allocations
 658452696 total faults taken
 659477064 traps
   5277424 device interrupts
 100576761 CPU context switches
  18016300 software interrupts
 1345006289 system calls
    373917 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1790842 forks total
   1271394 forks blocked parent
   1271394 forks shared address space with parent
  31164537 pagealloc zero wanted and avail
 329118733 pagealloc zero wanted and not avail
      1393 aborts of idle page zeroing
 445595179 pagealloc desired color avail
  45157246 pagealloc desired color not avail
 371587516 pagealloc local cpu avail
 119164909 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       898 faults had to wait on pages
         0 faults found released page
    570126 faults relock (566876 ok)
  34418333 anon page faults
    372371 anon retry faults
  69127256 amap copy faults
  24893785 neighbour anon page faults
 799615631 neighbour object page faults
 201827099 locked pager get faults
    195311 unlocked pager get faults
  22660274 anon faults
  11757541 anon copy on write faults
 161782418 object faults
  40039733 promote copy faults
 348058265 promote zero fill faults
       756 times daemon wokeup
       750 revolutions of the clock hand
   3259111 pages freed by daemon
   9391960 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2526303 object pages scanned by daemon
   2545805 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10410891 pages deactivated
 1825902110 total name lookups
 1730965392 good hits
  88626784 negative hits
   1066557 bad hits
    680883 false hits
   4254207 miss
    308287 too long
    647790 pass2 hits
   1125947 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 03:59:30 JST 2020
 load averages:  12.3,  11.1,  10.5;               up 0+19:33:16        03:59:31
 164 processes: 7 runnable, 153 sleeping, 4 on CPU
 CPU0 states: 70.3% user,  0.0% nice, 28.7% system,  1.0% interrupt,  0.0% idle
 CPU1 states: 94.1% user,  0.0% nice,  5.9% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 83.0% user,  0.0% nice, 17.0% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 93.1% user,  0.0% nice,  6.9% system,  0.0% interrupt,  0.0% idle
 Memory: 1318M Act, 754M Inact, 309M Wired, 320M Exec, 657M File, 295M Free
 Swap: 8972M Total, 1444M Used, 7528M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
   892 tsutsui   25    0  1385M  558M RUN/0     94:20 67.49% 64.31% firefox
     0 root     124    0     0K   30M syncer/0   9:29 57.37% 57.37% [system]
 11758 tsutsui   26    0   371M  129M CPU/1      9:24 43.72% 43.70% npviewer.bin
  1669 tsutsui   43    0   768M  273M parked/0  14:34 23.34% 23.34% firefox68
   881 tsutsui   25    0   245M   56M RUN/0     19:03  9.03%  9.03% X
 28938 tsutsui   79    0    34M   21M select/0   0:24  6.93%  6.93% xv
  1427 tsutsui   85    0    27M   11M select/0   0:18  0.10%  0.10% jwm
  1087 tsutsui   85    0   986M  247M select/1  37:47  0.00%  0.00% ruby26
   876 tsutsui   85    0    18M 6384K select/1   0:18  0.00%  0.00% mlterm
    77 tsutsui   85    0    28M 5544K select/1   0:17  0.00%  0.00% ibus-daemon

      4096 bytes per page
         8 page colors
    832700 pages managed
     75234 pages free
    337675 pages active
    192994 pages inactive
         0 pages paging
     79227 pages wired
      4235 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    142293 kernel pool pages
    359929 anonymous pages
    168105 cached file pages
     81863 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369748 swap pages in use
    392088 swap allocations
 660024818 total faults taken
 661052339 traps
   5298952 device interrupts
 100863542 CPU context switches
  18047106 software interrupts
 1347905740 system calls
    388130 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1793681 forks total
   1274027 forks blocked parent
   1274027 forks shared address space with parent
  31192620 pagealloc zero wanted and avail
 330112551 pagealloc zero wanted and not avail
      1394 aborts of idle page zeroing
 446732134 pagealloc desired color avail
  45249040 pagealloc desired color not avail
 372534473 pagealloc local cpu avail
 119446701 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       899 faults had to wait on pages
         0 faults found released page
    584665 faults relock (581294 ok)
  34449498 anon page faults
    386465 anon retry faults
  69263359 amap copy faults
  24901879 neighbour anon page faults
 801423251 neighbour object page faults
 202237860 locked pager get faults
    195636 unlocked pager get faults
  22686823 anon faults
  11762037 anon copy on write faults
 162147017 object faults
  40085891 promote copy faults
 349061752 promote zero fill faults
       757 times daemon wokeup
       750 revolutions of the clock hand
   3259111 pages freed by daemon
   9391960 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2526303 object pages scanned by daemon
   2545805 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10410891 pages deactivated
 1832408286 total name lookups
 1737229342 good hits
  88859280 negative hits
   1068820 bad hits
    682102 false hits
   4260317 miss
    308425 too long
    649237 pass2 hits
   1127739 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:00:01 JST 2020
 load averages:  12.2,  11.2,  10.5;               up 0+19:33:47        04:00:02
 156 processes: 6 runnable, 146 sleeping, 4 on CPU
 CPU0 states: 37.6% user,  0.0% nice, 61.4% system,  1.0% interrupt,  0.0% idle
 CPU1 states: 64.0% user,  0.0% nice, 33.0% system,  0.0% interrupt,  3.0% idle
 CPU2 states: 51.0% user,  0.0% nice, 40.0% system,  0.0% interrupt,  9.0% idle
 CPU3 states: 50.0% user,  0.0% nice, 49.0% system,  0.0% interrupt,  1.0% idle
 Memory: 1321M Act, 752M Inact, 309M Wired, 308M Exec, 689M File, 295M Free
 Swap: 8972M Total, 1444M Used, 7528M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   29M CPU/1      9:54 82.37% 82.37% [system]
 11758 tsutsui   29    0   371M  129M RUN/0      9:38 46.34% 46.34% npviewer.bin
   892 tsutsui   25    0  1384M  569M RUN/2     94:35 39.56% 39.16% firefox
   881 tsutsui   26    0   244M   56M RUN/0     19:07 11.13% 11.13% X
  1669 tsutsui   43    0   775M  276M parked/0  14:35 10.40% 10.40% firefox68
 28938 tsutsui   81    0    34M   21M select/0   0:25  3.13%  3.12% xv
 15660 tsutsui   85    0   155M   23M kqueue/0   0:07  2.83%  2.83% plugin-container
  1087 tsutsui   85    0   984M  223M select/0  37:49  0.98%  0.98% ruby26
  1427 tsutsui   85    0    27M   11M select/0   0:18  0.00%  0.00% jwm
   876 tsutsui   85    0    18M 6384K select/2   0:18  0.00%  0.00% mlterm

      4096 bytes per page
         8 page colors
    832700 pages managed
     75469 pages free
    338085 pages active
    192425 pages inactive
         0 pages paging
     79066 pages wired
       994 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    142449 kernel pool pages
    354560 anonymous pages
    176260 cached file pages
     78756 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369634 swap pages in use
    392444 swap allocations
 661612291 total faults taken
 662642910 traps
   5306836 device interrupts
 101130182 CPU context switches
  18062973 software interrupts
 1350987612 system calls
    388486 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1797297 forks total
   1276996 forks blocked parent
   1276996 forks shared address space with parent
  31221899 pagealloc zero wanted and avail
 331070354 pagealloc zero wanted and not avail
      1405 aborts of idle page zeroing
 447825296 pagealloc desired color avail
  45364173 pagealloc desired color not avail
 373459515 pagealloc local cpu avail
 119729954 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       938 faults had to wait on pages
         0 faults found released page
    585436 faults relock (582061 ok)
  34488137 anon page faults
    386856 anon retry faults
  69418921 amap copy faults
  24926857 neighbour anon page faults
 803345108 neighbour object page faults
 202683381 locked pager get faults
    196012 unlocked pager get faults
  22711402 anon faults
  11776093 anon copy on write faults
 162532642 object faults
  40145784 promote copy faults
 350025160 promote zero fill faults
       758 times daemon wokeup
       750 revolutions of the clock hand
   3259111 pages freed by daemon
   9391960 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2526303 object pages scanned by daemon
   2545805 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10410891 pages deactivated
 1839310302 total name lookups
 1743858862 good hits
  89120912 negative hits
   1071283 bad hits
    683079 false hits
   4267588 miss
    308578 too long
    650634 pass2 hits
   1129719 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:00:32 JST 2020
 load averages:  12.1,  11.3,  10.6;               up 0+19:34:18        04:00:33
 157 processes: 6 runnable, 147 sleeping, 4 on CPU
 CPU0 states: 28.7% user,  0.0% nice, 71.3% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 91.0% user,  0.0% nice,  8.0% system,  0.0% interrupt,  1.0% idle
 CPU2 states: 83.2% user,  0.0% nice, 16.8% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 44.6% user,  0.0% nice, 55.4% system,  0.0% interrupt,  0.0% idle
 Memory: 1358M Act, 755M Inact, 309M Wired, 308M Exec, 704M File, 254M Free
 Swap: 8972M Total, 1444M Used, 7529M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   29M CPU/3     10:26 94.34% 94.34% [system]
   892 tsutsui   31    0  1387M  594M CPU/1     94:54 65.67% 65.67% firefox
 11758 tsutsui   85    0   373M  132M RUN/3      9:53   128% 33.11% npviewer.bin
   881 tsutsui   26    0   245M   56M RUN/0     19:12 12.35% 12.35% X
 28938 tsutsui   79    0    34M   21M select/0   0:27  4.49%  4.49% xv
  1669 tsutsui   43    0   774M  279M parked/2  14:37  2.29%  2.29% firefox68
 15660 tsutsui   85    0   155M   23M kqueue/0   0:07  0.98%  0.98% plugin-container
 23170 root      77    0  8332K 3688K select/2   0:00  0.54%  0.20% nbmake
  1087 tsutsui   85    0   985M  224M select/0  37:49  0.00%  0.00% ruby26
  1427 tsutsui   85    0    27M   11M select/2   0:18  0.00%  0.00% jwm

      4096 bytes per page
         8 page colors
    832700 pages managed
     64679 pages free
    348088 pages active
    193106 pages inactive
         0 pages paging
     79075 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    142545 kernel pool pages
    361137 anonymous pages
    180199 cached file pages
     78934 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369537 swap pages in use
    392948 swap allocations
 663443808 total faults taken
 664478213 traps
   5314654 device interrupts
 101425360 CPU context switches
  18078523 software interrupts
 1354225779 system calls
    388990 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1801211 forks total
   1280613 forks blocked parent
   1280613 forks shared address space with parent
  31239192 pagealloc zero wanted and avail
 332191536 pagealloc zero wanted and not avail
      1407 aborts of idle page zeroing
 449028904 pagealloc desired color avail
  45545193 pagealloc desired color not avail
 374495742 pagealloc local cpu avail
 120078355 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       938 faults had to wait on pages
         0 faults found released page
    586290 faults relock (582897 ok)
  34511392 anon page faults
    387343 anon retry faults
  69595588 amap copy faults
  24938426 neighbour anon page faults
 805650018 neighbour object page faults
 203219302 locked pager get faults
    196362 unlocked pager get faults
  22728376 anon faults
  11782356 anon copy on write faults
 162998270 object faults
  40216075 promote copy faults
 351138273 promote zero fill faults
       758 times daemon wokeup
       750 revolutions of the clock hand
   3259111 pages freed by daemon
   9391960 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2526303 object pages scanned by daemon
   2545805 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10410891 pages deactivated
 1845345747 total name lookups
 1749703673 good hits
  89298569 negative hits
   1074415 bad hits
    683984 false hits
   4276053 miss
    309053 too long
    652565 pass2 hits
   1132049 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:01:03 JST 2020
 load averages:  10.6,  11.0,  10.5;               up 0+19:34:49        04:01:04
 157 processes: 6 runnable, 147 sleeping, 4 on CPU
 CPU0 states: 77.2% user,  0.0% nice, 21.8% system,  1.0% interrupt,  0.0% idle
 CPU1 states: 88.1% user,  0.0% nice, 11.9% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 80.2% user,  0.0% nice, 19.8% system,  0.0% interrupt,  0.0% idle
 CPU3 states:  3.0% user,  0.0% nice, 97.0% system,  0.0% interrupt,  0.0% idle
 Memory: 1349M Act, 751M Inact, 309M Wired, 308M Exec, 723M File, 265M Free
 Swap: 8972M Total, 1443M Used, 7529M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/3     10:57 97.66% 97.66% [system]
   892 tsutsui   26    0  1382M  559M RUN/0     95:13 55.86% 55.86% firefox
 11758 tsutsui   25    0   373M  132M RUN/1     10:08 45.52% 38.38% npviewer.bin
   881 tsutsui   26    0   246M   57M RUN/0     19:16 10.89% 10.89% X
  1669 tsutsui   43    0   773M  280M parked/2  14:39  3.66%  3.66% firefox68
 28938 tsutsui   79    0    34M   21M select/1   0:28  1.66%  1.66% xv
  1087 tsutsui   85    0   979M  225M select/1  37:51  0.44%  0.44% ruby26
 20434 root      26    0    26M   13M RUN/3      0:00  4.00%  0.20% cc1
  1427 tsutsui   85    0    27M   11M select/1   0:18  0.00%  0.00% jwm
   876 tsutsui   85    0    18M 6384K select/1   0:18  0.00%  0.00% mlterm

      4096 bytes per page
         8 page colors
    832700 pages managed
     68199 pages free
    345168 pages active
    192365 pages inactive
         0 pages paging
     79082 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    142676 kernel pool pages
    352677 anonymous pages
    185183 cached file pages
     78756 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369350 swap pages in use
    393458 swap allocations
 665097224 total faults taken
 666135539 traps
   5322729 device interrupts
 101697732 CPU context switches
  18094279 software interrupts
 1357125488 system calls
    389500 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1804917 forks total
   1283805 forks blocked parent
   1283805 forks shared address space with parent
  31262053 pagealloc zero wanted and avail
 333200077 pagealloc zero wanted and not avail
      1407 aborts of idle page zeroing
 450235741 pagealloc desired color avail
  45617430 pagealloc desired color not avail
 375500697 pagealloc local cpu avail
 120352474 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    587152 faults relock (583758 ok)
  34544063 anon page faults
    387856 anon retry faults
  69772121 amap copy faults
  24958305 neighbour anon page faults
 807565252 neighbour object page faults
 203683186 locked pager get faults
    196710 unlocked pager get faults
  22750500 anon faults
  11792903 anon copy on write faults
 163385468 object faults
  40292760 promote copy faults
 352144835 promote zero fill faults
       758 times daemon wokeup
       750 revolutions of the clock hand
   3259111 pages freed by daemon
   9391960 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2526303 object pages scanned by daemon
   2545805 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10410891 pages deactivated
 1849903959 total name lookups
 1754120822 good hits
  89427790 negative hits
   1077306 bad hits
    684720 false hits
   4283903 miss
    309418 too long
    654187 pass2 hits
   1134145 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:01:34 JST 2020
 load averages:  10.5,  10.9,  10.5;               up 0+19:35:20        04:01:35
 162 processes: 10 runnable, 148 sleeping, 4 on CPU
 CPU0 states: 43.6% user,  0.0% nice, 56.4% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 74.0% user,  0.0% nice, 26.0% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 53.5% user,  0.0% nice, 45.5% system,  0.0% interrupt,  1.0% idle
 CPU3 states: 70.0% user,  0.0% nice, 30.0% system,  0.0% interrupt,  0.0% idle
 Memory: 1451M Act, 748M Inact, 309M Wired, 308M Exec, 741M File, 166M Free
 Swap: 8972M Total, 1443M Used, 7530M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   29M CPU/3     11:28 97.85% 97.85% [system]
   892 tsutsui   43    0  1401M  620M RUN/0     95:29 50.63% 50.63% firefox
 11758 tsutsui   25    0   373M  132M RUN/1     10:23 41.21% 39.84% npviewer.bin
   881 tsutsui   26    0   247M   58M RUN/3     19:20 11.23% 11.23% X
 28938 tsutsui   31    0    29M   20M CPU/2      0:30  7.67%  7.67% xv
  1669 tsutsui   43    0   777M  284M RUN/0     14:41  3.76%  3.76% firefox68
  1087 tsutsui   85    0   956M  225M select/0  37:52  1.76%  1.76% ruby26
  1427 tsutsui   85    0    27M   11M select/3   0:18  0.00%  0.00% jwm
   876 tsutsui   85    0    18M 6384K select/3   0:18  0.00%  0.00% mlterm
    77 tsutsui   85    0    28M 5544K select/0   0:17  0.00%  0.00% ibus-daemon

      4096 bytes per page
         8 page colors
    832700 pages managed
     42579 pages free
    371489 pages active
    191471 pages inactive
         0 pages paging
     79075 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    142804 kernel pool pages
    373565 anonymous pages
    189696 cached file pages
     78774 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369282 swap pages in use
    393554 swap allocations
 666524793 total faults taken
 667566069 traps
   5329704 device interrupts
 101974350 CPU context switches
  18109428 software interrupts
 1359838714 system calls
    389596 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1808379 forks total
   1286595 forks blocked parent
   1286595 forks shared address space with parent
  31286308 pagealloc zero wanted and avail
 334056003 pagealloc zero wanted and not avail
      1407 aborts of idle page zeroing
 451246407 pagealloc desired color avail
  45707352 pagealloc desired color not avail
 376370280 pagealloc local cpu avail
 120583479 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    587670 faults relock (584271 ok)
  34582575 anon page faults
    387948 anon retry faults
  69914417 amap copy faults
  24984104 neighbour anon page faults
 809228406 neighbour object page faults
 204076014 locked pager get faults
    197132 unlocked pager get faults
  22774517 anon faults
  11807395 anon copy on write faults
 163718210 object faults
  40352846 promote copy faults
 353002569 promote zero fill faults
       758 times daemon wokeup
       750 revolutions of the clock hand
   3259111 pages freed by daemon
   9391960 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2526303 object pages scanned by daemon
   2545805 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10410891 pages deactivated
 1853928558 total name lookups
 1758016764 good hits
  89547332 negative hits
   1079426 bad hits
    685106 false hits
   4290344 miss
    309586 too long
    655079 pass2 hits
   1135612 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:02:05 JST 2020
 load averages:  10.9,  11.0,  10.5;               up 0+19:35:51        04:02:06
 153 processes: 5 runnable, 143 sleeping, 1 zombie, 4 on CPU
 CPU0 states:  7.9% user,  0.0% nice, 91.1% system,  1.0% interrupt,  0.0% idle
 CPU1 states: 81.0% user,  0.0% nice, 10.0% system,  0.0% interrupt,  9.0% idle
 CPU2 states: 82.2% user,  0.0% nice, 17.8% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 57.0% user,  0.0% nice, 43.0% system,  0.0% interrupt,  0.0% idle
 Memory: 1461M Act, 750M Inact, 309M Wired, 296M Exec, 769M File, 154M Free
 Swap: 8972M Total, 1442M Used, 7530M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   27M CPU/0     12:00 98.29% 98.29% [system]
   892 tsutsui   25    0  1422M  632M RUN/1     95:47 48.33% 47.61% firefox
 11758 tsutsui   85    0   373M  132M futex/0   10:36 42.92% 42.92% npviewer.bin
   881 tsutsui   26    0   247M   58M RUN/0     19:24 11.87% 11.87% X
 28938 tsutsui   80    0    24M   16M select/0   0:32  6.30%  6.30% xv
  1669 tsutsui   43    0   782M  288M parked/2  14:42  4.98%  4.98% firefox68
  1826 root      85    0  7040K 1408K pipe_w/2   0:03  1.17%  1.17% nbmake
  1427 tsutsui   85    0    27M   11M select/2   0:18  0.20%  0.20% jwm
  1087 tsutsui   85    0   959M  226M select/0  37:52  0.00%  0.00% ruby26
   876 tsutsui   85    0    18M 6384K select/2   0:18  0.00%  0.00% mlterm

      4096 bytes per page
         8 page colors
    832700 pages managed
     39205 pages free
    374191 pages active
    192111 pages inactive
         0 pages paging
     79048 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    142960 kernel pool pages
    372678 anonymous pages
    196944 cached file pages
     75728 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369219 swap pages in use
    393842 swap allocations
 668182384 total faults taken
 669226872 traps
   5336426 device interrupts
 102241642 CPU context switches
  18124578 software interrupts
 1362578154 system calls
    389884 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1811710 forks total
   1289567 forks blocked parent
   1289567 forks shared address space with parent
  31302128 pagealloc zero wanted and avail
 335119658 pagealloc zero wanted and not avail
      1416 aborts of idle page zeroing
 452392474 pagealloc desired color avail
  45836084 pagealloc desired color not avail
 377329012 pagealloc local cpu avail
 120899546 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    588183 faults relock (584776 ok)
  34607115 anon page faults
    388228 anon retry faults
  70065694 amap copy faults
  24997892 neighbour anon page faults
 811194167 neighbour object page faults
 204528944 locked pager get faults
    197357 unlocked pager get faults
  22791485 anon faults
  11814959 anon copy on write faults
 164115286 object faults
  40408699 promote copy faults
 354060224 promote zero fill faults
       758 times daemon wokeup
      1376 revolutions of the clock hand
   3259111 pages freed by daemon
   9391960 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2526303 object pages scanned by daemon
   2545805 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10411594 pages deactivated
 1858794322 total name lookups
 1762740523 good hits
  89678448 negative hits
   1082202 bad hits
    685633 false hits
   4297401 miss
    310115 too long
    656741 pass2 hits
   1137865 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:02:36 JST 2020
 load averages:  11.5,  11.1,  10.6;               up 0+19:36:22        04:02:37
 162 processes: 7 runnable, 151 sleeping, 4 on CPU
 CPU0 states: 48.5% user,  0.0% nice, 50.5% system,  1.0% interrupt,  0.0% idle
 CPU1 states: 79.2% user,  0.0% nice, 20.8% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 35.3% user,  0.0% nice, 64.7% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 93.1% user,  0.0% nice,  6.9% system,  0.0% interrupt,  0.0% idle
 Memory: 1470M Act, 739M Inact, 309M Wired, 320M Exec, 765M File, 156M Free
 Swap: 8972M Total, 1442M Used, 7530M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   29M CPU/0     12:31 97.41% 97.41% [system]
   892 tsutsui   27    0  1382M  580M CPU/2     96:06 47.32% 47.17% firefox
 11758 tsutsui   25    0   373M  132M RUN/3     10:51 41.27% 41.21% npviewer.bin
   881 tsutsui   25    0   247M   58M RUN/1     19:28 13.77% 13.77% X
 28938 tsutsui   83    0    24M   16M select/1   0:32  0.93%  0.93% xv
   525 root      25    0    30M   20M RUN/2      0:00 13.00%  0.63% cc1
  1669 tsutsui   43    0   781M  289M parked/2  14:44  0.44%  0.44% firefox68
 23880 root      25    0    30M   20M RUN/0      0:00  1.00%  0.05% cc1
  1087 tsutsui   85    0   962M  227M select/1  37:52  0.00%  0.00% ruby26
  1427 tsutsui   85    0    27M   11M select/1   0:18  0.00%  0.00% jwm

      4096 bytes per page
         8 page colors
    832700 pages managed
     39836 pages free
    376300 pages active
    189074 pages inactive
         0 pages paging
     79054 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    143200 kernel pool pages
    366636 anonymous pages
    195817 cached file pages
     81975 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369208 swap pages in use
    393994 swap allocations
 669801360 total faults taken
 670849286 traps
   5345230 device interrupts
 102506859 CPU context switches
  18141032 software interrupts
 1365567680 system calls
    390036 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1816768 forks total
   1293896 forks blocked parent
   1293896 forks shared address space with parent
  31342389 pagealloc zero wanted and avail
 335985714 pagealloc zero wanted and not avail
      1417 aborts of idle page zeroing
 453470260 pagealloc desired color avail
  45948404 pagealloc desired color not avail
 378253563 pagealloc local cpu avail
 121165101 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    588895 faults relock (585488 ok)
  34650904 anon page faults
    388380 anon retry faults
  70306286 amap copy faults
  25026093 neighbour anon page faults
 813376643 neighbour object page faults
 205099350 locked pager get faults
    197917 unlocked pager get faults
  22820492 anon faults
  11829741 anon copy on write faults
 164557501 object faults
  40536888 promote copy faults
 354933372 promote zero fill faults
       758 times daemon wokeup
      1376 revolutions of the clock hand
   3259111 pages freed by daemon
   9391960 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2526303 object pages scanned by daemon
   2545805 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10411594 pages deactivated
 1863140091 total name lookups
 1766940627 good hits
  89806770 negative hits
   1085977 bad hits
    686142 false hits
   4309460 miss
    311115 too long
    658486 pass2 hits
   1141074 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:03:07 JST 2020
 load averages:  11.3,  11.1,  10.6;               up 0+19:36:53        04:03:08
 164 processes: 7 runnable, 153 sleeping, 4 on CPU
 CPU0 states: 79.2% user,  0.0% nice, 20.8% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 89.0% user,  0.0% nice, 11.0% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 18.0% user,  0.0% nice, 82.0% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 67.0% user,  0.0% nice, 33.0% system,  0.0% interrupt,  0.0% idle
 Memory: 1500M Act, 769M Inact, 309M Wired, 321M Exec, 771M File, 94M Free
 Swap: 8972M Total, 1442M Used, 7530M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   29M CPU/2     13:03 98.10% 98.10% [system]
 11758 tsutsui   26    0   373M  132M CPU/1     11:04 49.82% 49.80% npviewer.bin
   892 tsutsui   32    0  1420M  622M RUN/0     96:23 93.70% 44.82% firefox
   881 tsutsui   25    0   248M   59M RUN/2     19:32 10.50% 10.50% X
  1669 tsutsui   43    0   784M  291M parked/1  14:45  5.91%  5.91% firefox68
 28938 tsutsui   81    0    29M   20M select/2   0:33  1.37%  1.37% xv
  1146 root     109    0    30M   20M RUN/2      0:00 21.00%  1.03% cc1
  1087 tsutsui   85    0   963M  227M select/3  37:52  0.00%  0.00% ruby26
  1427 tsutsui   85    0    27M   11M select/0   0:18  0.00%  0.00% jwm
   876 tsutsui   85    0    18M 6384K select/0   0:18  0.00%  0.00% mlterm

      4096 bytes per page
         8 page colors
    832700 pages managed
     24720 pages free
    383543 pages active
    196755 pages inactive
         0 pages paging
     79094 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    143322 kernel pool pages
    379806 anonymous pages
    197612 cached file pages
     81975 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369206 swap pages in use
    394108 swap allocations
 671161404 total faults taken
 672211581 traps
   5351304 device interrupts
 102750003 CPU context switches
  18155792 software interrupts
 1368000755 system calls
    390150 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1819018 forks total
   1296104 forks blocked parent
   1296104 forks shared address space with parent
  31370613 pagealloc zero wanted and avail
 336875365 pagealloc zero wanted and not avail
      1418 aborts of idle page zeroing
 454483510 pagealloc desired color avail
  46024117 pagealloc desired color not avail
 379142724 pagealloc local cpu avail
 121364903 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    589334 faults relock (585921 ok)
  34658844 anon page faults
    388489 anon retry faults
  70412006 amap copy faults
  25027709 neighbour anon page faults
 814788623 neighbour object page faults
 205422553 locked pager get faults
    198242 unlocked pager get faults
  22827547 anon faults
  11830621 anon copy on write faults
 164843604 object faults
  40573986 promote copy faults
 355836742 promote zero fill faults
       758 times daemon wokeup
      8267 revolutions of the clock hand
   3260321 pages freed by daemon
   9552109 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2527513 object pages scanned by daemon
   2648640 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10579229 pages deactivated
 1867840933 total name lookups
 1771479294 good hits
  89961758 negative hits
   1087699 bad hits
    686649 false hits
   4314270 miss
    311263 too long
    659577 pass2 hits
   1142678 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:03:38 JST 2020
 load averages:  11.5,  11.1,  10.6;               up 0+19:37:24        04:03:39
 164 processes: 7 runnable, 152 sleeping, 1 zombie, 4 on CPU
 CPU0 states: 50.5% user,  0.0% nice, 45.5% system,  4.0% interrupt,  0.0% idle
 CPU1 states: 87.0% user,  0.0% nice, 12.0% system,  0.0% interrupt,  1.0% idle
 CPU2 states: 90.1% user,  0.0% nice,  9.9% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 40.6% user,  0.0% nice, 59.4% system,  0.0% interrupt,  0.0% idle
 Memory: 1430M Act, 769M Inact, 309M Wired, 321M Exec, 740M File, 164M Free
 Swap: 8972M Total, 1442M Used, 7530M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   29M CPU/0     13:34 98.14% 98.14% [system]
   892 tsutsui   25    0  1382M  567M RUN/3     96:43 56.35% 56.35% firefox
 11758 tsutsui   25    0   373M  132M RUN/0     11:17 36.82% 36.82% npviewer.bin
   881 tsutsui   77    0   248M   60M select/2  19:36 10.21% 10.21% X
 28938 tsutsui   81    0    29M   20M select/0   0:35  3.91%  3.91% xv
  1669 tsutsui   43    0   786M  293M parked/1  14:47  0.54%  0.54% firefox68
  1427 tsutsui   85    0    27M   11M select/1   0:18  0.10%  0.10% jwm
  1087 tsutsui   41    0   968M  230M CPU/1     37:53  0.00%  0.00% ruby26
   876 tsutsui   85    0    18M 6384K select/1   0:18  0.00%  0.00% mlterm
    77 tsutsui   85    0    28M 5544K select/1   0:17  0.00%  0.00% ibus-daemon

      4096 bytes per page
         8 page colors
    832700 pages managed
     42025 pages free
    366122 pages active
    196836 pages inactive
         0 pages paging
     79048 pages wired
       372 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    143300 kernel pool pages
    370403 anonymous pages
    189423 cached file pages
     82180 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369196 swap pages in use
    394274 swap allocations
 672299468 total faults taken
 673351607 traps
   5357310 device interrupts
 102997261 CPU context switches
  18170193 software interrupts
 1370272853 system calls
    390316 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1820104 forks total
   1297190 forks blocked parent
   1297190 forks shared address space with parent
  31392886 pagealloc zero wanted and avail
 337640310 pagealloc zero wanted and not avail
      1418 aborts of idle page zeroing
 455403132 pagealloc desired color avail
  46082479 pagealloc desired color not avail
 379860808 pagealloc local cpu avail
 121624803 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    589673 faults relock (586252 ok)
  34662635 anon page faults
    388648 anon retry faults
  70479457 amap copy faults
  25027709 neighbour anon page faults
 815631622 neighbour object page faults
 205611269 locked pager get faults
    198415 unlocked pager get faults
  22831331 anon faults
  11830621 anon copy on write faults
 165016226 object faults
  40590080 promote copy faults
 356616524 promote zero fill faults
       758 times daemon wokeup
      8354 revolutions of the clock hand
   3268963 pages freed by daemon
   9569377 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2536155 object pages scanned by daemon
   2655421 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10596589 pages deactivated
 1870900111 total name lookups
 1774429136 good hits
  90067225 negative hits
   1088792 bad hits
    686817 false hits
   4316701 miss
    311440 too long
    660412 pass2 hits
   1143550 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:04:09 JST 2020
 load averages:  11.3,  11.1,  10.6;               up 0+19:37:55        04:04:10
 156 processes: 7 runnable, 144 sleeping, 1 zombie, 4 on CPU
 CPU0 states:  8.9% user,  0.0% nice, 88.1% system,  3.0% interrupt,  0.0% idle
 CPU1 states: 79.2% user,  0.0% nice, 18.8% system,  0.0% interrupt,  2.0% idle
 CPU2 states: 75.2% user,  0.0% nice, 22.8% system,  0.0% interrupt,  2.0% idle
 CPU3 states: 66.3% user,  0.0% nice, 32.7% system,  0.0% interrupt,  1.0% idle
 Memory: 1450M Act, 745M Inact, 309M Wired, 309M Exec, 762M File, 168M Free
 Swap: 8972M Total, 1442M Used, 7530M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/0     14:05 97.71% 97.71% [system]
   892 tsutsui   33    0  1405M  597M RUN/0     96:59 54.15% 54.15% firefox
 11758 tsutsui   39    0   370M  129M CPU/3     11:29   136% 30.18% npviewer.bin
   881 tsutsui   26    0   249M   60M RUN/0     19:40 11.57% 11.57% X
 28938 tsutsui   80    0    29M   20M select/0   0:36  0.49%  0.49% xv
 14192 root      78    0  6836K 2532K select/1   0:00  0.35%  0.15% nbmake
  1669 tsutsui   36    0   787M  291M parked/2  14:49  0.10%  0.10% firefox68
 12849 root      77    0  9492K 4484K select/1   0:00  0.54%  0.10% nbmake
  2287 root      26    0  7180K 2532K RUN/2      0:00  0.17%  0.05% nbmake
  1087 tsutsui   85    0   967M  230M select/3  37:53  0.00%  0.00% ruby26

      4096 bytes per page
         8 page colors
    832700 pages managed
     42843 pages free
    371303 pages active
    190595 pages inactive
         0 pages paging
     79118 pages wired
      1090 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    143568 kernel pool pages
    366822 anonymous pages
    195176 cached file pages
     79018 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369146 swap pages in use
    394394 swap allocations
 673838090 total faults taken
 674893198 traps
   5364085 device interrupts
 103251880 CPU context switches
  18185486 software interrupts
 1373080228 system calls
    390436 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1823622 forks total
   1300256 forks blocked parent
   1300256 forks shared address space with parent
  31420394 pagealloc zero wanted and avail
 338580088 pagealloc zero wanted and not avail
      1421 aborts of idle page zeroing
 456469807 pagealloc desired color avail
  46190429 pagealloc desired color not avail
 380718328 pagealloc local cpu avail
 121941908 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    590252 faults relock (586823 ok)
  34692600 anon page faults
    388761 anon retry faults
  70638137 amap copy faults
  25045311 neighbour anon page faults
 817487979 neighbour object page faults
 206052966 locked pager get faults
    198874 unlocked pager get faults
  22851763 anon faults
  11840147 anon copy on write faults
 165388865 object faults
  40659135 promote copy faults
 357560740 promote zero fill faults
       758 times daemon wokeup
      8358 revolutions of the clock hand
   3270307 pages freed by daemon
   9571726 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2537499 object pages scanned by daemon
   2656231 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10598942 pages deactivated
 1876753428 total name lookups
 1780077047 good hits
  90262646 negative hits
   1090900 bad hits
    687517 false hits
   4323741 miss
    311577 too long
    661420 pass2 hits
   1145324 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:04:40 JST 2020
 load averages:  11.9,  11.3,  10.7;               up 0+19:38:26        04:04:41
 164 processes: 7 runnable, 153 sleeping, 4 on CPU
 CPU0 states:  0.0% user,  0.0% nice, 98.0% system,  2.0% interrupt,  0.0% idle
 CPU1 states: 73.3% user,  0.0% nice, 26.7% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 88.1% user,  0.0% nice, 11.9% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 85.1% user,  0.0% nice, 14.9% system,  0.0% interrupt,  0.0% idle
 Memory: 1448M Act, 744M Inact, 309M Wired, 321M Exec, 756M File, 169M Free
 Swap: 8972M Total, 1442M Used, 7530M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   29M CPU/0     14:37 98.00% 98.00% [system]
   892 tsutsui   43    0  1384M  565M RUN/0     97:14 44.73% 44.73% firefox
 11758 tsutsui   85    0   370M  129M futex/1   11:43 30.57% 30.57% npviewer.bin
   881 tsutsui   25    0   250M   61M RUN/1     19:43 10.45% 10.45% X
 28938 tsutsui   30    0    29M   20M CPU/1      0:38  8.06%  8.06% xv
  1669 tsutsui   43    0   779M  288M parked/2  14:50  5.27%  5.27% firefox68
 27366 root      27    0    27M   15M RUN/0      0:00  2.56%  0.24% cc1
  1087 tsutsui   85    0   968M  232M select/1  37:53  0.00%  0.00% ruby26
  1427 tsutsui   85    0    27M   11M select/1   0:18  0.00%  0.00% jwm
   876 tsutsui   85    0    18M 6384K select/1   0:18  0.00%  0.00% mlterm

      4096 bytes per page
         8 page colors
    832700 pages managed
     43326 pages free
    370836 pages active
    190526 pages inactive
         0 pages paging
     79053 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    143675 kernel pool pages
    364846 anonymous pages
    193438 cached file pages
     82131 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369146 swap pages in use
    394454 swap allocations
 675202462 total faults taken
 676259903 traps
   5370313 device interrupts
 103492107 CPU context switches
  18199619 software interrupts
 1375714727 system calls
    390496 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1825738 forks total
   1302317 forks blocked parent
   1302317 forks shared address space with parent
  31437938 pagealloc zero wanted and avail
 339468764 pagealloc zero wanted and not avail
      1421 aborts of idle page zeroing
 457392071 pagealloc desired color avail
  46336978 pagealloc desired color not avail
 381528261 pagealloc local cpu avail
 122200788 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    590535 faults relock (587099 ok)
  34700600 anon page faults
    388815 anon retry faults
  70745403 amap copy faults
  25047454 neighbour anon page faults
 819002839 neighbour object page faults
 206394785 locked pager get faults
    199097 unlocked pager get faults
  22858588 anon faults
  11841316 anon copy on write faults
 165697211 object faults
  40692607 promote copy faults
 358453159 promote zero fill faults
       758 times daemon wokeup
      8549 revolutions of the clock hand
   3270307 pages freed by daemon
   9571726 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2537499 object pages scanned by daemon
   2656231 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10599134 pages deactivated
 1882750521 total name lookups
 1785860598 good hits
  90468610 negative hits
   1092712 bad hits
    688354 false hits
   4328547 miss
    311700 too long
    662721 pass2 hits
   1146931 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:05:11 JST 2020
 load averages:  11.4,  11.2,  10.7;               up 0+19:38:57        04:05:12
 164 processes: 6 runnable, 154 sleeping, 4 on CPU
 CPU0 states:  0.0% user,  0.0% nice, 99.0% system,  1.0% interrupt,  0.0% idle
 CPU1 states: 82.2% user,  0.0% nice, 13.9% system,  0.0% interrupt,  4.0% idle
 CPU2 states: 89.1% user,  0.0% nice,  8.9% system,  0.0% interrupt,  2.0% idle
 CPU3 states: 72.3% user,  0.0% nice, 22.8% system,  0.0% interrupt,  5.0% idle
 Memory: 1564M Act, 768M Inact, 309M Wired, 320M Exec, 684M File, 29M Free
 Swap: 8972M Total, 1442M Used, 7530M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/0     15:08 99.07% 99.07% [system]
   892 tsutsui   35    0  1555M  725M RUN/2     97:32   150% 33.11% firefox
 11758 tsutsui   85    0   371M  130M futex/2   11:54 30.62% 30.62% npviewer.bin
   881 tsutsui   29    0   250M   62M RUN/0     19:47  8.98%  8.98% X
 28938 tsutsui   82    0    29M   20M select/2   0:39  3.17%  3.17% xv
  1177 tsutsui   85    0   130M   32M select/2   0:10  1.95%  1.95% ibus-ui-gtk3
 21126 root      25    0    36M   26M RUN/1      0:00 18.00%  0.88% cc1
  1669 tsutsui   43    0   787M  293M parked/2  14:51  0.78%  0.78% firefox68
 18197 root      25    0    37M   27M RUN/1      0:00 16.00%  0.78% cc1
  1190 tsutsui   85    0    11M 3632K select/2   0:00  0.15%  0.15% kterm

      4096 bytes per page
         8 page colors
    832700 pages managed
      7298 pages free
    400598 pages active
    196672 pages inactive
         0 pages paging
     79054 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    143712 kernel pool pages
    419287 anonymous pages
    175174 cached file pages
     81863 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369090 swap pages in use
    394618 swap allocations
 676552206 total faults taken
 677611748 traps
   5376412 device interrupts
 103696158 CPU context switches
  18214995 software interrupts
 1377793025 system calls
    390660 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1827198 forks total
   1303743 forks blocked parent
   1303743 forks shared address space with parent
  31465383 pagealloc zero wanted and avail
 340437048 pagealloc zero wanted and not avail
      1421 aborts of idle page zeroing
 458433553 pagealloc desired color avail
  46431834 pagealloc desired color not avail
 382344966 pagealloc local cpu avail
 122520421 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    590805 faults relock (587369 ok)
  34707301 anon page faults
    388979 anon retry faults
  70829390 amap copy faults
  25048794 neighbour anon page faults
 820103676 neighbour object page faults
 206640877 locked pager get faults
    199203 unlocked pager get faults
  22864553 anon faults
  11842052 anon copy on write faults
 165922011 object faults
  40713895 promote copy faults
 359439016 promote zero fill faults
       758 times daemon wokeup
     14362 revolutions of the clock hand
   3290283 pages freed by daemon
   9624555 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2557475 object pages scanned by daemon
   2674622 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10658079 pages deactivated
 1887007404 total name lookups
 1789967935 good hits
  90612985 negative hits
   1094137 bad hits
    688751 false hits
   4331734 miss
    311862 too long
    663763 pass2 hits
   1148061 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:05:42 JST 2020
 load averages:  11.4,  11.2,  10.7;               up 0+19:39:28        04:05:43
 157 processes: 5 runnable, 148 sleeping, 4 on CPU
 CPU0 states: 14.9% user,  0.0% nice, 83.2% system,  2.0% interrupt,  0.0% idle
 CPU1 states: 84.0% user,  0.0% nice, 14.0% system,  0.0% interrupt,  2.0% idle
 CPU2 states: 76.0% user,  0.0% nice, 24.0% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 60.0% user,  0.0% nice, 39.0% system,  0.0% interrupt,  1.0% idle
 Memory: 1323M Act, 765M Inact, 309M Wired, 308M Exec, 637M File, 274M Free
 Swap: 8972M Total, 1441M Used, 7531M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   29M CPU/0     15:40 97.95% 97.95% [system]
   892 tsutsui   25    0  1392M  589M RUN/3     97:52 68.59% 57.28% firefox
 11758 tsutsui   85    0   371M  130M futex/3   12:04 29.00% 29.00% npviewer.bin
   881 tsutsui   26    0   253M   64M CPU/2     19:50  8.40%  8.40% X
  1669 tsutsui   43    0   790M  296M parked/0  14:53  7.96%  7.96% firefox68
 28938 tsutsui   82    0    29M   20M select/3   0:40  3.86%  3.86% xv
 17963 root      25    0    29M   20M RUN/1      0:00  6.00%  0.29% cc1
  1087 tsutsui   85    0   976M  240M select/0  37:55  0.05%  0.05% ruby26
  5391 root      26    0  7080K 2268K RUN/0      0:00  1.00%  0.05% nbmake
  1427 tsutsui   85    0    27M   11M select/3   0:18  0.00%  0.00% jwm

      4096 bytes per page
         8 page colors
    832700 pages managed
     78809 pages free
    330260 pages active
    195536 pages inactive
         0 pages paging
     79050 pages wired
      1430 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    143790 kernel pool pages
    363079 anonymous pages
    165954 cached file pages
     75813 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    369001 swap pages in use
    395160 swap allocations
 678092235 total faults taken
 679155011 traps
   5384015 device interrupts
 103951137 CPU context switches
  18232471 software interrupts
 1380265824 system calls
    391198 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1830639 forks total
   1306470 forks blocked parent
   1306470 forks shared address space with parent
  31495932 pagealloc zero wanted and avail
 341404526 pagealloc zero wanted and not avail
      1424 aborts of idle page zeroing
 459643373 pagealloc desired color avail
  46458088 pagealloc desired color not avail
 383263705 pagealloc local cpu avail
 122837756 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    591592 faults relock (588143 ok)
  34750041 anon page faults
    389506 anon retry faults
  70996985 amap copy faults
  25076433 neighbour anon page faults
 821630396 neighbour object page faults
 207033052 locked pager get faults
    199452 unlocked pager get faults
  22891948 anon faults
  11857386 anon copy on write faults
 166233803 object faults
  40794276 promote copy faults
 360414230 promote zero fill faults
       758 times daemon wokeup
     14404 revolutions of the clock hand
   3311503 pages freed by daemon
   9647599 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2578695 object pages scanned by daemon
   2675029 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10681390 pages deactivated
 1890317259 total name lookups
 1793158478 good hits
  90723539 negative hits
   1096294 bad hits
    688971 false hits
   4337884 miss
    312093 too long
    664647 pass2 hits
   1149245 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:06:14 JST 2020
 load averages:  10.8,  11.1,  10.7;               up 0+19:40:00        04:06:15
 180 processes: 8 runnable, 168 sleeping, 4 on CPU
 CPU0 states: 78.2% user,  0.0% nice, 21.8% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 85.1% user,  0.0% nice, 14.9% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 70.0% user,  0.0% nice, 30.0% system,  0.0% interrupt,  0.0% idle
 CPU3 states:  0.0% user,  0.0% nice,  100% system,  0.0% interrupt,  0.0% idle
 Memory: 1394M Act, 763M Inact, 309M Wired, 320M Exec, 646M File, 204M Free
 Swap: 8972M Total, 1439M Used, 7533M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/3     16:11 98.00% 98.00% [system]
   892 tsutsui   26    0  1388M  576M CPU/0     98:11 49.60% 47.95% firefox
 11758 tsutsui   26    0   371M  130M RUN/1     12:19 39.17% 39.11% npviewer.bin
   881 tsutsui   26    0   253M   65M RUN/0     19:54 10.64% 10.64% X
 25580 root      27    0    55M   46M RUN/3      0:01 38.81%  8.59% cc1
  1669 tsutsui   43    0   788M  295M parked/2  14:55  4.00%  4.00% firefox68
 28938 tsutsui   82    0    29M   20M select/3   0:41  0.83%  0.83% xv
  9736 root      26    0    32M   21M RUN/1      0:00  9.00%  0.44% cc1
 23054 root      26    0    30M   16M RUN/1      0:00  4.00%  0.20% cc1
  6522 root      25    0    29M   18M RUN/2      0:00  3.00%  0.15% cc1

      4096 bytes per page
         8 page colors
    832700 pages managed
     52075 pages free
    356875 pages active
    195269 pages inactive
         0 pages paging
     79084 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144011 kernel pool pages
    383972 anonymous pages
    165389 cached file pages
     81867 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    368331 swap pages in use
    395351 swap allocations
 679548092 total faults taken
 680613512 traps
   5391343 device interrupts
 104217561 CPU context switches
  18248543 software interrupts
 1383124989 system calls
    391387 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1833444 forks total
   1309037 forks blocked parent
   1309037 forks shared address space with parent
  31537814 pagealloc zero wanted and avail
 342310836 pagealloc zero wanted and not avail
      1425 aborts of idle page zeroing
 460776423 pagealloc desired color avail
  46502227 pagealloc desired color not avail
 384150809 pagealloc local cpu avail
 123127841 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    592532 faults relock (589083 ok)
  34767542 anon page faults
    389695 anon retry faults
  71136018 amap copy faults
  25085760 neighbour anon page faults
 823088869 neighbour object page faults
 207390468 locked pager get faults
    200203 unlocked pager get faults
  22904412 anon faults
  11862423 anon copy on write faults
 166528175 object faults
  40857319 promote copy faults
 361343974 promote zero fill faults
       758 times daemon wokeup
     14404 revolutions of the clock hand
   3311503 pages freed by daemon
   9647599 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2578695 object pages scanned by daemon
   2675029 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10681390 pages deactivated
 1895412603 total name lookups
 1798068208 good hits
  90892354 negative hits
   1098848 bad hits
    689342 false hits
   4351424 miss
    312427 too long
    668843 pass2 hits
   1155174 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:06:45 JST 2020
 load averages:  11.5,  11.2,  10.7;               up 0+19:40:31        04:06:46
 185 processes: 7 runnable, 174 sleeping, 4 on CPU
 CPU0 states: 76.2% user,  0.0% nice, 23.8% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 92.0% user,  0.0% nice,  8.0% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 44.6% user,  0.0% nice, 55.4% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 27.0% user,  0.0% nice, 73.0% system,  0.0% interrupt,  0.0% idle
 Memory: 1363M Act, 762M Inact, 309M Wired, 320M Exec, 664M File, 235M Free
 Swap: 8972M Total, 1439M Used, 7533M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   28M CPU/2     16:43 98.14% 98.14% [system]
   892 tsutsui   26    0  1392M  580M RUN/0     98:28 44.38% 44.38% firefox
 11758 tsutsui   26    0   371M  130M CPU/1     12:33 42.88% 42.87% npviewer.bin
   881 tsutsui   26    0   254M   65M RUN/0     19:58 10.69% 10.69% X
  1669 tsutsui   43    0   791M  300M parked/2  14:57  2.05%  2.05% firefox68
 23159 root      26    0    33M   23M RUN/2      0:00 10.76%  1.03% cc1
 12899 root      26    0    27M   15M RUN/0      0:00  5.00%  0.24% cc1
 15220 root      26    0    24M 9496K RUN/0      0:00  1.00%  0.05% cc1
  1087 tsutsui   85    0   976M  223M select/3  37:55  0.00%  0.00% ruby26
 28938 tsutsui   82    0    29M   20M select/0   0:42  0.00%  0.00% xv

      4096 bytes per page
         8 page colors
    832700 pages managed
     60516 pages free
    348491 pages active
    195096 pages inactive
         0 pages paging
     79118 pages wired
       951 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144090 kernel pool pages
    370589 anonymous pages
    170127 cached file pages
     81989 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    368329 swap pages in use
    395444 swap allocations
 680930041 total faults taken
 681997874 traps
   5397456 device interrupts
 104465457 CPU context switches
  18263457 software interrupts
 1385487665 system calls
    391480 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1835290 forks total
   1310703 forks blocked parent
   1310703 forks shared address space with parent
  31566148 pagealloc zero wanted and avail
 343257321 pagealloc zero wanted and not avail
      1425 aborts of idle page zeroing
 461883251 pagealloc desired color avail
  46566493 pagealloc desired color not avail
 385013855 pagealloc local cpu avail
 123435889 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    592867 faults relock (589415 ok)
  34781277 anon page faults
    389786 anon retry faults
  71236350 amap copy faults
  25092821 neighbour anon page faults
 824186164 neighbour object page faults
 207646739 locked pager get faults
    200445 unlocked pager get faults
  22913988 anon faults
  11866580 anon copy on write faults
 166750566 object faults
  40891197 promote copy faults
 362306494 promote zero fill faults
       758 times daemon wokeup
     14404 revolutions of the clock hand
   3311503 pages freed by daemon
   9647599 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2578695 object pages scanned by daemon
   2675029 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10681390 pages deactivated
 1899013772 total name lookups
 1801547461 good hits
  91006411 negative hits
   1100357 bad hits
    689583 false hits
   4357364 miss
    312596 too long
    669790 pass2 hits
   1157789 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:07:16 JST 2020
 load averages:  12.7,  11.6,  10.9;               up 0+19:41:02        04:07:17
 174 processes: 9 runnable, 161 sleeping, 4 on CPU
 CPU0 states: 42.6% user,  0.0% nice, 57.4% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 53.0% user,  0.0% nice, 46.0% system,  0.0% interrupt,  1.0% idle
 CPU2 states: 53.5% user,  0.0% nice, 41.6% system,  0.0% interrupt,  5.0% idle
 CPU3 states: 24.0% user,  0.0% nice, 74.0% system,  0.0% interrupt,  2.0% idle
 Memory: 1378M Act, 760M Inact, 309M Wired, 308M Exec, 706M File, 221M Free
 Swap: 8972M Total, 1439M Used, 7533M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   29M CPU/3     17:14 97.66% 97.66% [system]
   892 tsutsui   42    0  1390M  574M RUN/3     98:44  1095% 53.47% firefox
 11758 tsutsui   26    0   371M  130M RUN/1     12:47 40.63% 40.62% npviewer.bin
   881 tsutsui   26    0   254M   65M RUN/0     20:02 11.18% 11.18% X
 28938 tsutsui   81    0    29M   20M select/0   0:43  4.30%  4.30% xv
  1669 tsutsui   43    0   789M  300M RUN/0     14:58  3.47%  3.47% firefox68
   398 root      85    0  6456K 1480K select/0   0:00  1.76%  1.76% ypserv
   160 tsutsui   85    0    11M 3592K select/0   0:10  0.73%  0.73% kterm
  1087 tsutsui   85    0   958M  225M select/0  37:56  0.00%  0.00% ruby26
  1427 tsutsui   85    0    27M   11M RUN/0      0:19  0.00%  0.00% jwm

      4096 bytes per page
         8 page colors
    832700 pages managed
     56487 pages free
    352841 pages active
    194633 pages inactive
         0 pages paging
     79050 pages wired
      1503 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144407 kernel pool pages
    367021 anonymous pages
    180744 cached file pages
     78760 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    368316 swap pages in use
    395758 swap allocations
 682712072 total faults taken
 683783423 traps
   5404684 device interrupts
 104732397 CPU context switches
  18279758 software interrupts
 1388814349 system calls
    391794 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1840513 forks total
   1314938 forks blocked parent
   1314938 forks shared address space with parent
  31595200 pagealloc zero wanted and avail
 344272860 pagealloc zero wanted and not avail
      1446 aborts of idle page zeroing
 463099065 pagealloc desired color avail
  46664796 pagealloc desired color not avail
 385997107 pagealloc local cpu avail
 123766754 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    593987 faults relock (590529 ok)
  34840615 anon page faults
    390095 anon retry faults
  71439614 amap copy faults
  25131405 neighbour anon page faults
 826503431 neighbour object page faults
 208221254 locked pager get faults
    201251 unlocked pager get faults
  22951371 anon faults
  11888531 anon copy on write faults
 167218729 object faults
  40997544 promote copy faults
 363317236 promote zero fill faults
       758 times daemon wokeup
     14404 revolutions of the clock hand
   3311503 pages freed by daemon
   9647599 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2578695 object pages scanned by daemon
   2675029 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10681390 pages deactivated
 1905777782 total name lookups
 1808076153 good hits
  91227135 negative hits
   1103324 bad hits
    690459 false hits
   4367886 miss
    312825 too long
    671087 pass2 hits
   1159757 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:07:47 JST 2020
 load averages:  12.3,  11.6,  10.9;               up 0+19:41:33        04:07:48
 216 processes: 7 runnable, 205 sleeping, 4 on CPU
 CPU0 states: 90.1% user,  0.0% nice,  9.9% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 81.2% user,  0.0% nice, 18.8% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 88.2% user,  0.0% nice, 11.8% system,  0.0% interrupt,  0.0% idle
 CPU3 states:  0.0% user,  0.0% nice,  100% system,  0.0% interrupt,  0.0% idle
 Memory: 1483M Act, 767M Inact, 309M Wired, 320M Exec, 701M File, 106M Free
 Swap: 8972M Total, 1433M Used, 7539M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/3     17:45 99.27% 99.27% [system]
   892 tsutsui   26    0  1392M  577M RUN/3     99:03 66.10% 52.78% firefox
 11758 tsutsui   26    0   372M  131M RUN/0     13:00 30.23% 30.22% npviewer.bin
   881 tsutsui   26    0   254M   65M CPU/0     20:06 10.89% 10.89% X
 25169 root      25    0    42M   33M RUN/0      0:01 32.56%  5.91% cc1
 22599 root      25    0    47M   37M RUN/2      0:01 20.72%  3.76% cc1
  1669 tsutsui   43    0   791M  302M parked/0  15:00  3.27%  3.27% firefox68
  5442 root      25    0    40M   30M RUN/1      0:00 21.01%  2.00% cc1
  2984 root      27    0    32M   21M RUN/3      0:00 18.45%  1.76% cc1
 15919 root      26    0    38M   27M RUN/3      0:00 11.28%  1.07% cc1

      4096 bytes per page
         8 page colors
    832700 pages managed
     27100 pages free
    379741 pages active
    196360 pages inactive
         0 pages paging
     79068 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144779 kernel pool pages
    393785 anonymous pages
    179517 cached file pages
     81867 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    366957 swap pages in use
    396108 swap allocations
 684219450 total faults taken
 685294125 traps
   5413847 device interrupts
 105021120 CPU context switches
  18298166 software interrupts
 1392262652 system calls
    392139 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1843919 forks total
   1318191 forks blocked parent
   1318191 forks shared address space with parent
  31620593 pagealloc zero wanted and avail
 345202668 pagealloc zero wanted and not avail
      1446 aborts of idle page zeroing
 464256742 pagealloc desired color avail
  46711300 pagealloc desired color not avail
 386849393 pagealloc local cpu avail
 124118649 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    595436 faults relock (591973 ok)
  34856147 anon page faults
    390436 anon retry faults
  71651535 amap copy faults
  25137329 neighbour anon page faults
 828085416 neighbour object page faults
 208656142 locked pager get faults
    202355 unlocked pager get faults
  22963743 anon faults
  11891687 anon copy on write faults
 167543234 object faults
  41107927 promote copy faults
 364249509 promote zero fill faults
       758 times daemon wokeup
     16376 revolutions of the clock hand
   3317030 pages freed by daemon
   9654724 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2584222 object pages scanned by daemon
   2676323 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10690495 pages deactivated
 1913304221 total name lookups
 1815302317 good hits
  91499287 negative hits
   1106400 bad hits
    691065 false hits
   4391645 miss
    313507 too long
    678468 pass2 hits
   1174391 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:08:18 JST 2020
 load averages:  12.1,  11.6,  10.9;               up 0+19:42:04        04:08:19
 212 processes: 9 runnable, 200 sleeping, 3 on CPU
 CPU0 states: 66.3% user,  0.0% nice, 23.8% system,  1.0% interrupt,  8.9% idle
 CPU1 states:  0.0% user,  0.0% nice,  100% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 98.0% user,  0.0% nice,  2.0% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 69.3% user,  0.0% nice, 30.7% system,  0.0% interrupt,  0.0% idle
 Memory: 1459M Act, 757M Inact, 309M Wired, 320M Exec, 720M File, 141M Free
 Swap: 8972M Total, 1433M Used, 7539M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/1     18:17 98.34% 98.34% [system]
   892 tsutsui   43    0  1388M  575M parked/2  99:21 42.53% 42.53% firefox
 11758 tsutsui   26    0   373M  133M RUN/2     13:16 54.60% 41.16% npviewer.bin
   881 tsutsui   25    0   254M   65M RUN/3     20:09 10.21% 10.21% X
   532 tsutsui   85    0    11M 3572K select/3   0:09  9.72%  9.72% kterm
 29313 root      25    0    45M   35M RUN/2      0:00 25.91%  3.61% cc1
  1669 tsutsui   43    0   787M  298M RUN/1     15:02  3.37%  3.37% firefox68
  6167 root      25    0    36M   24M RUN/3      0:00 12.00%  0.59% cc1
  8211 root      27    0    23M   11M RUN/1      0:00  0.51%  0.05% cc1
 24253 root      77    0  7000K 2836K select/3   0:00  0.09%  0.05% nbmake

      4096 bytes per page
         8 page colors
    832700 pages managed
     36038 pages free
    373400 pages active
    193760 pages inactive
         0 pages paging
     79048 pages wired
       182 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144874 kernel pool pages
    380116 anonymous pages
    184225 cached file pages
     81867 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    366957 swap pages in use
    396172 swap allocations
 685412934 total faults taken
 686489747 traps
   5420301 device interrupts
 105267000 CPU context switches
  18313245 software interrupts
 1395033938 system calls
    392203 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1845802 forks total
   1319994 forks blocked parent
   1319994 forks shared address space with parent
  31640247 pagealloc zero wanted and avail
 346019255 pagealloc zero wanted and not avail
      1446 aborts of idle page zeroing
 465157315 pagealloc desired color avail
  46789250 pagealloc desired color not avail
 387598213 pagealloc local cpu avail
 124348352 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    595940 faults relock (592476 ok)
  34865518 anon page faults
    390499 anon retry faults
  71750882 amap copy faults
  25140625 neighbour anon page faults
 829135624 neighbour object page faults
 208909636 locked pager get faults
    202795 unlocked pager get faults
  22970857 anon faults
  11893943 anon copy on write faults
 167756433 object faults
  41148222 promote copy faults
 365072988 promote zero fill faults
       758 times daemon wokeup
     16376 revolutions of the clock hand
   3317030 pages freed by daemon
   9654724 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2584222 object pages scanned by daemon
   2676323 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10690495 pages deactivated
 1920018369 total name lookups
 1821741456 good hits
  91765159 negative hits
   1107603 bad hits
    691752 false hits
   4398763 miss
    313636 too long
    678884 pass2 hits
   1176230 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:08:49 JST 2020
 load averages:  12.0,  11.6,  11.0;               up 0+19:42:35        04:08:50
 209 processes: 7 runnable, 198 sleeping, 4 on CPU
 CPU0 states: 19.8% user,  0.0% nice, 76.2% system,  4.0% interrupt,  0.0% idle
 CPU1 states: 64.4% user,  0.0% nice, 34.7% system,  0.0% interrupt,  1.0% idle
 CPU2 states: 68.3% user,  0.0% nice, 31.7% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 46.5% user,  0.0% nice, 53.5% system,  0.0% interrupt,  0.0% idle
 Memory: 1457M Act, 754M Inact, 309M Wired, 320M Exec, 736M File, 145M Free
 Swap: 8972M Total, 1433M Used, 7539M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/3     18:48 98.24% 98.24% [system]
   892 tsutsui   26    0  1384M  566M CPU/1     99:37 44.99% 44.58% firefox
 11758 tsutsui   84    0   373M  133M RUN/2     13:30 40.80% 38.67% npviewer.bin
   881 tsutsui   26    0   254M   65M RUN/0     20:13  9.67%  9.67% X
   532 tsutsui   85    0    11M 3572K select/2   0:09  1.66%  1.66% kterm
  1669 tsutsui   43    0   792M  300M parked/0  15:03  1.32%  1.32% firefox68
  1087 tsutsui   85    0   994M  256M select/2  37:58  0.88%  0.88% ruby26
   127 root      77    0  7300K 2964K select/1   0:00  0.30%  0.24% nbmake
 28938 tsutsui   85    0    23M   15M select/2   0:43  0.00%  0.00% xv
  1427 tsutsui   85    0    27M   11M select/1   0:19  0.00%  0.00% jwm

      4096 bytes per page
         8 page colors
    832700 pages managed
     37143 pages free
    373106 pages active
    192925 pages inactive
         0 pages paging
     79048 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144936 kernel pool pages
    374864 anonymous pages
    188344 cached file pages
     81871 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    366799 swap pages in use
    396268 swap allocations
 687003787 total faults taken
 688083443 traps
   5427182 device interrupts
 105525054 CPU context switches
  18328904 software interrupts
 1399163829 system calls
    392299 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1849608 forks total
   1323780 forks blocked parent
   1323780 forks shared address space with parent
  31658394 pagealloc zero wanted and avail
 347003876 pagealloc zero wanted and not avail
      1446 aborts of idle page zeroing
 466214387 pagealloc desired color avail
  46917046 pagealloc desired color not avail
 388480770 pagealloc local cpu avail
 124650663 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    597258 faults relock (593793 ok)
  34875330 anon page faults
    390594 anon retry faults
  71914588 amap copy faults
  25141506 neighbour anon page faults
 831177427 neighbour object page faults
 209398465 locked pager get faults
    204017 unlocked pager get faults
  22980015 anon faults
  11894596 anon copy on write faults
 168163524 object faults
  41229958 promote copy faults
 366050958 promote zero fill faults
       758 times daemon wokeup
     16376 revolutions of the clock hand
   3317030 pages freed by daemon
   9654724 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2584222 object pages scanned by daemon
   2676323 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10690495 pages deactivated
 1933067099 total name lookups
 1834283019 good hits
  92258384 negative hits
   1109401 bad hits
    694522 false hits
   4408001 miss
    313772 too long
    679792 pass2 hits
   1177532 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:09:20 JST 2020
 load averages:  12.5,  11.8,  11.0;               up 0+19:43:06        04:09:21
 214 processes: 8 runnable, 203 sleeping, 3 on CPU
 CPU0 states: 52.5% user,  0.0% nice, 47.5% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 72.3% user,  0.0% nice, 19.8% system,  0.0% interrupt,  7.9% idle
 CPU2 states: 71.3% user,  0.0% nice, 28.7% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 36.6% user,  0.0% nice, 63.4% system,  0.0% interrupt,  0.0% idle
 Memory: 1528M Act, 759M Inact, 309M Wired, 308M Exec, 781M File, 70M Free
 Swap: 8972M Total, 1433M Used, 7539M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/0     19:20 97.66% 97.66% [system]
   892 tsutsui   29    0  1384M  570M CPU/3     99:55 51.12% 51.03% firefox
 11758 tsutsui   27    0   373M  133M RUN/3     13:45 41.08% 40.62% npviewer.bin
   881 tsutsui   26    0   254M   66M RUN/0     20:17 11.38% 11.38% X
  1669 tsutsui   43    0   793M  302M parked/1  15:05  5.08%  5.08% firefox68
 28938 tsutsui   83    0    29M   21M select/1   0:44  1.86%  1.86% xv
 13981 root      25    0    31M   20M RUN/2      0:00 25.00%  1.22% cc1
 15720 root      26    0    37M   27M RUN/2      0:00 23.00%  1.12% cc1
  1299 root      26    0    10M 4176K RUN/0      0:00  5.00%  0.24% nbmake
 20161 root      77    0  9480K 5048K select/3   0:00  0.14%  0.10% nbmake

      4096 bytes per page
         8 page colors
    832700 pages managed
     17360 pages free
    391518 pages active
    194371 pages inactive
         0 pages paging
     79053 pages wired
       137 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144804 kernel pool pages
    386293 anonymous pages
    196786 cached file pages
     81863 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    366799 swap pages in use
    396334 swap allocations
 688614116 total faults taken
 689696957 traps
   5434979 device interrupts
 105792091 CPU context switches
  18345978 software interrupts
 1403177065 system calls
    392365 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1853272 forks total
   1327430 forks blocked parent
   1327430 forks shared address space with parent
  31700340 pagealloc zero wanted and avail
 347933958 pagealloc zero wanted and not avail
      1451 aborts of idle page zeroing
 467393107 pagealloc desired color avail
  47003730 pagealloc desired color not avail
 389512203 pagealloc local cpu avail
 124884634 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    598360 faults relock (594893 ok)
  34885649 anon page faults
    390660 anon retry faults
  72149591 amap copy faults
  25142094 neighbour anon page faults
 832923208 neighbour object page faults
 209884989 locked pager get faults
    205053 unlocked pager get faults
  22989937 anon faults
  11894993 anon copy on write faults
 168523891 object faults
  41356110 promote copy faults
 366998094 promote zero fill faults
       758 times daemon wokeup
     17732 revolutions of the clock hand
   3317030 pages freed by daemon
   9654724 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2584222 object pages scanned by daemon
   2676323 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10691975 pages deactivated
 1943459060 total name lookups
 1844251597 good hits
  92665472 negative hits
   1110664 bad hits
    696418 false hits
   4420454 miss
    314455 too long
    683511 pass2 hits
   1185176 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:09:51 JST 2020
 load averages:  11.8,  11.6,  11.0;               up 0+19:43:37        04:09:52
 214 processes: 8 runnable, 200 sleeping, 6 on CPU
 CPU0 states: 95.0% user,  0.0% nice,  5.0% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 61.4% user,  0.0% nice, 38.6% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 24.0% user,  0.0% nice, 76.0% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 63.4% user,  0.0% nice, 36.6% system,  0.0% interrupt,  0.0% idle
 Memory: 1468M Act, 766M Inact, 310M Wired, 308M Exec, 692M File, 121M Free
 Swap: 8972M Total, 1433M Used, 7539M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/2     19:51 97.41% 97.41% [system]
   892 tsutsui   26    0  1403M  612M RUN/2    100:14 78.26% 53.52% firefox
 11758 tsutsui   26    0   373M  133M CPU/0     13:57 33.87% 33.79% npviewer.bin
   881 tsutsui   25    0   256M   67M RUN/3     20:20  9.23%  9.23% X
  1669 tsutsui   43    0   792M  301M parked/2  15:07  2.69%  2.69% firefox68
 19144 root      26    0    37M   25M RUN/2      0:00 18.00%  0.88% cc1
 11433 root      26    0    29M   19M RUN/2      0:00  2.80%  0.39% cc1
 15661 root      25    0    28M   16M RUN/0      0:00  4.00%  0.20% cc1
  1087 tsutsui   85    0   993M  250M select/0  37:58  0.00%  0.00% ruby26
 28938 tsutsui   83    0    29M   21M select/3   0:45  0.00%  0.00% xv

      4096 bytes per page
         8 page colors
    832700 pages managed
     30896 pages free
    375782 pages active
    196223 pages inactive
         0 pages paging
     79318 pages wired
       396 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144861 kernel pool pages
    395333 anonymous pages
    177202 cached file pages
     78788 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    366799 swap pages in use
    396405 swap allocations
 690234703 total faults taken
 691320693 traps
   5441489 device interrupts
 106045640 CPU context switches
  18361980 software interrupts
 1406999823 system calls
    392436 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1856185 forks total
   1330332 forks blocked parent
   1330332 forks shared address space with parent
  31723879 pagealloc zero wanted and avail
 349002188 pagealloc zero wanted and not avail
      1451 aborts of idle page zeroing
 468633690 pagealloc desired color avail
  47048688 pagealloc desired color not avail
 390375469 pagealloc local cpu avail
 125306909 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    598863 faults relock (595396 ok)
  34893518 anon page faults
    390731 anon retry faults
  72306365 amap copy faults
  25142499 neighbour anon page faults
 834448980 neighbour object page faults
 210284453 locked pager get faults
    205485 unlocked pager get faults
  22997544 anon faults
  11895255 anon copy on write faults
 168840756 object faults
  41438706 promote copy faults
 368069680 promote zero fill faults
       758 times daemon wokeup
     19612 revolutions of the clock hand
   3343669 pages freed by daemon
   9711432 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2610861 object pages scanned by daemon
   2692763 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10750573 pages deactivated
 1954896907 total name lookups
 1855174172 good hits
  93166091 negative hits
   1112390 bad hits
    698707 false hits
   4430958 miss
    314589 too long
    684074 pass2 hits
   1188992 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:10:22 JST 2020
 load averages:  12.0,  11.7,  11.1;               up 0+19:44:08        04:10:23
 207 processes: 6 runnable, 197 sleeping, 4 on CPU
 CPU0 states:  0.0% user,  0.0% nice,  100% system,  0.0% interrupt,  0.0% idle
 CPU1 states: 96.0% user,  0.0% nice,  4.0% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 69.0% user,  0.0% nice, 17.0% system,  0.0% interrupt, 14.0% idle
 CPU3 states: 41.6% user,  0.0% nice, 14.9% system,  0.0% interrupt, 43.6% idle
 Memory: 1421M Act, 765M Inact, 309M Wired, 308M Exec, 718M File, 171M Free
 Swap: 8972M Total, 1433M Used, 7539M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/0     20:22 98.14% 98.14% [system]
   892 tsutsui   25    0  1395M  568M CPU/2    100:32 49.22% 49.22% firefox
 11758 tsutsui   28    0   373M  133M CPU/1     14:14 40.55% 40.53% npviewer.bin
   881 tsutsui   77    0   255M   67M radfen/2  20:24  9.03%  9.03% X
  1669 tsutsui   43    0   797M  306M parked/0  15:09  1.12%  1.12% firefox68
  8408 root      26    0    26M   13M RUN/0      0:00  2.05%  0.20% cc1
  1087 tsutsui   85    0   993M  250M select/3  37:59  0.00%  0.00% ruby26
 28938 tsutsui   84    0    28M   20M select/2   0:45  0.00%  0.00% xv
  1427 tsutsui   85    0    27M   11M select/2   0:19  0.00%  0.00% jwm
   876 tsutsui   85    0    18M 6384K select/1   0:18  0.00%  0.00% mlterm

      4096 bytes per page
         8 page colors
    832700 pages managed
     43864 pages free
    363718 pages active
    195744 pages inactive
         0 pages paging
     79076 pages wired
      3512 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144799 kernel pool pages
    376069 anonymous pages
    183671 cached file pages
     78798 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    366792 swap pages in use
    396486 swap allocations
 691629886 total faults taken
 692718291 traps
   5449291 device interrupts
 106322095 CPU context switches
  18378919 software interrupts
 1410675152 system calls
    392517 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1858880 forks total
   1333001 forks blocked parent
   1333001 forks shared address space with parent
  31751480 pagealloc zero wanted and avail
 349849314 pagealloc zero wanted and not avail
      1453 aborts of idle page zeroing
 469674439 pagealloc desired color avail
  47112412 pagealloc desired color not avail
 391176692 pagealloc local cpu avail
 125610159 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    599565 faults relock (596098 ok)
  34901626 anon page faults
    390812 anon retry faults
  72445942 amap copy faults
  25143578 neighbour anon page faults
 835861400 neighbour object page faults
 210646430 locked pager get faults
    206106 unlocked pager get faults
  23004905 anon faults
  11896002 anon copy on write faults
 169129866 object faults
  41511573 promote copy faults
 368926471 promote zero fill faults
       758 times daemon wokeup
     19612 revolutions of the clock hand
   3343669 pages freed by daemon
   9711432 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2610861 object pages scanned by daemon
   2692763 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10750573 pages deactivated
 1964642810 total name lookups
 1864495481 good hits
  93578537 negative hits
   1113994 bad hits
    700413 false hits
   4439661 miss
    314724 too long
    685101 pass2 hits
   1190823 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:10:53 JST 2020
 load averages:  12.3,  11.8,  11.1;               up 0+19:44:39        04:10:54
 215 processes: 7 runnable, 204 sleeping, 4 on CPU
 CPU0 states:  0.0% user,  0.0% nice, 99.0% system,  1.0% interrupt,  0.0% idle
 CPU1 states: 79.2% user,  0.0% nice, 18.8% system,  0.0% interrupt,  2.0% idle
 CPU2 states: 87.0% user,  0.0% nice, 10.0% system,  0.0% interrupt,  3.0% idle
 CPU3 states: 87.1% user,  0.0% nice, 10.9% system,  0.0% interrupt,  2.0% idle
 Memory: 1455M Act, 764M Inact, 309M Wired, 320M Exec, 720M File, 138M Free
 Swap: 8972M Total, 1432M Used, 7540M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/0     20:54 98.34% 98.34% [system]
 11758 tsutsui   84    0   373M  133M RUN/3     14:28 49.91% 49.90% npviewer.bin
   892 tsutsui   37    0  1379M  552M CPU/1    100:48 46.26% 45.61% firefox
   881 tsutsui   27    0   255M   75M RUN/0     20:28  9.91%  9.91% X
 28938 tsutsui   83    0    28M   20M select/0   0:46  4.64%  4.64% xv
  1669 tsutsui   43    0   791M  303M parked/1  15:11  3.86%  3.86% firefox68
  1087 tsutsui   85    0   978M  244M select/0  38:01  3.08%  3.08% ruby26
 29340 root      25    0    37M   27M CPU/2      0:00 47.00%  2.29% cc1
 13376 root      25    0    35M   24M RUN/3      0:00 16.00%  0.78% cc1
  1427 tsutsui   85    0    27M   11M select/2   0:19  0.00%  0.00% jwm

      4096 bytes per page
         8 page colors
    832700 pages managed
     35238 pages free
    372419 pages active
    195515 pages inactive
         0 pages paging
     79065 pages wired
       773 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144910 kernel pool pages
    380721 anonymous pages
    184383 cached file pages
     81895 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    366680 swap pages in use
    399121 swap allocations
 692911627 total faults taken
 694002639 traps
   5458780 device interrupts
 106574350 CPU context switches
  18397996 software interrupts
 1414062681 system calls
    393569 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1860803 forks total
   1334921 forks blocked parent
   1334921 forks shared address space with parent
  31782921 pagealloc zero wanted and avail
 350653164 pagealloc zero wanted and not avail
      1454 aborts of idle page zeroing
 470612819 pagealloc desired color avail
  47204057 pagealloc desired color not avail
 391920366 pagealloc local cpu avail
 125896510 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    602732 faults relock (599264 ok)
  34908861 anon page faults
    391863 anon retry faults
  72547548 amap copy faults
  25143718 neighbour anon page faults
 837086018 neighbour object page faults
 210935714 locked pager get faults
    208221 unlocked pager get faults
  23012012 anon faults
  11896129 anon copy on write faults
 169380596 object faults
  41550126 promote copy faults
 369748726 promote zero fill faults
       758 times daemon wokeup
     19632 revolutions of the clock hand
   3343669 pages freed by daemon
   9711432 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2610861 object pages scanned by daemon
   2692763 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10750593 pages deactivated
 1973614865 total name lookups
 1873116589 good hits
  93919972 negative hits
   1115270 bad hits
    701997 false hits
   4445457 miss
    315580 too long
    686412 pass2 hits
   1193110 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:11:24 JST 2020
 load averages:  11.9,  11.8,  11.1;               up 0+19:45:10        04:11:25
 214 processes: 7 runnable, 203 sleeping, 4 on CPU
 CPU0 states: 54.0% user,  0.0% nice, 41.0% system,  5.0% interrupt,  0.0% idle
 CPU1 states: 79.0% user,  0.0% nice,  3.0% system,  0.0% interrupt, 18.0% idle
 CPU2 states: 38.4% user,  0.0% nice, 61.6% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 69.0% user,  0.0% nice, 26.0% system,  0.0% interrupt,  5.0% idle
 Memory: 1495M Act, 765M Inact, 309M Wired, 319M Exec, 728M File, 96M Free
 Swap: 8972M Total, 1432M Used, 7541M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/0     21:25 98.54% 98.54% [system]
   892 tsutsui   26    0  1393M  586M RUN/2    101:07 46.38% 46.24% firefox
 11758 tsutsui   27    0   374M  133M CPU/1     14:39 30.23% 30.22% npviewer.bin
 15660 tsutsui   85    0   155M   23M kqueue/0   0:10 12.45% 12.45% plugin-container
   881 tsutsui   26    0   255M   75M RUN/0     20:31 10.99% 10.99% X
  1669 tsutsui   43    0   798M  307M parked/0  15:13  7.96%  7.96% firefox68
 16397 root      26    0    33M   23M RUN/2      0:00 15.38%  1.46% cc1
 13871 root      25    0    30M   18M RUN/0      0:00 15.00%  0.73% cc1
 21032 root      26    0    29M   19M CPU/2      0:00  6.66%  0.63% cc1
 20671 root      25    0    23M   17M RUN/0      0:00  9.00%  0.44% nbmake

      4096 bytes per page
         8 page colors
    832700 pages managed
     24553 pages free
    382711 pages active
    195926 pages inactive
         0 pages paging
     79051 pages wired
       142 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144862 kernel pool pages
    389734 anonymous pages
    186295 cached file pages
     81659 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    366467 swap pages in use
    399291 swap allocations
 694295221 total faults taken
 695388555 traps
   5465861 device interrupts
 106821559 CPU context switches
  18414060 software interrupts
 1417353485 system calls
    393739 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1863353 forks total
   1337452 forks blocked parent
   1337452 forks shared address space with parent
  31812136 pagealloc zero wanted and avail
 351498488 pagealloc zero wanted and not avail
      1455 aborts of idle page zeroing
 471655626 pagealloc desired color avail
  47270655 pagealloc desired color not avail
 392740253 pagealloc local cpu avail
 126186028 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       942 faults had to wait on pages
         0 faults found released page
    603539 faults relock (600064 ok)
  34916714 anon page faults
    392026 anon retry faults
  72682285 amap copy faults
  25144463 neighbour anon page faults
 838428979 neighbour object page faults
 211276434 locked pager get faults
    208858 unlocked pager get faults
  23019413 anon faults
  11896574 anon copy on write faults
 169655319 object faults
  41616120 promote copy faults
 370606386 promote zero fill faults
       758 times daemon wokeup
     20241 revolutions of the clock hand
   3346666 pages freed by daemon
   9714997 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2613858 object pages scanned by daemon
   2692976 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10754774 pages deactivated
 1981869145 total name lookups
 1881039649 good hits
  94239751 negative hits
   1117051 bad hits
    703419 false hits
   4453446 miss
    315829 too long
    687252 pass2 hits
   1194735 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:11:55 JST 2020
 load averages:  11.2,  11.6,  11.1;               up 0+19:45:41        04:11:56
 203 processes: 2 runnable, 197 sleeping, 4 on CPU
 CPU0 states: 58.4% user,  0.0% nice, 39.6% system,  2.0% interrupt,  0.0% idle
 CPU1 states: 20.8% user,  0.0% nice, 78.2% system,  0.0% interrupt,  1.0% idle
 CPU2 states: 67.0% user,  0.0% nice, 32.0% system,  0.0% interrupt,  1.0% idle
 CPU3 states: 81.2% user,  0.0% nice, 12.9% system,  0.0% interrupt,  5.9% idle
 Memory: 1148M Act, 681M Inact, 290M Wired, 234M Exec, 798M File, 549M Free
 Swap: 8972M Total, 1123M Used, 7850M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/1     21:56 99.07% 99.07% [system]
   892 tsutsui   26    0  1346M  544M CPU/0    101:16 25.29% 25.29% firefox
   881 tsutsui   85    0   250M   82M select/0  20:34  9.13%  9.13% X
  1087 tsutsui   85    0   964M  238M select/3  38:02  1.61%  1.61% ruby26
 22141 root      25    0    36M   25M CPU/2      0:00  9.00%  0.44% cc1
 27598 root     109    0    28M   16M tstile/2   0:00  5.00%  0.24% cc1
  1427 tsutsui   85    0    27M   11M select/3   0:19  0.00%  0.00% jwm
   876 tsutsui   85    0    18M 6444K select/0   0:18  0.00%  0.00% mlterm
    77 tsutsui   85    0    28M 5776K select/2   0:17  0.00%  0.00% ibus-daemon
  1177 tsutsui   85    0   130M   32M select/1   0:11  0.00%  0.00% ibus-ui-gtk3

      4096 bytes per page
         8 page colors
    832700 pages managed
    148132 pages free
    288804 pages active
    171749 pages inactive
         0 pages paging
     74125 pages wired
      9761 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144840 kernel pool pages
    270584 anonymous pages
    204285 cached file pages
     59810 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    282779 swap pages in use
    415758 swap allocations
 696230474 total faults taken
 697328711 traps
   5491317 device interrupts
 107088736 CPU context switches
  18449867 software interrupts
 1421369571 system calls
    410183 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1867881 forks total
   1341931 forks blocked parent
   1341931 forks shared address space with parent
  31869056 pagealloc zero wanted and avail
 352598195 pagealloc zero wanted and not avail
      1455 aborts of idle page zeroing
 473013211 pagealloc desired color avail
  47342348 pagealloc desired color not avail
 393894316 pagealloc local cpu avail
 126461243 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       949 faults had to wait on pages
         0 faults found released page
    664184 faults relock (660437 ok)
  34949117 anon page faults
    408206 anon retry faults
  72910784 amap copy faults
  25146332 neighbour anon page faults
 840917647 neighbour object page faults
 211931395 locked pager get faults
    253053 unlocked pager get faults
  23050563 anon faults
  11897556 anon copy on write faults
 170200979 object faults
  41725405 promote copy faults
 371734091 promote zero fill faults
       758 times daemon wokeup
     20241 revolutions of the clock hand
   3346666 pages freed by daemon
   9714997 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2613858 object pages scanned by daemon
   2692976 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10754774 pages deactivated
 1993505841 total name lookups
 1892267549 good hits
  94624951 negative hits
   1120466 bad hits
    707421 false hits
   4468657 miss
    316797 too long
    689844 pass2 hits
   1199426 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:12:26 JST 2020
 load averages:  9.17,  11.1,  10.9;               up 0+19:46:12        04:12:27
 208 processes: 5 runnable, 198 sleeping, 1 zombie, 4 on CPU
 CPU0 states: 66.3% user,  0.0% nice, 31.7% system,  2.0% interrupt,  0.0% idle
 CPU1 states: 64.0% user,  0.0% nice, 36.0% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 70.3% user,  0.0% nice, 29.7% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 13.0% user,  0.0% nice, 87.0% system,  0.0% interrupt,  0.0% idle
 Memory: 1112M Act, 663M Inact, 289M Wired, 234M Exec, 820M File, 608M Free
 Swap: 8972M Total, 1098M Used, 7874M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   29M CPU/0     22:28 99.46% 99.46% [system]
   892 tsutsui   43    0  1323M  472M parked/2 101:17  6.10%  6.10% firefox
   881 tsutsui   85    0   249M   82M RUN/0     20:37  5.42%  5.42% X
   160 tsutsui   83    0    11M 3648K select/3   0:11  1.17%  1.17% kterm
 15953 root      25    0    33M   22M RUN/2      0:00 16.00%  0.78% cc1
  1087 tsutsui   85    0   955M  238M select/3  38:02  0.00%  0.00% ruby26
  1427 tsutsui   85    0    27M   11M select/3   0:19  0.00%  0.00% jwm
   876 tsutsui   85    0    18M 6444K select/1   0:18  0.00%  0.00% mlterm
    77 tsutsui   85    0    28M 5728K select/3   0:17  0.00%  0.00% ibus-daemon
  1177 tsutsui   85    0   130M   32M select/0   0:11  0.00%  0.00% ibus-ui-gtk3

      4096 bytes per page
         8 page colors
    832700 pages managed
    155647 pages free
    284775 pages active
    169610 pages inactive
         0 pages paging
     74104 pages wired
     12836 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    144032 kernel pool pages
    258765 anonymous pages
    209914 cached file pages
     59810 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    281073 swap pages in use
    416580 swap allocations
 698132088 total faults taken
 699232398 traps
   5499844 device interrupts
 107241861 CPU context switches
  18463626 software interrupts
 1427430452 system calls
    410874 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1872010 forks total
   1345990 forks blocked parent
   1345990 forks shared address space with parent
  31895231 pagealloc zero wanted and avail
 353800922 pagealloc zero wanted and not avail
      1455 aborts of idle page zeroing
 474422363 pagealloc desired color avail
  47342348 pagealloc desired color not avail
 395263730 pagealloc local cpu avail
 126500981 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       949 faults had to wait on pages
         0 faults found released page
    665690 faults relock (661943 ok)
  34962476 anon page faults
    408898 anon retry faults
  73108748 amap copy faults
  25149082 neighbour anon page faults
 843396452 neighbour object page faults
 212513684 locked pager get faults
    253867 unlocked pager get faults
  23062422 anon faults
  11899057 anon copy on write faults
 170700515 object faults
  41808157 promote copy faults
 372936741 promote zero fill faults
       758 times daemon wokeup
     20241 revolutions of the clock hand
   3346666 pages freed by daemon
   9714997 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2613858 object pages scanned by daemon
   2692976 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10754774 pages deactivated
 2006964418 total name lookups
 1905236974 good hits
  95093632 negative hits
   1123490 bad hits
    712061 false hits
   4480686 miss
    317575 too long
    692129 pass2 hits
   1203480 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:12:57 JST 2020
 load averages:  8.10,  10.6,  10.8;               up 0+19:46:43        04:12:58
 189 processes: 17 runnable, 168 sleeping, 4 on CPU
 CPU0 states: 10.9% user,  0.0% nice, 88.1% system,  1.0% interrupt,  0.0% idle
 CPU1 states: 49.5% user,  0.0% nice, 49.5% system,  0.0% interrupt,  1.0% idle
 CPU2 states: 68.3% user,  0.0% nice, 31.7% system,  0.0% interrupt,  0.0% idle
 CPU3 states: 56.4% user,  0.0% nice, 37.6% system,  0.0% interrupt,  5.9% idle
 Memory: 1159M Act, 661M Inact, 289M Wired, 222M Exec, 868M File, 564M Free
 Swap: 8972M Total, 1096M Used, 7876M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   30M CPU/3     22:59 99.17% 99.17% [system]
  1087 tsutsui   85    0   973M  259M select/3  38:03  2.10%  2.10% ruby26
   881 tsutsui   35    0   245M   81M CPU/0     20:38  1.76%  1.76% X
   892 tsutsui   43    0  1323M  473M parked/2 101:17  0.63%  0.63% firefox
  1427 tsutsui   85    0    27M   11M select/3   0:19  0.00%  0.00% jwm
   876 tsutsui   85    0    18M 6444K select/3   0:18  0.00%  0.00% mlterm
    77 tsutsui   85    0    28M 5736K select/3   0:17  0.00%  0.00% ibus-daemon
   160 tsutsui   85    0    11M 3664K select/1   0:12  0.00%  0.00% kterm
  1177 tsutsui   85    0   130M   32M select/3   0:11  0.00%  0.00% ibus-ui-gtk3
   730 tsutsui   43    0    11M 3640K CPU/0      0:10  0.00%  0.00% kterm

      4096 bytes per page
         8 page colors
    832700 pages managed
    144145 pages free
    297118 pages active
    169129 pages inactive
         0 pages paging
     74044 pages wired
     24488 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    143814 kernel pool pages
    261129 anonymous pages
    219176 cached file pages
     59986 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    280551 swap pages in use
    416637 swap allocations
 699845316 total faults taken
 700947055 traps
   5506865 device interrupts
 107377653 CPU context switches
  18476038 software interrupts
 1431339049 system calls
    410931 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1876098 forks total
   1349954 forks blocked parent
   1349954 forks shared address space with parent
  31970070 pagealloc zero wanted and avail
 354810525 pagealloc zero wanted and not avail
      1464 aborts of idle page zeroing
 475664204 pagealloc desired color avail
  47342348 pagealloc desired color not avail
 396428671 pagealloc local cpu avail
 126577881 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       949 faults had to wait on pages
         0 faults found released page
    667009 faults relock (663262 ok)
  34976525 anon page faults
    408955 anon retry faults
  73296725 amap copy faults
  25153903 neighbour anon page faults
 845809967 neighbour object page faults
 213078657 locked pager get faults
    255129 unlocked pager get faults
  23074037 anon faults
  11901491 anon copy on write faults
 171184972 object faults
  41888665 promote copy faults
 373994971 promote zero fill faults
       758 times daemon wokeup
     20241 revolutions of the clock hand
   3346666 pages freed by daemon
   9714997 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2613858 object pages scanned by daemon
   2692976 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10754774 pages deactivated
 2020203196 total name lookups
 1917958807 good hits
  95591782 negative hits
   1126477 bad hits
    716877 false hits
   4491634 miss
    317619 too long
    693278 pass2 hits
   1205709 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 Fri Jan 17 04:13:28 JST 2020
 load averages:  7.54,  10.2,  10.6;               up 0+19:47:14        04:13:29
 188 processes: 5 runnable, 178 sleeping, 1 zombie, 4 on CPU
 CPU0 states: 62.4% user,  0.0% nice, 36.6% system,  1.0% interrupt,  0.0% idle
 CPU1 states: 13.9% user,  0.0% nice, 86.1% system,  0.0% interrupt,  0.0% idle
 CPU2 states: 73.3% user,  0.0% nice, 25.7% system,  0.0% interrupt,  1.0% idle
 CPU3 states: 60.4% user,  0.0% nice, 38.6% system,  0.0% interrupt,  1.0% idle
 Memory: 1184M Act, 661M Inact, 289M Wired, 221M Exec, 889M File, 540M Free
 Swap: 8972M Total, 1096M Used, 7876M Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root     126    0     0K   29M CPU/3     23:31 98.34% 98.34% [system]
  6607 root      85    0  7028K 1408K select/2   0:05  5.71%  5.71% nbmake
   881 tsutsui   82    0   245M   82M select/0  20:40  5.08%  5.08% X
   892 tsutsui   43    0  1318M  474M RUN/2    101:20  4.79%  4.79% firefox
   160 tsutsui   85    0    11M 3664K select/2   0:12  2.69%  2.69% kterm
  5611 root      85    0  7076K 1408K select/2   0:02  1.22%  1.22% nbmake
  1087 tsutsui   85    0   970M  256M select/0  38:03  0.83%  0.83% ruby26
  1826 root      85    0  7040K 1408K select/2   0:04  0.83%  0.83% nbmake
 12113 root      26    0    32M   21M CPU/1      0:00  7.00%  0.34% cc1
  2862 root      85    0  6244K 2224K select/2   0:00  0.34%  0.34% nbmake

      4096 bytes per page
         8 page colors
    832700 pages managed
    138294 pages free
    303061 pages active
    169107 pages inactive
         0 pages paging
     74055 pages wired
     36075 zero pages
         1 reserve pagedaemon pages
        20 reserve kernel pages
     13825 boot kernel pages
    143690 kernel pool pages
    261897 anonymous pages
    227664 cached file pages
     56662 cached executable pages
      1024 minimum free pages
      1365 target free pages
    277566 maximum wired pages
         1 swap devices
   2296837 swap pages
    280489 swap pages in use
    417157 swap allocations
 701659529 total faults taken
 702762669 traps
   5513822 device interrupts
 107525189 CPU context switches
  18489171 software interrupts
 1435012775 system calls
    411366 pagein requests
     46336 pageout requests
         0 pages swapped in
    740137 pages swapped out
   1880161 forks total
   1353889 forks blocked parent
   1353889 forks shared address space with parent
  32024513 pagealloc zero wanted and avail
 355901966 pagealloc zero wanted and not avail
      1505 aborts of idle page zeroing
 476996970 pagealloc desired color avail
  47342632 pagealloc desired color not avail
 397697592 pagealloc local cpu avail
 126642010 pagealloc local cpu not avail
       373 faults with no memory
         0 faults with no anons
       949 faults had to wait on pages
         0 faults found released page
    668244 faults relock (664496 ok)
  34992959 anon page faults
    409389 anon retry faults
  73477383 amap copy faults
  25158918 neighbour anon page faults
 848266399 neighbour object page faults
 213650157 locked pager get faults
    255929 unlocked pager get faults
  23087685 anon faults
  11904276 anon copy on write faults
 171679113 object faults
  41966024 promote copy faults
 375114930 promote zero fill faults
       758 times daemon wokeup
     20241 revolutions of the clock hand
   3346666 pages freed by daemon
   9714997 pages scanned by daemon
    732808 anonymous pages scanned by daemon
   2613858 object pages scanned by daemon
   2692976 pages reactivated
      1509 pages found busy by daemon
    693801 total pending pageouts
  10754774 pages deactivated
 2032824746 total name lookups
 1930102248 good hits
  96051775 negative hits
   1129490 bad hits
    721213 false hits
   4502280 miss
    317740 too long
    694191 pass2 hits
   1207744 2passes
           cache hits (94% pos + 4% neg) system 0% per-process
           deletions 0%, falsehits 0%, toolong 0%

 ---
 Izumi Tsutsui

From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
To: ad@netbsd.org
Cc: gnats-bugs@netbsd.org, martin@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Sat, 18 Jan 2020 14:40:56 +0900

 > > > 	http://www.netbsd.org/~ad/2020/uvm-9.diff
 > > 
 > > This version successfully builds. I'll check how it goes.
 > 
 > The pgdaemon spin still occurs with this patch.

 Now I also get the following panic
 (not sure if the spin occurs at this moment):

 ---
 kernel: supervisor trap page fault, code=0
 Stopped in pid 0.98 (system) at netbsd:uvmpdpol_selectvictim+0xdb:     cmpl    $
 0xc0f0ff54,4(%esi)
 db{1}> bt
 uvmpdpol_selectvictim(c13128ac,9f000,0,a0000,0,9,c1043bcf,0,0,9f5a87) at netbsd: uvmpdpol_selectvictim+0xdb
 uvm_pageout(c5435240,1542000,154c000,0,c01003fd,0,0,0,0,0) at netbsd:uvm_pageout+0x1df
 db{1}> 

 ---

 ---
 Izumi Tsutsui

From: Havard Eidnes <he@NetBSD.org>
To: gnats-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp, ad@NetBSD.org
Cc: kern-bug-people@netbsd.org, netbsd-bugs@netbsd.org
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Mon, 03 Feb 2020 16:19:42 +0100 (CET)

 > Note I forgot to note in the first mail, it happened also on NetBSD 8=
 .1.

 Just FYI, I'm seeing the same issue on NetBSD/amd64 8.1_RC1, even
 though my host has 16G memory.  Firefox bloats and easily
 consumes half of that, and that's probably at least part of the
 trigger (I have 3G of 32G swap in use).

 Is it of any interest to collect any information from this
 system, or would that just add more noise at this stage?

 Sincerely,

 - H=E5vard

Responsible-Changed-From-To: kern-bug-people->ad
Responsible-Changed-By: ad@NetBSD.org
Responsible-Changed-When: Wed, 26 Feb 2020 22:00:16 +0000
Responsible-Changed-Why:
take for now


From: Havard Eidnes <he@uninett.no>
To: ad@netbsd.org, tsutsui@ceres.dti.ne.jp
Cc: gnats-bugs@netbsd.org
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Mon, 02 Mar 2020 10:42:34 +0100 (CET)

 >  > > Thank you for the vmstat output, that was very helpful.  I
 >  > > think it trying to balance the queues but your other
 >  > > processes are competing with it, and because of
 >  > > uvm_pageqlock nobody can make progress.  This is not a
 >  > > problem on -current any more because uvm_pageqlock is
 >  > > gone..  I have updated the patch to try and help this
 >  > > situation.  I have not compiled it because I don't have a
 >  > > 9.0 system at the moment.  Could you please try it again?
 >  > >
 >  > > 	http://www.netbsd.org/~ad/2020/uvm-9.diff =

 >  >
 >  > Thanks, but unfortunately it fails because this change includes
 >  > "TAILQ_ENTRY(vm_page) pdqueue" not available in netbsd-9:
 >
 >  Oops.  Updated patch is here:
 >
 >  	http://www.netbsd.org/~ad/2020/uvm-9.diff

 I've manually applied this diff to my 8.1_RC1 src tree, and the
 build succeeded.  Any predictions whether this has any chance of
 improving the situation?  It's completely "cargo cult" to me...
 I see that Izumi says the pdaemon looping still happens though.
 Despite this, I'll test this kernel myself as well, and if it
 re-occurs I can try to collect information as best I can, similar
 to what Izumi has done here.

 I noticed in my latest wedge, the X11 server was stuck in "vmem",
 the pdaemon was spinning, and "top" showed 117MB free memory.
 The end result was that the X11 session was stuck, and I had to
 login remotely to reboot the machine (and that even eventually
 required a push of the power butteon).

 Regards,

 - H=E5vard

From: mlelstv@serpens.de (Michael van Elst)
To: gnats-bugs@netbsd.org
Cc: 
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Mon, 2 Mar 2020 12:05:48 -0000 (UTC)

 he@uninett.no (Havard Eidnes) writes:

 > >  	http://www.netbsd.org/~ad/2020/uvm-9.diff
 > 
 > I've manually applied this diff to my 8.1_RC1 src tree, and the
 > build succeeded.  Any predictions whether this has any chance of
 > improving the situation?

 I'd say no. If there is (enough) free memory the pagedaemon should
 only be running if there is a kernel VA shortage and the diff doesn't
 influence that.

 -- 
 -- 
                                 Michael van Elst
 Internet: mlelstv@serpens.de
                                 "A potential Snark may lurk in every tree."

From: Havard Eidnes <he@uninett.no>
To: gnats-bugs@netbsd.org, mlelstv@serpens.de
Cc: ad@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Mon, 02 Mar 2020 15:34:00 +0100 (CET)

 >  > >  	http://www.netbsd.org/~ad/2020/uvm-9.diff
 >  >
 >  > I've manually applied this diff to my 8.1_RC1 src tree, and the
 >  > build succeeded.  Any predictions whether this has any chance of
 >  > improving the situation?
 >
 >  I'd say no. If there is (enough) free memory the pagedaemon should
 >  only be running if there is a kernel VA shortage and the diff doesn'=
 t
 >  influence that.

 Hm, I guess hat jives with Izumi's observations that the loop
 persists.

 However...  This leaves two questions which IMO needs validation:

 1) Is there "enough" free memory?
 2) Is there a kernel VA shortage?

 Is there any information I can collect to answer one or both of
 these?  Is just "vmstat -s" enough (only answers #1?), or is more
 needed?

 Regards,

 - H=E5vard

From: Michael van Elst <mlelstv@serpens.de>
To: Havard Eidnes <he@uninett.no>
Cc: gnats-bugs@netbsd.org, ad@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Mon, 2 Mar 2020 16:10:27 +0100

 On Mon, Mar 02, 2020 at 03:34:00PM +0100, Havard Eidnes wrote:
 > 
 > However...  This leaves two questions which IMO needs validation:
 > 
 > 1) Is there "enough" free memory?
 > 2) Is there a kernel VA shortage?
 > 
 > Is there any information I can collect to answer one or both of
 > these?  Is just "vmstat -s" enough (only answers #1?), or is more
 > needed?

 Once kicked, the pagedaemon keeps running until free memory ("pages
 free") exceeds reserved kernel memory ("reserve kernel pages") and there
 is no kernel VA shortage.

 Of course it might be kicked again immediately if some memory allocation
 still fails. this is counted ("times daemon wokeup").


 I use a local patch that reports when VA is starved and that
 slows down the busy loop somewhat.

 http://ftp.netbsd.org/pub/NetBSD/misc/mlelstv/uvm_pdaemon.patch

 But I haven't reproduced the issue so far.


 Greetings,
 -- 
                                 Michael van Elst
 Internet: mlelstv@serpens.de
                                 "A potential Snark may lurk in every tree."

From: Havard Eidnes <he@uninett.no>
To: mlelstv@serpens.de
Cc: gnats-bugs@netbsd.org, ad@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Mon, 02 Mar 2020 23:02:07 +0100 (CET)

 > I use a local patch that reports when VA is starved and that
 > slows down the busy loop somewhat.
 >
 > http://ftp.netbsd.org/pub/NetBSD/misc/mlelstv/uvm_pdaemon.patch
 >
 > But I haven't reproduced the issue so far.

 Thanks, I've added this to my kernel now as well, we'll see if it
 fires.

 Regards,

 - H=E5vard

From: Havard Eidnes <he@uninett.no>
To: mlelstv@serpens.de
Cc: gnats-bugs@netbsd.org, ad@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Wed, 04 Mar 2020 12:09:22 +0100 (CET)

 >> I use a local patch that reports when VA is starved and that
 >> slows down the busy loop somewhat.
 >>
 >> http://ftp.netbsd.org/pub/NetBSD/misc/mlelstv/uvm_pdaemon.patch
 >>
 >> But I haven't reproduced the issue so far.
 >
 > Thanks, I've added this to my kernel now as well, we'll see if it
 > fires.

 The "out of KVA" check fired.

 It was seemingly triggered by the X server; I moved the mouse
 between windows and it froze; "top" shows "Xorg" in "vmem"
 status, and the kernel started printing

 pagedaemon: Out of KVA, awaiting doom...

 I could log in over the network even though the X server was
 wedged, and collect some information -- it follows here below.
 If there is other information I should collect, please inform
 me.

 The question remains: is there something I can do to prevent this
 from happening again?

 Does the number of files on this host perhaps impact this?
 There's quite a lot of small files on this host, and they're
 being backed up periodically via rsync:

 : {4} ; df -hi
 Filesystem   Size    Used   Avail %Cap    iUsed   iAvail %iCap Mounted on
 /dev/wd0a    886G    247G    595G  29% 10276968 47861398   17% /
 ...

 "vmstat -m" shows quite a lot of "Npage" in buf32k and buf4k, and
 it doesn't appear to change between runs of "vmstat -m".  They
 way I read it, "vmstat -m" sum says 2.4G is "in use" for pools.


 Collected information:

 : {1} ; vmstat -s
      4096 bytes per page
         8 page colors
   4041631 pages managed
     98743 pages free
   1432740 pages active
    745029 pages inactive
         0 pages paging
     19548 pages wired
     95597 zero pages
         1 reserve pagedaemon pages
        40 reserve kernel pages
   1019897 anonymous pages
   1081586 cached file pages
     95834 cached executable pages
      2048 minimum free pages
      2730 target free pages
   1347210 maximum wired pages
         1 swap devices
   8388826 swap pages
         1 swap pages in use
         0 swap allocations
  67574827 total faults taken
  65285274 traps
   7438388 device interrupts
  89847588 CPU context switches
  31287654 software interrupts
 1028430556 system calls
         0 pagein requests
         1 pageout requests
         0 pages swapped in
         1 pages swapped out
     25909 forks total
     11413 forks blocked parent
     11413 forks shared address space with parent
  42867536 pagealloc zero wanted and avail
   6576374 pagealloc zero wanted and not avail
         8 aborts of idle page zeroing
  61931077 pagealloc desired color avail
   1331859 pagealloc desired color not avail
  39920782 pagealloc local cpu avail
  23342154 pagealloc local cpu not avail
         0 faults with no memory
         0 faults with no anons
         0 faults had to wait on pages
         0 faults found released page
     82160 faults relock (81368 ok)
  12283881 anon page faults
         0 anon retry faults
   2326273 amap copy faults
   1395819 neighbour anon page faults
  10192517 neighbour object page faults
   4374440 locked pager get faults
     82163 unlocked pager get faults
  12035606 anon faults
    248293 anon copy on write faults
   3643458 object faults
    730202 promote copy faults
  47162585 promote zero fill faults
   1240808 times daemon wokeup
       219 revolutions of the clock hand
   1831677 pages freed by daemon
   2876033 pages scanned by daemon
         1 anonymous pages scanned by daemon
   1831676 object pages scanned by daemon
    144539 pages reactivated
         0 pages found busy by daemon
         0 total pending pageouts
   3551154 pages deactivated
 114882354 total name lookups
  80840025 good hits
   1944977 negative hits
     14969 bad hits
      2006 false hits
  31366087 miss
    714290 too long
  28095446 pass2 hits
  28168723 2passes
           cache hits (70% pos + 1% neg) system 24% per-process
           deletions 0%, falsehits 0%, toolong 0%

 : {2} ;  

 : {2} ; vmstat -s
      4096 bytes per page
         8 page colors
   4041631 pages managed
     98665 pages free
   1432818 pages active
    745029 pages inactive
         0 pages paging
     19548 pages wired
     95426 zero pages
         1 reserve pagedaemon pages
        40 reserve kernel pages
   1019975 anonymous pages
   1081586 cached file pages
     95834 cached executable pages
      2048 minimum free pages
      2730 target free pages
   1347210 maximum wired pages
         1 swap devices
   8388826 swap pages
         1 swap pages in use
         0 swap allocations
  67575069 total faults taken
  65285516 traps
   7439911 device interrupts
  89851413 CPU context switches
  31294275 software interrupts
 1028450495 system calls
         0 pagein requests
         1 pageout requests
         0 pages swapped in
         1 pages swapped out
     25910 forks total
     11414 forks blocked parent
     11414 forks shared address space with parent
  42867707 pagealloc zero wanted and avail
   6576374 pagealloc zero wanted and not avail
         8 aborts of idle page zeroing
  61932177 pagealloc desired color avail
   1331859 pagealloc desired color not avail
  39921874 pagealloc local cpu avail
  23342162 pagealloc local cpu not avail
         0 faults with no memory
         0 faults with no anons
         0 faults had to wait on pages
         0 faults found released page
     82160 faults relock (81368 ok)
  12283896 anon page faults
         0 anon retry faults
   2326313 amap copy faults
   1395819 neighbour anon page faults
  10192727 neighbour object page faults
   4374513 locked pager get faults
     82163 unlocked pager get faults
  12035621 anon faults
    248293 anon copy on write faults
   3643514 object faults
    730219 promote copy faults
  47162739 promote zero fill faults
   1240960 times daemon wokeup
       219 revolutions of the clock hand
   1831677 pages freed by daemon
   2876033 pages scanned by daemon
         1 anonymous pages scanned by daemon
   1831676 object pages scanned by daemon
    144539 pages reactivated
         0 pages found busy by daemon
         0 total pending pageouts
   3551154 pages deactivated
 114882523 total name lookups
  80840186 good hits
   1944985 negative hits
     14969 bad hits
      2006 false hits
  31366087 miss
    714290 too long
  28095446 pass2 hits
  28168723 2passes
           cache hits (70% pos + 1% neg) system 24% per-process
           deletions 0%, falsehits 0%, toolong 0%

 : {3} ;  dmesg
 ...
 pagedaemon: Out of KVA, awaiting doom...
 pagedaemon: Out of KVA, awaiting doom...
 pagedaemon: Out of KVA, awaiting doom...
 pagedaemon: Out of KVA, awaiting doom...
 pagedaemon: Out of KVA, awaiting doom...
 pagedaemon: Out of KVA, awaiting doom...
 pagedaemon: Out of KVA, awaiting doom...
 pagedaemon: Out of KVA, awaiting doom...
 pagedaemon: Out of KVA, awaiting doom...
 pagedaemon: Out of KVA, awaiting doom...
 pagedaemon: Out of KVA, awaiting doom...
 pagedaemon: Out of KVA, awaiting doom...
 ...


 load averages:  0.02,  0.13,  0.25;               up 1+03:13:39        11:43:01
 130 processes: 128 sleeping, 2 on CPU
 CPU states:  0.0% user,  0.0% nice,  0.0% system,  0.0% interrupt,  100% idle
 Memory: 5601M Act, 2910M Inact, 76M Wired, 374M Exec, 4225M File, 381M Free
 Swap: 32G Total, 4K Used, 32G Free

   PID USERNAME PRI NICE   SIZE   RES STATE      TIME   WCPU    CPU COMMAND
     0 root       0    0     0K   26M CPU/7     10:31  0.00%  0.34% [system]
   689 he        43    0  4689M 3169M parked/2 112:09  0.00%  0.00% firefox
   564 he        85    0   250M   90M vmem/0    29:52  0.00%  0.00% Xorg
   330 he        85    0   478M  240M select/2   1:16  0.00%  0.00% emacs-26.2
   437 haldaemo  85    0    70M 5028K select/2   0:48  0.00%  0.00% hald
   209 root      43    0   439M  264M parked/1   0:35  0.00%  0.00% named

 load averages:  0.02,  0.12,  0.24;               up 1+03:13:51        11:43:13
 368 threads: 33 idle, 320 sleeping, 7 zombie, 8 on CPU
 CPU states:  0.0% user,  0.0% nice,  0.2% system,  0.0% interrupt, 99.7% idle
 Memory: 5601M Act, 2910M Inact, 76M Wired, 374M Exec, 4225M File, 381M Free
 Swap: 32G Total, 4K Used, 32G Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
   689     1 he        43 parked/0  90:01  0.00%  0.00% -         firefox
   564     1 he        85 vmem/0    29:52  0.00%  0.00% -         Xorg
   689    22 he        85 select/7  12:15  0.00%  0.00% Composito firefox
     0   118 root     124 syncer/0   8:52  0.00%  0.00% ioflush   [system]
   689    21 he        43 parked/5   1:23  0.00%  0.00% SoftwareV firefox
   330     1 he        85 select/1   1:16  0.00%  0.00% -         emacs-26.2
   437     1 haldaemo  85 select/5   0:48  0.00%  0.00% -         hald
     0     9 root     125 vdrain/3   0:40  0.00%  0.00% vdrain    [system]
   689    12 he        43 parked/7   0:39  0.00%  0.00% Timer     firefox
   689    15 he        43 parked/3   0:36  0.00%  0.00% Cache2 I/ firefox
     0   117 root     126 pgdaem/1   0:30  0.00%  0.00% pgdaemon  [system]
   689    13 he        85 select/0   0:30  0.00%  0.00% Socket Th firefox
   689    71 he        43 parked/6   0:20  0.00%  0.00% URL Class firefox
   689   102 he        43 parked/4   0:18  0.00%  0.00% DOM Worke firefox
   689    10 he        43 parked/1   0:16  0.00%  0.00% JS Helper firefox
   689    11 he        43 parked/6   0:15  0.00%  0.00% JS Helper firefox
   689     9 he        43 parked/2   0:15  0.00%  0.00% JS Helper firefox
   689     8 he        43 parked/6   0:15  0.00%  0.00% JS Helper firefox
   689     4 he        43 parked/6   0:15  0.00%  0.00% JS Helper firefox
   689     6 he        43 parked/3   0:14  0.00%  0.00% JS Helper firefox
   689     7 he        43 parked/4   0:14  0.00%  0.00% JS Helper firefox
   689     5 he        43 parked/7   0:14  0.00%  0.00% JS Helper firefox
  1399     1 he        85 select/0   0:11  0.00%  0.00% -         pidgin


 : {8} ; vmstat -s
      4096 bytes per page
         8 page colors
   4041631 pages managed
     97594 pages free
   1433886 pages active
    745029 pages inactive
         0 pages paging
     19548 pages wired
     92514 zero pages
         1 reserve pagedaemon pages
        40 reserve kernel pages
   1020969 anonymous pages
   1081660 cached file pages
     95834 cached executable pages
      2048 minimum free pages
      2730 target free pages
   1347210 maximum wired pages
         1 swap devices
   8388826 swap pages
         1 swap pages in use
         0 swap allocations
  67609683 total faults taken
  65321077 traps
   7448571 device interrupts
  89910985 CPU context switches
  31337985 software interrupts
 1028698866 system calls
         0 pagein requests
         1 pageout requests
         0 pages swapped in
         1 pages swapped out
     25951 forks total
     11434 forks blocked parent
     11434 forks shared address space with parent
  42892955 pagealloc zero wanted and avail
   6576643 pagealloc zero wanted and not avail
         8 aborts of idle page zeroing
  61965858 pagealloc desired color avail
   1331888 pagealloc desired color not avail
  39952332 pagealloc local cpu avail
  23345414 pagealloc local cpu not avail
         0 faults with no memory
         0 faults with no anons
         0 faults had to wait on pages
         0 faults found released page
     82163 faults relock (81371 ok)
  12288293 anon page faults
         0 anon retry faults
   2327745 amap copy faults
   1396621 neighbour anon page faults
  10208555 neighbour object page faults
   4379512 locked pager get faults
     82166 unlocked pager get faults
  12039675 anon faults
    248636 anon copy on write faults
   3647242 object faults
    731490 promote copy faults
  47186892 promote zero fill faults
   1251810 times daemon wokeup
       219 revolutions of the clock hand
   1831677 pages freed by daemon
   2876033 pages scanned by daemon
         1 anonymous pages scanned by daemon
   1831676 object pages scanned by daemon
    144539 pages reactivated
         0 pages found busy by daemon
         0 total pending pageouts
   3551154 pages deactivated
 114903588 total name lookups
  80859506 good hits
   1946665 negative hits
     14980 bad hits
      2006 false hits
  31366141 miss
    714290 too long
  28095450 pass2 hits
  28168728 2passes
           cache hits (70% pos + 1% neg) system 24% per-process
           deletions 0%, falsehits 0%, toolong 0%

 : {9} ;  

 : {9} ; vmstat -m
 Memory resource pool statistics
 Name        Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
 amappl        80    22924    0      208   455     0   455   455     0   inf    0
 anonpl        32  1305084    0   210496  9459     0  9459  9459     0   inf  221
 ataspl        96  2337417    0  2337417     1     0     1     1     0   inf    1
 biopl        288      931    0      760    55     0    55    55     0   inf   42
 buf16k      16384    1411    0     1270   241   205    36    93     1     1    0
 buf1k       1024        2    0        2     1     0     1     1     1     1    1
 buf2k       2048        9    0        9     5     4     1     5     1     1    1
 buf32k      32768  223292    0   193503 92993 73624 19369 37065     1     1    0
 buf4k       4096   491370    0   391560 491371 391560 99811 179873  1     1    1
 buf64k      65536       4    0        0     5     0     5     5     1     1    1
 buf8k       8192     1865    0     1613   160   128    32    63     1     1    0
 bufpl        288   210502    0    80506 15026     0 15026 15026     0   inf  111
 cwdi          64      335    0      135     5     0     5     5     0   inf    0
 ehcixfer     368        3    0        1     1     0     1     1     0   inf    0
 ehcixfer     368        4    0        2     1     0     1     1     0   inf    0
 execargs    262144  23091    0    23091     6     4     2     4     0    16    2
 extent        40        4    0        4     1     1     0     1     0   inf    0
 fcrpl        168       39    0       38     3     0     3     3     3   inf    2
 fdfile        64     8690    0     5571   127     0   127   127     0   inf   12
 ffsdino2     256   844103    0    94293 52739     0 52739 52739     0   inf  192
 ffsino       256   844103    0    94293 52739     0 52739 52739     0   inf  194
 file         128     1976    0      764    56     0    56    56     0   inf    0
 filedesc     832      339    0      139    65     0    65    65     0   inf    9
 i915_gem_object 448   534    0        0    60     0    60    60     0   inf    0
 icmp          24     3582    0     3582     1     0     1     1     0   inf    1
 icmp6         24    52790    0    52790     1     0     1     1     0   inf    1
 in6pcbpl     272    81137    0    81089    18     0    18    18     0   inf   13
 inmltpl       48        2    0        0     1     0     1     1     0   inf    0
 inpcbpl      232    64972    0    64849    12     0    12    12     0   inf    3
 ipfrenpl      64        3    0        0     1     0     1     1     0   inf    0
 kcpuset       64      497    0        0     8     0     8     8     0   inf    0
 kcredpl      192     1034    0        0    50     0    50    50     0   inf    0
 kmem-1024   1024    10596    0        0  2649     0  2649  2649     0   inf    0
 kmem-112     112     2501    0        0    70     0    70    70     0   inf    0
 kmem-128     128     8109    0        0   254     0   254   254     0   inf    0
 kmem-16       16    16292    0        0    64     0    64    64     0   inf    0
 kmem-160     160     1521    0        0    61     0    61    61     0   inf    0
 kmem-192     192      504    0        0    24     0    24    24     0   inf    0
 kmem-2048   2048     5976    0        0  2988     0  2988  2988     0   inf    0
 kmem-224     224      584    0        0    33     0    33    33     0   inf    0
 kmem-24       24     1467    0        0     9     0     9     9     0   inf    0
 kmem-256     256      894    0        0    56     0    56    56     0   inf    0
 kmem-32       32     2407    0        0    19     0    19    19     0   inf    0
 kmem-320     320     1197    0        0   100     0   100   100     0   inf    0
 kmem-384     384      604    0        0    61     0    61    61     0   inf    0
 kmem-40       40     6085    0        0    60     0    60    60     0   inf    0
 kmem-4096   4096      192    0        0   192     0   192   192     0   inf    0
 kmem-448     448      390    0        0    44     0    44    44     0   inf    0
 kmem-48       48     4270    0        0    51     0    51    51     0   inf    0
 kmem-512     512      326    0        0    41     0    41    41     0   inf    0
 kmem-56       56     1520    0        0    21     0    21    21     0   inf    0
 kmem-64       64     9339    0        0   146     0   146   146     0   inf    0
 kmem-768     768      553    0        0   111     0   111   111     0   inf    0
 kmem-8         8    34248    0        0    67     0    67    67     0   inf    0
 kmem-80       80     5964    0        0   117     0   117   117     0   inf    0
 kmem-96       96      736    0        0    18     0    18    18     0   inf    0
 ksiginfo      72      188    0        0     4     0     4     4     0   inf    0
 kva-12288   12288      35    0        0     2     0     2     2     0   inf    0
 kva-16384   16384      17    0        0     2     0     2     2     0   inf    0
 kva-20480   20480      84    0        0     7     0     7     7     0   inf    0
 kva-24576   24576       9    0        0     1     0     1     1     0   inf    0
 kva-28672   28672       3    0        0     1     0     1     1     0   inf    0
 kva-32768   32768       1    0        0     1     0     1     1     0   inf    0
 kva-36864   36864       3    0        0     1     0     1     1     0   inf    0
 kva-40960   40960     108    0        0    18     0    18    18     0   inf    0
 kva-49152   49152       1    0        0     1     0     1     1     0   inf    0
 kva-65536   65536   63924    0        0 15981     0 15981 15981     0   inf    0
 kva-8192    8192       52    0        0     2     0     2     2     0   inf    0
 llentrypl    272        3    0        0     1     0     1     1     0   inf    0
 lockf        112      152    0        0     5     0     5     5     0   inf    0
 lwppl       1056      528    0        0   176     0   176   176     0   inf    0
 mbpl         512     1730    0        0   219     0   219   219     2   inf    2
 mclpl       2048      816    0        0   412     0   412   412     4 260389   4
 mutex         64   863162    0        0 13701     0 13701 13701     0   inf    0
 ncache       192   833333    0        0 39683     0 39683 39683     0   inf    0
 pcache      2688       90    0        4    86     0    86    86     0   inf    0
 pcachecpu     64      658    0        0    11     0    11    11     0   inf    0
 pcglarge    1024     7101    0     4459  1342     0  1342  1342     0   inf  681
 pcgnormal    256    48895    0    20117  3031     0  3031  3031     0   inf  432
 pdict16       72     1757    0     1702     2     0     2     2     0   inf    1
 pdict32       88      367    0      356     1     0     1     1     0   inf    0
 pdppl       4096      265    0        0   265     0   265   265     0   inf    0
 phpool-0      56   702605    0   465521  4284     0  4284  4284     0   inf   32
 phpool-128    64      151    0        0     3     0     3     3     0   inf    0
 phpool-256    80       73    0        0     2     0     2     2     0   inf    0
 phpool-512   112       67    0        0     2     0     2     2     0   inf    0
 phpool-64     56    16551    0        4   230     0   230   230     0   inf    0
 piperd       320      211    0        0    18     0    18    18     0   inf    0
 pipewr       320      232    0        0    20     0    20    20     0   inf    0
 plimitpl     232      180    0        0    11     0    11    11     0   inf    0
 pmappl       408      265    0        0    27     0    27    27     0   inf    0
 pnbufpl     1024      160    0        0    40     0    40    40     0   inf    0
 procpl       720      272    0        0    55     0    55    55     0   inf    0
 proparay      48      132    0        0     2     0     2     2     0   inf    0
 propdict      48     1596    0     1208     5     0     5     5     0   inf    0
 propnmbr      56   102776    0   102734     1     0     1     1     0   inf    0
 propstng      40     1660    0      988     7     0     7     7     0   inf    0
 pstatspl     448      272    0        0    31     0    31    31     0   inf    0
 ptimerpl     264     4390    0     4351     7     0     7     7     0   inf    4
 ptimerspl    304     4390    0     4351     7     0     7     7     0   inf    4
 pvpl          40   160325    0        0  1588     0  1588  1588     0   inf    0
 ractx         32   257700    0        0  2046     0  2046  2046     0   inf    0
 rndctx        16       15    0        0     1     0     1     1     0   inf    0
 rndsample    536       78    0        0    12     0    12    12     0   586    0
 rndtemp      512        8    0        0     1     0     1     1     0   inf    0
 rtentpl      320       31    0        2     3     0     3     3     0   inf    0
 rwlock        64        6    0        0     1     0     1     1     0   inf    0
 sackholepl    32       70    0       70     1     0     1     1     0   inf    1
 scxspl       256    39116    0    39116     1     0     1     1     1   inf    1
 sigacts     3088      259    0        0   259     0   259   259     0   inf    0
 socket       592     1027    0        0   172     0   172   172     0   inf    0
 synpl        312     1823    0     1823     1     0     1     1     0   inf    1
 tcpcbpl      832    14302    0    14245    54     0    54    54     0   inf   37
 tcpipqepl     80     7924    0     7924     1     0     1     1     0   inf    1
 tmpfs_dirent  48    34615    0    34600     1     0     1     1     0   inf    0
 tmpfs_node   216    34616    0    34370    18     0    18    18     0   inf    3
 tstilepl      96      528    0        0    13     0    13    13     0   inf    0
 uarea       16384     528    0        0   528     0   528   528     0   inf    0
 ufsdir       264       17    0        0     2     0     2     2     0   inf    0
 vcachepl     336   844075    0        0 70340     0 70340 70340     0   inf    0
 vmembt        56    67918    0        0   944     0   944   944     0   inf    0
 vmmpepl      144    46483    0        0  1661     0  1661  1661     0   inf    0
 vmsppl       368      270    0        0    25     0    25    25     0   inf    0
 wapbldealloc  32      954    0      954     1     0     1     1     0   inf    1
 wapblentrypl  40    10687    0    10687     1     0     1     1     0   inf    1
 wapblinopl    32    53816    0    53815     1     0     1     1     0   inf    0
 xhcixfer     624       13    0        0     3     0     3     3     0   inf    0
 Totals           10874793    0  4441386880310465526414784

 In use 2385264K, total allocated 2770048K; utilization 86.1%

 : {10} ; 

 : {10} ; vmstat -m
 Memory resource pool statistics
 Name        Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
 amappl        80    22924    0      208   455     0   455   455     0   inf    0
 anonpl        32  1305084    0   210496  9459     0  9459  9459     0   inf  221
 ataspl        96  2337458    0  2337458     1     0     1     1     0   inf    1
 biopl        288      931    0      760    55     0    55    55     0   inf   42
 buf16k      16384    1411    0     1270   241   205    36    93     1     1    0
 buf1k       1024        2    0        2     1     0     1     1     1     1    1
 buf2k       2048        9    0        9     5     4     1     5     1     1    1
 buf32k      32768  223292    0   193503 92993 73624 19369 37065     1     1    0
 buf4k       4096   491370    0   391560 491371 391560 99811 179873  1     1    1
 buf64k      65536       4    0        0     5     0     5     5     1     1    1
 buf8k       8192     1865    0     1613   160   128    32    63     1     1    0
 bufpl        288   210502    0    80506 15026     0 15026 15026     0   inf  111
 cwdi          64      335    0      135     5     0     5     5     0   inf    0
 ehcixfer     368        3    0        1     1     0     1     1     0   inf    0
 ehcixfer     368        4    0        2     1     0     1     1     0   inf    0
 execargs    262144  23092    0    23092     6     4     2     4     0    16    2
 extent        40        4    0        4     1     1     0     1     0   inf    0
 fcrpl        168       39    0       38     3     0     3     3     3   inf    2
 fdfile        64     8690    0     5571   127     0   127   127     0   inf   12
 ffsdino2     256   844103    0    94293 52739     0 52739 52739     0   inf  192
 ffsino       256   844103    0    94293 52739     0 52739 52739     0   inf  194
 file         128     1976    0      764    56     0    56    56     0   inf    0
 filedesc     832      339    0      139    65     0    65    65     0   inf    9
 i915_gem_object 448   534    0        0    60     0    60    60     0   inf    0
 icmp          24     3582    0     3582     1     0     1     1     0   inf    1
 icmp6         24    52812    0    52812     1     0     1     1     0   inf    1
 in6pcbpl     272    81137    0    81089    18     0    18    18     0   inf   13
 inmltpl       48        2    0        0     1     0     1     1     0   inf    0
 inpcbpl      232    64972    0    64849    12     0    12    12     0   inf    3
 ipfrenpl      64        3    0        0     1     0     1     1     0   inf    0
 kcpuset       64      497    0        0     8     0     8     8     0   inf    0
 kcredpl      192     1034    0        0    50     0    50    50     0   inf    0
 kmem-1024   1024    10596    0        0  2649     0  2649  2649     0   inf    0
 kmem-112     112     2501    0        0    70     0    70    70     0   inf    0
 kmem-128     128     8109    0        0   254     0   254   254     0   inf    0
 kmem-16       16    16292    0        0    64     0    64    64     0   inf    0
 kmem-160     160     1521    0        0    61     0    61    61     0   inf    0
 kmem-192     192      504    0        0    24     0    24    24     0   inf    0
 kmem-2048   2048     5976    0        0  2988     0  2988  2988     0   inf    0
 kmem-224     224      584    0        0    33     0    33    33     0   inf    0
 kmem-24       24     1467    0        0     9     0     9     9     0   inf    0
 kmem-256     256      894    0        0    56     0    56    56     0   inf    0
 kmem-32       32     2407    0        0    19     0    19    19     0   inf    0
 kmem-320     320     1197    0        0   100     0   100   100     0   inf    0
 kmem-384     384      604    0        0    61     0    61    61     0   inf    0
 kmem-40       40     6085    0        0    60     0    60    60     0   inf    0
 kmem-4096   4096      192    0        0   192     0   192   192     0   inf    0
 kmem-448     448      390    0        0    44     0    44    44     0   inf    0
 kmem-48       48     4270    0        0    51     0    51    51     0   inf    0
 kmem-512     512      326    0        0    41     0    41    41     0   inf    0
 kmem-56       56     1520    0        0    21     0    21    21     0   inf    0
 kmem-64       64     9339    0        0   146     0   146   146     0   inf    0
 kmem-768     768      553    0        0   111     0   111   111     0   inf    0
 kmem-8         8    34248    0        0    67     0    67    67     0   inf    0
 kmem-80       80     5964    0        0   117     0   117   117     0   inf    0
 kmem-96       96      736    0        0    18     0    18    18     0   inf    0
 ksiginfo      72      188    0        0     4     0     4     4     0   inf    0
 kva-12288   12288      35    0        0     2     0     2     2     0   inf    0
 kva-16384   16384      17    0        0     2     0     2     2     0   inf    0
 kva-20480   20480      84    0        0     7     0     7     7     0   inf    0
 kva-24576   24576       9    0        0     1     0     1     1     0   inf    0
 kva-28672   28672       3    0        0     1     0     1     1     0   inf    0
 kva-32768   32768       1    0        0     1     0     1     1     0   inf    0
 kva-36864   36864       3    0        0     1     0     1     1     0   inf    0
 kva-40960   40960     108    0        0    18     0    18    18     0   inf    0
 kva-49152   49152       1    0        0     1     0     1     1     0   inf    0
 kva-65536   65536   63924    0        0 15981     0 15981 15981     0   inf    0
 kva-8192    8192       52    0        0     2     0     2     2     0   inf    0
 llentrypl    272        3    0        0     1     0     1     1     0   inf    0
 lockf        112      152    0        0     5     0     5     5     0   inf    0
 lwppl       1056      528    0        0   176     0   176   176     0   inf    0
 mbpl         512     1730    0        0   219     0   219   219     2   inf    2
 mclpl       2048      816    0        0   412     0   412   412     4 260389   4
 mutex         64   863162    0        0 13701     0 13701 13701     0   inf    0
 ncache       192   833333    0        0 39683     0 39683 39683     0   inf    0
 pcache      2688       90    0        4    86     0    86    86     0   inf    0
 pcachecpu     64      658    0        0    11     0    11    11     0   inf    0
 pcglarge    1024     7101    0     4459  1342     0  1342  1342     0   inf  681
 pcgnormal    256    48895    0    20117  3031     0  3031  3031     0   inf  432
 pdict16       72     1757    0     1702     2     0     2     2     0   inf    1
 pdict32       88      367    0      356     1     0     1     1     0   inf    0
 pdppl       4096      265    0        0   265     0   265   265     0   inf    0
 phpool-0      56   702605    0   465521  4284     0  4284  4284     0   inf   32
 phpool-128    64      151    0        0     3     0     3     3     0   inf    0
 phpool-256    80       73    0        0     2     0     2     2     0   inf    0
 phpool-512   112       67    0        0     2     0     2     2     0   inf    0
 phpool-64     56    16551    0        4   230     0   230   230     0   inf    0
 piperd       320      211    0        0    18     0    18    18     0   inf    0
 pipewr       320      232    0        0    20     0    20    20     0   inf    0
 plimitpl     232      180    0        0    11     0    11    11     0   inf    0
 pmappl       408      265    0        0    27     0    27    27     0   inf    0
 pnbufpl     1024      160    0        0    40     0    40    40     0   inf    0
 procpl       720      272    0        0    55     0    55    55     0   inf    0
 proparay      48      132    0        0     2     0     2     2     0   inf    0
 propdict      48     1596    0     1208     5     0     5     5     0   inf    0
 propnmbr      56   102795    0   102753     1     0     1     1     0   inf    0
 propstng      40     1660    0      988     7     0     7     7     0   inf    0
 pstatspl     448      272    0        0    31     0    31    31     0   inf    0
 ptimerpl     264     4390    0     4351     7     0     7     7     0   inf    4
 ptimerspl    304     4390    0     4351     7     0     7     7     0   inf    4
 pvpl          40   160325    0        0  1588     0  1588  1588     0   inf    0
 ractx         32   257700    0        0  2046     0  2046  2046     0   inf    0
 rndctx        16       15    0        0     1     0     1     1     0   inf    0
 rndsample    536       78    0        0    12     0    12    12     0   586    0
 rndtemp      512        8    0        0     1     0     1     1     0   inf    0
 rtentpl      320       31    0        2     3     0     3     3     0   inf    0
 rwlock        64        6    0        0     1     0     1     1     0   inf    0
 sackholepl    32       70    0       70     1     0     1     1     0   inf    1
 scxspl       256    39124    0    39124     1     0     1     1     1   inf    1
 sigacts     3088      259    0        0   259     0   259   259     0   inf    0
 socket       592     1027    0        0   172     0   172   172     0   inf    0
 synpl        312     1823    0     1823     1     0     1     1     0   inf    1
 tcpcbpl      832    14302    0    14245    54     0    54    54     0   inf   37
 tcpipqepl     80     7924    0     7924     1     0     1     1     0   inf    1
 tmpfs_dirent  48    34615    0    34600     1     0     1     1     0   inf    0
 tmpfs_node   216    34616    0    34370    18     0    18    18     0   inf    3
 tstilepl      96      528    0        0    13     0    13    13     0   inf    0
 uarea       16384     528    0        0   528     0   528   528     0   inf    0
 ufsdir       264       17    0        0     2     0     2     2     0   inf    0
 vcachepl     336   844075    0        0 70340     0 70340 70340     0   inf    0
 vmembt        56    67918    0        0   944     0   944   944     0   inf    0
 vmmpepl      144    46483    0        0  1661     0  1661  1661     0   inf    0
 vmsppl       368      270    0        0    25     0    25    25     0   inf    0
 wapbldealloc  32      954    0      954     1     0     1     1     0   inf    1
 wapblentrypl  40    10689    0    10689     1     0     1     1     0   inf    1
 wapblinopl    32    53816    0    53815     1     0     1     1     0   inf    0
 xhcixfer     624       13    0        0     3     0     3     3     0   inf    0
 Totals           10874886    0  4441479880310465526414784

 In use 2385264K, total allocated 2770048K; utilization 86.1%

 : {11} ; 

From: mlelstv@serpens.de (Michael van Elst)
To: gnats-bugs@netbsd.org
Cc: 
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Wed, 4 Mar 2020 12:09:53 -0000 (UTC)

 he@uninett.no (Havard Eidnes) writes:

 > Does the number of files on this host perhaps impact this?

 The number of files touched determines how many vnodes and 
 associated data structures are cached. And that's in KVA.

 > vcachepl     336   844075    0        0 70340     0 70340 70340     0   inf    0

 The vnode cache acquired 70340 pages (about 844000 entries) and never
 released anything. That's not special for such a machine. But there
 is more data referenced by the cache that can only be evicted if the
 cache entry is removed. The real memory footprint can be much larger.

 It's possible that kern.maxvnodes is too large nowadays, but also
 important is that vnodes are not evicted under memory pressure.
 Maybe the page daemon should invoke vfs_drainvnodes(), even when it
 is not an issue here, I suspect maxvnodes to be larger than 844000.
 Then the page daemon must be able to shrink the cache below the
 maxvnodes value.

 -- 
 -- 
                                 Michael van Elst
 Internet: mlelstv@serpens.de
                                 "A potential Snark may lurk in every tree."

From: Lars Reichardt <lars@paradoxon.info>
To: gnats-bugs@netbsd.org, ad@netbsd.org, gnats-admin@netbsd.org,
 netbsd-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp
Cc: 
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Wed, 4 Mar 2020 16:32:28 +0100

 On 2020-03-04 12:10, Havard Eidnes wrote:
 > The following reply was made to PR kern/54818; it has been noted by GNATS.
 >
 > From: Havard Eidnes <he@uninett.no>
 > To: mlelstv@serpens.de
 > Cc: gnats-bugs@netbsd.org, ad@netbsd.org, tsutsui@ceres.dti.ne.jp
 > Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
 > Date: Wed, 04 Mar 2020 12:09:22 +0100 (CET)
 ...
 >   The "out of KVA" check fired.
 >   
 >   It was seemingly triggered by the X server; I moved the mouse
 >   between windows and it froze; "top" shows "Xorg" in "vmem"
 >   status, and the kernel started printing
 >   
 >   pagedaemon: Out of KVA, awaiting doom...
 >   
 >   I could log in over the network even though the X server was
 >   wedged, and collect some information -- it follows here below.
 >   If there is other information I should collect, please inform
 >   me.
 >   
 >   The question remains: is there something I can do to prevent this
 >   from happening again?
 >   
 >   
 >   
 >   : {9} ; vmstat -m
 >   Memory resource pool statistics
 >   Name        Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
 >   amappl        80    22924    0      208   455     0   455   455     0   inf    0
 >   anonpl        32  1305084    0   210496  9459     0  9459  9459     0   inf  221
 >   ataspl        96  2337417    0  2337417     1     0     1     1     0   inf    1
 >   biopl        288      931    0      760    55     0    55    55     0   inf   42

 <<<<< these don't allocate via kmem but directly from kernel_map

 >   buf16k      16384    1411    0     1270   241   205    36    93     1     1    0
 >   buf1k       1024        2    0        2     1     0     1     1     1     1    1
 >   buf2k       2048        9    0        9     5     4     1     5     1     1    1
 >   buf32k      32768  223292    0   193503 92993 73624 19369 37065     1     1    0
 >   buf4k       4096   491370    0   391560 491371 391560 99811 179873  1     1    1
 >   buf64k      65536       4    0        0     5     0     5     5     1     1    1
 >   buf8k       8192     1865    0     1613   160   128    32    63     1     1    0
 >   bufpl        288   210502    0    80506 15026     0 15026 15026     0   inf  111

 >>>> these are very interesting:

 These are the quantum caches for allocation virtual address space.

 No 4k allocation as the direct map is used (that's expected) and most pools have a pool page size of 4k
 but a lot of 64k allocations with the backing pool page size 256k.

 That is 64*63924 4091136kb worse of allocations
 (15981 pool pages each 256k)

 and no releases at all seems like some leak to me.

 Does that happen when starting X?
 Seems to be an intel drmkms judged from the list of pools.

 The kmem arena is most likely a bit more than this mentioned 4g as the machine seems to have 16gb?
 It should be the second entry of the output of "pmap 0".

 >   kva-12288   12288      35    0        0     2     0     2     2     0   inf    0
 >   kva-16384   16384      17    0        0     2     0     2     2     0   inf    0
 >   kva-20480   20480      84    0        0     7     0     7     7     0   inf    0
 >   kva-24576   24576       9    0        0     1     0     1     1     0   inf    0
 >   kva-28672   28672       3    0        0     1     0     1     1     0   inf    0
 >   kva-32768   32768       1    0        0     1     0     1     1     0   inf    0
 >   kva-36864   36864       3    0        0     1     0     1     1     0   inf    0
 >   kva-40960   40960     108    0        0    18     0    18    18     0   inf    0
 >   kva-49152   49152       1    0        0     1     0     1     1     0   inf    0
 >   kva-65536   65536   63924    0        0 15981     0 15981 15981     0   inf    0
 >   kva-8192    8192       52    0        0     2     0     2     2     0   inf    0

 ...
 I'm not aware of any pool that allocates from the 64k quantum cache so it doesn't surprise me that that pagedaemon/pool_drain
 isn't able to free anything.

 Kind regards,
 Lars

From: Havard Eidnes <he@NetBSD.org>
To: lars@paradoxon.info
Cc: gnats-bugs@netbsd.org, ad@netbsd.org, gnats-admin@netbsd.org,
 netbsd-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Thu, 05 Mar 2020 16:41:39 +0100 (CET)

 >>   buf16k 16384 1411 0 1270 241 205 36 93 1 1 0
 >>   buf1k 1024 2 0 2 1 0 1 1 1 1 1
 >>   buf2k 2048 9 0 9 5 4 1 5 1 1 1
 >>   buf32k 32768 223292 0 193503 92993 73624 19369 37065 1 1 0
 >>   buf4k 4096 491370 0 391560 491371 391560 99811 179873 1 1 1
 >>   buf64k 65536 4 0 0 5 0 5 5 1 1 1
 >>   buf8k 8192 1865 0 1613 160 128 32 63 1 1 0
 >>   bufpl 288 210502 0 80506 15026 0 15026 15026 0 inf 111
 >
 >>>>> these are very interesting:
 >
 > These are the quantum caches for allocation virtual address space.
 >
 > No 4k allocation as the direct map is used (that's expected) and most
 > pools have a pool page size of 4k
 > but a lot of 64k allocations with the backing pool page size 256k.
 >
 > That is 64*63924 4091136kb worse of allocations
 > (15981 pool pages each 256k)
 >  and no releases at all seems like some leak to me.
 >
 > Does that happen when starting X?

 No.  It typically happens after a few days running.

 > Seems to be an intel drmkms judged from the list of pools.

 Correct:

 i915drmkms0 at pci0 dev 2 function 0: vendor 8086 product 0412 (rev. 0x06)
 drm: Memory usable by graphics device = 2048M
 drm: Supports vblank timestamp caching Rev 2 (21.10.2013).
 drm: Driver supports precise vblank timestamp query.
 i915drmkms0: interrupting at ioapic0 pin 16 (i915)
 intelfb0 at i915drmkms0
 i915drmkms0: info: registered panic notifier
 i915drmkms0: More than 8 outputs detected via ACPI
 intelfb0: framebuffer at 0xffff80013bc7d000, size 1920x1200, depth 32, stride 7680
 wsdisplay0 at intelfb0 kbdmux 1: console (default, vt100 emulation)
 wsmux1: connecting to wsdisplay0

 > The kmem arena is most likely a bit more than this mentioned 4g as the
 > machine seems to have 16gb?

 Physical ram is 16GB, yes.

 > It should be the second entry of the output of "pmap 0".

 Currently the first three lines from "pmap 0" is

 FFFF800000000000 473672K read/write/exec     [ anon ]
 FFFF80001CE92000 4166220K read/write/exec     [ anon ]
 FFFF80011B325000 524288K read/write/exec     [ pager_map ]

 >>   kva-12288 12288 35 0 0 2 0 2 2 0 inf 0
 >>   kva-16384 16384 17 0 0 2 0 2 2 0 inf 0
 >>   kva-20480 20480 84 0 0 7 0 7 7 0 inf 0
 >>   kva-24576 24576 9 0 0 1 0 1 1 0 inf 0
 >>   kva-28672 28672 3 0 0 1 0 1 1 0 inf 0
 >>   kva-32768 32768 1 0 0 1 0 1 1 0 inf 0
 >>   kva-36864 36864 3 0 0 1 0 1 1 0 inf 0
 >>   kva-40960 40960 108 0 0 18 0 18 18 0 inf 0
 >>   kva-49152 49152 1 0 0 1 0 1 1 0 inf 0
 >>   kva-65536 65536 63924 0 0 15981 0 15981 15981 0 inf 0
 >>   kva-8192 8192 52 0 0 2 0 2 2 0 inf 0
 >
 > ...
 > I'm not aware of any pool that allocates from the 64k quantum cache so
 > it doesn't surprise me that that pagedaemon/pool_drain
 > isn't able to free anything.

 Hm.  I know too little about the mechanism, but having something
 allocate from it and nothing to release when there's pressure
 looks like a recipe for disaster.

 Regards,

 - Havard

From: Havard Eidnes <he@uninett.no>
To: lars@paradoxon.info
Cc: gnats-bugs@netbsd.org, ad@netbsd.org, gnats-admin@netbsd.org,
 netbsd-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Thu, 05 Mar 2020 23:45:39 +0100 (CET)

 Oh, BTW, I also appear to have this problem on my old Lenovo T60
 laptop, which runs 8.1/i386 and has 3070MB usable physical memory, and
 it's apparently easier to get this one to trigger.

 I'm assuming it's the same issue.  I've not dragged over the diffs to
 this host (yet).

 Anything more relevant to collect?


 NetBSD 8.1 (GENERIC) #4: Fri May 31 19:20:18 CEST 2019
         he@jonsvatnet.uninett.no:/usr/obj/sys/arch/i386/compile/GENERIC
 total memory = 3070 MB
 avail memory = 3000 MB
 ...
 cpu0 at mainbus0 apid 0
 cpu0: Intel(R) Core(TM)2 CPU         T7400  @ 2.16GHz, id 0x6f6
 cpu0: package 0, core 0, smt 0
 cpu1 at mainbus0 apid 1
 cpu1: Intel(R) Core(TM)2 CPU         T7400  @ 2.16GHz, id 0x6f6
 cpu1: package 0, core 1, smt 0


 load averages:  1.37,  1.39,  1.85;               up 12+06:27:51       22:12:34
 189 threads: 10 idle, 3 runnable, 171 sleeping, 3 zombie, 2 on CPU
 CPU states:  0.3% user,  0.0% nice, 50.7% system,  0.2% interrupt, 48.8% idle
 Memory: 1603M Act, 822M Inact, 2232K Wired, 330M Exec, 1274M File, 46M Free
 Swap: 2048M Total, 168M Used, 1881M Free

   PID   LID USERNAME PRI STATE      TIME   WCPU    CPU NAME      COMMAND
     0    69 root     126 CPU/0     32.0H 99.02% 99.02% pgdaemon  [system]
     0    70 root     124 syncer/1  68:45  0.00%  0.00% ioflush   [system]
  6010     1 he        85 select/1  63:10  0.00%  0.00% -         firefox
   935     1 he        85 select/1  27:15  0.00%  0.00% -         Xorg

 "pmap 0" shows (first few lines):

 C0000000  72416K read/write/exec     [ anon ]
 C46B8000 368640K read/write/exec     [ anon ]
 DAEB8000  16384K read/write/exec     [ pager_map ]
 DBEB8000    136K read/write          [ anon ]
 DBEDA000   1200K read/write/exec     [ phys_map ]


 # vmstat -s
      4096 bytes per page
        64 page colors
    768257 pages managed
     11263 pages free
    410923 pages active
    210357 pages inactive
         0 pages paging
       564 pages wired
      7893 zero pages
         1 reserve pagedaemon pages
        10 reserve kernel pages
    211102 anonymous pages
    326183 cached file pages
     84559 cached executable pages
       512 minimum free pages
       682 target free pages
    256085 maximum wired pages
         1 swap devices
 ...
      5255 times daemon wokeup
     14374 revolutions of the clock hand
  12168425 pages freed by daemon
  21614865 pages scanned by daemon
    373482 anonymous pages scanned by daemon
  11794943 object pages scanned by daemon
   2944803 pages reactivated
         0 pages found busy by daemon
    349016 total pending pageouts
  21923470 pages deactivated
 ...

 # vmstat -s
      4096 bytes per page
        64 page colors
    768257 pages managed
     11049 pages free
    411160 pages active
    210352 pages inactive
         0 pages paging
       558 pages wired
      7893 zero pages
         1 reserve pagedaemon pages
        10 reserve kernel pages
    211328 anonymous pages
    326219 cached file pages
     84523 cached executable pages
       512 minimum free pages
       682 target free pages
    256085 maximum wired pages
         1 swap devices
 ...
      5255 times daemon wokeup
     14374 revolutions of the clock hand
  12168425 pages freed by daemon
  21614865 pages scanned by daemon
    373482 anonymous pages scanned by daemon
  11794943 object pages scanned by daemon
   2944803 pages reactivated
         0 pages found busy by daemon
    349016 total pending pageouts
  21923470 pages deactivated
 ...

 # vmstat -m
 Memory resource pool statistics
 Name        Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
 amappl        48    85115    0    80705   157    28   129   157     0   inf    0
 anonpl        16  1615944    0  1371415  2401   981  1420  2358     0   inf    0
 ataspl        56 11316431    0 11316431  2437  2436     1     1     0   inf    1
 biopl        180    12405    0    12405   816   816     0    27     0   inf    0
 buf16k      16384  539666    0   532479 38906 37109  1797  2924     1     1    0
 buf1k       1024    21200    0    20325   312   298    14    14     1     1    0
 buf2k       2048  1228730    0  1184215 21613 20140  1473  1582     1     1    0
 buf4k       4096    22608    0    22290   439   416    23    25     1     1    0
 buf64k      65536       8    0        0     9     0     9     9     1     1    1
 buf8k       8192    14451    0    14182   756   720    36    41     1     1    0
 bufpl        180   164745    0   111573  2558    14  2544  2558     0   inf    0
 cwdi          64     2330    0     2265     2     0     2     2     0   inf    0
 ehcixfer     204        5    0        4     1     0     1     1     0   inf    0
 execargs    262144 254658    0   254658   492   491     1     3     0    16    1
 extent        20      126    0      120     1     0     1     1     0   inf    0
 fcrpl        104       12    0       11     2     0     2     2     2   inf    1
 fdfile        64    14765    0    13783    25     3    22    23     0   inf    0
 ffsdino1     128  1480577    0  1378332  3717   104  3613  3716     0   inf    0
 ffsino       180  1515152    0  1412907  5242   169  5073  5235     0   inf    0
 file          64    15963    0    15471    13     0    13    13     0   inf    0
 filedesc     704     2310    0     2245    35    17    18    21     0   inf    0
 icmp          16   149635    0   149635  1859  1859     0     1     0   inf    0
 icmp6         16    47634    0    47634  2385  2385     0     1     0   inf    0
 in6pcbpl     176    42267    0    42257    12    11     1     3     0   inf    0
 inmltpl       32        4    0        2     1     0     1     1     0   inf    0
 inpcbpl      140   154708    0   154690    12    11     1     6     0   inf    0
 ipfrenpl      64       71    0       71    30    30     0     1     0   inf    0
 kcpuset       64     4316    0     4154     4     0     4     4     0   inf    0
 kcredpl      192     7592    0     7325    23     6    17    20     0   inf    0
 kmem-1024   1024    32123    0    28877  2691  1864   827  1864     0   inf    0
 kmem-112     112    15593    0    15435     8     2     6     7     0   inf    0
 kmem-128     128    13084    0    12476    38    18    20    37     0   inf    0
 kmem-16       16    79743    0    78083    13     0    13    13     0   inf    0
 kmem-160     160    11499    0    11059    27     8    19    21     0   inf    0
 kmem-192     192     6444    0     6288    15     6     9    12     0   inf    0
 kmem-2048   2048    11150    0    11051   711   660    51    70     0   inf    0
 kmem-224     224     3935    0     3860    14     7     7     7     0   inf    0
 kmem-24       24   265875    0   263425   148   111    37    56     0   inf    0
 kmem-256     256    14346    0    13887    47    14    33    45     0   inf    0
 kmem-32       32   101314    0    98408   107    70    37    42     0   inf    0
 kmem-320     320    14197    0    14080    19     6    13    18     0   inf    0
 kmem-384     384     4007    0     3835    38    19    19    27     0   inf    0
 kmem-40       40   405967    0   401765   190   112    78    89     0   inf    0
 kmem-4096   4096    81574    0    81432 11262 11120   142   330     0   inf    0
 kmem-448     448     2992    0     2914    21    10    11    15     0   inf    0
 kmem-48       48     8778    0     6050    36     3    33    34     0   inf    0
 kmem-512     512    12862    0    12786    23    10    13    18     0   inf    0
 kmem-56       56    49170    0    48684    29    20     9    19     0   inf    0
 kmem-64       64    24694    0    22986    81    39    42    81     0   inf    0
 kmem-768     768    56667    0    56196   412   295   117   278     0   inf    0
 kmem-8         8   330457    0   323799    69    11    58    68     0   inf    0
 kmem-80       80    95736    0    94743    64    30    34    56     0   inf    0
 kmem-96       96    43696    0    41730    81    31    50    69     0   inf    0
 ksiginfo      48     1913    0     1911     1     0     1     1     0   inf    0
 kva-12288   12288   10333    0    10322    16    15     1     2     0   inf    0
 kva-16384   16384    9358    0     9355    13    12     1     3     0   inf    0
 kva-20480   20480       4    0        3     1     0     1     1     0   inf    0
 kva-24576   24576       4    0        0     1     0     1     1     0   inf    0
 kva-32768   32768      18    0       14     1     0     1     1     0   inf    0
 kva-4096    4096    60886    0    33977   467    14   453   465     0   inf    0
 kva-49152   49152       1    0        0     1     0     1     1     0   inf    0
 kva-65536   65536    3265    0       77   804     6   798   798     0   inf    0
 kva-8192    8192     5967    0     5956    12    11     1     2     0   inf    0
 llentrypl    168      214    0      192     1     0     1     1     0   inf    0
 lockf         64     2783    0     2765     1     0     1     1     0   inf    0
 lwppl        672     3098    0     2907    54    19    35    43     0   inf    0
 mbpl         256   203243    0   202857    73    43    30    34     1   inf    0
 mclpl       2048    98622    0    98621 10173 10168     5    65     4 16384    4
 mutex         64  1668843    0  1561445  1928    29  1899  1928     0   inf    0
 ncache       128  1561202    0  1456515  3585    17  3568  3569     0   inf    0
 pcache       576       91    0        4    13     0    13    13     0   inf    0
 pcachecpu     64       95    0        0     2     0     2     2     0   inf    0
 pcglarge     512    42741    0    42740  5848  5847     1  1133     0   inf    0
 pcgnormal    128  1486090    0  1486046  8524  8522     2   516     0   inf    0
 pdict16       44      264    0      206     1     0     1     1     0   inf    0
 pdict32       60       18    0        5     1     0     1     1     0   inf    0
 pdppl       4096     2517    0     2452  1762  1697    65   105     0   inf    0
 phpool-0      32    96617    0    91747    57     6    51    57     0   inf    0
 phpool-128    44      362    0      205     2     0     2     2     0   inf    0
 phpool-256    60      161    0      111     2     1     1     2     0   inf    0
 phpool-512    92       69    0       11     2     0     2     2     0   inf    0
 phpool-64     36     4189    0     2739    14     0    14    14     0   inf    0
 piperd       192     1325    0     1297     8     5     3     3     0   inf    0
 pipewr       192     1299    0     1273     3     0     3     3     0   inf    0
 plimitpl     216      210    0      196     4     3     1     2     0   inf    0
 pmappl       120     2281    0     2216    11     8     3     4     0   inf    0
 pnbufpl     1024    39603    0    39597  1151  1149     2    10     0   inf    0
 procpl       484     2184    0     2119    17     6    11    13     0   inf    0
 proparay      32      111    0        0     1     0     1     1     0   inf    0
 propdict      32      435    0      102     3     0     3     3     0   inf    0
 propnmbr      32   665139    0   665095     1     0     1     1     0   inf    0
 propstng      20      831    0      221     4     0     4     4     0   inf    0
 pstatspl     268     2175    0     2110    10     4     6     7     0   inf    0
 ptimerpl     164      299    0      282     1     0     1     1     0   inf    0
 ptimerspl    152      299    0      282     1     0     1     1     0   inf    0
 pvpl          20   518678    0   479543   285     8   277   285     0   inf    0
 ractx         24   546362    0   502084   718   356   362   434     0   inf    0
 rndctx         8       42    0       41     1     0     1     1     0   inf    0
 rndsample    528     1199    0     1178    17    12     5     8     0   586    0
 rndtemp      512       40    0       40    21    21     0     1     0   inf    0
 rtentpl      208      132    0      103     2     0     2     2     0   inf    0
 rwlock        64        4    0        1     1     0     1     1     0   inf    0
 sigacts     2056     2250    0     2185  1769  1704    65   104     0   inf    0
 socket       336     6649    0     6450    40    15    25    32     0   inf    0
 tcpcbpl      528    45020    0    45003    86    83     3    26     0   inf    0
 tcpipqepl     44     8142    0     8142   177   177     0     1     0   inf    0
 tmpfs_dirent  24    92621    0    92621    58    58     0     1     0   inf    0
 tmpfs_node   140    92622    0    92604    45    42     3     6     0   inf    0
 tstilepl      52     2992    0     2801     4     0     4     4     0   inf    0
 uaoeltpl      84       99    0       87     2     0     2     2     0   inf    0
 uarea       8192     3064    0     2873  2381  2190   191   257     0   inf    0
 ufsdir       264      663    0      663   388   388     0     1     0   inf    0
 uhcixfer     208        2    0        1     1     0     1     1     0   inf    0
 uhcixfer     208        3    0        2     1     0     1     1     0   inf    0
 uhcixfer     208        4    0        3     1     0     1     1     0   inf    0
 uhcixfer     208        3    0        2     1     0     1     1     0   inf    0
 vcachepl     196  1532480    0  1430187  5784   225  5559  5761     0   inf    0
 vmembt        28    25565    0    20827    60     0    60    60     0   inf    0
 vmmpepl       88   190987    0   183462   525   136   389   512     0   inf    0
 vmsppl       212     2264    0     2199    11     6     5     6     0   inf    0
 wapbldealloc  20    27902    0    27902   256   256     0     1     0   inf    0
 wapblentrypl  20   114424    0   114424  2384  2384     0     1     0   inf    0
 wapblinopl    20   424009    0   424009   479   479     0     1     0   inf    0
 Totals           30016712    0 28972843150477118632 31845

 In use 303551K, total allocated 334304K; utilization 90.8%

 # 

From: Lars Reichardt <lars@paradoxon.info>
To: Havard Eidnes <he@NetBSD.org>
Cc: gnats-bugs@netbsd.org, ad@netbsd.org, gnats-admin@netbsd.org,
 netbsd-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Fri, 6 Mar 2020 07:34:38 +0100

 On 05.03.2020 16:41, Havard Eidnes wrote:
 >>>    buf16k 16384 1411 0 1270 241 205 36 93 1 1 0
 >>>    buf1k 1024 2 0 2 1 0 1 1 1 1 1
 >>>    buf2k 2048 9 0 9 5 4 1 5 1 1 1
 >>>    buf32k 32768 223292 0 193503 92993 73624 19369 37065 1 1 0
 >>>    buf4k 4096 491370 0 391560 491371 391560 99811 179873 1 1 1
 >>>    buf64k 65536 4 0 0 5 0 5 5 1 1 1
 >>>    buf8k 8192 1865 0 1613 160 128 32 63 1 1 0
 >>>    bufpl 288 210502 0 80506 15026 0 15026 15026 0 inf 111
 >>>>>> these are very interesting:
 >> These are the quantum caches for allocation virtual address space.
 >>
 >> No 4k allocation as the direct map is used (that's expected) and most
 >> pools have a pool page size of 4k
 >> but a lot of 64k allocations with the backing pool page size 256k.
 >>
 >> That is 64*63924 4091136kb worse of allocations
 >> (15981 pool pages each 256k)
 >>   and no releases at all seems like some leak to me.
 >>
 >> Does that happen when starting X?
 > No.  It typically happens after a few days running.
 >
 >> Seems to be an intel drmkms judged from the list of pools.
 > Correct:
 >
 > i915drmkms0 at pci0 dev 2 function 0: vendor 8086 product 0412 (rev. 0x06)
 > drm: Memory usable by graphics device = 2048M
 > drm: Supports vblank timestamp caching Rev 2 (21.10.2013).
 > drm: Driver supports precise vblank timestamp query.
 > i915drmkms0: interrupting at ioapic0 pin 16 (i915)
 > intelfb0 at i915drmkms0
 > i915drmkms0: info: registered panic notifier
 > i915drmkms0: More than 8 outputs detected via ACPI
 > intelfb0: framebuffer at 0xffff80013bc7d000, size 1920x1200, depth 32, stride 7680
 > wsdisplay0 at intelfb0 kbdmux 1: console (default, vt100 emulation)
 > wsmux1: connecting to wsdisplay0
 >
 >> The kmem arena is most likely a bit more than this mentioned 4g as the
 >> machine seems to have 16gb?
 > Physical ram is 16GB, yes.
 >
 >> It should be the second entry of the output of "pmap 0".
 > Currently the first three lines from "pmap 0" is
 >
 > FFFF800000000000 473672K read/write/exec     [ anon ]
 > FFFF80001CE92000 4166220K read/write/exec     [ anon ]
 > FFFF80011B325000 524288K read/write/exec     [ pager_map ]
 >
 >>>    kva-12288 12288 35 0 0 2 0 2 2 0 inf 0
 >>>    kva-16384 16384 17 0 0 2 0 2 2 0 inf 0
 >>>    kva-20480 20480 84 0 0 7 0 7 7 0 inf 0
 >>>    kva-24576 24576 9 0 0 1 0 1 1 0 inf 0
 >>>    kva-28672 28672 3 0 0 1 0 1 1 0 inf 0
 >>>    kva-32768 32768 1 0 0 1 0 1 1 0 inf 0
 >>>    kva-36864 36864 3 0 0 1 0 1 1 0 inf 0
 >>>    kva-40960 40960 108 0 0 18 0 18 18 0 inf 0
 >>>    kva-49152 49152 1 0 0 1 0 1 1 0 inf 0
 >>>    kva-65536 65536 63924 0 0 15981 0 15981 15981 0 inf 0
 >>>    kva-8192 8192 52 0 0 2 0 2 2 0 inf 0
 >> ...
 >> I'm not aware of any pool that allocates from the 64k quantum cache so
 >> it doesn't surprise me that that pagedaemon/pool_drain
 >> isn't able to free anything.
 > Hm.  I know too little about the mechanism, but having something
 > allocate from it and nothing to release when there's pressure
 > looks like a recipe for disaster.
 >
 Those kva pools are caches that cache some virtual address space 
 allocated from the kmem arena no memory is directly allocated for them.

 See: https://nxr.netbsd.org/xref/src/sys/uvm/uvm_km.c#783

 uvm_km_kmem_alloc is use by most pools allocators to get memory for 
 their pool_pages.

 vmstat -mvW does show non used pools as well as their pool_pagesize.

 As the only pools with 64k pool_pagesize are the buf pools and they have 
 a different pool_allocator not allocation from kmem but from the kernel_map.

 As far as I can guess currently the most likely candidate is kmem_alloc 
 with a size of >48k and <= 64k as we only have kmem caches up to 
 page_size and large allocations are done directly via uvm_km_kmem_alloc.


 Monitoring those large allocation via dtrace should show us the consumer 
 that doesn't free it's memory.

 I'll try to come up a dtrace script for that, feel free to beat me on 
 that my time is a bit rare at the moment.


 he@: Yes, the report of NetBSD 8.1/i386 looks very much the same.

 There is no direct map so the kva-4096 is used for all the pool_sized 
 allocations but a lot of 64k allocations with almost no releases again.


 Regards,

 Lars


 -- 
 You will continue to suffer
 if you have an emotional reaction to everything that is said to you.
 True power is sitting back and observing everything with logic.
 If words control you that means everyone else can control you.
 Breathe and allow things to pass.

 --- Bruce Lee

From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
To: mlelstv@serpens.de
Cc: gnats-bugs@netbsd.org, he@uninett.no, ad@netbsd.org,
        tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Sun, 8 Mar 2020 15:50:28 +0900

 > http://ftp.netbsd.org/pub/NetBSD/misc/mlelstv/uvm_pdaemon.patch

 I've tried a modifyed patch for NetBSD 9.0 today
 (not sure if it's correct) and it fires soon with
 tons of log messages:

 ---
 Mar  8 15:40:43 mirage /netbsd: [ 8930.4150611] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8930.4150611] pagedaemon: Out of70.2942777] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8970.2942vaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8930.4150611] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 2 times
 Mar  8 15:40:43 mirage /netbsd: [ 893
 Mar  8 15:40:43 mirage /netbsd: [ 9268.5974764] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 926811] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8930.4150611] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8930.4150611] pagedaemon...
 qe
 Mar  8 15:40:43 mirage /netbsd: [ 8971.3547885] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 6 times
 Mar  8 15:40:43 mirage /netbsd: [ 8971.3547885] pagedaemon: Out of KVA, avaitgedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8930.4150611] pagedaemom...
 Mar  8 15:40:43 mirage /netbsd: [ 8971.8550302] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8930.4150611] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 3 times
 Mar  8 15:40:43 mirage /netbsd: [ 8930.7452203] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8930.7952444] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8930.7952444] pting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8973.9060171] pagedaemon: Out of KVA, avaiting doon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8930.8552735] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.1454075] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 2 times
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 4 times
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Ou[ 8976.1270867] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8976. of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9270.0881898] pagedaemon: Out of KVAting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Out of KVA, avaiting doaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8977.1275694] pagedaemon:  8931.4355476] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476]aiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8978.1280504] pagedaemon: Out of KVA, avaiting emon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Out of KVA, ava7] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8979.5887537] pageoom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 2 times
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476vaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8980.0289657] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8980.0289657] pagedaemon: Out of KVA, avaiting doom..Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Out of 0.5292066] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8980.5292066] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8980.9193953] pagdoom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Out of KVA, avaiting doom...: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8981.1595101] pagedaemon: Out o.4355476] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.4355476] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.7256930] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8931.8457509] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.0458473] pagedaemon: Out of KVA, a918] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8982.6602327] pa doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.0558524] pagedaemon: Out of KVA, avaiting doom..n: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8983.1604735] pagedaemon: Out 2.4560452] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.45604 avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8983.6607144] pagedaemon: Out of KVA, avaitiedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.4560452] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 15 times
 Mar  8 15:40:43 mirage /netbsd: [ 8932.456A, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8986.2119428] pagedaemon: Out of KVA, avaiagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.4560452] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.4560452] pagedaemon: Out 8986.6621595] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8986.6KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9067.1008507] pagedaemon: Out of KVA, aving doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.4560452] pagedaemon: Out of KVA, avaiting dooemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8986.6621595] pagedaemon: O8932.4560452] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.4560452] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 9 times
 Mar  8 15:40:43 mirage /netbsd: [ 8932.4560452]aiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8987.8527327] pagedaemon: Out of KVA, avaiting emon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.4560452] pagedaemon: O
 Mar  8 15:40:43 mirage /netbsd: [ 8987.8527327] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8987VA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.4560452] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.4560452] pagedaemon: Out of KVA, avaiting ddaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8988.3529735] pagedaemon:[ 8932.4560452] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8932.8362284] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8933.2364212] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 2 times
 Mar  8 15:40:43 mirage /netbsd: [ 8933.2364212] pagedaemon: Out of 8.3529735] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8988.35297aiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8933.4765370] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8933.6065998] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8933.8567205] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8933.8567205] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8934.3469574] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8934.3469574] pageg doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8989.3534551] pagedaemon: Out of KVA, avaiting doom. Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8934.8171833] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 2 times
 Mar  8 15:40:43 mirage /netbsd: [ 8935.3174244] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8935.3174244] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 893of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8990.8541776] pagedaemon: Out of KVA,84] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8936.4879895] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8936.5180031] pagedaemon...
 Mar  8 15:40:43 mirage /netbsd: [ 8991.3544192] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8f KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8936.5180031] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 7 times
 Mar  8 15:40:43 mirage /netbsd: [ 8936.5280034] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 4 times
 Mar  8 15:40:43 mirage /netbsd: [ 8t of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8991.9747170] pagedaemon: Out of KV0034] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8936.5280034] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 20 times
 Mar  8 15:40:43 mirage /netbsd: [ 8936.5280034] pagedaemon: Out of K.2667830] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8996.767023iting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8936.5280034] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8936.5780321] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 3 times
 Mar  8 15:40:43 mirage /netbsd: [ 8936.6880852] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8936.6880852] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8936.6980901] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8936.6980901] pagedaemon: Out of KVA, av61] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8998.2677461] pagdoom...
 Mar  8 15:40:43 mirage /netbsd: [ 8936.6980901] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8936.6980901] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8937[ 9277.0915414] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9277.1] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8937.0282491] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8937.5184798] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8937.5384895] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 5 times
 Mar  8 15:40:43 mirage /netbsd: [ 8937.53848.0824940] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9279.582733edaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8937.5384895] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 30 times
 Mar  8 15:40:43 mirage /netbsd: [ 8938.0387306] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8938.1787981] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 12 times
 Mar  8 15:40:43 mirage /netbsd: [ 8938.1888030] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 30 times
 Mar  8 15:40:43 mirage /netbsd: [ 8938.6890440] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8938.8191068] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 2 times
 Mar  8 15:40:43 mirage /netbsd: [ 8938.819A, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9014.1053676] pagedaemon: Out of KVA, avaiagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8938.8191068] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 65 times
 Mar  8 15:40:43 mirage /netbsd: [ 8939.0592282] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 4 times
 Mar  8 15:40:43 mirage /netbsd: [ 8939.5494644] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8939.5494644] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8939.5794788] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 2 times
 Mar  8 15:40:43 mirage /netbsd: [ 8939.5894791] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8939.7395560] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 6 times
 Mar  8 15:40:43 mirage /netbsd: [ 8939.7495560] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 8 times
 Mar  8 15:40:43 mirage /netbsd: [ 8939.7495560] pagedaemon: Out of 4.2554356] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9014.25543aiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8939.7495560] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8940.2397971] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8940.2397971] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8940.5999715] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8940.6900141] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 2 times
 Mar  8 15:40:43 mirage /netbsd: [ 8940.8500912] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8941.3403276] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8941.6404723] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 7 times
 Mar  8 15:40:43 mirage /netbsd: [ 8942.0006458] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8942.0006458] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8942.4908821] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8942.5709206] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 2 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.0611572] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.5613981] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.5613981] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6014173] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 77 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.60301.0830203] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9301.083pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6014173] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 33 times
 Mar  8 15:40:43 mirage /netbsd: [ 894
 Mar  8 15:40:43 mirage /netbsd: [ 9302.5737333] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 930273] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6014173] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 18 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.69.0862150] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9099.08621 pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6014173] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6014173] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6014173] pagedaemon: Out of Kf KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9099.0862150] pagedaemon: Out of KVA, iting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6014173] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 24 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6014173] pagedaemon: Out o Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9305.5751681] pagedaemon: Out ofavaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6014173] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 32 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6114174] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 6 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6214220] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 20 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6314319] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 7 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6314319] paaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9236.0919145] pagedaemon: Out of KVA, avaiting n: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6314319] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 10 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6414325] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 3 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.64f KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 9236.0919145] pagedaemon: Out of KVA, pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6414325] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 50 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6514368] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage syslogd[469]: last message repeated 9 times
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6614418] pagedaemon: Out of KVA, avaiting doom...
 Mar  8 15:40:43 mirage /netbsd: [ 8943.6614418] pagedaemon: Out of KVA, avaiting doom...

 ---

 And the kernel hung up.
 (the above log is taken from /var/log/messages after reboot)

 ---
 Index: uvm_pdaemon.c
 ===================================================================
 RCS file: /cvsroot/src/sys/uvm/uvm_pdaemon.c,v
 retrieving revision 1.110
 diff -u -p -d -r1.110 uvm_pdaemon.c
 --- uvm_pdaemon.c	21 Apr 2019 15:32:18 -0000	1.110
 +++ uvm_pdaemon.c	8 Mar 2020 06:47:50 -0000
 @@ -239,6 +239,7 @@ uvm_pageout(void *arg)
  {
  	int npages = 0;
  	int extrapages = 0;
 +	int timo = 0;

  	UVMHIST_FUNC("uvm_pageout"); UVMHIST_CALLED(pdhist);

 @@ -270,15 +271,19 @@ uvm_pageout(void *arg)
  		bool needsscan, needsfree, kmem_va_starved;

  		kmem_va_starved = uvm_km_va_starved_p();
 +		if (timo > 0 && kmem_va_starved)
 +			printf("pagedaemon: Out of KVA, avaiting doom...\n");

  		mutex_spin_enter(&uvm_fpageqlock);
 -		if ((uvm_pagedaemon_waiters == 0 || uvmexp.paging > 0) &&
 -		    !kmem_va_starved) {
 +		if (timo > 0 ||
 +		    ((uvm_pagedaemon_waiters == 0 || uvmexp.paging > 0) &&
 +		    !kmem_va_starved)) {
  			UVMHIST_LOG(pdhist,"  <<SLEEPING>>",0,0,0,0);
  			UVM_UNLOCK_AND_WAIT(&uvm.pagedaemon,
 -			    &uvm_fpageqlock, false, "pgdaemon", 0);
 +			    &uvm_fpageqlock, false, "pgdaemon", timo);
  			uvmexp.pdwoke++;
  			UVMHIST_LOG(pdhist,"  <<WOKE UP>>",0,0,0,0);
 +			timo = 0;
  		} else {
  			mutex_spin_exit(&uvm_fpageqlock);
  		}
 @@ -336,17 +341,16 @@ uvm_pageout(void *arg)
  		mutex_exit(&uvm_pageqlock);

  		/*
 -		 * if we don't need free memory, we're done.
 +		 * if we don't need free memory, kick the poll drainer thread
  		 */

 -		if (!needsfree && !kmem_va_starved)
 -			continue;

 +		if (needsfree || kmem_va_starved)
 +			uvmpd_pool_drain_wakeup();
  		/*
 -		 * kick the pool drainer thread.
 +		 * sleep before next run
  		 */
 -
 -		uvmpd_pool_drain_wakeup();
 +		timo = hz/2;
  	}
  	/*NOTREACHED*/
  }

 ---
 Izumi Tsutsui

From: Michael van Elst <mlelstv@serpens.de>
To: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
Cc: gnats-bugs@netbsd.org, he@uninett.no, ad@netbsd.org
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Sun, 8 Mar 2020 09:00:34 +0100

 On Sun, Mar 08, 2020 at 03:50:28PM +0900, Izumi Tsutsui wrote:

 > > http://ftp.netbsd.org/pub/NetBSD/misc/mlelstv/uvm_pdaemon.patch
 > 
 > I've tried a modifyed patch for NetBSD 9.0 today
 > (not sure if it's correct) and it fires soon with
 > tons of log messages:

 Is this a 32bit or 64bit system ? Anything running that would eat KVA ?


 -- 
                                 Michael van Elst
 Internet: mlelstv@serpens.de
                                 "A potential Snark may lurk in every tree."

From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
To: mlelstv@serpens.de
Cc: gnats-bugs@netbsd.org, he@uninett.no, ad@netbsd.org,
        tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Sun, 8 Mar 2020 18:38:59 +0900

 > Is this a 32bit or 64bit system ? Anything running that would eat KVA ?

 - NetBSD/i386 9.0
   https://mail-index.netbsd.org/netbsd-bugs/2020/01/01/msg065292.html
   https://dmesgd.nycbug.org/index.cgi?do=view&id=5276

 - build.sh -U -j 4 release, Youtube video on Firefox 73,
   Xserver on RADEON HD5450, and raidframe RAID1 with wd0 and wd1 etc.
   Is there any good way to see KVA usage on userland?
   (getting ddb prompt is a bit annoying)

 ---
 Izumi Tsutsui  

From: Havard Eidnes <he@uninett.no>
To: lars@paradoxon.info
Cc: gnats-bugs@netbsd.org, ad@netbsd.org, gnats-admin@netbsd.org,
 netbsd-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Wed, 11 Mar 2020 09:47:06 +0100 (CET)

 Hi,

 and thanks for the dtrace script!  Via hints, I managed to get it
 to run using "-x nolib", and it has now traced a number of 64k
 allocations and free operations -- it seems these are all audio
 related.  Counting the "alloc"s vs. the "free"s, it seems I have
 81 allocs but only 45 frees.  Somewhat correspondingly, I appear
 to have 72 'audioopen' but only 9 'audioclose'.  The only audio-
 using application I use is firefox, but it's not collecting audio
 file descriptors -- I only find one open at the moment (and I
 don't use pulseaudio):

 : {16} ; fstat -n -p 1412 | grep audio
 he       firefox     1412  179* audio 0xfffffe805f1f3240
 : {17} ; 

 The dtrace script I got is

 ---
 #pragma D option flowindent

 fbt::uvm_km_kmem_alloc:entry
 /arg1 > 49152 && arg1 <= 65536 /
 {
         printf("size: %d", arg1);
         stack();
 }

 fbt::uvm_km_kmem_free:entry
 /arg2 > 49152 && arg2 <= 65536/
 {
         printf("size: %d", arg2);
         stack();
 }
 ---

 and the trace collected so far is shown below.

 Inspecting the code of audiosetinfo() doesn't reveal a
 kmem_intr_zalloc anywhere in sight, but dissassembling finds
 there is a call to kmem_zalloc() on the expected location:

    0xffffffff80791195 <audiosetinfo+4334>:      
     callq  0xffffffff8098c3be <kmem_zalloc>

 There is one in audio_stream_ctor(), though, called from
 audio_setup_rfilters(), which are both static functions, so both
 may have been hoisted up into audiosetinfo() (?)

 Anyway, the collected info so far is:


 CPU FUNCTION                                 
   7  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7          -> uvm_km_kmem_alloc             size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7              -> uvm_km_kmem_free          size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_free                      size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audio_destroy_pfilters+0x51
               netbsd`audio_close.part.31+0xc9
               netbsd`audioclose+0x116
               netbsd`closef+0x63
               netbsd`fd_close+0x108
               netbsd`sys_close+0x1f
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0          -> uvm_km_kmem_alloc             size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0              -> uvm_km_kmem_free          size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   3  -> uvm_km_kmem_free                      size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audio_destroy_pfilters+0x51
               netbsd`audio_close.part.31+0xc9
               netbsd`audioclose+0x116
               netbsd`closef+0x63
               netbsd`fd_close+0x108
               netbsd`sys_close+0x1f
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7          -> uvm_km_kmem_free              size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1          -> uvm_km_kmem_free              size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1          -> uvm_km_kmem_alloc             size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   1              -> uvm_km_kmem_free          size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_free                      size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audio_destroy_pfilters+0x51
               netbsd`audio_close.part.31+0xc9
               netbsd`audioclose+0x116
               netbsd`closef+0x63
               netbsd`fd_close+0x108
               netbsd`sys_close+0x1f
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_free                      size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audio_destroy_pfilters+0x51
               netbsd`audio_close.part.31+0xc9
               netbsd`audioclose+0x116
               netbsd`closef+0x63
               netbsd`fd_close+0x108
               netbsd`sys_close+0x1f
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6          -> uvm_km_kmem_alloc             size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6              -> uvm_km_kmem_free          size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6          -> uvm_km_kmem_free              size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0          -> uvm_km_kmem_free              size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   3  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   3    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   3      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   3        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   3          -> uvm_km_kmem_alloc             size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   3            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   3              -> uvm_km_kmem_free          size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7  -> uvm_km_kmem_free                      size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audio_destroy_pfilters+0x51
               netbsd`audio_close.part.31+0xc9
               netbsd`audioclose+0x116
               netbsd`closef+0x63
               netbsd`fd_close+0x108
               netbsd`sys_close+0x1f
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   5  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   5    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   5      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   5        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   5          -> uvm_km_kmem_alloc             size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   5            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   5              -> uvm_km_kmem_free          size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_free                      size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audio_destroy_pfilters+0x51
               netbsd`audio_close.part.31+0xc9
               netbsd`audioclose+0x116
               netbsd`closef+0x63
               netbsd`fd_close+0x108
               netbsd`sys_close+0x1f
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0          -> uvm_km_kmem_alloc             size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0              -> uvm_km_kmem_free          size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_free                      size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audio_destroy_pfilters+0x51
               netbsd`audio_close.part.31+0xc9
               netbsd`audioclose+0x116
               netbsd`closef+0x63
               netbsd`fd_close+0x108
               netbsd`sys_close+0x1f
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   2  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   2    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   2      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   2        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   2          -> uvm_km_kmem_free              size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   2            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0          -> uvm_km_kmem_alloc             size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0              -> uvm_km_kmem_free          size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_free                      size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audio_destroy_pfilters+0x51
               netbsd`audio_close.part.31+0xc9
               netbsd`audioclose+0x116
               netbsd`closef+0x63
               netbsd`fd_close+0x108
               netbsd`sys_close+0x1f
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6          -> uvm_km_kmem_free              size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   6            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0          -> uvm_km_kmem_free              size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0          -> uvm_km_kmem_free              size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0          -> uvm_km_kmem_free              size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x10f3
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.33+0x44c
               netbsd`audioopen+0xf5
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7          -> uvm_km_kmem_alloc             size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audiosetinfo+0x645
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   7              -> uvm_km_kmem_free          size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x1579
               netbsd`audio_ioctl+0x21c
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0  -> uvm_km_kmem_free                      size: 65536
               netbsd`audio_stream_dtor+0x1d
               netbsd`audio_destroy_pfilters+0x51
               netbsd`audio_close.part.31+0xc9
               netbsd`audioclose+0x116
               netbsd`closef+0x63
               netbsd`fd_close+0x108
               netbsd`sys_close+0x1f
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

From: mlelstv@serpens.de (Michael van Elst)
To: gnats-bugs@netbsd.org
Cc: 
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Wed, 11 Mar 2020 09:04:45 -0000 (UTC)

 he@uninett.no (Havard Eidnes) writes:

 >There is one in audio_stream_ctor(), though, called from
 >audio_setup_rfilters(), which are both static functions, so both
 >may have been hoisted up into audiosetinfo() (?)

 audio_stream_ctor is netbsd-8. The code in netbsd-9 is different.

 -- 
 -- 
                                 Michael van Elst
 Internet: mlelstv@serpens.de
                                 "A potential Snark may lurk in every tree."

From: Havard Eidnes <he@uninett.no>
To: gnats-bugs@netbsd.org, ""@uninett.no
Cc: ad@netbsd.org, netbsd-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Sun, 15 Mar 2020 11:38:20 +0100 (CET)

 Hi,

 (just to remind folks, this is with NetBSD 8.1)

 It seems that the behaviour from the audio.c driver is that each time
 I start a playback in firefox, the dtrace script logs 4 instances of
 uvm_km_kmem_alloc(), but only two of uvm_km_kmem_free().

 On a hunch, I tried with this diff:

 --- audio.c     7 Jun 2018 18:24:16 -0000       1.357.2.13
 +++ audio.c     15 Mar 2020 09:18:57 -0000
 @@ -1491,6 +1491,9 @@ audio_stream_ctor(audio_stream_t *stream
         int frame_size;

         size = min(size, AU_RING_SIZE);
 +       /* Will overwrite, may need to release */
 +       if (stream->start != NULL)
 +               kmem_free(stream->start, stream->bufsize);
         stream->bufsize = size;
         stream->start = kmem_zalloc(size, KM_SLEEP);
         frame_size = (param->precision + 7) / 8 * param->channels;

 but that doesn't appear to have any effect.  The trace changes
 slightly with the above patch in place, in that the audio_stream_ctor
 is now visible in the dtrace output as

   0        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audio_stream_ctor.constprop.39+0x34
               netbsd`audiosetinfo+0x10a0
               netbsd`audio_set_defaults+0xeb

 There appears to be two places in audiosetinfo() which calls
 audio_stream_ctor():

               netbsd`audio_stream_ctor.constprop.39+0x34
               netbsd`audiosetinfo+0x636
 and
               netbsd`audio_stream_ctor.constprop.39+0x34
               netbsd`audiosetinfo+0x10a0

 and each of them allocates twice.  auidio_stream_dtor() is always
 called from the same spot:

               netbsd`audio_stream_dtor+0x1d
               netbsd`audiosetinfo+0x14d5

 The two places in "audiosetinfo()" which calls audio_stream_ctor()
 decodes to

 (gdb) x/i audiosetinfo+0x636
    0xffffffff8079076d <audiosetinfo+1590>:      test   %eax,%eax
 (gdb) i li *0xffffffff8079076d
 Line 1307 of "/usr/src/sys/dev/audio.c"
    starts at address 0xffffffff80790762 <audiosetinfo+1579>
    and ends at 0xffffffff80790775 <audiosetinfo+1598>.
 (gdb) x/i audiosetinfo+0x10a0
    0xffffffff807911d7 <audiosetinfo+4256>:      test   %eax,%eax
 (gdb) i li *0xffffffff807911d7
 Line 1389 of "/usr/src/sys/dev/audio.c"
    starts at address 0xffffffff807911c5 <audiosetinfo+4238>
    and ends at 0xffffffff807911df <audiosetinfo+4264>.
 (gdb)

 The first one is the audio_stream_ctor() inside
 audio_setup_pfilters(), the second is in audio_setup_rfilters().
 Both appear to have been hoisted into audiosetinfo() by the
 optimizer.

 The visible trace of audio_stream_dtor() invocation is near the end of
 audio_setup_pfilters().  However, the "destroy old filters" part in
 audio_setup_rfilters() appears to never call the audio_stream_dtor(),
 despite there being a loop near the end:

         /* Destroy old filters. */
         for (i = 0; i < onfilters; i++) {
                 of[i]->dtor(of[i]);
                 audio_stream_dtor(&os[i]);
         }

         return 0;

 onfilters is set from vc->sc_nrfilters, and that field is subsequently
 set to rfilters->req_size, and the preceding loop which contains the
 call to audio_stream_ctor() is over rfilters->req_size.

 So ... I can't quite figure out what is going on and why it's leaking
 two allocations on each audio device activation.  It would seem that
 vc->sc_nrfilters is zero(?)

 Anyone have any hints?  (I'll have opportunity to test a new kernel
 tomorrow.)

 Regards,

 - Havard

From: Izumi Tsutsui <tsutsui@ceres.dti.ne.jp>
To: gnats-bugs@netbsd.org
Cc: ad@netbsd.org, he@uninett.no, mlelstv@serpens.de, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Thu, 19 Mar 2020 23:57:08 +0900

 When "pagedaemon: Out of KVA, avaiting doom.." logs are fired
 on NetBSD/i386 9.0, vmstat -m shows kmem-96 leaks (sorted manually):

 ---
 Memory resource pool statistics
 Name        Size Requests Fail Releases Pgreq Pgrel Npage Hiwat Minpg Maxpg Idle
 kmem-96       96  1591574    0    17237 37485     0 37485 37485     0   inf    0
 ffsdino2     256   291458    0   181327  7349   443  6906  6909     0   inf    0
 vcachepl     196   314493    0   203865  5928   396  5532  5532     0   inf    0
 ffsino       180   288959    0   178828  5332   309  5023  5025     0   inf    0
 ncache       128   234935    0   139902  3573     3  3570  3570     0   inf    0
 mutex         64   570138    0   435669  2723    28  2695  2695     0   inf    0
 anonpl        16  3921211    0  3556675  2451    18  2433  2434     0   inf    0
 kmem-1024   1024    59572    0    51137  5604  3495  2109  3658     0   inf    0
 buf32k      32768   28519    0    24786  7922  6055  1867  3214     1     1    0
 kmem-768     768   134145    0   125129  5367  3563  1804  4080     0   inf    0
 vmmpepl       88   388286    0   337405  2981  1296  1685  2089     0   inf    0
 kva-4096    4096    88288    0    11244  1204     0  1204  1204     0   inf    0
 pvpl          20   622361    0   519223   712     1   711   711     0   inf    0
 bufpl        184    27623    0    17690   815   145   670   814     0   inf    0
 kmem-80       80   217907    0   202970   706   132   574   672     0   inf    0
 amappl        48   175874    0   145683   565     3   562   564     0   inf    0
 uarea       8192     4781    0     4283  1480   982   498   638     0   inf    0
 ractx         24   302615    0   247814   607   111   496   496     0   inf    1
 buf4k       4096    37533    0    31424  1177   776   401   887     1     1    0
 kmem-320     320    23822    0    20157   528   135   393   521     0   inf    0
 kmem-160     160    31201    0    25516   415    30   385   414     0   inf    0
 kmem-4096   4096     3057    0     2780  1024   747   277   285     0   inf    0
 kmem-384     384    15927    0    14104   530   264   266   520     0   inf    0
 kmem-512     512     9472    0     8238   348   107   241   344     0   inf    0
 kmem-256     256    16466    0    14069   272    33   239   272     0   inf    0
 kmem-448     448    11841    0    10316   416   184   232   413     0   inf    0
 kmem-112     112   106045    0   102605   439   229   210   354     0   inf    0
 kmem-48       48   133544    0   123603   216    12   204   216     0   inf    0
 mclpl       2048    23995    0    23753  1373  1194   179   277     4 16384   58
 kmem-192     192    16168    0    13580   195    17   178   195     0   inf    0
 kmem-224     224    13859    0    11551   210    33   177   208     0   inf    0
 pcgnormal    128   492419    0   488858  4968  4793   175   796     0   inf   16
 kmem-24       24   128903    0   110574   151     1   150   151     0   inf    0
 kmem-32       32   156296    0   141977   141     0   141   141     0   inf    0
 kmem-16       16   164512    0   147615   156    21   135   155     0   inf    0
 vmembt        28    82234    0    71120   129     1   128   129     0   inf    0
 kmem-2048   2048     3194    0     2941   444   317   127   166     0   inf    0
 phpool-32     32    34069    0    20948   127     4   123   126     0   inf    0
 pdppl       4096     2546    0     2454   563   456   107   223     0   inf   15
 sigacts     2056     2467    0     2375   588   482   106   220     0   inf   14
 kmem-8         8   394845    0   363684   105     0   105   105     0   inf    0
 lwppl        704     4659    0     4201   158    56   102   128     0   inf    0
 kmem-64       64    40752    0    36887   185    87    98   162     0   inf    0
 kmem-56       56    82044    0    76610   125    28    97   109     0   inf    0
 kmem-40       40   136641    0   130194    85     1    84    85     0   inf    0
 kmem-128     128    15685    0    13763    83     0    83    83     0   inf    0
 pcglarge     512   105205    0   104757  5836  5772    64   788     0   inf    8
 mbpl         256    44924    0    44480   163   114    49    62     1   inf    3
 socket       336     4396    0     3956    63    17    46    51     0   inf    1
 kcredpl      192     3994    0     3322    41     7    34    35     0   inf    0
 fdfile        64    17315    0    15511    39     6    33    35     0   inf    0
 biopl        184    30965    0    30485   257   235    22    74     0   inf    0
 filedesc     704     2508    0     2402    89    67    22    45     0   inf    0
 file          64    10910    0     9701    25     3    22    25     0   inf    0
 rf_dagnode_pl 444 26486417   0 26486417   431   410    21    21    15    57   21
 procpl       564     2277    0     2185    49    32    17    31     0   inf    0
 pcache       576      100    0        4    14     0    14    14     0   inf    0
 buf16k      16384     634    0      588    61    49    12    21     1     1    0
 phpool-64     36     1205    0        0    11     0    11    11     0   inf    0
 tcpcbpl      528    87359    0    87306   143   133    10    28     0   inf    0
 uaoeltpl      84      902    0      788    12     2    10    11     0   inf    0
 pstatspl     268     2242    0     2150    19    10     9    15     0   inf    0
 buf64k      65536      14    0        6    15     6     9    13     1     1    1
 tstilepl      52     4723    0     4225     9     0     9     9     0   inf    0
 pnbufpl     1024     4788    0     4770   111   103     8    21     0   inf    3
 vmsppl       212     2500    0     2393    17    10     7    12     0   inf    0
 pipewr       188     2513    0     2416    16     9     7    10     0   inf    0
 piperd       188     2394    0     2295    16     9     7    10     0   inf    0
 kcpuset       64     3561    0     3198    11     4     7     9     0   inf    0
 buf8k       8192     1138    0     1102    41    35     6    20     1     1    0
 pmappl       156     2521    0     2429    12     6     6     9     0   inf    0
 rf_asm_pl    288  6400719    0  6400719    81    75     6     6     5    14    6
 phpool-128    44      569    0       42     6     0     6     6     0   inf    0
 rndsample    528      819    0      797    16    11     5     9     0   586    0
 inpcbpl      140   400916    0   400836    14     9     5     7     0   inf    1
 phpool-256    60      307    0       22     5     0     5     5     0   inf    0
 pcachecpu     64      312    0        0     5     0     5     5     0   inf    0
 tmpfs_node   140   121248    0   121142    58    54     4    11     0   inf    0
 propstng      20     1688    0      883     5     1     4     5     0   inf    0
 rf_rad_pl    376  4656534    0  4656534     4     0     4     4     4    13    4
 rf_alloclist_pl 168 6400720  0  6400719    77    73     4     4     3    11    3
 propdict      32      970    0      490     4     0     4     4     0   inf    0
 kva-12288   12288    1616    0     1581    25    22     3     5     0   inf    0
 cwdi          64     2502    0     2396     4     1     3     4     0   inf    0
 rf_dqd_pl    128  7284260    0  7284260     3     0     3     3     3     9    3
 rf_daglist_pl 272 6400719    0  6400719     3     0     3     3     3     9    3
 phpool-512    92      105    0        0     3     0     3     3     0   inf    0
 plimitpl     216     1526    0     1497    14    12     2     7     0   inf    0
 kva-8192    8192     1057    0     1027    15    13     2     4     0   inf    0
 kva-24576   24576     352    0      345    17    15     2     3     0   inf    1
 in6pcbpl     176     5119    0     5096    37    35     2     3     0   inf    1
 kva-32768   32768     239    0      231     9     7     2     3     0   inf    0
 ksiginfo     108     4370    0     4308    15    13     2     3     0   inf    0
 fcrpl        104       38    0       33     2     0     2     2     2   inf    1
 rtentpl      208      264    0      230     2     0     2     2     0   inf    0
 proparay      32      231    0       55     2     0     2     2     0   inf    0
 lockf         64     2965    0     2899     2     0     2     2     0   inf    0
 kva-65536   65536       7    0        0     2     0     2     2     0   inf    0
 kva-36864   36864     667    0      664    62    61     1     8     0   inf    0
 kva-20480   20480     463    0      455    19    18     1     7     0   inf    0
 kva-16384   16384     801    0      788    24    23     1     5     0   inf    0
 execargs    262144 3860849   0  3860849    79    78     1     4     0    16    1
 wapbldealloc  20    23783    0    23783    40    39     1     3     0   inf    1
 kva-28672   28672     190    0      186    10     9     1     3     0   inf    0
 buf512b      512     2553    0     2552     2     1     1     2     1     1    0
 ataspl       100 12600587    0 12600587    36    35     1     2     0   inf    1
 ptimerpl     228     1096    0     1083    14    13     1     2     0   inf    0
 ehcixfer     204       60    0       58     2     1     1     2     0   inf    0
 scxspl       164    60882    0    60882     1     0     1     1     1   inf    1
 rf_dagh_pl    76  6400719    0  6400719     1     0     1     1     1     3    1
 rf_stripelock_p 28 6400719   0  6400719     1     0     1     1     1     2    1
 rf_pda_pl     40 12801438    0 12801438     1     0     1     1     1     2    1
 rf_funclist_pl 8  6400719    0  6400719     1     0     1     1     1     2    1
 rf_asmhdr_pl  16  4656534    0  4656534     1     0     1     1     1     2    1
 rf_vple_pl     8       30    0        0     1     0     1     1     1     2    0
 buf2k       2048       81    0       81     1     0     1     1     1     1    1
 buf1k       1024       35    0       35     1     0     1     1     1     1    1
 wapblentrypl  20    56652    0    56652     8     7     1     1     0   inf    1
 tcpipqepl     32    31363    0    31363    56    55     1     1     0   inf    1
 icmp6         16    29302    0    29302    54    53     1     1     0   inf    1
 icmp          16     4651    0     4651    34    33     1     1     0   inf    1
 wapblinopl    20 11842840    0 11842834     3     2     1     1     0   inf    0
 ufsdir       264      321    0      317    13    12     1     1     0   inf    0
 tmpfs_dirent  24   121246    0   121245     1     0     1     1     0   inf    0
 rwlock        64        4    0        0     1     0     1     1     0   inf    0
 rndtemp      512      243    0      241    17    16     1     1     0   inf    0
 rndctx         8      290    0      279     1     0     1     1     0   inf    0
 ptimerspl    152     1096    0     1083     1     0     1     1     0   inf    0
 propnmbr      32       74    0       32     1     0     1     1     0   inf    0
 propdata      20        1    0        0     1     0     1     1     0   inf    0
 pdict32       60       14    0        2     1     0     1     1     0   inf    0
 pdict16       44      635    0      585     1     0     1     1     0   inf    0
 ohcixfer     184        3    0        2     1     0     1     1     0   inf    0
 ohcixfer     184        2    0        1     1     0     1     1     0   inf    0
 ohcixfer     184        3    0        2     1     0     1     1     0   inf    0
 ohcixfer     184        2    0        1     1     0     1     1     0   inf    0
 ohcixfer     184        3    0        2     1     0     1     1     0   inf    0
 ohcixfer     184        2    0        1     1     0     1     1     0   inf    0
 ohcixfer     184        4    0        1     1     0     1     1     0   inf    0
 nfsvapl      136        2    0        0     1     0     1     1     0   inf    0
 nfsnodepl    208        2    0        0     1     0     1     1     0   inf    0
 llentrypl    168      108    0      105     1     0     1     1     0   inf    0
 kva-49152   49152       3    0        0     1     0     1     1     0   inf    0
 inmltpl       32        3    0        0     1     0     1     1     0   inf    0
 extent        20      133    0      119     1     0     1     1     0   inf    0
 ehcixfer     204        2    0        1     1     0     1     1     0   inf    0
 ffsdino1     128       90    0       90     4     4     0     3     0   inf    0
 synpl        208       72    0       72    28    28     0     1     0   inf    0
 sackholepl    20      211    0      211    13    13     0     1     0   inf    0
 rttmrpl       36        3    0        3     3     3     0     1     0   inf    0
 msdosnopl    120        4    0        4     1     1     0     1     0   inf    0
 kva-40960   40960       2    0        2     2     2     0     1     0   inf    0
 Totals           135264204   0 132221846 116441 34875 81566

 In use 432751K, total allocated 463388K; utilization 93.4%
 ---
 Izumi Tsutsui

From: Havard Eidnes <he@NetBSD.org>
To: gnats-bugs@netbsd.org
Cc: ad@netbsd.org, netbsd-bugs@netbsd.org, tsutsui@ceres.dti.ne.jp
Subject: Re: kern/54818: 9.0_RC1 pagedaemon spins
Date: Wed, 20 May 2020 22:11:56 +0200 (CEST)

 Hi,

 as we've found, this isn't the exact same bug as I'm seeing on
 8.1 related to audio, but since I started logging state here I
 thought I'd continue...

 I've added the below diff to sys/dev/audio.c, and done a couple
 of "play video with firefox via youtube" cycles.  One such cycle
 might look like this:

 audio_open: flags=0x2 sc=0xfffffe811e23d000 hdl=0xfffffe811e22d130
 audio_initbufs: mode=0x5
 audio_init_ringbuffer: MI blksize=8192
 audio_init_ringbuffer: final blksize=8192
 audio_init_ringbuffer: MI blksize=8192
 audio_init_ringbuffer: final blksize=8192
 audio_init_ringbuffer: MI blksize=8192
 audio_init_ringbuffer: final blksize=8192
 audio_init_ringbuffer: MI blksize=8192
 audio_init_ringbuffer: final blksize=8192
 audiosetinfo sc=0xfffffe811e23d000 ai=0xffff800141296af0
 audiosetinfo() Setting record params: enc=1 1ch 8/8bit 8000Hz
 audiosetinfo() Setting play params: enc=1 1ch 8/8bit 8000Hz
 audio_setup_pfilters: new 2, old 0
 audio_setup_pfilters: HW-buffer=0xfffffe83df618ac0 pustream=0xfffffe83df618838
 [0] enc=1 1ch 8/8bit 8000Hz
 [1] enc=6 1ch 16/16bit 8000Hz
 [HW] enc=6 2ch 16/16bit 48000Hz
 audio_setup_rfilters: HW-buffer=0xfffffe83df618b68 rustream=0xfffffe83df618c80
 [HW] enc=6 2ch 16/16bit 48000Hz
 [0] enc=6 1ch 16/16bit 8000Hz
 [1] enc=1 1ch 8/8bit 8000Hz
 audio_setup_rfilters: new 2, old 0
 audiosetinfo: filter setup is completed.
 audio_initbufs: mode=0x5
 audio_init_ringbuffer: MI blksize=336
 audio_init_ringbuffer: final blksize=512
 audio_calcwater: plow=49152 phigh=65536 rlow=0 rhigh=65195
 audio_calcwater: plow=49152 phigh=65536 rlow=0 rhigh=65195
 audio_open: done sc_mode = 0x5
 AUDIO_SETINFO mode=0x5
 audiosetinfo sc=0xfffffe811e23d000 ai=0xfffffe83fb636008
 audiosetinfo() Setting play params: enc=10 2ch 16/16bit 48000Hz
 audio_setup_pfilters: new 0, old 2
 audio_stream_dtor: non-null start, size 65536
 audio_stream_dtor: non-null start, size 65536
 audio_setup_pfilters: HW-buffer=0xfffffe83df618ac0 pustream=0xfffffe83df618ac0
 [HW] enc=6 2ch 16/16bit 48000Hz
 audiosetinfo: filter setup is completed.
 audio_initbufs: mode=0x1
 audio_init_ringbuffer: MI blksize=8192
 audio_init_ringbuffer: final blksize=8192
 audio_calcwater: plow=49152 phigh=65536 rlow=0 rhigh=65195
 AUDIO_GETINFO
 audio_pint: used < blksize vc=0xfffffe83df618800 used=0 blksize=8192
 audio_pint: used < blksize vc=0xfffffe83df618800 used=0 blksize=8192
 mix_write: call trigger_output

 (audio playing here, until closing firefox window)

 audio_pint: used < blksize vc=0xfffffe83df618800 used=128 blksize=8192
 audio_pint: used < blksize vc=0xfffffe83df618800 used=0 blksize=8192
 audio_pint: used < blksize vc=0xfffffe83df618800 used=0 blksize=8192

 etc.

 audio_close: sc=0xfffffe811e23d000
 audio_destroy_pfilters: npfilters 0
 audio_destroy_rfilters: nrfilters 2


 Now...

 What's "missing" here at the end is printf() from
 audio_stream_dtor() indicating that stream->start was non-NULL,
 so nothing gets actually released in this audio_stream_dtor()
 invocation for the "rfilters".  So the allocations made for
 audio_setup_rfilters() by audio_stream_ctor(), ref. the

 audio_setup_rfilters: new 2, old 0

 logged line, appear to get lost, two for each cycle of
 open/close.

 The releases shown below correspond to

 audio_setup_pfilters: new 0, old 2
 audio_stream_dtor: non-null start, size 65536
 audio_stream_dtor: non-null start, size 65536


 The diff below also changes a few of the key functions from
 statics to non-static -- this makes them visible in backtrace
 from dtrace, and an open/close cycle looks like this in the
 dtrace log:


 CPU FUNCTION                                 
   0  -> uvm_km_kmem_alloc                     size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audio_stream_ctor+0x5e
               netbsd`audio_setup_pfilters+0xc4
               netbsd`audiosetinfo+0x64e
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.32+0x488
               netbsd`audioopen+0x101
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0    -> uvm_km_kmem_alloc                   size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audio_stream_ctor+0x5e
               netbsd`audio_setup_pfilters+0xc4
               netbsd`audiosetinfo+0x64e
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.32+0x488
               netbsd`audioopen+0x101
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0      -> uvm_km_kmem_alloc                 size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audio_stream_ctor+0x5e
               netbsd`audio_setup_rfilters+0x180
               netbsd`audiosetinfo+0xea8
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.32+0x488
               netbsd`audioopen+0x101
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0        -> uvm_km_kmem_alloc               size: 65536
               netbsd`kmem_intr_alloc+0x6d
               netbsd`kmem_intr_zalloc+0xf
               netbsd`audio_stream_ctor+0x5e
               netbsd`audio_setup_rfilters+0x180
               netbsd`audiosetinfo+0xea8
               netbsd`audio_set_defaults+0xeb
               netbsd`audio_open.part.32+0x488
               netbsd`audioopen+0x101
               netbsd`spec_open+0x276
               netbsd`VOP_OPEN+0x4c
               netbsd`vn_open+0x1ed
               netbsd`do_open+0x112
               netbsd`do_sys_openat+0x68
               netbsd`sys_open+0x24
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0          -> uvm_km_kmem_free              size: 65536
               netbsd`audio_stream_dtor+0x37
               netbsd`audio_setup_pfilters+0x47b
               netbsd`audiosetinfo+0x64e
               netbsd`audio_ioctl+0x2a6
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d

   0            -> uvm_km_kmem_free            size: 65536
               netbsd`audio_stream_dtor+0x37
               netbsd`audio_setup_pfilters+0x47b
               netbsd`audiosetinfo+0x64e
               netbsd`audio_ioctl+0x2a6
               netbsd`audioioctl+0xe6
               netbsd`sys_ioctl+0x11a
               netbsd`syscall+0x1ec
               netbsd`handle_syscall+0x2d


 So -- 4 allocations and 2 releases.

 However, I'll have to admit that I'm still not able to connect
 the dots and see what's actually causing this memory leak, so
 I've here just summarized the debugging done so far.


 For the record, the dtrace script is

 ------------------------------
 #pragma D option flowindent

 fbt::uvm_km_kmem_alloc:entry
 /arg1 > 49152 && arg1 <= 65536 /
 {
         printf("size: %d", arg1);
         stack();
 }

 fbt::uvm_km_kmem_free:entry
 /arg2 > 49152 && arg2 <= 65536/
 {
         printf("size: %d", arg2);
         stack();
 }
 ------------------------------

 and run via "dtrace -s kmem.d -x nolibs > logfile 2>&1".

 And lastly, here's the diff to audio.c.  I've made a few of the
 ouputs which appear to come "as the audio plays" dependent on
 AUDIO_DEBUG >= 2, so that the setup debug info isn't drowned out.


 Index: sys/dev/audio.c
 ===================================================================
 RCS file: /cvsroot/src/sys/dev/Attic/audio.c,v
 retrieving revision 1.357.2.13
 diff -u -p -U 5 -r1.357.2.13 audio.c
 --- sys/dev/audio.c	7 Jun 2018 18:24:16 -0000	1.357.2.13
 +++ sys/dev/audio.c	20 May 2020 19:54:26 -0000
 @@ -195,11 +195,11 @@ __KERNEL_RCSID(0, "$NetBSD: audio.c,v 1.

  #include <uvm/uvm.h>

  #include "ioconf.h"

 -/* #define AUDIO_DEBUG	1 */
 +#define AUDIO_DEBUG	1
  #ifdef AUDIO_DEBUG
  #define DPRINTF(x)	if (audiodebug) printf x
  #define DPRINTFN(n,x)	if (audiodebug>(n)) printf x
  int	audiodebug = AUDIO_DEBUG;
  #else
 @@ -297,18 +297,18 @@ void	audio_clear(struct audio_softc *, s
  void	audio_clear_intr_unlocked(struct audio_softc *sc,
  				  struct virtual_channel *);
  int	audio_alloc_ring(struct audio_softc *, struct audio_ringbuffer *, int,
  			 size_t);
  void	audio_free_ring(struct audio_softc *, struct audio_ringbuffer *);
 -static int audio_setup_pfilters(struct audio_softc *, const audio_params_t *,
 +int audio_setup_pfilters(struct audio_softc *, const audio_params_t *,
  			      stream_filter_list_t *, struct virtual_channel *);
 -static int audio_setup_rfilters(struct audio_softc *, const audio_params_t *,
 +int audio_setup_rfilters(struct audio_softc *, const audio_params_t *,
  			      stream_filter_list_t *, struct virtual_channel *);
  static void audio_destroy_pfilters(struct virtual_channel *);
  static void audio_destroy_rfilters(struct virtual_channel *);
 -static void audio_stream_dtor(audio_stream_t *);
 -static int audio_stream_ctor(audio_stream_t *, const audio_params_t *, int);
 +void audio_stream_dtor(audio_stream_t *);
 +int audio_stream_ctor(audio_stream_t *, const audio_params_t *, int);
  static void stream_filter_list_append(stream_filter_list_t *,
  		stream_filter_factory_t, const audio_params_t *);
  static void stream_filter_list_prepend(stream_filter_list_t *,
  	    	stream_filter_factory_t, const audio_params_t *);
  static void stream_filter_list_set(stream_filter_list_t *, int,
 @@ -1279,11 +1279,11 @@ audio_free_ring(struct audio_softc *sc, 
  	}

  	r->s.start = NULL;
  }

 -static int
 +int
  audio_setup_pfilters(struct audio_softc *sc, const audio_params_t *pp,
  		     stream_filter_list_t *pfilters, struct virtual_channel *vc)
  {
  	stream_filter_t *pf[AUDIO_MAX_FILTERS], *of[AUDIO_MAX_FILTERS];
  	audio_stream_t ps[AUDIO_MAX_FILTERS], os[AUDIO_MAX_FILTERS];
 @@ -1339,10 +1339,12 @@ audio_setup_pfilters(struct audio_softc 
  		vc->sc_mpr.s.param = pfilters->filters[0].param;
  		vc->sc_pustream = &vc->sc_pstreams[0];
  	}
  	HW_UNLOCK(vc);

 +	printf("%s: new %d, old %d\n",
 +		__func__, pfilters->req_size, onfilters);
  	/* Destroy old filters. */
  	for (i = 0; i < onfilters; i++) {
  		of[i]->dtor(of[i]);
  		audio_stream_dtor(&os[i]);
  	}
 @@ -1361,11 +1363,11 @@ audio_setup_pfilters(struct audio_softc 
  #endif /* AUDIO_DEBUG */

  	return 0;
  }

 -static int
 +int
  audio_setup_rfilters(struct audio_softc *sc, const audio_params_t *rp,
  		     stream_filter_list_t *rfilters, struct virtual_channel *vc)
  {
  	stream_filter_t *rf[AUDIO_MAX_FILTERS], *of[AUDIO_MAX_FILTERS];
  	audio_stream_t rs[AUDIO_MAX_FILTERS], os[AUDIO_MAX_FILTERS];
 @@ -1439,10 +1441,12 @@ audio_setup_rfilters(struct audio_softc 
  			audio_print_params(num, &vc->sc_rstreams[i].param);
  		}
  	}
  #endif /* AUDIO_DEBUG */

 +	printf("%s: new %d, old %d\n",
 +		__func__, rfilters->req_size, onfilters);
  	/* Destroy old filters. */
  	for (i = 0; i < onfilters; i++) {
  		of[i]->dtor(of[i]);
  		audio_stream_dtor(&os[i]);
  	}
 @@ -1453,10 +1457,12 @@ audio_setup_rfilters(struct audio_softc 
  static void
  audio_destroy_pfilters(struct virtual_channel *vc)
  {
  	int i;

 +	printf("%s: npfilters %d\n",
 +		__func__, vc->sc_npfilters);
  	for (i = 0; i < vc->sc_npfilters; i++) {
  		vc->sc_pfilters[i]->dtor(vc->sc_pfilters[i]);
  		vc->sc_pfilters[i] = NULL;
  		audio_stream_dtor(&vc->sc_pstreams[i]);
  	}
 @@ -1466,33 +1472,44 @@ audio_destroy_pfilters(struct virtual_ch
  static void
  audio_destroy_rfilters(struct virtual_channel *vc)
  {
  	int i;

 +	printf("%s: nrfilters %d\n",
 +		__func__, vc->sc_nrfilters);
  	for (i = 0; i < vc->sc_nrfilters; i++) {
  		vc->sc_rfilters[i]->dtor(vc->sc_rfilters[i]);
  		vc->sc_rfilters[i] = NULL;
  		audio_stream_dtor(&vc->sc_pstreams[i]);
  	}
  	vc->sc_nrfilters = 0;
  }

 -static void
 +void
  audio_stream_dtor(audio_stream_t *stream)
  {

 -	if (stream->start != NULL)
 +	if (stream->start != NULL) {
 +		printf("%s: non-null start, size %zd\n",
 +			__func__, stream->bufsize);
  		kmem_free(stream->start, stream->bufsize);
 +	}
  	memset(stream, 0, sizeof(audio_stream_t));
  }

 -static int
 +int
  audio_stream_ctor(audio_stream_t *stream, const audio_params_t *param, int size)
  {
  	int frame_size;

  	size = min(size, AU_RING_SIZE);
 +	/* Will overwrite, may need to release */
 +	if (stream->start != NULL) {
 +		printf("%s: non-null start, size %zd\n",
 +			__func__, stream->bufsize);
 +		kmem_free(stream->start, stream->bufsize);
 +	}
  	stream->bufsize = size;
  	stream->start = kmem_zalloc(size, KM_SLEEP);
  	frame_size = (param->precision + 7) / 8 * param->channels;
  	size = (size / frame_size) * frame_size;
  	stream->end = stream->start + size;
 @@ -2432,11 +2449,11 @@ audio_drain(struct audio_softc *sc, stru

  	KASSERT(mutex_owned(sc->sc_lock));
  	KASSERT(mutex_owned(sc->sc_intr_lock));

  	error = 0;
 -	DPRINTF(("audio_drain: enter busy=%d\n", vc->sc_pbus));
 +	DPRINTFN(2, ("audio_drain: enter busy=%d\n", vc->sc_pbus));
  	cb = &vc->sc_mpr;
  	if (cb->mmapped)
  		return 0;

  	used = audio_stream_get_used(&cb->s);
 @@ -2501,11 +2518,11 @@ audio_drain(struct audio_softc *sc, stru
  	else if (sc->sc_usemixer)
  		drops += sc->sc_mixring.sc_mpr.blksize * PREFILL_BLOCKS;

  	error = 0;
  	while (cb->drops <= drops && !error) {
 -		DPRINTF(("audio_drain: vc=%p used=%d, drops=%ld\n",
 +		DPRINTFN(2, ("audio_drain: vc=%p used=%d, drops=%ld\n",
  			vc,
  			audio_stream_get_used(&vc->sc_mpr.s),
  			cb->drops));
  		mutex_exit(sc->sc_intr_lock);
  		error = audio_waitio(sc, &sc->sc_wchan, vc);
 @@ -2998,11 +3015,11 @@ audio_write(struct audio_softc *sc, stru
  		return 0;
  	}

  	if (!(vc->sc_mode & AUMODE_PLAY_ALL) && vc->sc_playdrop > 0) {
  		m = min(vc->sc_playdrop, uio->uio_resid);
 -		DPRINTF(("audio_write: playdrop %d\n", m));
 +		DPRINTFN(2, ("audio_write: playdrop %d\n", m));
  		uio->uio_offset += m;
  		uio->uio_resid -= m;
  		vc->sc_playdrop -= m;
  		if (uio->uio_resid == 0)
  			return 0;
 @@ -3128,11 +3145,11 @@ audio_ioctl(dev_t dev, struct audio_soft
  	if (!sc->sc_usemixer || chan->deschan != 0)
  		vc = pchan->vc;
  	else
  		vc = &sc->sc_mixring;

 -	DPRINTF(("audio_ioctl(%lu,'%c',%lu)\n",
 +	DPRINTFN(2, ("audio_ioctl(%lu,'%c',%lu)\n",
  		 IOCPARM_LEN(cmd), (char)IOCGROUP(cmd), cmd&0xff));
  	hw = sc->hw_if;
  	if (hw == NULL)
  		return ENXIO;
  	error = 0;
 @@ -3157,18 +3174,18 @@ audio_ioctl(dev_t dev, struct audio_soft
  		if (*(int *)addr) {
  			if (sc->sc_async_audio != 0)
  				error = EBUSY;
  			else
  				sc->sc_async_audio = pchan->chan;
 -			DPRINTF(("audio_ioctl: FIOASYNC chan %d\n",
 +			DPRINTFN(2, ("audio_ioctl: FIOASYNC chan %d\n",
  			    pchan->chan));
  		} else
  			sc->sc_async_audio = 0;
  		break;

  	case AUDIO_FLUSH:
 -		DPRINTF(("AUDIO_FLUSH\n"));
 +		DPRINTFN(2, ("AUDIO_FLUSH\n"));
  		rbus = vc->sc_rbus;
  		pbus = vc->sc_pbus;
  		mutex_enter(sc->sc_intr_lock);
  		audio_clear(sc, vc);
  		error = audio_initbufs(sc, vc);
 @@ -3316,11 +3333,11 @@ audio_ioctl(dev_t dev, struct audio_soft
  			DPRINTF(("audio_ioctl: unknown ioctl\n"));
  			error = EINVAL;
  		}
  		break;
  	}
 -	DPRINTF(("audio_ioctl(%lu,'%c',%lu) result %d\n",
 +	DPRINTFN(2, ("audio_ioctl(%lu,'%c',%lu) result %d\n",
  		 IOCPARM_LEN(cmd), (char)IOCGROUP(cmd), cmd&0xff, error));
  	return error;
  }

  int
 @@ -3582,11 +3599,11 @@ audiostartr(struct audio_softc *sc, stru
  {
  	int error;

  	KASSERT(mutex_owned(sc->sc_lock));

 -	DPRINTF(("audiostartr: start=%p used=%d(hi=%d) mmapped=%d\n",
 +	DPRINTFN(2, ("audiostartr: start=%p used=%d(hi=%d) mmapped=%d\n",
  		 vc->sc_mrr.s.start, audio_stream_get_used(&vc->sc_mrr.s),
  		 vc->sc_mrr.usedhigh, vc->sc_mrr.mmapped));

  	if (!audio_can_capture(sc))
  		return EINVAL;
 @@ -3613,11 +3630,11 @@ audiostartp(struct audio_softc *sc, stru

  	KASSERT(mutex_owned(sc->sc_lock));

  	error = 0;
  	used = audio_stream_get_used(&vc->sc_mpr.s);
 -	DPRINTF(("audiostartp: start=%p used=%d(hi=%d blk=%d) mmapped=%d\n",
 +	DPRINTFN(2, ("audiostartp: start=%p used=%d(hi=%d blk=%d) mmapped=%d\n",
  		 vc->sc_mpr.s.start, used, vc->sc_mpr.usedhigh,
  		 vc->sc_mpr.blksize, vc->sc_mpr.mmapped));

  	if (!audio_can_playback(sc))
  		return EINVAL;
 @@ -3630,11 +3647,11 @@ audiostartp(struct audio_softc *sc, stru
  	else
  		blksize = vc->sc_mpr.blksize;

  	if (!vc->sc_mpr.mmapped && used < blksize) {
  		cv_broadcast(&sc->sc_wchan);
 -		DPRINTF(("%s: wakeup and return\n", __func__));
 +		DPRINTFN(2, ("%s: wakeup and return\n", __func__));
  		return 0;
  	}

  	vc->sc_pbus = true;
  	if (sc->sc_trigger_started == false) {
 @@ -3773,11 +3790,11 @@ audio_mix(void *v)
  	uint8_t *inp;
  	int cc, cc1, used, blksize;

  	sc = v;

 -	DPRINTF(("PINT MIX\n"));
 +	DPRINTFN(2, ("PINT MIX\n"));
  	sc->schedule_rih = false;
  	sc->schedule_wih = false;
  	sc->sc_writeme = false;

  	if (sc->sc_dying == true)

>Unformatted:

NetBSD Home
NetBSD PR Database Search

(Contact us) $NetBSD: query-full-pr,v 1.46 2020/01/03 16:35:01 leot Exp $
$NetBSD: gnats_config.sh,v 1.9 2014/08/02 14:16:04 spz Exp $
Copyright © 1994-2020 The NetBSD Foundation, Inc. ALL RIGHTS RESERVED.