NetBSD Problem Report #60820

From www@netbsd.org  Mon Sep 28 21:37:11 2026
Return-Path: <www@netbsd.org>
Received: from mail.netbsd.org (mail.netbsd.org [199.233.217.200])
	(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
	 key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256
	 client-signature RSA-PSS (2048 bits) client-digest SHA256)
	(Client CN "mail.netbsd.org", Issuer "YR1" (not verified))
	by mollari.NetBSD.org (Postfix) with ESMTPS id A6C931A923B
	for <gnats-bugs@gnats.NetBSD.org>; Mon, 28 Sep 2026 21:37:11 +0000 (UTC)
Message-Id: <20260928213709.E72521A923C@mollari.NetBSD.org>
Date: Mon, 28 Sep 2026 21:37:09 +0000 (UTC)
From: clare@csel.org
Reply-To: clare@csel.org
To: gnats-bugs@NetBSD.org
Subject: LFS in-kernel cleaner can enter deadlock with build.sh
X-Send-Pr-Version: www-1.0
X-From4GNATS: "clare@csel.org via gnats" <gnats-admin@NetBSD.org>

>Number:         60820
>Category:       kern
>Synopsis:       LFS in-kernel cleaner can enter deadlock with build.sh
>Confidential:   no
>Severity:       serious
>Priority:       medium
>Responsible:    kern-bug-people
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Mon Sep 28 21:40:00 +0000 2026
>Last-Modified:  Tue Oct 06 18:35:01 +0000 2026
>Originator:     Shinichi Doyashiki
>Release:        11.99.8 around 2026-09-28 04:00 JST
>Organization:
at home
>Environment:
NetBSD kanade2.hq.csel.org 11.99.8 NetBSD 11.99.8 (KANADE2_12) #27: Mon Sep 28 04:34:19 JST 2026  clare@kanade2.hq.csel.org:/export/netbsd/debuglfs/src/sys/arch/amd64/compile/KANADE2_12 amd64
>Description:
LFS in-kernel cleaner can enter deadlock with build.sh
reproducibility is currently unknown

db{0}> ps
PID     LID S CPU     FLAGS       STRUCT LWP *               NAME WAIT
3229   3229 3   0         0   ffff89ac014a4800               find prelockcv
10122 10122 3   1         0   ffff89ac5c118400                 rm prelockcv
0     21402 3   2       200   ffff89a80059f400        lfs_cleaner vnode
0      8556 3   3       200   ffff89abe64d2000         lfs_writer lfswrite
0       169 3   3       240   ffff89a7f8561c00            ioflush prelockcv
0        12 3   0       200   ffff89af30ab7800             lfsswq lfsswq
0        11 3   2       200   ffff89af30ab7400             lfscwq lfscwq

find process

db{0}> bt /a ffff89ac014a4800
trace: pid 3229 lid 3229 at 0xffff9c894c8ccd20
sleepq_block() at netbsd:sleepq_block+0xea
cv_wait() at netbsd:cv_wait+0xca
lfs_prelock() at netbsd:lfs_prelock+0x59
lfs_itimes() at netbsd:lfs_itimes+0x3c6
lfs_close() at netbsd:lfs_close+0x81
VOP_CLOSE() at netbsd:VOP_CLOSE+0xa0
vn_close() at netbsd:vn_close+0x34
closef() at netbsd:closef+0xa1
fd_close() at netbsd:fd_close+0xf3
sys_close() at netbsd:sys_close+0x22
syscall() at netbsd:syscall+0x9d
--- syscall (number 6) ---
netbsd:syscall+0x9d:

rm process

db{0}> bt /a ffff89ac5c118400
trace: pid 10122 lid 10122 at 0xffff9c895e5fbc00
sleepq_block() at netbsd:sleepq_block+0xea
cv_wait() at netbsd:cv_wait+0xca
lfs_prelock() at netbsd:lfs_prelock+0x59
lfs_vfree() at netbsd:lfs_vfree+0xc2
lfs_reclaim() at netbsd:lfs_reclaim+0x275
VOP_RECLAIM() at netbsd:VOP_RECLAIM+0x60
vcache_reclaim() at netbsd:vcache_reclaim+0x19e
vrelel() at netbsd:vrelel+0x24d
lfs_remove() at netbsd:lfs_remove+0xca
VOP_REMOVE() at netbsd:VOP_REMOVE+0xe7
do_sys_unlinkat() at netbsd:do_sys_unlinkat+0x2bc
syscall() at netbsd:syscall+0x9d
--- syscall (number 10) ---
netbsd:syscall+0x9d:

lfs_cleaner thread

db{0}> bt /a ffff89a80059f400
trace: pid 0 lid 21402 at 0xffff9c8949164a80
sleepq_block() at netbsd:sleepq_block+0xea
cv_wait() at netbsd:cv_wait+0xca
vstate_assert_wait_stable() at netbsd:vstate_assert_wait_stable+0x4b
vcache_vget() at netbsd:vcache_vget+0x34
vcache_get() at netbsd:vcache_get+0x31b
lfs_vget() at netbsd:lfs_vget+0x23
finfo_func_checkempty() at netbsd:finfo_func_checkempty+0xe2
lfs_parse_pseg() at netbsd:lfs_parse_pseg+0x39f
lfs_checkempty() at netbsd:lfs_checkempty+0xee
lfs_markclean() at netbsd:lfs_markclean+0x3e
lfs_auto_segclean() at netbsd:lfs_auto_segclean+0x24b
lfs_segunlock() at netbsd:lfs_segunlock+0x781
lfs_segwrite() at netbsd:lfs_segwrite+0x531
lfs_cleanerd() at netbsd:lfs_cleanerd+0x89e

db{0}> bt /a ffff89abe64d2000
trace: pid 0 lid 8556 at 0xffff9c895448bf50
sleepq_block() at netbsd:sleepq_block+0x233
cv_timedwait() at netbsd:cv_timedwait+0xd4
lfs_writerd() at netbsd:lfs_writerd+0x3ad

ioflush

db{0}> bt /a ffff89a7f8561c00
trace: pid 0 lid 169 at 0xffff9c8946d5ce20
sleepq_block() at netbsd:sleepq_block+0xea
cv_wait() at netbsd:cv_wait+0xca
lfs_prelock() at netbsd:lfs_prelock+0x59
lfs_seglock() at netbsd:lfs_seglock+0x16
lfs_segwrite() at netbsd:lfs_segwrite+0x71
lfs_sync() at netbsd:lfs_sync+0xc6
VFS_SYNC() at netbsd:VFS_SYNC+0x22
sched_sync() at netbsd:sched_sync+0xd6

db{0}> bt /a ffff89af30ab7800
trace: pid 0 lid 12 at 0xffff9c8936382f60
sleepq_block() at netbsd:sleepq_block+0xea
cv_wait() at netbsd:cv_wait+0xca
workqueue_worker() at netbsd:workqueue_worker+0x90

db{0}> bt /a ffff89af30ab7400
trace: pid 0 lid 11 at 0xffff9c893637bf60
sleepq_block() at netbsd:sleepq_block+0xea
cv_wait() at netbsd:cv_wait+0xca
workqueue_worker() at netbsd:workqueue_worker+0x90

db{0}> call fstrans_dump(0)
Fstrans locks by lwp:
[ 88245.5130885] 3229.322 (/lfs) shared 1 cow 0 alias 0
[ 88245.5130885] 10122.10 (/lfs) shared 2 cow 0 alias 0
[ 88245.5130885] 0.21402  (/lfs) shared 1 cow 0 alias 0
[ 88245.5130885] 0.169    (/lfs) shared 1 cow 0 alias 0
[ 88245.5130885] Fstrans state by mount:
0

db{0}> show all locks
PID     LID CPU       STRUCT LWP *              OWNER BITS LOCK


>How-To-Repeat:
$ cat /run-build.sh 
#!/bin/sh
TMPDIR=/tmp
export TMPDIR

OBJMACHINE=yes
export OBJMACHINE

testdir=/lfs/netbsd/current

while true; do
  if fsck_lfs -nf /lfs; then
    echo "===> FSCK OK";
  else
    echo "===> FSCK ERROR";
    exit 1
  fi
  if mount /lfs; then
    echo "===> MOUNT OK";
  else
    echo "===> MOUNT ERROR";
    exit 1
  fi
  rm ${testdir}/build.log
  for machine in amd64; do
    (
    cd ${testdir}/src || exit
    tooldir=${testdir}/tools.${machine}
    destdir=${testdir}/destdir.${machine}
    objdir=${testdir}/objdir.${machine}
    releasedir=${testdir}/release.test
    rm -rf ${tooldir}
    rm -rf ${destdir}
    rm -rf ${objdir}
    rm -rf ${releasedir}
    mkdir -p ${objdir}
    ./build.sh -m ${machine} \
        -j 5 \
        -U \
        -O ${objdir} \
        -D ${destdir} \
        -R ${releasedir} \
        -T ${tooldir} \
        release
    ) 2>&1 | tee ${testdir}/build.log
  done
  sync
  if umount /lfs; then
    echo "===> UMOUNT OK";
  else
    echo "===> UMOUNT ERROR";
    exit 1
  fi
done

>Fix:
unknown

>Audit-Trail:
From: Shinichi Doyashiki <clare@csel.org>
To: gnats-bugs@netbsd.org
Cc: 
Subject:  Re: kern/60820: LFS in-kernel cleaner can enter deadlock with build.sh
Date: Wed, 07 Oct 2026 03:32:59 +0900

 A deadlock was being observed, likely the reproduced.
 the kernel version was 11.99.9 around 2026-10-06 05:44 JST.

 db{0}> ps
 PID     LID S CPU     FLAGS       STRUCT LWP *               NAME WAIT
 1117   1117 3   3   1000000   ffff8509fda7dc00                 rm prelockcv
 0      1078 3   3       200   ffff8509f8b4c800        lfs_cleaner vnode
 0      2227 3   2       200   ffff8509fcbfbc00         lfs_writer lfswrite
 0       170 3   0       200   ffff8509f8891c00            ioflush prelockcv

 db{0}> show all locks
 [Locks tracked through LWPs]

 ****** LWP 1117.1117 (rm) @ 0xffff8509fda7dc00, l_stat=3

 *** Locks held:

 * Lock 0 (initialized at netbsd:vcache_alloc+0x6b)
 lock address : ffff850a41e1bf00
 type         : sleep/adaptive
 initialized  : netbsd:vcache_alloc+0x6b
 shared holds :                  0 exclusive:                  1
 shares wanted:                  0 exclusive:                  0
 relevant cpu :                  3 last held:                  2
 relevant lwp : 0xffff8509fda7dc00 last held: 0xffff8509fda7dc00
 last locked* : netbsd:genfs_lock+0x89
 unlocked     : netbsd:genfs_unlock+0x14
 owner/count  : 0xffff8509fda7dc00 flags    : 0x0000000000000004
 Turnstile: no active turnstile for this lock.

 *** Locks wanted: none

 ****** LWP 1443.1443 (ntpd) @ 0xffff8509fbba4000, l_stat=3

 *** Locks held: none

 *** Locks wanted:

 * Lock 0 (initialized at netbsd:module_hook_init+0x17)
 lock address : netbsd:module_hook
 type         : sleep/adaptive
 initialized  : netbsd:module_hook_init+0x17
 shared holds :                  0 exclusive:                  0
 shares wanted:                  0 exclusive:                  0
 relevant cpu :                  3 last held:                  0
 relevant lwp : 0xffff8509fbba4000 last held: 000000000000000000
 last locked  : 0
 unlocked*    : 0
 owner field  : 000000000000000000 wait/spin:                0/0
 Turnstile: no active turnstile for this lock.

 ****** LWP 389.389 (rpcbind) @ 0xffff8509faf96400, l_stat=3

 *** Locks held: none

 *** Locks wanted:

 * Lock 0 (initialized at netbsd:module_hook_init+0x17)
 lock address : netbsd:module_hook
 type         : sleep/adaptive
 initialized  : netbsd:module_hook_init+0x17
 shared holds :                  0 exclusive:                  0
 shares wanted:                  0 exclusive:                  0
 relevant cpu :                  1 last held:                  0
 relevant lwp : 0xffff8509faf96400 last held: 000000000000000000
 last locked  : 0
 unlocked*    : 0
 owner field  : 000000000000000000 wait/spin:                0/0
 Turnstile: no active turnstile for this lock.

 ****** LWP 692.692 (dhcpcd) @ 0xffff8509f921e800, l_stat=3

 *** Locks held: none

 *** Locks wanted:

 * Lock 0 (initialized at netbsd:module_hook_init+0x17)
 lock address : netbsd:module_hook
 type         : sleep/adaptive
 initialized  : netbsd:module_hook_init+0x17
 shared holds :                  0 exclusive:                  0
 shares wanted:                  0 exclusive:                  0
 relevant cpu :                  3 last held:                  0
 relevant lwp : 0xffff8509f921e800 last held: 000000000000000000
 last locked  : 0
 unlocked*    : 0
 owner field  : 000000000000000000 wait/spin:                0/0
 Turnstile: no active turnstile for this lock.

 ****** LWP 679.679 (dhcpcd) @ 0xffff8509f98c7c00, l_stat=3

 *** Locks held: none

 *** Locks wanted:

 * Lock 0 (initialized at netbsd:module_hook_init+0x17)
 lock address : netbsd:module_hook
 type         : sleep/adaptive
 initialized  : netbsd:module_hook_init+0x17
 shared holds :                  0 exclusive:                  0
 shares wanted:                  0 exclusive:                  0
 relevant cpu :                  2 last held:                  0
 relevant lwp : 0xffff8509f98c7c00 last held: 000000000000000000
 last locked  : 0
 unlocked*    : 0
 owner field  : 000000000000000000 wait/spin:                0/0
 Turnstile: no active turnstile for this lock.

 ****** LWP 0.1078 (lfs_cleaner) @ 0xffff8509f8b4c800, l_stat=3

 *** Locks held:

 * Lock 0 (initialized at netbsd:lfs_mountfs+0x537)
 lock address : ffff850a07f822b8
 type         : sleep/adaptive
 initialized  : netbsd:lfs_mountfs+0x537
 shared holds :                  0 exclusive:                  1
 shares wanted:                  0 exclusive:                  0
 relevant cpu :                  3 last held:                  3
 relevant lwp : 0xffff8509f8b4c800 last held: 0xffff8509f8b4c800
 last locked* : netbsd:lfs_prelock+0xaa
 unlocked     : netbsd:lfs_preunlock+0x77
 owner/count  : 0xffff8509f8b4c800 flags    : 0x0000000000000004
 Turnstile: no active turnstile for this lock.

 *** Locks wanted: none

 ****** LWP 0.28 (softclk/1) @ 0xffff8509f715f800, l_stat=1

 *** Locks held: none

 *** Locks wanted:

 * Lock 0 (initialized at netbsd:module_hook_init+0x17)
 lock address : netbsd:module_hook
 type         : sleep/adaptive
 initialized  : netbsd:module_hook_init+0x17
 shared holds :                  0 exclusive:                  0
 shares wanted:                  0 exclusive:                  0
 relevant cpu :                  1 last held:                  0
 relevant lwp : 0xffff8509f715f800 last held: 000000000000000000
 last locked  : 0
 unlocked*    : 0
 owner field  : 000000000000000000 wait/spin:                0/0
 Turnstile: no active turnstile for this lock.

 ****** LWP 0.13 (iflnkst) @ 0xffff851130aa9c00, l_stat=3

 *** Locks held: none

 *** Locks wanted:

 * Lock 0 (initialized at netbsd:module_hook_init+0x17)
 lock address : netbsd:module_hook
 type         : sleep/adaptive
 initialized  : netbsd:module_hook_init+0x17
 shared holds :                  0 exclusive:                  0
 shares wanted:                  0 exclusive:                  0
 relevant cpu :                  2 last held:                  0
 relevant lwp : 0xffff851130aa9c00 last held: 000000000000000000
 last locked  : 0
 unlocked*    : 0
 owner field  : 000000000000000000 wait/spin:                0/0
 Turnstile: no active turnstile for this lock.

 ****** LWP 0.3 (softnet/0) @ 0xffff851133b13400, l_stat=1

 *** Locks held: none

 *** Locks wanted:

 * Lock 0 (initialized at netbsd:module_hook_init+0x17)
 lock address : netbsd:module_hook
 type         : sleep/adaptive
 initialized  : netbsd:module_hook_init+0x17
 shared holds :                  0 exclusive:                  0
 shares wanted:                  0 exclusive:                  0
 relevant cpu :                  0 last held:                  0
 relevant lwp : 0xffff851133b13400 last held: 000000000000000000
 last locked  : 0
 unlocked*    : 0
 owner field  : 000000000000000000 wait/spin:                0/0
 Turnstile: no active turnstile for this lock.

 ****** LWP 0.0 (swapper) @ 0xffffffff814648c0, l_stat=3

 *** Locks held: none

 *** Locks wanted:

 * Lock 0 (initialized at netbsd:module_hook_init+0x17)
 lock address : netbsd:module_hook
 type         : sleep/adaptive
 initialized  : netbsd:module_hook_init+0x17
 shared holds :                  0 exclusive:                  0
 shares wanted:                  0 exclusive:                  0
 relevant cpu :                  2 last held:                  0
 relevant lwp : 0xffffffff814648c0 last held: 000000000000000000
 last locked  : 0
 unlocked*    : 0
 owner field  : 000000000000000000 wait/spin:                0/0
 Turnstile: no active turnstile for this lock.

 [Locks tracked through CPUs]

 ******* Locks held on cpu0:

 * Lock 0 (initialized at netbsd:com_attach_subr+0x18c)
 lock address : ffff851134b807b0
 type         : spin
 initialized  : netbsd:com_attach_subr+0x18c
 shared holds :                  0 exclusive:                  1
 shares wanted:                  0 exclusive:                  0
 relevant cpu :                  0 last held:                  0
 relevant lwp : 0xffff851133b13000 last held: 0xffff851133b13000
 last locked* : netbsd:comintr+0x52
 unlocked     : netbsd:comintr+0x891
 owner field  : 0x0000000000010800 wait/spin:                0/1

 db{0}> show proc /a ffff8509f8b4c800
 lwp_t ffff8509f8b4c800
 system: pid 0 proc ffffffff81465040 vmspace/map ffffffff8155a2e0 flags 20002
   lwp 1078 [lfs_cleaner] ffff8509f8b4c800 pcb ffff948946d4a000
     stat 3 flags 200 cpu 3 pri 123 ref 0
     wmesg vnode wchan ffff850a39e3d580
   lwp 2227 [lfs_writer] ffff8509fcbfbc00 pcb ffff948947625000
     stat 3 flags 200 cpu 2 pri 123 ref 0
     wmesg lfswrite wchan ffffffff814f3e50
   lwp 170 [ioflush] ffff8509f8891c00 pcb ffff948946d58000
     stat 3 flags 200 cpu 0 pri 124 ref 0
     wmesg prelockcv wchan ffff850a07f82228
   lwp 12 [lfsswq] ffff851130aa9800 pcb ffff94893637e000
     stat 3 flags 200 cpu 1 pri 123 ref 0
     wmesg lfsswq wchan ffff851133b34448
   lwp 11 [lfscwq] ffff851130aa9400 pcb ffff948936377000
     stat 3 flags 200 cpu 2 pri 123 ref 0
     wmesg lfscwq wchan ffff851133b34308


 the cleaner thread

 db{0}> trace /a ffff8509f8b4c800
 trace: pid 0 lid 1078 at 0xffff948946d4ea80
 sleepq_block() at netbsd:sleepq_block+0xea
 cv_wait() at netbsd:cv_wait+0xca
 vstate_assert_wait_stable() at netbsd:vstate_assert_wait_stable+0x4b
 vcache_vget() at netbsd:vcache_vget+0x34
 vcache_get() at netbsd:vcache_get+0x31b
 lfs_vget() at netbsd:lfs_vget+0x23
 finfo_func_checkempty() at netbsd:finfo_func_checkempty+0xf4
 lfs_parse_pseg() at netbsd:lfs_parse_pseg+0x3f4
 lfs_checkempty() at netbsd:lfs_checkempty+0xfc
 lfs_markclean() at netbsd:lfs_markclean+0x4c
 lfs_auto_segclean() at netbsd:lfs_auto_segclean+0x259
 lfs_segunlock() at netbsd:lfs_segunlock+0x727
 lfs_segwrite() at netbsd:lfs_segwrite+0x53f
 lfs_cleanerd() at netbsd:lfs_cleanerd+0x8d0

 db{0}> trace /a ffff8509fcbfbc00
 trace: pid 0 lid 2227 at 0xffff948947629f50
 sleepq_block() at netbsd:sleepq_block+0x233
 cv_timedwait() at netbsd:cv_timedwait+0xd4
 lfs_writerd() at netbsd:lfs_writerd+0x3bb

 db{0}> trace /a ffff8509f8891c00
 trace: pid 0 lid 170 at 0xffff948946d5ce20
 sleepq_block() at netbsd:sleepq_block+0xea
 cv_wait() at netbsd:cv_wait+0xca
 lfs_prelock() at netbsd:lfs_prelock+0x59
 lfs_seglock() at netbsd:lfs_seglock+0x16
 lfs_segwrite() at netbsd:lfs_segwrite+0x7f
 lfs_sync() at netbsd:lfs_sync+0xc6
 VFS_SYNC() at netbsd:VFS_SYNC+0x22
 sched_sync() at netbsd:sched_sync+0xe4


 the rm process

 db{0}> show proc /a ffff8509fda7dc00
 lwp_t ffff8509fda7dc00
 rm: pid 1117 proc ffff8509f93477c0 vmspace/map ffff8509fb3b9900 flags 4000
   lwp 1117 ffff8509fda7dc00 pcb ffff948947544000
     stat 3 flags 1000000 cpu 3 pri 63 ref 0
     wmesg prelockcv wchan ffff850a07f82228

 db{0}> trace /a ffff8509fda7dc00
 trace: pid 1117 lid 1117 at 0xffff948947548c20
 sleepq_block() at netbsd:sleepq_block+0xea
 cv_wait() at netbsd:cv_wait+0xca
 lfs_prelock() at netbsd:lfs_prelock+0x59
 lfs_vfree() at netbsd:lfs_vfree+0xd4
 lfs_reclaim() at netbsd:lfs_reclaim+0x275
 VOP_RECLAIM() at netbsd:VOP_RECLAIM+0x60
 vcache_reclaim() at netbsd:vcache_reclaim+0x1cc
 vrelel() at netbsd:vrelel+0x251
 lfs_rmdir() at netbsd:lfs_rmdir+0xc3
 VOP_RMDIR() at netbsd:VOP_RMDIR+0xe0
 do_sys_unlinkat() at netbsd:do_sys_unlinkat+0x2eb
 syscall() at netbsd:syscall+0xaf
 --- syscall (number 137) ---
 netbsd:syscall+0xaf:

 db{0}> call fstrans_dump(0)
 Fstrans locks by lwp:
 [ 71281.6153000] 1117.111 (/lfs) shared 2 cow 0 alias 0
 [ 71281.6153000] 0.1078   (/lfs) shared 1 cow 0 alias 0
 [ 71281.6153000] 0.170    (/lfs) shared 1 cow 0 alias 0
 [ 71281.6153000] Fstrans state by mount:
 0

Home	PR Database Search