NetBSD Problem Report #59912

From clare@csel.org  Mon Jan 12 09:11:06 2026
Return-Path: <clare@csel.org>
Received: from mail.netbsd.org (mail.netbsd.org [199.233.217.200])
	(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
	 key-exchange X25519 server-signature RSA-PSS (2048 bits)
	 client-signature RSA-PSS (2048 bits))
	(Client CN "mail.netbsd.org", Issuer "R13" (verified OK))
	by mollari.NetBSD.org (Postfix) with ESMTPS id 1C1FC1A9244
	for <gnats-bugs@gnats.NetBSD.org>; Mon, 12 Jan 2026 09:11:06 +0000 (UTC)
Message-Id: <20260112091057.C0B7199886@mail.csel.org>
Date: Mon, 12 Jan 2026 18:10:57 +0900 (JST)
From: Shinichi Doyashiki <clare@csel.org>
Reply-To: Shinichi Doyashiki <clare@csel.org>
To: gnats-bugs@NetBSD.org
Subject: running rm -rf on LFS hangs the rm process forever on prelock state
X-Send-Pr-Version: 3.95
X-From4GNATS: "Shinichi Doyashiki via gnats" <gnats-admin@NetBSD.org>

>Number:         59912
>Category:       kern
>Synopsis:       running rm -rf on LFS hangs the rm process forever on prelock state
>Confidential:   no
>Severity:       serious
>Priority:       medium
>Responsible:    kern-bug-people
>State:          open
>Class:          sw-bug
>Submitter-Id:   net
>Arrival-Date:   Mon Jan 12 09:15:00 +0000 2026
>Last-Modified:  Sat Jan 17 14:30:02 +0000 2026
>Originator:     Shinichi Doyashiki
>Release:        NetBSD 11.99.4 as of 2026-01-12 (JST)
>Organization:
	at home
>Environment:
System: NetBSD kanade.hq.csel.org 11.99.4 NetBSD 11.99.4 (KANADE_12) #3: Mon Jan 12 04:25:44 JST 2026 clare@kanade.hq.csel.org:/work/current/src/sys/arch/amd64/compile/KANADE_12 amd64
Architecture: x86_64
Machine: amd64
>Description:
	running rm -rf on filled LFS filesystem can hang
	the rm process forever on prelock state.

	the rm process:
(gdb) kvm proc 0xffff99f0031fc800
0xffffffff80960c23 in mi_switch (l=l@entry=0xffff99f0031fc800)
    at ../../../../kern/kern_synch.c:811
811                     prevlwp = cpu_switchto(l, newl, returning);
(gdb) bt
#0  0xffffffff80960c23 in mi_switch (l=l@entry=0xffff99f0031fc800)
    at ../../../../kern/kern_synch.c:811
#1  0xffffffff8095d263 in sleepq_block (timo=timo@entry=0,
    catch_p=catch_p@entry=false,
    syncobj=syncobj@entry=0xffffffff80da3480 <cv_syncobj>,
    nlocks=nlocks@entry=1) at ../../../../kern/kern_sleepq.c:392
#2  0xffffffff80914f91 in cv_wait (cv=0xffff99f15f73b228,
    mtx=0xffffffff810c80d8 <lfs_lock>) at ../../../../kern/kern_condvar.c:182
#3  0xffffffff808a897f in lfs_prelock (fs=fs@entry=0xffff99f15f73b000,
    flags=flags@entry=0) at ../../../../ufs/lfs/lfs_subr.c:683
#4  0xffffffff808a89d1 in lfs_fraglock_enter (fs=fs@entry=0xffff99f15f73b000,
    enter_exit=enter_exit@entry=0) at ../../../../ufs/lfs/lfs_subr.c:284
#5  0xffffffff80889b8c in lfs_orphan (fs=0xffff99f15f73b000,
    vp=vp@entry=0xffff99f24b358d00) at ../../../../ufs/lfs/lfs_alloc.c:968
#6  0xffffffff808b6298 in lfs_remove (v=0xffffa7094ab3be98)
    at ../../../../ufs/lfs/lfs_vnops.c:1056
#7  0xffffffff80a24af8 in VOP_REMOVE (dvp=0xffff99f306611a80,
    vp=0xffff99f24b358d00, cnp=cnp@entry=0xffffa7094ab3bf50)
    at ../../../../kern/vnode_if.c:1218
#8  0xffffffff80a0ad5d in do_sys_unlinkat (l=<optimized out>,
    fdat=<optimized out>, arg=<optimized out>, flags=<optimized out>,
    seg=<optimized out>) at ../../../../kern/vfs_syscalls.c:2963
#9  0xffffffff80526a0b in sy_call (sy=0xffffffff81059e90 <sysent+240>,
    l=0xffff99f0031fc800, uap=0xffffa7094ab3c000, rval=0xffffa7094ab3bfb0)
    at ../../../../sys/syscallvar.h:65
#10 sy_invoke (sy=0xffffffff81059e90 <sysent+240>, l=0xffff99f0031fc800,
    uap=0xffffa7094ab3c000, rval=0xffffa7094ab3bfb0, code=10)
    at ../../../../sys/syscallvar.h:94
#11 syscall (frame=0xffffa7094ab3c000)
    at ../../../../arch/x86/x86/syscall.c:137
#12 0xffffffff8021025d in handle_syscall ()
(gdb)

	the cleaner:
(gdb) kvm proc 0xffff99f002d69000
0xffffffff80960c23 in mi_switch (l=l@entry=0xffff99f002d69000)
    at ../../../../kern/kern_synch.c:811
811                     prevlwp = cpu_switchto(l, newl, returning);
(gdb) bt
#0  0xffffffff80960c23 in mi_switch (l=l@entry=0xffff99f002d69000)
    at ../../../../kern/kern_synch.c:811
#1  0xffffffff8095d263 in sleepq_block (timo=timo@entry=0,
    catch_p=catch_p@entry=false,
    syncobj=syncobj@entry=0xffffffff80da4940 <sleep_syncobj>,
    nlocks=nlocks@entry=2) at ../../../../kern/kern_sleepq.c:392
#2  0xffffffff809605d7 in mtsleep (ident=ident@entry=0xffff99f15f73b23c,
    priority=priority@entry=529,
    wmesg=wmesg@entry=0xffffffff80e07384 "clean1", timo=timo@entry=0,
    mtx=0xffffffff810c80d8 <lfs_lock>) at ../../../../kern/kern_synch.c:232
#3  0xffffffff808b32a5 in lfs_strategy (v=<optimized out>)
    at ../../../../ufs/lfs/lfs_vnops.c:1533
#4  0xffffffff80a256c2 in VOP_STRATEGY (vp=vp@entry=0xffff99f012474540,
    bp=bp@entry=0xffff99f1a75e0da8) at ../../../../kern/vnode_if.c:1733
#5  0xffffffff809f61ad in bio_doread (vp=0xffff99f012474540,
    blkno=<optimized out>, size=<optimized out>, async=async@entry=0)
    at ../../../../kern/vfs_bio.c:714
#6  0xffffffff809f630d in bread (vp=<optimized out>, blkno=<optimized out>,
    size=<optimized out>, flags=flags@entry=0,
    bpp=bpp@entry=0xffffa70946dedc28) at ../../../../kern/vfs_bio.c:754
#7  0xffffffff808a7bf2 in lfs_auto_segclean (fs=fs@entry=0xffff99f15f73b000)
    at ../../../../ufs/lfs/lfs_subr.c:440
#8  0xffffffff808a8527 in lfs_segunlock (fs=0xffff99f15f73b000)
    at ../../../../ufs/lfs/lfs_subr.c:581
#9  0xffffffff808aa2fb in lfs_markv (fsidp=fsidp@entry=0xffff99f166139118,
    blkiov=blkiov@entry=0xffffa70042d21000, blkcnt=blkcnt@entry=331,
    l=0xffff99f002d69000) at ../../../../ufs/lfs/lfs_syscalls.c:504
#10 0xffffffff808aac84 in lfs_markv (l=l@entry=0xffff99f002d69000,
    fsidp=fsidp@entry=0xffff99f166139118,
    blkiov=blkiov@entry=0xffffa70042d21000, blkcnt=blkcnt@entry=331)
    at ../../../../ufs/lfs/lfs_syscalls.c:541
#11 0xffffffff808b79e3 in lfs_fcntl (v=0xffffa70946dede40)
    at ../../../../ufs/lfs/lfs_vnops.c:2010
#12 lfs_fcntl (v=0xffffa70946dede40) at ../../../../ufs/lfs/lfs_vnops.c:1851
#13 0xffffffff80a2464e in VOP_FCNTL (vp=0xffff99f012474540,
    command=<optimized out>, data=<optimized out>, fflag=<optimized out>,
    cred=<optimized out>) at ../../../../kern/vnode_if.c:972
#14 0xffffffff809ad664 in fcntl_forfs (fd=<optimized out>, fp=<optimized out>,
    cmd=<optimized out>, arg=0x7f7fff42e1a0)
    at ../../../../kern/sys_descrip.c:226
#15 sys_fcntl (l=<optimized out>, uap=<optimized out>, retval=<optimized out>)
    at ../../../../kern/sys_descrip.c:385
#16 0xffffffff80526a0b in sy_call (sy=0xffffffff8105a640 <sysent+2208>,
    l=0xffff99f002d69000, uap=0xffffa70946dee000, rval=0xffffa70946dedfb0)
    at ../../../../sys/syscallvar.h:65
#17 sy_invoke (sy=0xffffffff8105a640 <sysent+2208>, l=0xffff99f002d69000,
    uap=0xffffa70946dee000, rval=0xffffa70946dedfb0, code=92)
    at ../../../../sys/syscallvar.h:94
#18 syscall (frame=0xffffa70946dee000)
    at ../../../../arch/x86/x86/syscall.c:137
#19 0xffffffff8021025d in handle_syscall ()
(gdb)

	ioflush process:
(gdb) kvm proc 0xffff99eff8799c00
0xffffffff80960c23 in mi_switch (l=l@entry=0xffff99eff8799c00)
    at ../../../../kern/kern_synch.c:811
811                     prevlwp = cpu_switchto(l, newl, returning);
(gdb) bt
#0  0xffffffff80960c23 in mi_switch (l=l@entry=0xffff99eff8799c00)
    at ../../../../kern/kern_synch.c:811
#1  0xffffffff8095d263 in sleepq_block (timo=timo@entry=0,
    catch_p=catch_p@entry=false,
    syncobj=syncobj@entry=0xffffffff80da4940 <sleep_syncobj>,
    nlocks=nlocks@entry=1) at ../../../../kern/kern_sleepq.c:392
#2  0xffffffff809605d7 in mtsleep (ident=ident@entry=0xffff99f15f73b244,
    priority=priority@entry=17,
    wmesg=wmesg@entry=0xffffffff80e0718f "lfs_wsync", timo=timo@entry=0,
    mtx=0xffffffff810c80d8 <lfs_lock>) at ../../../../kern/kern_synch.c:232
#3  0xffffffff808a8cc4 in lfs_writer_enter (fs=0xffff99f15f73b000,
    wmesg=0xffffffff80e0718f "lfs_wsync") at ../../../../ufs/lfs/lfs_subr.c:744
#4  0xffffffff808ad591 in lfs_sync (mp=0xffff99f166139000, waitfor=3,
    cred=<optimized out>) at ../../../../ufs/lfs/lfs_vfsops.c:1653
#5  0xffffffff80a08385 in VFS_SYNC (mp=0xffff99f166139000, a=<optimized out>,
    b=<optimized out>) at ../../../../kern/vfs_subr.c:1590
#6  0xffffffff80a084a3 in sched_sync (arg=<optimized out>)
    at ./machine/cpu.h:80
#7  0xffffffff80210327 in lwp_trampoline ()
#8  0x0000000000000000 in ?? ()
(gdb)

>How-To-Repeat:
	0. boot kernel with serial console attached
	1. create 7GiB of LFS and mount it
           (avoid automount at next boot time)
	2. copy src tree into the LFS
	3. umount the LFS; and run fsck_lfs -nf;
	   (at this time, no errors reported)
	4. mount the LFS again
	5. run rm -rf src &; on the LFS

>Fix:
	Sorry, unknown.

>Audit-Trail:
From: Shinichi Doyashiki <clare@csel.org>
To: gnats-bugs@netbsd.org
Cc: 
Subject: Re: kern/59912: running rm -rf on LFS hangs the rm process forever on
 prelock state
Date: Tue, 13 Jan 2026 03:45:52 +0900

 it seems to be missing wakeup(&fs->lfs_seglock) sleeping "clean1".

From: Shinichi Doyashiki <clare@csel.org>
To: gnats-bugs@netbsd.org
Cc: 
Subject: Re: kern/59912: running rm -rf on LFS hangs the rm process forever on
 prelock state
Date: Sat, 17 Jan 2026 23:26:07 +0900

 I don't understand the LFS code yet.

 After providing the necessary information to LLM and investigating,
 I was able to come up with a starting patch for the lfs_cleanerd
 process, as shown below. This code removes the sleep in the clean1
 state and instead adds a counter-based wait code to control the
 progress of lfs_cleanerd. However, even with this patch, fcntl
 still causes a panic.  According to LLM, this may be due
 to stack corruption or structure corruption around fcntl.

 $ cvs diff -u
 cvs diff: Diffing .
 Index: lfs.h
 ===================================================================
 RCS file: /cvsroot/src/sys/ufs/lfs/lfs.h,v
 retrieving revision 1.219
 diff -u -r1.219 lfs.h
 --- lfs.h       5 Jan 2026 05:02:47 -0000       1.219
 +++ lfs.h       17 Jan 2026 13:57:31 -0000
 @@ -1050,6 +1050,8 @@
          uint32_t  lfs_seglock;          /* segment lock counter */

          uint32_t lfs_iocount;           /* number of ios pending */
 +       uint64_t lfs_seg_epoch;
 +       kcondvar_t lfs_seg_epoch_cv;
          uint32_t lfs_writer;            /* don't allow any dirops to start */
          uint32_t lfs_dirops;            /* count of active directory ops */
          kcondvar_t lfs_diropscv;        /* condvar of active directory ops */
 Index: lfs_subr.c
 ===================================================================
 RCS file: /cvsroot/src/sys/ufs/lfs/lfs_subr.c,v
 retrieving revision 1.110
 diff -u -r1.110 lfs_subr.c
 --- lfs_subr.c  5 Jan 2026 05:02:47 -0000       1.110
 +++ lfs_subr.c  17 Jan 2026 13:57:32 -0000
 @@ -546,6 +546,15 @@
                          LFS_ENTER_LOG("segunlock_std", __FILE__, __LINE__, 0, 0, curproc->p_pid);

                          --fs->lfs_seglock;
 +
 +                       /*
 +                        * Segment writer reached a stable point.
 +                        * Wake up readers waiting for cleaner progress.
 +                        */
 +                       mutex_enter(&lfs_lock);
 +                       fs->lfs_seg_epoch++;
 +                       cv_broadcast(&fs->lfs_seg_epoch_cv);
 +                       mutex_exit(&lfs_lock);
                  }
                  /*
                   * We let checkpoints happen asynchronously.  That means
 @@ -588,6 +597,14 @@
                          LFS_ENTER_LOG("segunlock_ckp", __FILE__, __LINE__, 0, 0, curproc->p_pid);

                          --fs->lfs_seglock;
 +
 +                       /*
 +                        * Checkpoint completed: filesystem state is stable.
 +                        */
 +                       mutex_enter(&lfs_lock);
 +                       fs->lfs_seg_epoch++;
 +                       cv_broadcast(&fs->lfs_seg_epoch_cv);
 +                       mutex_exit(&lfs_lock);
                  }
                  if (do_unmark_dirop)
                          lfs_unmark_dirop(fs);
 Index: lfs_vfsops.c
 ===================================================================
 RCS file: /cvsroot/src/sys/ufs/lfs/lfs_vfsops.c,v
 retrieving revision 1.399
 diff -u -r1.399 lfs_vfsops.c
 --- lfs_vfsops.c        5 Jan 2026 05:02:47 -0000       1.399
 +++ lfs_vfsops.c        17 Jan 2026 13:57:33 -0000
 @@ -1147,6 +1147,8 @@
          cv_init(&fs->lfs_cleanercv, "cleancv");
          cv_init(&fs->lfs_prelockcv, "prelockcv");
          cv_init(&fs->lfs_cleanquitcv, "cleanquit");
 +       cv_init(&fs->lfs_seg_epoch_cv, "segepoch");
 +       fs->lfs_seg_epoch = 0;

          /* Set the file system readonly/modify bits. */
          fs->lfs_ronly = ronly;
 @@ -1488,6 +1490,7 @@
          cv_destroy(&fs->lfs_diropscv);
          cv_destroy(&fs->lfs_stopcv);
          cv_destroy(&fs->lfs_nextsegsleep);
 +       cv_destroy(&fs->lfs_seg_epoch_cv);

          rw_destroy(&fs->lfs_fraglock);
          rw_destroy(&fs->lfs_iflock);
 Index: lfs_vnops.c
 ===================================================================
 RCS file: /cvsroot/src/sys/ufs/lfs/lfs_vnops.c,v
 retrieving revision 1.352
 diff -u -r1.352 lfs_vnops.c
 --- lfs_vnops.c 5 Jan 2026 05:02:47 -0000       1.352
 +++ lfs_vnops.c 17 Jan 2026 13:57:34 -0000
 @@ -1436,7 +1436,8 @@
    * We inline ulfs_strategy to make sure that the VOP_BMAP occurs *before*
    * the active cleaner test.
    *
 - * XXX This code assumes that lfs_markv makes synchronous checkpoints.
 + * XXX This code assumes that cleaner progress is observable via
 + *     lfs_seg_epoch updates (i.e., no silent segment reuse).
    */
   int
   lfs_strategy(void *v)
 @@ -1484,9 +1485,17 @@

          slept = 1;
          loopcount = 0;
 -       mutex_enter(&lfs_lock);
 -       while (slept && fs->lfs_seglock) {
 +       for (;;) {
 +               uint64_t epoch;
 +
 +               mutex_enter(&lfs_lock);
 +               if (!slept || !fs->lfs_seglock) {
 +                       mutex_exit(&lfs_lock);
 +                       break;
 +               }
 +               epoch = fs->lfs_seg_epoch;
                  mutex_exit(&lfs_lock);
 +
                  /*
                   * Look through list of intervals.
                   * There will only be intervals to look through
 @@ -1529,25 +1538,26 @@
                                          slept = 1;
                                          ++loopcount;
                                          break;
 -                               } else if (fs->lfs_seglock) {
 -                                       mtsleep(&fs->lfs_seglock,
 -                                               (PRIBIO + 1) | PNORELOCK,
 -                                               "clean1", 0,
 -                                               &lfs_lock);
 +                               } else if (fs->lfs_seglock &&
 +                                          curlwp != fs->lfs_cleaner_thread) {
 +                                       /*
 +                                        * Do not block the cleaner thread itself:
 +                                        * it would deadlock waiting for progress
 +                                        * that only it can make.
 +                                        */
 +                                       while (epoch == fs->lfs_seg_epoch && fs->lfs_seglock)
 +                                               cv_wait(&fs->lfs_seg_epoch_cv, &lfs_lock);
                                          slept = 1;
                                          break;
                                  }
                                  mutex_exit(&lfs_lock);
                          }
                  }
 -               mutex_enter(&lfs_lock);
                  if (loopcount > MAXLOOP) {
                          printf("lfs_strategy: breaking out of clean2 loop\n");
                          break;
                  }
          }
 -       mutex_exit(&lfs_lock);
 -
          vp = ip->i_devvp;
          return VOP_STRATEGY(vp, bp);
   }

 (gdb) bt
 #0  0xffffffff80239db5 in cpu_reboot (howto=howto@entry=260,
      bootstr=bootstr@entry=0x0) at ../../../../arch/amd64/amd64/machdep.c:709
 #1  0xffffffff8094f3cf in kern_reboot (howto=howto@entry=260,
      bootstr=bootstr@entry=0x0) at ../../../../kern/kern_reboot.c:91
 #2  0xffffffff8099be5d in vpanic (fmt=fmt@entry=0xffffffff80dca941 "trap",
      ap=ap@entry=0xffffc489472ebb38) at ../../../../kern/subr_prf.c:288
 #3  0xffffffff8099bf32 in panic (fmt=fmt@entry=0xffffffff80dca941 "trap")
      at ../../../../kern/subr_prf.c:209
 #4  0xffffffff8023cb7e in trap (frame=0xffffc489472ebc80)
      at ../../../../arch/amd64/amd64/trap.c:325
 #5  0xffffffff80234ad4 in alltraps ()
 #6  0xffffffff808b7836 in lfs_fcntl (v=0xffffc489472ebe40)
      at ../../../../ufs/lfs/lfs_vnops.c:2047
 #7  lfs_fcntl (v=0xffffc489472ebe40) at ../../../../ufs/lfs/lfs_vnops.c:1861
 #8  0xffffffff80a246fe in VOP_FCNTL (vp=0xffff8a098b7faa40,
      command=<optimized out>, data=<optimized out>, fflag=<optimized out>,
      cred=<optimized out>) at ../../../../kern/vnode_if.c:972
 #9  0xffffffff809adcc6 in fcntl_forfs (fd=<optimized out>, fp=<optimized out>,
      cmd=-1610036220, arg=0x0) at ../../../../kern/sys_descrip.c:226
 #10 sys_fcntl (l=<optimized out>, uap=<optimized out>, retval=<optimized out>)
      at ../../../../kern/sys_descrip.c:385
 #11 0xffffffff80526a0b in sy_call (sy=0xffffffff8105a640 <sysent+2208>,
      l=0xffff8a097f3fec00, uap=0xffffc489472ec000, rval=0xffffc489472ebfb0)
      at ../../../../sys/syscallvar.h:65
 #12 sy_invoke (sy=0xffffffff8105a640 <sysent+2208>, l=0xffff8a097f3fec00,
      uap=0xffffc489472ec000, rval=0xffffc489472ebfb0, code=92)
      at ../../../../sys/syscallvar.h:94
 #13 syscall (frame=0xffffc489472ec000)
      at ../../../../arch/x86/x86/syscall.c:137
 #14 0xffffffff8021025d in handle_syscall ()

Home	PR Database Search